Update Mojo to v24.5.0

README.md

@@ -129,9 +129,7 @@ latest:
 
 ```bash
 cd base && docker build \
-  --build-arg MODULAR_VERSION=0.9.2 \
-  --build-arg MODULAR_AUTH_KEY=<your-modular-auth-key> \
-  --build-arg MOJO_VERSION=24.4.0 \
+  --build-arg MOJO_VERSION=24.5.0 \
   --build-arg PYTHON_VERSION=3.12.7 \
   -t jupyterlab/mojo/base \
   -f latest.Dockerfile .
@@ -141,7 +139,6 @@ version:
 
 ```bash
 cd base && docker build \
-  --build-arg MODULAR_AUTH_KEY=<your-modular-auth-key> \
   -t jupyterlab/mojo/base:MAJOR.MINOR.PATCH \
   -f MAJOR.MINOR.PATCH.Dockerfile .
 ```

VERSION_MATRIX.md

@@ -8,6 +8,7 @@ Topmost entry = Tag `latest`
 
 | Mojo   | Python     | Jupyter Hub | Jupyter Lab | code‑server (Code) | Neovim | Git    | Git LFS | Pandoc | Linux distro |
 |:-------|:-----------|:------------|:------------|:-------------------|:-------|:-------|:--------|:-------|:-------------|
+| 24.5.0 | 3.12.7     | 5.2.0       | 4.2.5       | 4.93.1 (1.93.1)    | 0.10.2 | 2.46.2 | 3.5.1   | 3.2    | Debian 12    |
 | 24.4.0 | 3.12.6     | 5.1.0       | 4.2.5       | 4.92.2 (1.92.2)    | 0.10.1 | 2.46.1 | 3.5.1   | 3.2    | Debian 12    |
 | 24.3.0 | 3.12.4     | 5.0.0       | 4.2.1       | 4.89.1 (1.89.1)    | n/a    | 2.45.2 | 3.5.1   | 3.1.11 | Debian 12    |
 
@@ -15,5 +16,6 @@ Topmost entry = Tag `latest`
 
 | Mojo   | CTAN date  | Quarto  |
 |:-------|:-----------|:--------|
+| 24.5.0 |            | 1.5.57  |
 | 24.4.0 | 2024-09-13 | 1.5.57  |
 | 24.3.0 | 2024-06-07 | 1.4.555 |

base/.gitlab-ci.yml

 .env:
   variables:
-    MODULAR_VERSION: "0.9.2"
-    MOJO_VERSION: "24.4.0"
+    MOJO_VERSION: "24.5.0"
+    MOJO_EXTENSION_VERSION: "24.5.2"
     PYTHON_VERSION: "3.12.7"
     TAG_VERSION_MAJ_MIN: "false"
     TAG_VERSION_MAJ: "false"
@@ -68,8 +68,8 @@ build:latest-linux-amd64:
       --build-arg BASE_IMAGE
       --build-arg BASE_IMAGE_TAG
       --build-arg BUILD_ON_IMAGE
-      --build-arg MODULAR_VERSION
       --build-arg MOJO_VERSION
+      --build-arg MOJO_EXTENSION_VERSION
       --build-arg INSTALL_MAX
       --build-arg PYTHON_VERSION
       --build-arg CUDA_IMAGE_FLAVOR
@@ -117,8 +117,8 @@ build:latest-linux-arm64v8:
       --build-arg BASE_IMAGE
       --build-arg BASE_IMAGE_TAG
       --build-arg BUILD_ON_IMAGE
-      --build-arg MODULAR_VERSION
       --build-arg MOJO_VERSION
+      --build-arg MOJO_EXTENSION_VERSION
       --build-arg INSTALL_MAX
       --build-arg PYTHON_VERSION
       --build-arg CUDA_IMAGE_FLAVOR

base/conf/jupyterlab/usr/local/etc/jupyter/jupyter_server_config.d/mojo-lsp-server.json

@@ -3,7 +3,7 @@
     "language_servers": {
       "mojo": {
         "argv": [
-          "/opt/modular/pkg/packages.modular.com_mojo/bin/mojo-lsp-server",
+          "/opt/modular/bin/mojo-lsp-server",
           "--log=error"
         ],
         "version": 2,

base/conf/shell/etc/profile.d/00-reset-path.sh

 # Reset default PATH settings
-PATH="/opt/modular/pkg/packages.modular.com_mojo/bin:/opt/code-server/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+PATH="/opt/modular/bin:/opt/code-server/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 export PATH

base/latest.Dockerfile

 ARG BASE_IMAGE=debian
 ARG BASE_IMAGE_TAG=12
 ARG BUILD_ON_IMAGE=glcr.b-data.ch/python/ver
-ARG MODULAR_VERSION
 ARG MOJO_VERSION
+ARG MOJO_EXTENSION_VERSION=${MOJO_VERSION}
 ARG PYTHON_VERSION
 ARG CUDA_IMAGE_FLAVOR
 
@@ -19,8 +19,6 @@ ARG PANDOC_VERSION=3.2
 
 FROM ${BUILD_ON_IMAGE}${PYTHON_VERSION:+:}${PYTHON_VERSION}${CUDA_IMAGE_FLAVOR:+-}${CUDA_IMAGE_FLAVOR} AS files
 
-ARG INSTALL_MAX
-
 ARG NB_UID
 ENV NB_GID=100
 
@@ -44,13 +42,6 @@ RUN cp -a /files/etc/skel/. /files/var/backups/skel \
     /files/usr/local/share/jupyter/lab/static/assets \
   && cp -a /files/opt/code-server/src/browser/media/fonts \
     /files/usr/local/share/jupyter/lab/static/assets \
-  ## Update Modular setup
-  && if [ "${INSTALL_MAX}" = "1" -o "${INSTALL_MAX}" = "true" ]; then \
-    sed -i s/packages.modular.com_mojo/packages.modular.com_max/g \
-      /files/usr/local/etc/jupyter/jupyter_server_config.d/mojo-lsp-server.json; \
-    sed -i s/packages.modular.com_mojo/packages.modular.com_max/g \
-      /files/etc/profile.d/00-reset-path.sh; \
-  fi \
   && if [ -n "${CUDA_VERSION}" ]; then \
     ## Use entrypoint of CUDA image
     mv /opt/nvidia/entrypoint.d /opt/nvidia/nvidia_entrypoint.sh \
@@ -103,7 +94,7 @@ LABEL org.opencontainers.image.licenses="$IMAGE_LICENSE" \
       org.opencontainers.image.authors="$IMAGE_AUTHORS"
 
 ENV PARENT_IMAGE=${BUILD_ON_IMAGE}${PYTHON_VERSION:+:}${PYTHON_VERSION}${CUDA_IMAGE_FLAVOR:+-}${CUDA_IMAGE_FLAVOR} \
-    MODULAR_HOME=/opt/modular \
+    MODULAR_HOME=/opt/modular/share/max \
     MOJO_VERSION=${MOJO_VERSION%%-*} \
     NB_USER=${NB_USER} \
     NB_UID=${NB_UID} \
@@ -192,10 +183,16 @@ RUN dpkgArch="$(dpkg --print-architecture)" \
       setuptools \
       wheel; \
   fi \
-  ## Modular: Additional runtime dependencies
-  && apt-get -y install --no-install-recommends \
-    libtinfo-dev \
-    libxml2-dev \
+  ## Mojo/MAX: Additional runtime dependency
+  && apt-get -y install --no-install-recommends libncurses-dev \
+  ## mblack: Additional Python dependencies
+  && export PIP_BREAK_SYSTEM_PACKAGES=1 \
+  && pip install \
+    click \
+    mypy-extensions \
+    packaging \
+    pathspec \
+    platformdirs \
   ## Install font MesloLGS NF
   && mkdir -p /usr/share/fonts/truetype/meslo \
   && curl -sL https://github.com/romkatv/powerlevel10k-media/raw/master/MesloLGS%20NF%20Regular.ttf -o "/usr/share/fonts/truetype/meslo/MesloLGS NF Regular.ttf" \
@@ -239,56 +236,119 @@ RUN dpkgArch="$(dpkg --print-architecture)" \
   && rm -rf /var/lib/apt/lists/* \
     ${HOME}/.cache
 
-FROM base as modular
+FROM base AS modular
+
+ARG NB_GID=100
 
-ARG MODULAR_VERSION
-ARG MODULAR_NO_AUTH
-ARG MODULAR_AUTH_KEY
 ARG MOJO_VERSION
-ARG MOJO_VERSION_FULL=${MOJO_VERSION}
 ARG INSTALL_MAX
 
-## Install Modular
-RUN dpkgArch="$(dpkg --print-architecture)" \
-  && apt-get update \
-  && apt-get -y install --no-install-recommends \
-    libtinfo-dev \
-    libxml2-dev \
-  && curl -sSL https://dl.modular.com/public/installer/deb/debian/pool/any-version/main/m/mo/modular_${MODULAR_VERSION}/modular-v${MODULAR_VERSION}-${dpkgArch}.deb \
-    -o modular.deb \
-  && dpkg --ignore-depends=python3,python3-pip,python3-venv -i modular.deb \
-  && rm modular.deb \
+  ## Install Magic
+RUN curl -ssL https://magic.modular.com | bash \
+  && mv $HOME/.modular/bin/magic /usr/local/bin \
   ## Clean up
-  && rm -rf /var/lib/apt/lists/*
-
-## Install Mojo or MAX
-RUN modular config-set telemetry.enabled=false \
-  && modular config-set crash_reporting.enabled=false \
-  && if [ "${MODULAR_NO_AUTH}" != "1" ] && [ "${MODULAR_NO_AUTH}" != "true" ]; then \
-    modular auth \
-      "${MODULAR_AUTH_KEY:-$(echo -n "${NB_USER}" | sha256sum | cut -c -8)}"; \
-  fi \
+  && rm -rf $HOME/.modular \
+  && rm -rf /usr/local/lib/python${PYTHON_VERSION%.*}/site-packages/* \
+  ## Install Mojo/MAX
+  && cd /tmp \
   && if [ "${INSTALL_MAX}" = "1" ] || [ "${INSTALL_MAX}" = "true" ]; then \
-    modular install --install-version "${MOJO_VERSION}" max; \
+    if [ "${MOJO_VERSION}" = "nightly" ]; then \
+      magic init -c conda-forge -c https://conda.modular.com/max-nightly; \
+      magic add max; \
+    else \
+      magic init -c conda-forge -c https://conda.modular.com/max; \
+      magic add max==${MOJO_VERSION}; \
+    fi \
   else \
-    modular install --install-version "${MOJO_VERSION_FULL}" mojo; \
+    if [ "${MOJO_VERSION}" = "nightly" ]; then \
+      magic init -c conda-forge -c https://conda.modular.com/max-nightly; \
+      magic add mojo-jupyter; \
+    else \
+      magic init -c conda-forge -c https://conda.modular.com/max; \
+      magic add mojo-jupyter==${MOJO_VERSION}; \
+    fi \
   fi \
-  && chown -R root:${NB_GID} ${MODULAR_HOME} \
-  && chmod -R g+w ${MODULAR_HOME} \
-  && chmod -R g+rx ${MODULAR_HOME}/crashdb \
-  ## Clean up
-  && rm -rf ${MODULAR_HOME}/.*_cache
+  ## Disable telemetry
+  && magic telemetry --disable \
+  ## Get rid of all the unnecessary stuff
+  ## and move installation to /opt/modular
+  && mkdir -p /opt/modular/bin \
+  && mkdir -p /opt/modular/lib \
+  && mkdir -p /opt/modular/share \
+  && cd /tmp/.magic/envs \
+  && if [ "${INSTALL_MAX}" = "1" ] || [ "${INSTALL_MAX}" = "true" ]; then \
+    cp -a default/bin/max /opt/modular/bin; \
+    cp -a default/lib/libDevice* \
+      default/lib/libGenericMLSupport* \
+      default/lib/libmodular* \
+      default/lib/libmof.so \
+      default/lib/*MOGG* \
+      default/lib/libmonnx.so \
+      default/lib/libmtorch.so \
+      default/lib/libServe* \
+      default/lib/libStock* \
+      default/lib/libTorch* \
+      /opt/modular/lib; \
+  fi \
+  && if [ "${INSTALL_MAX}" = "1" ] || [ "${INSTALL_MAX}" = "true" ]; then \
+    cp -a default/lib/python${PYTHON_VERSION%.*}/site-packages/max* \
+      /usr/local/lib/python${PYTHON_VERSION%.*}/site-packages; \
+  fi \
+  && cp -a default/bin/lldb* \
+    default/bin/mblack \
+    default/bin/modular* \
+    default/bin/mojo* \
+    /opt/modular/bin \
+  && cp -a default/lib/libAsyncRT* \
+    default/lib/libCUDA* \
+    default/lib/libKGENCompilerRT* \
+    default/lib/liblldb* \
+    default/lib/libMojo* \
+    default/lib/libMSupport* \
+    default/lib/liborc_rt.a \
+    default/lib/lldb* \
+    default/lib/mojo* \
+    /opt/modular/lib \
+  && cp -a default/lib/python${PYTHON_VERSION%.*}/site-packages/*mblack* \
+    default/lib/python${PYTHON_VERSION%.*}/site-packages/mblib* \
+    /usr/local/lib/python${PYTHON_VERSION%.*}/site-packages \
+  && cp -a default/share/max /opt/modular/share \
+  && cp -a default/test /opt/modular \
+  && mkdir ${MODULAR_HOME}/crashdb \
+  && rm ${MODULAR_HOME}/firstActivation \
+  ## Fix Modular home for Mojo
+  && sed -i "s|/tmp/.magic/envs/default|/opt/modular|g" \
+    ${MODULAR_HOME}/modular.cfg \
+  ## Fix Python path for mblack
+  && sed -i "s|/tmp/.magic/envs/default|/usr/local|g" \
+    /opt/modular/bin/mblack \
+  ## Fix permissions
+  && chown -R root:${NB_GID} /opt/modular \
+  && chmod -R g+w /opt/modular \
+  && chmod -R g+rx ${MODULAR_HOME}/crashdb
 
 ## Install the Mojo kernel for Jupyter
 RUN mkdir -p /usr/local/share/jupyter/kernels \
-  && mv ${HOME}/.local/share/jupyter/kernels/mojo* \
+  && mv /tmp/.magic/envs/default/share/jupyter/kernels/mojo* \
     /usr/local/share/jupyter/kernels/ \
   ## Fix Modular home in the Mojo kernel for Jupyter
-  && grep -rl ${HOME}/.local /usr/local/share/jupyter/kernels/mojo* | \
-    xargs sed -i "s|${HOME}/.local|/usr/local|g"
+  && grep -rl /tmp/.magic/envs/default/share/jupyter /usr/local/share/jupyter/kernels/mojo* | \
+    xargs sed -i "s|/tmp/.magic/envs/default|/usr/local|g" \
+  && grep -rl /usr/local/share/max /usr/local/share/jupyter/kernels/mojo* | \
+    xargs sed -i "s|/usr/local/share/max|/opt/modular/share/max|g" \
+  ## Change display name in the Mojo kernel for Jupyter
+  && sed -i "s|\"display_name\".*|\"display_name\": \"Mojo $MOJO_VERSION ${INSTALL_MAX:+(MAX)}\",|g" \
+    /usr/local/share/jupyter/kernels/mojo*/kernel.json \
+  && if [ "${MOJO_VERSION}" = "nightly" ]; then \
+    cp -a /usr/local/share/jupyter/kernels/mojo*/nightly-logo-64x64.png \
+      /usr/local/share/jupyter/kernels/mojo*/logo-64x64.png; \
+    cp -a /usr/local/share/jupyter/kernels/mojo*/nightly-logo.svg \
+      /usr/local/share/jupyter/kernels/mojo*/logo.svg; \
+  fi
 
 FROM base
 
+ARG MOJO_EXTENSION_VERSION
 ARG INSTALL_MAX
 
 ENV PATH=/opt/code-server/bin:$PATH \
@@ -313,8 +373,8 @@ RUN mkdir /opt/code-server \
   && code-server --extensions-dir ${CODE_BUILTIN_EXTENSIONS_DIR} --install-extension piotrpalarz.vscode-gitignore-generator-1.0.3.vsix \
   && curl -sLO https://dl.b-data.ch/vsix/mutantdino.resourcemonitor-1.0.7.vsix \
   && code-server --extensions-dir ${CODE_BUILTIN_EXTENSIONS_DIR} --install-extension mutantdino.resourcemonitor-1.0.7.vsix \
-  && curl -sLO https://dl.b-data.ch/vsix/modular-mojotools.vscode-mojo-${MOJO_VERSION}.vsix \
-  && code-server --extensions-dir ${CODE_BUILTIN_EXTENSIONS_DIR} --install-extension modular-mojotools.vscode-mojo-${MOJO_VERSION}.vsix \
+  && curl -sLO https://dl.b-data.ch/vsix/modular-mojotools.vscode-mojo-${MOJO_EXTENSION_VERSION}.vsix \
+  && code-server --extensions-dir ${CODE_BUILTIN_EXTENSIONS_DIR} --install-extension modular-mojotools.vscode-mojo-${MOJO_EXTENSION_VERSION}.vsix \
   && code-server --extensions-dir ${CODE_BUILTIN_EXTENSIONS_DIR} --install-extension alefragnani.project-manager \
   && code-server --extensions-dir ${CODE_BUILTIN_EXTENSIONS_DIR} --install-extension GitHub.vscode-pull-request-github \
   && code-server --extensions-dir ${CODE_BUILTIN_EXTENSIONS_DIR} --install-extension GitLab.gitlab-workflow \
@@ -369,21 +429,24 @@ RUN export PIP_BREAK_SYSTEM_PACKAGES=1 \
   && rm -rf /tmp/* \
     ${HOME}/.cache
 
-ARG MODULAR_PKG_BIN=${INSTALL_MAX:+$MODULAR_HOME/pkg/packages.modular.com_max/bin}
-ARG MODULAR_PKG_BIN=${MODULAR_PKG_BIN:-$MODULAR_HOME/pkg/packages.modular.com_mojo/bin}
-
-ENV PATH=${MODULAR_PKG_BIN}:$PATH
+ENV PATH=/opt/modular/bin:$PATH
 
-## Install Mojo or MAX
+## Install Mojo/MAX
 COPY --from=modular /opt /opt
 ## Install the Mojo kernel for Jupyter
 COPY --from=modular /usr/local/share/jupyter /usr/local/share/jupyter
+## Install Python packages to the site library
+COPY --from=modular /usr/local/lib/python${PYTHON_VERSION%.*}/site-packages \
+  /usr/local/lib/python${PYTHON_VERSION%.*}/site-packages
 
-## Install the MAX Engine Python package or numpy
+## Mojo/MAX: Install Python dependencies
 RUN export PIP_BREAK_SYSTEM_PACKAGES=1 \
   && if [ "${INSTALL_MAX}" = "1" ] || [ "${INSTALL_MAX}" = "true" ]; then \
-    pip install --find-links \
-      ${MODULAR_HOME}/pkg/packages.modular.com_max/wheels max-engine; \
+    packages=$(grep "Requires-Dist:" \
+      /usr/local/lib/python${PYTHON_VERSION%.*}/site-packages/max*.dist-info/METADATA | \
+      sed "s|Requires-Dist: \(.*\)|\1|" | \
+      tr -d "[:blank:]"); \
+    pip install $packages; \
   else \
     pip install numpy; \
   fi \

base/scripts/usr/local/bin/before-notebook.d/12-mojo.sh

+#!/bin/bash
+# Copyright (c) 2020 b-data GmbH.
+# Distributed under the terms of the MIT License.
+
+set -e
+
+# MAX SDK: Evaluate and set version
+if [ "${MOJO_VERSION}" = "nightly" ]; then
+  extDataDir=/home/$NB_USER${DOMAIN:+@$DOMAIN}/.local/share/code-server/User/globalStorage/modular-mojotools.vscode-mojo-nightly
+  sdkVersion=$(jq -r '.sdkVersion' /opt/code-server/lib/vscode/extensions/modular-mojotools.vscode-mojo*/package.json)
+else
+  extDataDir=/home/$NB_USER${DOMAIN:+@$DOMAIN}/.local/share/code-server/User/globalStorage/modular-mojotools.vscode-mojo
+  sdkVersion=$MOJO_VERSION
+fi
+
+# MAX SDK: Create symlink to /opt/modular
+if [ "$(id -u)" == 0 ] ; then
+  run_user_group mkdir -p "$extDataDir/magic-data-home/envs"
+  run_user_group ln -s /opt/modular "$extDataDir/magic-data-home/envs/max"
+  run_user_group mkdir -p "$extDataDir/versionDone/$sdkVersion"
+else
+  mkdir -p "$extDataDir/magic-data-home/envs"
+  ln -s /opt/modular "$extDataDir/magic-data-home/envs/max"
+  mkdir -p "$extDataDir/versionDone/$sdkVersion"
+fi

scipy/.gitlab-ci.yml

 .env:
   variables:
-    MOJO_VERSION: "24.4.0"
+    MOJO_VERSION: "24.5.0"
     CTAN_REPO_BUILD_LATEST: "https://mirror.init7.net/ctan/systems/texlive/tlnet"
     TAG_VERSION_MAJ_MIN: "false"
     TAG_VERSION_MAJ: "false"