From f508bf167e815a47e7e53136803d77ff0a68f854 Mon Sep 17 00:00:00 2001 From: Madhur Tandon Date: Tue, 20 Apr 2021 13:04:08 +0530 Subject: [PATCH] replace mamba & conda with micromamba --- repo2docker/buildpacks/conda/__init__.py | 12 ++-- .../buildpacks/conda/activate-conda.sh | 10 ++-- .../buildpacks/conda/install-miniforge.bash | 60 +++++++------------ tests/conda/binder-dir/verify | 4 +- tests/conda/default-env/postBuild | 2 +- tests/conda/simple-py2/verify | 4 +- 6 files changed, 40 insertions(+), 52 deletions(-) diff --git a/repo2docker/buildpacks/conda/__init__.py b/repo2docker/buildpacks/conda/__init__.py index 17af70cc..f4400e4a 100644 --- a/repo2docker/buildpacks/conda/__init__.py +++ b/repo2docker/buildpacks/conda/__init__.py @@ -337,9 +337,9 @@ class CondaBuildPack(BaseImage): "${NB_USER}", r""" TIMEFORMAT='time: %3R' \ - bash -c 'time mamba env update -p {0} -f "{1}" && \ - time mamba clean --all -f -y && \ - mamba list -p {0} \ + bash -c 'time /tmp/bin/micromamba install -p {0} -f "{1}" -c conda-forge && \ + # time /tmp/bin/micromamba clean --all -y && \ + /tmp/bin/micromamba list -p {0} \ ' """.format( env_prefix, environment_yml @@ -356,9 +356,9 @@ class CondaBuildPack(BaseImage): ( "${NB_USER}", r""" - mamba install -p {0} r-base{1} r-irkernel=1.2 r-devtools -y && \ - mamba clean --all -f -y && \ - mamba list -p {0} + /tmp/bin/micromamba install -p {0} r-base{1} r-irkernel={2} r-devtools -y -c conda-forge && \ + # /tmp/bin/micromamba clean --all -f -y && \ + /tmp/bin/micromamba list -p {0} """.format( env_prefix, r_pin ), diff --git a/repo2docker/buildpacks/conda/activate-conda.sh b/repo2docker/buildpacks/conda/activate-conda.sh index 2af21a11..58996058 100755 --- a/repo2docker/buildpacks/conda/activate-conda.sh +++ b/repo2docker/buildpacks/conda/activate-conda.sh @@ -1,11 +1,13 @@ # enable conda and activate the notebook environment -CONDA_PROFILE="${CONDA_DIR}/etc/profile.d/conda.sh" +export MAMBA_EXE="/tmp/bin/micromamba" +export MAMBA_ROOT_PREFIX="/srv/conda" +CONDA_PROFILE="${CONDA_DIR}/etc/profile.d/mamba.sh" test -f $CONDA_PROFILE && . $CONDA_PROFILE if [[ "${KERNEL_PYTHON_PREFIX}" != "${NB_PYTHON_PREFIX}" ]]; then # if the kernel is a separate env, stack them # so both are on PATH, notebook first - conda activate ${KERNEL_PYTHON_PREFIX} - conda activate --stack ${NB_PYTHON_PREFIX} + micromamba activate ${KERNEL_PYTHON_PREFIX} + micromamba activate ${NB_PYTHON_PREFIX} --stack # even though it's second on $PATH # make sure CONDA_DEFAULT_ENV is the *kernel* env @@ -14,5 +16,5 @@ if [[ "${KERNEL_PYTHON_PREFIX}" != "${NB_PYTHON_PREFIX}" ]]; then # which only contains UI when the two are different export CONDA_DEFAULT_ENV="${KERNEL_PYTHON_PREFIX}" else - conda activate ${NB_PYTHON_PREFIX} + micromamba activate ${NB_PYTHON_PREFIX} fi diff --git a/repo2docker/buildpacks/conda/install-miniforge.bash b/repo2docker/buildpacks/conda/install-miniforge.bash index 3343378c..8e2395c7 100755 --- a/repo2docker/buildpacks/conda/install-miniforge.bash +++ b/repo2docker/buildpacks/conda/install-miniforge.bash @@ -5,50 +5,40 @@ set -ex cd $(dirname $0) -MINIFORGE_VERSION=4.9.2-2 -MAMBA_VERSION=0.7.4 -# SHA256 for installers can be obtained from https://github.com/conda-forge/miniforge/releases -SHA256SUM="7a7bfaff87680298304a97ba69bcf92f66c810995a7155a2918b99fafb8ca1dc" -URL="https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Mambaforge-${MINIFORGE_VERSION}-Linux-x86_64.sh" -INSTALLER_PATH=/tmp/miniforge-installer.sh +URL="https://micro.mamba.pm/api/micromamba/linux-64/latest" # make sure we don't do anything funky with user's $HOME # since this is run as root unset HOME -time wget --quiet $URL -O ${INSTALLER_PATH} -chmod +x ${INSTALLER_PATH} +mkdir -p ${CONDA_DIR} -# check sha256 checksum -if ! echo "${SHA256SUM} ${INSTALLER_PATH}" | sha256sum --quiet -c -; then - echo "sha256 mismatch for ${INSTALLER_PATH}, exiting!" - exit 1 -fi +time wget -qO- ${URL} | tar -xvj bin/micromamba +export MAMBA_ROOT_PREFIX=${CONDA_DIR} +eval "$(./bin/micromamba shell hook -p ${CONDA_DIR} -s posix)" +./bin/micromamba shell init -s bash -p ${CONDA_DIR} +# source ~/.bashrc -time bash ${INSTALLER_PATH} -b -p ${CONDA_DIR} -export PATH="${CONDA_DIR}/bin:$PATH" +micromamba activate +# micromamba install conda -n base -c conda-forge -# Preserve behavior of miniconda - packages come from conda-forge + defaults -conda config --system --append channels defaults +export PATH="${PWD}/bin:$PATH" -# Do not attempt to auto update conda or dependencies -conda config --system --set auto_update_conda false -conda config --system --set show_channel_urls true - -# bug in conda 4.3.>15 prevents --set update_dependencies -echo 'update_dependencies: false' >> ${CONDA_DIR}/.condarc - -# avoid future changes to default channel_priority behavior -conda config --system --set channel_priority "flexible" - -time mamba install -y mamba==${MAMBA_VERSION} +cat <> ${CONDA_DIR}/.condarc +channels: + - defaults +auto_update_conda: false +show_channel_urls: true +update_dependencies: false +channel_priority: flexible +EOT echo "installing notebook env:" cat "${NB_ENVIRONMENT_FILE}" -time mamba create -p ${NB_PYTHON_PREFIX} --file "${NB_ENVIRONMENT_FILE}" +time micromamba create -p ${NB_PYTHON_PREFIX} -f "${NB_ENVIRONMENT_FILE}" if [[ ! -z "${NB_REQUIREMENTS_FILE:-}" ]]; then echo "installing pip requirements" @@ -65,7 +55,7 @@ if [[ ! -z "${KERNEL_ENVIRONMENT_FILE:-}" ]]; then # install kernel env and register kernelspec echo "installing kernel env:" cat "${KERNEL_ENVIRONMENT_FILE}" - time mamba create -p ${KERNEL_PYTHON_PREFIX} --file "${KERNEL_ENVIRONMENT_FILE}" + time micromamba create -p ${KERNEL_PYTHON_PREFIX} -f "${KERNEL_ENVIRONMENT_FILE}" if [[ ! -z "${KERNEL_REQUIREMENTS_FILE:-}" ]]; then echo "installing pip requirements for kernel" @@ -75,22 +65,18 @@ if [[ ! -z "${KERNEL_ENVIRONMENT_FILE:-}" ]]; then ${KERNEL_PYTHON_PREFIX}/bin/ipython kernel install --prefix "${NB_PYTHON_PREFIX}" echo '' > ${KERNEL_PYTHON_PREFIX}/conda-meta/history - mamba list -p ${KERNEL_PYTHON_PREFIX} + micromamba list -p ${KERNEL_PYTHON_PREFIX} fi # Clean things out! -time mamba clean --all -f -y - -# Remove the big installer so we don't increase docker image size too much -rm ${INSTALLER_PATH} +time micromamba clean --all -y # Remove the pip cache created as part of installing miniforge rm -rf /root/.cache chown -R $NB_USER:$NB_USER ${CONDA_DIR} -mamba list -n root -mamba list -p ${NB_PYTHON_PREFIX} +micromamba list -p ${NB_PYTHON_PREFIX} # Set NPM config ${NB_PYTHON_PREFIX}/bin/npm config --global set prefix ${NPM_DIR} diff --git a/tests/conda/binder-dir/verify b/tests/conda/binder-dir/verify index b5e017b3..ad20e3c7 100755 --- a/tests/conda/binder-dir/verify +++ b/tests/conda/binder-dir/verify @@ -4,7 +4,7 @@ from subprocess import check_output assert sys.version_info[:2] == (3, 5), sys.version -out = check_output(["conda", "--version"]).decode("utf8").strip() -assert out == "conda 4.9.2", out +out = check_output(["/tmp/bin/micromamba", "--version"]).decode("utf8").strip() +assert out == "0.12.0", out import numpy diff --git a/tests/conda/default-env/postBuild b/tests/conda/default-env/postBuild index 980d00a0..c8db24eb 100644 --- a/tests/conda/default-env/postBuild +++ b/tests/conda/default-env/postBuild @@ -1,5 +1,5 @@ #!/bin/bash # install pytest with conda in the default env (should be $KERNEL_PYTHON_PREFIX) -conda install -yq pytest +/tmp/bin/micromamba install -yq pytest -p $KERNEL_PYTHON_PREFIX -c conda-forge # install there with pip (should be the same) pip install there diff --git a/tests/conda/simple-py2/verify b/tests/conda/simple-py2/verify index e6efc73b..9a843017 100755 --- a/tests/conda/simple-py2/verify +++ b/tests/conda/simple-py2/verify @@ -14,13 +14,13 @@ assert sorted(specs) == ["python2", "python3"], specs.keys() import json from subprocess import check_output -envs = json.loads(check_output(["conda", "env", "list", "--json"]).decode("utf8")) +envs = json.loads(check_output(["/tmp/bin/micromamba", "env", "list", "--json"]).decode("utf8")) assert envs == { "envs": ["/srv/conda", "/srv/conda/envs/kernel", "/srv/conda/envs/notebook"] }, envs pkgs = json.loads( - check_output(["conda", "list", "-n", "kernel", "--json"]).decode("utf8") + check_output(["/tmp/bin/micromamba", "list", "-n", "kernel", "--json"]).decode("utf8") ) pkg_names = [pkg["name"] for pkg in pkgs] assert "ipykernel" in pkg_names, pkg_names