replace mamba & conda with micromamba

pull/1062/head
Madhur Tandon 2021-04-20 13:04:08 +05:30 zatwierdzone przez Sylvain Corlay
rodzic a37a205c0e
commit f508bf167e
6 zmienionych plików z 40 dodań i 52 usunięć

Wyświetl plik

@ -337,9 +337,9 @@ class CondaBuildPack(BaseImage):
"${NB_USER}",
r"""
TIMEFORMAT='time: %3R' \
bash -c 'time mamba env update -p {0} -f "{1}" && \
time mamba clean --all -f -y && \
mamba list -p {0} \
bash -c 'time /tmp/bin/micromamba install -p {0} -f "{1}" -c conda-forge && \
# time /tmp/bin/micromamba clean --all -y && \
/tmp/bin/micromamba list -p {0} \
'
""".format(
env_prefix, environment_yml
@ -356,9 +356,9 @@ class CondaBuildPack(BaseImage):
(
"${NB_USER}",
r"""
mamba install -p {0} r-base{1} r-irkernel=1.2 r-devtools -y && \
mamba clean --all -f -y && \
mamba list -p {0}
/tmp/bin/micromamba install -p {0} r-base{1} r-irkernel={2} r-devtools -y -c conda-forge && \
# /tmp/bin/micromamba clean --all -f -y && \
/tmp/bin/micromamba list -p {0}
""".format(
env_prefix, r_pin
),

Wyświetl plik

@ -1,11 +1,13 @@
# enable conda and activate the notebook environment
CONDA_PROFILE="${CONDA_DIR}/etc/profile.d/conda.sh"
export MAMBA_EXE="/tmp/bin/micromamba"
export MAMBA_ROOT_PREFIX="/srv/conda"
CONDA_PROFILE="${CONDA_DIR}/etc/profile.d/mamba.sh"
test -f $CONDA_PROFILE && . $CONDA_PROFILE
if [[ "${KERNEL_PYTHON_PREFIX}" != "${NB_PYTHON_PREFIX}" ]]; then
# if the kernel is a separate env, stack them
# so both are on PATH, notebook first
conda activate ${KERNEL_PYTHON_PREFIX}
conda activate --stack ${NB_PYTHON_PREFIX}
micromamba activate ${KERNEL_PYTHON_PREFIX}
micromamba activate ${NB_PYTHON_PREFIX} --stack
# even though it's second on $PATH
# make sure CONDA_DEFAULT_ENV is the *kernel* env
@ -14,5 +16,5 @@ if [[ "${KERNEL_PYTHON_PREFIX}" != "${NB_PYTHON_PREFIX}" ]]; then
# which only contains UI when the two are different
export CONDA_DEFAULT_ENV="${KERNEL_PYTHON_PREFIX}"
else
conda activate ${NB_PYTHON_PREFIX}
micromamba activate ${NB_PYTHON_PREFIX}
fi

Wyświetl plik

@ -5,50 +5,40 @@ set -ex
cd $(dirname $0)
MINIFORGE_VERSION=4.9.2-2
MAMBA_VERSION=0.7.4
# SHA256 for installers can be obtained from https://github.com/conda-forge/miniforge/releases
SHA256SUM="7a7bfaff87680298304a97ba69bcf92f66c810995a7155a2918b99fafb8ca1dc"
URL="https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Mambaforge-${MINIFORGE_VERSION}-Linux-x86_64.sh"
INSTALLER_PATH=/tmp/miniforge-installer.sh
URL="https://micro.mamba.pm/api/micromamba/linux-64/latest"
# make sure we don't do anything funky with user's $HOME
# since this is run as root
unset HOME
time wget --quiet $URL -O ${INSTALLER_PATH}
chmod +x ${INSTALLER_PATH}
mkdir -p ${CONDA_DIR}
# check sha256 checksum
if ! echo "${SHA256SUM} ${INSTALLER_PATH}" | sha256sum --quiet -c -; then
echo "sha256 mismatch for ${INSTALLER_PATH}, exiting!"
exit 1
fi
time wget -qO- ${URL} | tar -xvj bin/micromamba
export MAMBA_ROOT_PREFIX=${CONDA_DIR}
eval "$(./bin/micromamba shell hook -p ${CONDA_DIR} -s posix)"
./bin/micromamba shell init -s bash -p ${CONDA_DIR}
# source ~/.bashrc
time bash ${INSTALLER_PATH} -b -p ${CONDA_DIR}
export PATH="${CONDA_DIR}/bin:$PATH"
micromamba activate
# micromamba install conda -n base -c conda-forge
# Preserve behavior of miniconda - packages come from conda-forge + defaults
conda config --system --append channels defaults
export PATH="${PWD}/bin:$PATH"
# Do not attempt to auto update conda or dependencies
conda config --system --set auto_update_conda false
conda config --system --set show_channel_urls true
# bug in conda 4.3.>15 prevents --set update_dependencies
echo 'update_dependencies: false' >> ${CONDA_DIR}/.condarc
# avoid future changes to default channel_priority behavior
conda config --system --set channel_priority "flexible"
time mamba install -y mamba==${MAMBA_VERSION}
cat <<EOT >> ${CONDA_DIR}/.condarc
channels:
- defaults
auto_update_conda: false
show_channel_urls: true
update_dependencies: false
channel_priority: flexible
EOT
echo "installing notebook env:"
cat "${NB_ENVIRONMENT_FILE}"
time mamba create -p ${NB_PYTHON_PREFIX} --file "${NB_ENVIRONMENT_FILE}"
time micromamba create -p ${NB_PYTHON_PREFIX} -f "${NB_ENVIRONMENT_FILE}"
if [[ ! -z "${NB_REQUIREMENTS_FILE:-}" ]]; then
echo "installing pip requirements"
@ -65,7 +55,7 @@ if [[ ! -z "${KERNEL_ENVIRONMENT_FILE:-}" ]]; then
# install kernel env and register kernelspec
echo "installing kernel env:"
cat "${KERNEL_ENVIRONMENT_FILE}"
time mamba create -p ${KERNEL_PYTHON_PREFIX} --file "${KERNEL_ENVIRONMENT_FILE}"
time micromamba create -p ${KERNEL_PYTHON_PREFIX} -f "${KERNEL_ENVIRONMENT_FILE}"
if [[ ! -z "${KERNEL_REQUIREMENTS_FILE:-}" ]]; then
echo "installing pip requirements for kernel"
@ -75,22 +65,18 @@ if [[ ! -z "${KERNEL_ENVIRONMENT_FILE:-}" ]]; then
${KERNEL_PYTHON_PREFIX}/bin/ipython kernel install --prefix "${NB_PYTHON_PREFIX}"
echo '' > ${KERNEL_PYTHON_PREFIX}/conda-meta/history
mamba list -p ${KERNEL_PYTHON_PREFIX}
micromamba list -p ${KERNEL_PYTHON_PREFIX}
fi
# Clean things out!
time mamba clean --all -f -y
# Remove the big installer so we don't increase docker image size too much
rm ${INSTALLER_PATH}
time micromamba clean --all -y
# Remove the pip cache created as part of installing miniforge
rm -rf /root/.cache
chown -R $NB_USER:$NB_USER ${CONDA_DIR}
mamba list -n root
mamba list -p ${NB_PYTHON_PREFIX}
micromamba list -p ${NB_PYTHON_PREFIX}
# Set NPM config
${NB_PYTHON_PREFIX}/bin/npm config --global set prefix ${NPM_DIR}

Wyświetl plik

@ -4,7 +4,7 @@ from subprocess import check_output
assert sys.version_info[:2] == (3, 5), sys.version
out = check_output(["conda", "--version"]).decode("utf8").strip()
assert out == "conda 4.9.2", out
out = check_output(["/tmp/bin/micromamba", "--version"]).decode("utf8").strip()
assert out == "0.12.0", out
import numpy

Wyświetl plik

@ -1,5 +1,5 @@
#!/bin/bash
# install pytest with conda in the default env (should be $KERNEL_PYTHON_PREFIX)
conda install -yq pytest
/tmp/bin/micromamba install -yq pytest -p $KERNEL_PYTHON_PREFIX -c conda-forge
# install there with pip (should be the same)
pip install there

Wyświetl plik

@ -14,13 +14,13 @@ assert sorted(specs) == ["python2", "python3"], specs.keys()
import json
from subprocess import check_output
envs = json.loads(check_output(["conda", "env", "list", "--json"]).decode("utf8"))
envs = json.loads(check_output(["/tmp/bin/micromamba", "env", "list", "--json"]).decode("utf8"))
assert envs == {
"envs": ["/srv/conda", "/srv/conda/envs/kernel", "/srv/conda/envs/notebook"]
}, envs
pkgs = json.loads(
check_output(["conda", "list", "-n", "kernel", "--json"]).decode("utf8")
check_output(["/tmp/bin/micromamba", "list", "-n", "kernel", "--json"]).decode("utf8")
)
pkg_names = [pkg["name"] for pkg in pkgs]
assert "ipykernel" in pkg_names, pkg_names