replace mamba & conda with micromamba

pull/1062/head
Madhur Tandon 2021-04-20 13:04:08 +05:30 zatwierdzone przez Sylvain Corlay
rodzic a37a205c0e
commit f508bf167e
6 zmienionych plików z 40 dodań i 52 usunięć

Wyświetl plik

@ -337,9 +337,9 @@ class CondaBuildPack(BaseImage):
"${NB_USER}", "${NB_USER}",
r""" r"""
TIMEFORMAT='time: %3R' \ TIMEFORMAT='time: %3R' \
bash -c 'time mamba env update -p {0} -f "{1}" && \ bash -c 'time /tmp/bin/micromamba install -p {0} -f "{1}" -c conda-forge && \
time mamba clean --all -f -y && \ # time /tmp/bin/micromamba clean --all -y && \
mamba list -p {0} \ /tmp/bin/micromamba list -p {0} \
' '
""".format( """.format(
env_prefix, environment_yml env_prefix, environment_yml
@ -356,9 +356,9 @@ class CondaBuildPack(BaseImage):
( (
"${NB_USER}", "${NB_USER}",
r""" r"""
mamba install -p {0} r-base{1} r-irkernel=1.2 r-devtools -y && \ /tmp/bin/micromamba install -p {0} r-base{1} r-irkernel={2} r-devtools -y -c conda-forge && \
mamba clean --all -f -y && \ # /tmp/bin/micromamba clean --all -f -y && \
mamba list -p {0} /tmp/bin/micromamba list -p {0}
""".format( """.format(
env_prefix, r_pin env_prefix, r_pin
), ),

Wyświetl plik

@ -1,11 +1,13 @@
# enable conda and activate the notebook environment # enable conda and activate the notebook environment
CONDA_PROFILE="${CONDA_DIR}/etc/profile.d/conda.sh" export MAMBA_EXE="/tmp/bin/micromamba"
export MAMBA_ROOT_PREFIX="/srv/conda"
CONDA_PROFILE="${CONDA_DIR}/etc/profile.d/mamba.sh"
test -f $CONDA_PROFILE && . $CONDA_PROFILE test -f $CONDA_PROFILE && . $CONDA_PROFILE
if [[ "${KERNEL_PYTHON_PREFIX}" != "${NB_PYTHON_PREFIX}" ]]; then if [[ "${KERNEL_PYTHON_PREFIX}" != "${NB_PYTHON_PREFIX}" ]]; then
# if the kernel is a separate env, stack them # if the kernel is a separate env, stack them
# so both are on PATH, notebook first # so both are on PATH, notebook first
conda activate ${KERNEL_PYTHON_PREFIX} micromamba activate ${KERNEL_PYTHON_PREFIX}
conda activate --stack ${NB_PYTHON_PREFIX} micromamba activate ${NB_PYTHON_PREFIX} --stack
# even though it's second on $PATH # even though it's second on $PATH
# make sure CONDA_DEFAULT_ENV is the *kernel* env # make sure CONDA_DEFAULT_ENV is the *kernel* env
@ -14,5 +16,5 @@ if [[ "${KERNEL_PYTHON_PREFIX}" != "${NB_PYTHON_PREFIX}" ]]; then
# which only contains UI when the two are different # which only contains UI when the two are different
export CONDA_DEFAULT_ENV="${KERNEL_PYTHON_PREFIX}" export CONDA_DEFAULT_ENV="${KERNEL_PYTHON_PREFIX}"
else else
conda activate ${NB_PYTHON_PREFIX} micromamba activate ${NB_PYTHON_PREFIX}
fi fi

Wyświetl plik

@ -5,50 +5,40 @@ set -ex
cd $(dirname $0) cd $(dirname $0)
MINIFORGE_VERSION=4.9.2-2
MAMBA_VERSION=0.7.4
# SHA256 for installers can be obtained from https://github.com/conda-forge/miniforge/releases
SHA256SUM="7a7bfaff87680298304a97ba69bcf92f66c810995a7155a2918b99fafb8ca1dc"
URL="https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Mambaforge-${MINIFORGE_VERSION}-Linux-x86_64.sh" URL="https://micro.mamba.pm/api/micromamba/linux-64/latest"
INSTALLER_PATH=/tmp/miniforge-installer.sh
# make sure we don't do anything funky with user's $HOME # make sure we don't do anything funky with user's $HOME
# since this is run as root # since this is run as root
unset HOME unset HOME
time wget --quiet $URL -O ${INSTALLER_PATH} mkdir -p ${CONDA_DIR}
chmod +x ${INSTALLER_PATH}
# check sha256 checksum time wget -qO- ${URL} | tar -xvj bin/micromamba
if ! echo "${SHA256SUM} ${INSTALLER_PATH}" | sha256sum --quiet -c -; then export MAMBA_ROOT_PREFIX=${CONDA_DIR}
echo "sha256 mismatch for ${INSTALLER_PATH}, exiting!" eval "$(./bin/micromamba shell hook -p ${CONDA_DIR} -s posix)"
exit 1 ./bin/micromamba shell init -s bash -p ${CONDA_DIR}
fi # source ~/.bashrc
time bash ${INSTALLER_PATH} -b -p ${CONDA_DIR} micromamba activate
export PATH="${CONDA_DIR}/bin:$PATH" # micromamba install conda -n base -c conda-forge
# Preserve behavior of miniconda - packages come from conda-forge + defaults export PATH="${PWD}/bin:$PATH"
conda config --system --append channels defaults
# Do not attempt to auto update conda or dependencies cat <<EOT >> ${CONDA_DIR}/.condarc
conda config --system --set auto_update_conda false channels:
conda config --system --set show_channel_urls true - defaults
auto_update_conda: false
# bug in conda 4.3.>15 prevents --set update_dependencies show_channel_urls: true
echo 'update_dependencies: false' >> ${CONDA_DIR}/.condarc update_dependencies: false
channel_priority: flexible
# avoid future changes to default channel_priority behavior EOT
conda config --system --set channel_priority "flexible"
time mamba install -y mamba==${MAMBA_VERSION}
echo "installing notebook env:" echo "installing notebook env:"
cat "${NB_ENVIRONMENT_FILE}" cat "${NB_ENVIRONMENT_FILE}"
time mamba create -p ${NB_PYTHON_PREFIX} --file "${NB_ENVIRONMENT_FILE}" time micromamba create -p ${NB_PYTHON_PREFIX} -f "${NB_ENVIRONMENT_FILE}"
if [[ ! -z "${NB_REQUIREMENTS_FILE:-}" ]]; then if [[ ! -z "${NB_REQUIREMENTS_FILE:-}" ]]; then
echo "installing pip requirements" echo "installing pip requirements"
@ -65,7 +55,7 @@ if [[ ! -z "${KERNEL_ENVIRONMENT_FILE:-}" ]]; then
# install kernel env and register kernelspec # install kernel env and register kernelspec
echo "installing kernel env:" echo "installing kernel env:"
cat "${KERNEL_ENVIRONMENT_FILE}" cat "${KERNEL_ENVIRONMENT_FILE}"
time mamba create -p ${KERNEL_PYTHON_PREFIX} --file "${KERNEL_ENVIRONMENT_FILE}" time micromamba create -p ${KERNEL_PYTHON_PREFIX} -f "${KERNEL_ENVIRONMENT_FILE}"
if [[ ! -z "${KERNEL_REQUIREMENTS_FILE:-}" ]]; then if [[ ! -z "${KERNEL_REQUIREMENTS_FILE:-}" ]]; then
echo "installing pip requirements for kernel" echo "installing pip requirements for kernel"
@ -75,22 +65,18 @@ if [[ ! -z "${KERNEL_ENVIRONMENT_FILE:-}" ]]; then
${KERNEL_PYTHON_PREFIX}/bin/ipython kernel install --prefix "${NB_PYTHON_PREFIX}" ${KERNEL_PYTHON_PREFIX}/bin/ipython kernel install --prefix "${NB_PYTHON_PREFIX}"
echo '' > ${KERNEL_PYTHON_PREFIX}/conda-meta/history echo '' > ${KERNEL_PYTHON_PREFIX}/conda-meta/history
mamba list -p ${KERNEL_PYTHON_PREFIX} micromamba list -p ${KERNEL_PYTHON_PREFIX}
fi fi
# Clean things out! # Clean things out!
time mamba clean --all -f -y time micromamba clean --all -y
# Remove the big installer so we don't increase docker image size too much
rm ${INSTALLER_PATH}
# Remove the pip cache created as part of installing miniforge # Remove the pip cache created as part of installing miniforge
rm -rf /root/.cache rm -rf /root/.cache
chown -R $NB_USER:$NB_USER ${CONDA_DIR} chown -R $NB_USER:$NB_USER ${CONDA_DIR}
mamba list -n root micromamba list -p ${NB_PYTHON_PREFIX}
mamba list -p ${NB_PYTHON_PREFIX}
# Set NPM config # Set NPM config
${NB_PYTHON_PREFIX}/bin/npm config --global set prefix ${NPM_DIR} ${NB_PYTHON_PREFIX}/bin/npm config --global set prefix ${NPM_DIR}

Wyświetl plik

@ -4,7 +4,7 @@ from subprocess import check_output
assert sys.version_info[:2] == (3, 5), sys.version assert sys.version_info[:2] == (3, 5), sys.version
out = check_output(["conda", "--version"]).decode("utf8").strip() out = check_output(["/tmp/bin/micromamba", "--version"]).decode("utf8").strip()
assert out == "conda 4.9.2", out assert out == "0.12.0", out
import numpy import numpy

Wyświetl plik

@ -1,5 +1,5 @@
#!/bin/bash #!/bin/bash
# install pytest with conda in the default env (should be $KERNEL_PYTHON_PREFIX) # install pytest with conda in the default env (should be $KERNEL_PYTHON_PREFIX)
conda install -yq pytest /tmp/bin/micromamba install -yq pytest -p $KERNEL_PYTHON_PREFIX -c conda-forge
# install there with pip (should be the same) # install there with pip (should be the same)
pip install there pip install there

Wyświetl plik

@ -14,13 +14,13 @@ assert sorted(specs) == ["python2", "python3"], specs.keys()
import json import json
from subprocess import check_output from subprocess import check_output
envs = json.loads(check_output(["conda", "env", "list", "--json"]).decode("utf8")) envs = json.loads(check_output(["/tmp/bin/micromamba", "env", "list", "--json"]).decode("utf8"))
assert envs == { assert envs == {
"envs": ["/srv/conda", "/srv/conda/envs/kernel", "/srv/conda/envs/notebook"] "envs": ["/srv/conda", "/srv/conda/envs/kernel", "/srv/conda/envs/notebook"]
}, envs }, envs
pkgs = json.loads( pkgs = json.loads(
check_output(["conda", "list", "-n", "kernel", "--json"]).decode("utf8") check_output(["/tmp/bin/micromamba", "list", "-n", "kernel", "--json"]).decode("utf8")
) )
pkg_names = [pkg["name"] for pkg in pkgs] pkg_names = [pkg["name"] for pkg in pkgs]
assert "ipykernel" in pkg_names, pkg_names assert "ipykernel" in pkg_names, pkg_names