diff --git a/repo2docker/buildpacks/conda/__init__.py b/repo2docker/buildpacks/conda/__init__.py index 56e525e6..4b014612 100644 --- a/repo2docker/buildpacks/conda/__init__.py +++ b/repo2docker/buildpacks/conda/__init__.py @@ -18,7 +18,7 @@ class CondaBuildPack(BuildPack): build_script_files = { 'conda/install-miniconda.bash': '/tmp/install-miniconda.bash', - 'conda/environment.yml': '/tmp/environment.yml' + 'conda/environment.frozen.yml': '/tmp/environment.yml' } build_scripts = [ diff --git a/repo2docker/buildpacks/conda/environment.frozen.yml b/repo2docker/buildpacks/conda/environment.frozen.yml new file mode 100644 index 00000000..78cc21f3 --- /dev/null +++ b/repo2docker/buildpacks/conda/environment.frozen.yml @@ -0,0 +1,61 @@ +name: r2d +channels: +- conda-forge +- defaults +dependencies: +- bleach=2.0.0=py36_0 +- ca-certificates=2017.11.5=0 +- certifi=2017.11.5=py36_0 +- decorator=4.1.2=py36_0 +- entrypoints=0.2.3=py36_1 +- gmp=6.1.2=0 +- html5lib=0.999999999=py36_0 +- ipykernel=4.6.1=py36_0 +- ipython=6.2.1=py36_0 +- ipython_genutils=0.2.0=py36_0 +- ipywidgets=6.0.1=py36_0 +- jedi=0.10.2=py36_0 +- jinja2=2.10=py36_0 +- jsonschema=2.6.0=py36_0 +- jupyter_client=5.1.0=py36_0 +- jupyter_core=4.4.0=py_0 +- jupyterlab=0.29.2=py36_0 +- jupyterlab_launcher=0.5.5=py36_0 +- libsodium=1.0.15=1 +- markupsafe=1.0=py36_0 +- mistune=0.8=py36_0 +- nbconvert=5.3.1=py_1 +- nbformat=4.4.0=py36_0 +- ncurses=5.9=10 +- notebook=5.2.2=py36_1 +- openssl=1.0.2m=0 +- pandoc=2.0.4=0 +- pandocfilters=1.4.1=py36_0 +- pexpect=4.3.0=py36_0 +- pickleshare=0.7.4=py36_0 +- pip=9.0.1=py36_0 +- prompt_toolkit=1.0.15=py36_0 +- ptyprocess=0.5.2=py36_0 +- pygments=2.2.0=py36_0 +- python=3.6.1=3 +- python-dateutil=2.6.1=py36_0 +- pyzmq=16.0.2=py36_2 +- readline=6.2=0 +- setuptools=38.2.3=py36_0 +- simplegeneric=0.8.1=py36_0 +- six=1.11.0=py36_1 +- sqlite=3.13.0=1 +- terminado=0.8.1=py36_0 +- testpath=0.3.1=py36_0 +- tk=8.5.19=2 +- tornado=4.5.2=py36_0 +- traitlets=4.3.2=py36_0 +- wcwidth=0.1.7=py36_0 +- webencodings=0.5=py36_0 +- wheel=0.30.0=py_1 +- widgetsnbextension=2.0.1=py36_0 +- xz=5.2.3=0 +- zeromq=4.2.1=1 +- zlib=1.2.11=0 +prefix: /opt/conda/envs/r2d + diff --git a/repo2docker/buildpacks/conda/environment.yml b/repo2docker/buildpacks/conda/environment.yml index f1b817d3..fa6459a8 100644 --- a/repo2docker/buildpacks/conda/environment.yml +++ b/repo2docker/buildpacks/conda/environment.yml @@ -1,7 +1,7 @@ dependencies: - python==3.6.1 - - notebook==5.2.2 - ipython==6.2.1 - ipykernel==4.6.1 - ipywidgets==6.0.1 - jupyterlab==0.29.2 + - notebook==5.2.2 diff --git a/repo2docker/buildpacks/conda/freeze.py b/repo2docker/buildpacks/conda/freeze.py new file mode 100644 index 00000000..75d503f9 --- /dev/null +++ b/repo2docker/buildpacks/conda/freeze.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +""" +Freeze the conda environment.yml + +Run in a continuumio/miniconda3 image to ensure portability +""" + +from datetime import datetime +import os +import pathlib +from subprocess import check_call + +from ruamel.yaml import YAML + + +MINICONDA_VERSION = '4.3.27' + +HERE = pathlib.Path(os.path.dirname(os.path.abspath(__file__))) + +ENV_FILE = 'environment.yml' +FROZEN_FILE = 'environment.frozen.yml' + +yaml = YAML(typ='rt') + + +def fixup(frozen_file): + """Fixup a frozen environment file + + Conda export has a bug! + https://github.com/conda/conda/pull/6391 + """ + with open(frozen_file) as f: + env = yaml.load(f) + + # scrub spurious pip dependencies + # due to conda #6391 + + # note: this scrubs *all* pip dependencies, + # so be more careful if we ever *want* conda to call + # out to pip. + pip_found = False + for idx, dep in enumerate(env['dependencies']): + if isinstance(dep, dict) and 'pip' in dep: + pip_found = True + break + + if pip_found: + env['dependencies'].pop(idx) + + with open(frozen_file, 'w') as f: + yaml.dump(env, f) + + +def freeze(env_file, frozen_file): + """Freeze a conda environment.yml + + By running in docker: + + conda env create + conda env export + + Result will be stored in frozen_file + """ + + with open(HERE / frozen_file, 'w') as f: + f.write(f"# AUTO GENERATED FROM {env_file}, DO NOT MANUALLY MODIFY\n") + f.write(f"# Frozen on {datetime.utcnow():%Y-%m-%d %H:%M:%S UTC}\n") + + check_call([ + 'docker', + 'run', + '--rm', + '-v' f"{HERE}:/r2d", + '-it', + f"continuumio/miniconda3:{MINICONDA_VERSION}", + "sh", "-c", + '; '.join([ + 'conda config --add channels conda-forge', + 'conda config --system --set auto_update_conda false', + f"conda env create -v -f /r2d/{env_file} -n r2d", + f"conda env export -n r2d > /r2d/{frozen_file}", + ]) + ]) + fixup(HERE / frozen_file) + + +if __name__ == '__main__': + freeze(ENV_FILE, FROZEN_FILE) diff --git a/repo2docker/buildpacks/conda/install-miniconda.bash b/repo2docker/buildpacks/conda/install-miniconda.bash index 58a67783..95d47d6b 100755 --- a/repo2docker/buildpacks/conda/install-miniconda.bash +++ b/repo2docker/buildpacks/conda/install-miniconda.bash @@ -3,7 +3,7 @@ set -ex cd $(dirname $0) -CONDA_VERSION=4.3.14 +CONDA_VERSION=4.3.30 URL="https://repo.continuum.io/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh" INSTALLER_PATH=/tmp/miniconda-installer.sh @@ -12,7 +12,7 @@ chmod +x ${INSTALLER_PATH} # Only MD5 checksums are available for miniconda # Can be obtained from https://repo.continuum.io/miniconda/ -MD5SUM="fc6fc37479e3e3fcf3f9ba52cae98991" +MD5SUM="0b80a152332a4ce5250f3c09589c7a81" if ! echo "${MD5SUM} ${INSTALLER_PATH}" | md5sum --quiet -c -; then echo "md5sum mismatch for ${INSTALLER_PATH}, exiting!" @@ -26,7 +26,8 @@ ${CONDA_DIR}/bin/conda config --system --add channels conda-forge # Do not attempt to auto update conda or dependencies ${CONDA_DIR}/bin/conda config --system --set auto_update_conda false -${CONDA_DIR}/bin/conda config --system --set update_dependencies false +# bug in conda 4.3.>15 prevents --set update_dependencies +echo 'update_dependencies: false' >> ${CONDA_DIR}/.condarc ${CONDA_DIR}/bin/conda config --system --set show_channel_urls true ${CONDA_DIR}/bin/conda env update -n root -f /tmp/environment.yml