Merge pull request #163 from minrk/freeze-conda

Freeze conda environment as well

repo2docker/buildpacks/conda/__init__.py

@@ -18,7 +18,7 @@ class CondaBuildPack(BuildPack):
 
     build_script_files = {
         'conda/install-miniconda.bash': '/tmp/install-miniconda.bash',
-        'conda/environment.yml': '/tmp/environment.yml'
+        'conda/environment.frozen.yml': '/tmp/environment.yml'
     }
 
     build_scripts = [

repo2docker/buildpacks/conda/environment.frozen.yml

@@ -0,0 +1,61 @@
+name: r2d
+channels:
+- conda-forge
+- defaults
+dependencies:
+- bleach=2.0.0=py36_0
+- ca-certificates=2017.11.5=0
+- certifi=2017.11.5=py36_0
+- decorator=4.1.2=py36_0
+- entrypoints=0.2.3=py36_1
+- gmp=6.1.2=0
+- html5lib=0.999999999=py36_0
+- ipykernel=4.6.1=py36_0
+- ipython=6.2.1=py36_0
+- ipython_genutils=0.2.0=py36_0
+- ipywidgets=6.0.1=py36_0
+- jedi=0.10.2=py36_0
+- jinja2=2.10=py36_0
+- jsonschema=2.6.0=py36_0
+- jupyter_client=5.1.0=py36_0
+- jupyter_core=4.4.0=py_0
+- jupyterlab=0.29.2=py36_0
+- jupyterlab_launcher=0.5.5=py36_0
+- libsodium=1.0.15=1
+- markupsafe=1.0=py36_0
+- mistune=0.8=py36_0
+- nbconvert=5.3.1=py_1
+- nbformat=4.4.0=py36_0
+- ncurses=5.9=10
+- notebook=5.2.2=py36_1
+- openssl=1.0.2m=0
+- pandoc=2.0.4=0
+- pandocfilters=1.4.1=py36_0
+- pexpect=4.3.0=py36_0
+- pickleshare=0.7.4=py36_0
+- pip=9.0.1=py36_0
+- prompt_toolkit=1.0.15=py36_0
+- ptyprocess=0.5.2=py36_0
+- pygments=2.2.0=py36_0
+- python=3.6.1=3
+- python-dateutil=2.6.1=py36_0
+- pyzmq=16.0.2=py36_2
+- readline=6.2=0
+- setuptools=38.2.3=py36_0
+- simplegeneric=0.8.1=py36_0
+- six=1.11.0=py36_1
+- sqlite=3.13.0=1
+- terminado=0.8.1=py36_0
+- testpath=0.3.1=py36_0
+- tk=8.5.19=2
+- tornado=4.5.2=py36_0
+- traitlets=4.3.2=py36_0
+- wcwidth=0.1.7=py36_0
+- webencodings=0.5=py36_0
+- wheel=0.30.0=py_1
+- widgetsnbextension=2.0.1=py36_0
+- xz=5.2.3=0
+- zeromq=4.2.1=1
+- zlib=1.2.11=0
+prefix: /opt/conda/envs/r2d
+

repo2docker/buildpacks/conda/environment.yml

@@ -1,7 +1,7 @@
 dependencies:
   - python==3.6.1
-  - notebook==5.2.2
   - ipython==6.2.1
   - ipykernel==4.6.1
   - ipywidgets==6.0.1
   - jupyterlab==0.29.2
+  - notebook==5.2.2

repo2docker/buildpacks/conda/freeze.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""
+Freeze the conda environment.yml
+
+Run in a continuumio/miniconda3 image to ensure portability
+"""
+
+from datetime import datetime
+import os
+import pathlib
+from subprocess import check_call
+
+from ruamel.yaml import YAML
+
+
+MINICONDA_VERSION = '4.3.27'
+
+HERE = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))
+
+ENV_FILE = 'environment.yml'
+FROZEN_FILE = 'environment.frozen.yml'
+
+yaml = YAML(typ='rt')
+
+
+def fixup(frozen_file):
+    """Fixup a frozen environment file
+
+    Conda export has a bug!
+    https://github.com/conda/conda/pull/6391
+    """
+    with open(frozen_file) as f:
+        env = yaml.load(f)
+
+    # scrub spurious pip dependencies
+    # due to conda #6391
+
+    # note: this scrubs *all* pip dependencies,
+    # so be more careful if we ever *want* conda to call
+    # out to pip.
+    pip_found = False
+    for idx, dep in enumerate(env['dependencies']):
+        if isinstance(dep, dict) and 'pip' in dep:
+            pip_found = True
+            break
+
+    if pip_found:
+        env['dependencies'].pop(idx)
+
+    with open(frozen_file, 'w') as f:
+        yaml.dump(env, f)
+
+
+def freeze(env_file, frozen_file):
+    """Freeze a conda environment.yml
+
+    By running in docker:
+
+        conda env create
+        conda env export
+
+    Result will be stored in frozen_file
+    """
+
+    with open(HERE / frozen_file, 'w') as f:
+        f.write(f"# AUTO GENERATED FROM {env_file}, DO NOT MANUALLY MODIFY\n")
+        f.write(f"# Frozen on {datetime.utcnow():%Y-%m-%d %H:%M:%S UTC}\n")
+
+    check_call([
+        'docker',
+        'run',
+        '--rm',
+        '-v' f"{HERE}:/r2d",
+        '-it',
+        f"continuumio/miniconda3:{MINICONDA_VERSION}",
+        "sh", "-c",
+        '; '.join([
+            'conda config --add channels conda-forge',
+            'conda config --system --set auto_update_conda false',
+            f"conda env create -v -f /r2d/{env_file} -n r2d",
+            f"conda env export -n r2d > /r2d/{frozen_file}",
+        ])
+    ])
+    fixup(HERE / frozen_file)
+
+
+if __name__ == '__main__':
+    freeze(ENV_FILE, FROZEN_FILE)

repo2docker/buildpacks/conda/install-miniconda.bash

@@ -3,7 +3,7 @@
 set -ex
 
 cd $(dirname $0)
-CONDA_VERSION=4.3.14
+CONDA_VERSION=4.3.30
 URL="https://repo.continuum.io/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh"
 INSTALLER_PATH=/tmp/miniconda-installer.sh
 
@@ -12,7 +12,7 @@ chmod +x ${INSTALLER_PATH}
 
 # Only MD5 checksums are available for miniconda
 # Can be obtained from https://repo.continuum.io/miniconda/
-MD5SUM="fc6fc37479e3e3fcf3f9ba52cae98991"
+MD5SUM="0b80a152332a4ce5250f3c09589c7a81"
 
 if ! echo "${MD5SUM}  ${INSTALLER_PATH}" | md5sum  --quiet -c -; then
     echo "md5sum mismatch for ${INSTALLER_PATH}, exiting!"
@@ -26,7 +26,8 @@ ${CONDA_DIR}/bin/conda config --system --add channels conda-forge
 
 # Do not attempt to auto update conda or dependencies
 ${CONDA_DIR}/bin/conda config --system --set auto_update_conda false
-${CONDA_DIR}/bin/conda config --system --set update_dependencies false
+# bug in conda 4.3.>15 prevents --set update_dependencies
+echo 'update_dependencies: false' >> ${CONDA_DIR}/.condarc
 ${CONDA_DIR}/bin/conda config --system --set show_channel_urls true
 
 ${CONDA_DIR}/bin/conda env update -n root -f /tmp/environment.yml