Merge pull request #182 from minrk/conda-python-runtime

conda: separate base envs by Python version

repo2docker/buildpacks/conda/__init__.py

@@ -1,26 +1,31 @@
 """
-Generates a variety of Dockerfiles based on an input matrix
+Buildpack for conda environments
 """
-from traitlets import default
+import glob
 import os
+import re
+
+from ruamel.yaml import YAML
+from traitlets import default, Unicode
+
 from ..base import BuildPack
 
+# pattern for parsing conda dependency line
+PYTHON_REGEX = re.compile(r'python\s*=+\s*([\d\.]*)')
+# current directory
+HERE = os.path.dirname(os.path.abspath(__file__))
+
 
 class CondaBuildPack(BuildPack):
     name = "conda"
     version = "0.1"
     env = [
         ('CONDA_DIR', '${APP_BASE}/conda'),
-        ('NB_PYTHON_PREFIX', '${CONDA_DIR}')
+        ('NB_PYTHON_PREFIX', '${CONDA_DIR}'),
     ]
 
     path = ['${CONDA_DIR}/bin']
 
-    build_script_files = {
-        'conda/install-miniconda.bash': '/tmp/install-miniconda.bash',
-        'conda/environment.frozen.yml': '/tmp/environment.yml'
-    }
-
     build_scripts = [
         (
             "root",
@@ -31,19 +36,90 @@ class CondaBuildPack(BuildPack):
         )
     ]
 
+    major_pythons = {
+        '2': '2.7',
+        '3': '3.6',
+    }
+
+    @default('build_script_files')
+    def setup_build_script_files(self):
+        files = {
+            'conda/install-miniconda.bash': '/tmp/install-miniconda.bash',
+        }
+        py_version = self.python_version
+        self.log.info("Building conda environment for python=%s" % py_version)
+        # Select the frozen base environment based on Python version.
+        # avoids expensive and possibly conflicting upgrades when changing
+        # major Python versions during upgrade.
+        # If no version is specified or no matching X.Y version is found,
+        # the default base environment is used.
+        frozen_name = 'environment.frozen.yml'
+        if py_version:
+            if self.py2:
+                # python 2 goes in a different env
+                files['conda/environment.py-2.7.frozen.yml'] = '/tmp/kernel-environment.yml'
+            else:
+                py_frozen_name = \
+                    'environment.py-{py}.frozen.yml'.format(py=py_version)
+                if os.path.exists(os.path.join(HERE, py_frozen_name)):
+                    frozen_name = py_frozen_name
+                else:
+                    self.log.warning("No frozen env: %s", py_frozen_name)
+        files['conda/' + frozen_name] = '/tmp/environment.yml'
+        return files
+
+    python_version = Unicode()
+    @default('python_version')
+    def detect_python_version(self):
+        """Detect the Python version for a given environment.yml
+
+        Will return 'x.y' if found, or Falsy '' if not.
+        """
+        py_version = None
+        environment_yml = self.binder_path('environment.yml')
+        if not os.path.exists(environment_yml):
+            return ''
+        with open(environment_yml) as f:
+            env = YAML().load(f)
+            for dep in env.get('dependencies', []):
+                if not isinstance(dep, str):
+                    continue
+                match = PYTHON_REGEX.match(dep)
+                if not match:
+                    continue
+                py_version = match.group(1)
+                break
+
+        # extract major.minor
+        if py_version:
+            if len(py_version) == 1:
+                return self.major_pythons.get(py_version[0])
+            else:
+                # return major.minor
+                return '.'.join(py_version[:2])
+
+        return ''
+
+    @property
+    def py2(self):
+        """Am I building a Python 2 kernel environment?"""
+        return self.python_version and self.python_version.split('.')[0] == '2'
+
     @default('assemble_scripts')
     def setup_assembly(self):
         assembly_scripts = []
         environment_yml = self.binder_path('environment.yml')
+        env_name = 'kernel' if self.py2 else 'root'
         if os.path.exists(environment_yml):
             assembly_scripts.append((
                 '${NB_USER}',
                 r"""
-                conda env update -v -n root -f "{}" && \
+                conda env update -v -n {} -f "{}" && \
                 conda clean -tipsy
-                """.format(environment_yml)
+                """.format(env_name, environment_yml)
             ))
         return assembly_scripts
 
+
     def detect(self):
         return os.path.exists(self.binder_path('environment.yml')) and super().detect()

repo2docker/buildpacks/conda/environment.frozen.yml

@@ -1,3 +1,5 @@
+# AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY
+# Frozen on 2018-01-03 15:33:55 UTC
 name: r2d
 channels:
 - conda-forge
@@ -11,13 +13,13 @@ dependencies:
 - gmp=6.1.2=0
 - html5lib=1.0.1=py_0
 - ipykernel=4.7.0=py36_0
-- ipython=6.2.1=py36_0
+- ipython=6.2.1=py36_1
 - ipython_genutils=0.2.0=py36_0
 - ipywidgets=6.0.1=py36_0
 - jedi=0.10.2=py36_0
 - jinja2=2.10=py36_0
 - jsonschema=2.6.0=py36_0
-- jupyter_client=5.1.0=py36_0
+- jupyter_client=5.2.0=py36_0
 - jupyter_core=4.4.0=py_0
 - jupyterlab=0.30.6=py36_0
 - jupyterlab_launcher=0.6.0=py36_0
@@ -31,23 +33,23 @@ dependencies:
 - openssl=1.0.2n=0
 - pandoc=2.0.5=0
 - pandocfilters=1.4.1=py36_0
-- pexpect=4.3.0=py36_0
+- pexpect=4.3.1=py36_0
 - pickleshare=0.7.4=py36_0
-- pip=9.0.1=py36_0
+- pip=9.0.1=py36_1
 - prompt_toolkit=1.0.15=py36_0
 - ptyprocess=0.5.2=py36_0
 - pygments=2.2.0=py36_0
-- python=3.6.1=3
+- python=3.6.4=0
 - python-dateutil=2.6.1=py36_0
 - pyzmq=16.0.2=py36_2
-- readline=6.2=0
+- readline=7.0=0
 - setuptools=38.2.4=py36_0
 - simplegeneric=0.8.1=py36_0
 - six=1.11.0=py36_1
-- sqlite=3.13.0=1
+- sqlite=3.20.1=2
 - terminado=0.8.1=py36_0
 - testpath=0.3.1=py36_0
-- tk=8.5.19=2
+- tk=8.6.7=0
 - tornado=4.5.2=py36_0
 - traitlets=4.3.2=py36_0
 - wcwidth=0.1.7=py36_0

repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml

@@ -0,0 +1,49 @@
+# AUTO GENERATED FROM environment.py-2.7.yml, DO NOT MANUALLY MODIFY
+# Frozen on 2018-01-03 16:56:23 UTC
+name: r2d
+channels:
+- conda-forge
+- defaults
+dependencies:
+- backports=1.0=py27_1
+- backports.shutil_get_terminal_size=1.0.0=py_3
+- backports_abc=0.5=py27_0
+- ca-certificates=2017.11.5=0
+- certifi=2017.11.5=py27_0
+- decorator=4.1.2=py27_0
+- enum34=1.1.6=py27_1
+- ipykernel=4.7.0=py27_0
+- ipython=5.5.0=py27_0
+- ipython_genutils=0.2.0=py27_0
+- jupyter_client=5.2.0=py27_0
+- jupyter_core=4.4.0=py_0
+- libsodium=1.0.15=1
+- ncurses=5.9=10
+- openssl=1.0.2n=0
+- pathlib2=2.3.0=py27_0
+- pexpect=4.3.1=py27_0
+- pickleshare=0.7.4=py27_0
+- pip=9.0.1=py27_1
+- prompt_toolkit=1.0.15=py27_0
+- ptyprocess=0.5.2=py27_0
+- pygments=2.2.0=py27_0
+- python=2.7.14=4
+- python-dateutil=2.6.1=py27_0
+- pyzmq=16.0.2=py27_2
+- readline=7.0=0
+- scandir=1.6=py27_0
+- setuptools=38.2.4=py27_0
+- simplegeneric=0.8.1=py27_0
+- singledispatch=3.4.0.3=py27_0
+- six=1.11.0=py27_1
+- sqlite=3.20.1=2
+- ssl_match_hostname=3.5.0.1=py27_1
+- tk=8.6.7=0
+- tornado=4.5.2=py27_0
+- traitlets=4.3.2=py27_0
+- wcwidth=0.1.7=py27_0
+- wheel=0.30.0=py_1
+- zeromq=4.2.1=1
+- zlib=1.2.11=0
+prefix: /opt/conda/envs/r2d
+

repo2docker/buildpacks/conda/environment.py-2.7.yml

@@ -0,0 +1,3 @@
+dependencies:
+- python=2.7.*
+- ipykernel==4.7.0

repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml

@@ -0,0 +1,63 @@
+# AUTO GENERATED FROM environment.py-3.5.yml, DO NOT MANUALLY MODIFY
+# Frozen on 2018-01-03 15:31:22 UTC
+name: r2d
+channels:
+- conda-forge
+- defaults
+dependencies:
+- bleach=2.0.0=py35_0
+- ca-certificates=2017.11.5=0
+- certifi=2017.11.5=py35_0
+- decorator=4.1.2=py35_0
+- entrypoints=0.2.3=py35_1
+- gmp=6.1.2=0
+- html5lib=1.0.1=py_0
+- ipykernel=4.7.0=py35_0
+- ipython=6.2.1=py35_1
+- ipython_genutils=0.2.0=py35_0
+- ipywidgets=6.0.1=py35_0
+- jedi=0.10.2=py35_0
+- jinja2=2.10=py35_0
+- jsonschema=2.6.0=py35_0
+- jupyter_client=5.2.0=py35_0
+- jupyter_core=4.4.0=py_0
+- jupyterlab=0.30.6=py35_0
+- jupyterlab_launcher=0.6.0=py35_0
+- libsodium=1.0.15=1
+- markupsafe=1.0=py35_0
+- mistune=0.8.3=py_0
+- nbconvert=5.3.1=py_1
+- nbformat=4.4.0=py35_0
+- ncurses=5.9=10
+- notebook=5.2.2=py35_1
+- openssl=1.0.2n=0
+- pandoc=2.0.5=0
+- pandocfilters=1.4.1=py35_0
+- pexpect=4.3.1=py35_0
+- pickleshare=0.7.4=py35_0
+- pip=9.0.1=py35_1
+- prompt_toolkit=1.0.15=py35_0
+- ptyprocess=0.5.2=py35_0
+- pygments=2.2.0=py35_0
+- python=3.5.4=3
+- python-dateutil=2.6.1=py35_0
+- pyzmq=16.0.2=py35_2
+- readline=7.0=0
+- setuptools=38.2.4=py35_0
+- simplegeneric=0.8.1=py35_0
+- six=1.11.0=py35_1
+- sqlite=3.20.1=2
+- terminado=0.8.1=py35_0
+- testpath=0.3.1=py35_0
+- tk=8.6.7=0
+- tornado=4.5.2=py35_0
+- traitlets=4.3.2=py35_0
+- wcwidth=0.1.7=py35_0
+- webencodings=0.5=py35_0
+- wheel=0.30.0=py_1
+- widgetsnbextension=2.0.1=py35_0
+- xz=5.2.3=0
+- zeromq=4.2.1=1
+- zlib=1.2.11=0
+prefix: /opt/conda/envs/r2d
+

repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml

@@ -0,0 +1,63 @@
+# AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY
+# Frozen on 2018-01-03 15:33:55 UTC
+name: r2d
+channels:
+- conda-forge
+- defaults
+dependencies:
+- bleach=2.0.0=py36_0
+- ca-certificates=2017.11.5=0
+- certifi=2017.11.5=py36_0
+- decorator=4.1.2=py36_0
+- entrypoints=0.2.3=py36_1
+- gmp=6.1.2=0
+- html5lib=1.0.1=py_0
+- ipykernel=4.7.0=py36_0
+- ipython=6.2.1=py36_1
+- ipython_genutils=0.2.0=py36_0
+- ipywidgets=6.0.1=py36_0
+- jedi=0.10.2=py36_0
+- jinja2=2.10=py36_0
+- jsonschema=2.6.0=py36_0
+- jupyter_client=5.2.0=py36_0
+- jupyter_core=4.4.0=py_0
+- jupyterlab=0.30.6=py36_0
+- jupyterlab_launcher=0.6.0=py36_0
+- libsodium=1.0.15=1
+- markupsafe=1.0=py36_0
+- mistune=0.8.3=py_0
+- nbconvert=5.3.1=py_1
+- nbformat=4.4.0=py36_0
+- ncurses=5.9=10
+- notebook=5.2.2=py36_1
+- openssl=1.0.2n=0
+- pandoc=2.0.5=0
+- pandocfilters=1.4.1=py36_0
+- pexpect=4.3.1=py36_0
+- pickleshare=0.7.4=py36_0
+- pip=9.0.1=py36_1
+- prompt_toolkit=1.0.15=py36_0
+- ptyprocess=0.5.2=py36_0
+- pygments=2.2.0=py36_0
+- python=3.6.4=0
+- python-dateutil=2.6.1=py36_0
+- pyzmq=16.0.2=py36_2
+- readline=7.0=0
+- setuptools=38.2.4=py36_0
+- simplegeneric=0.8.1=py36_0
+- six=1.11.0=py36_1
+- sqlite=3.20.1=2
+- terminado=0.8.1=py36_0
+- testpath=0.3.1=py36_0
+- tk=8.6.7=0
+- tornado=4.5.2=py36_0
+- traitlets=4.3.2=py36_0
+- wcwidth=0.1.7=py36_0
+- webencodings=0.5=py36_0
+- wheel=0.30.0=py_1
+- widgetsnbextension=2.0.1=py36_0
+- xz=5.2.3=0
+- zeromq=4.2.1=1
+- zlib=1.2.11=0
+prefix: /opt/conda/envs/r2d
+

repo2docker/buildpacks/conda/environment.yml

@@ -1,5 +1,5 @@
 dependencies:
-  - python==3.6.1
+  - python==3.6.*
   - ipywidgets==6.0.1
   - jupyterlab==0.30.6
   - notebook==5.2.2

repo2docker/buildpacks/conda/freeze.py

@@ -8,6 +8,7 @@ Run in a continuumio/miniconda3 image to ensure portability
 from datetime import datetime
 import os
 import pathlib
+import shutil
 from subprocess import check_call
 
 from ruamel.yaml import YAML
@@ -18,7 +19,10 @@ MINICONDA_VERSION = '4.3.27'
 HERE = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))
 
 ENV_FILE = 'environment.yml'
-FROZEN_FILE = 'environment.frozen.yml'
+FROZEN_FILE = os.path.splitext(ENV_FILE)[0] + '.frozen.yml'
+
+ENV_FILE_T = 'environment.py-{py}.yml'
+FROZEN_FILE_T = os.path.splitext(ENV_FILE_T)[0] + '.frozen.yml'
 
 yaml = YAML(typ='rt')
 
@@ -78,11 +82,40 @@ def freeze(env_file, frozen_file):
             'conda config --add channels conda-forge',
             'conda config --system --set auto_update_conda false',
             f"conda env create -v -f /r2d/{env_file} -n r2d",
-            f"conda env export -n r2d > /r2d/{frozen_file}",
+            f"conda env export -n r2d >> /r2d/{frozen_file}",
         ])
     ])
     fixup(HERE / frozen_file)
 
 
+def set_python(py_env_file, py):
+    """Set the Python version in an env file"""
+    if os.path.exists(py_env_file):
+        # only clobber auto-generated files
+        with open(py_env_file) as f:
+            text = f.read()
+            if text and 'GENERATED' not in text:
+                return
+    with open(ENV_FILE) as f:
+        env = yaml.load(f)
+    for idx, dep in enumerate(env['dependencies']):
+        if dep.split('=')[0] == 'python':
+            env['dependencies'][idx] = f'python={py}.*'
+            break
+    else:
+        raise ValueError(f"python dependency not found in {env['dependencies']}")
+    # update python dependency
+    with open(py_env_file, 'w') as f:
+        f.write(f"# AUTO GENERATED FROM {ENV_FILE}, DO NOT MANUALLY MODIFY\n")
+        f.write(f"# Generated on {datetime.utcnow():%Y-%m-%d %H:%M:%S UTC}\n")
+        yaml.dump(env, f)
+
+
 if __name__ == '__main__':
-    freeze(ENV_FILE, FROZEN_FILE)
+    for py in ('2.7', '3.5', '3.6'):
+        env_file = ENV_FILE_T.format(py=py)
+        set_python(env_file, py)
+        frozen_file = os.path.splitext(env_file)[0] + '.frozen.yml'
+        freeze(env_file, frozen_file)
+    # use last version as default
+    shutil.copy(frozen_file, FROZEN_FILE)

repo2docker/buildpacks/conda/install-miniconda.bash

@@ -32,6 +32,12 @@ ${CONDA_DIR}/bin/conda config --system --set show_channel_urls true
 
 ${CONDA_DIR}/bin/conda env update -n root -f /tmp/environment.yml
 
+if [[ -f /tmp/kernel-environment.yml ]]; then
+    # install kernel env and register kernelspec
+    ${CONDA_DIR}/bin/conda env create -n kernel -f /tmp/kernel-environment.yml
+    ${CONDA_DIR}/envs/kernel/bin/ipython kernel install --prefix "${CONDA_DIR}"
+fi
+
 # Clean things out!
 ${CONDA_DIR}/bin/conda clean -tipsy
 

setup.py

@@ -8,7 +8,8 @@ setup(
         'traitlets',
         'python-json-logger',
         'escapism',
-        'jinja2'
+        'jinja2',
+        'ruamel.yaml>=0.15',
     ],
     python_requires='>=3.4',
     author='Yuvi Panda',

tests/conda/simple-py2/README.rst

@@ -0,0 +1,5 @@
+Conda Environment
+-----------------
+
+Conda environments files may allow for more complex builds and dependencies. You
+can specify them in the standard `environment.yml` files.

tests/conda/simple-py2/environment.yml

@@ -0,0 +1,3 @@
+dependencies:
+  - python=2
+  - numpy

tests/conda/simple-py2/verify

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+import sys
+
+assert sys.version_info[:2] == (3, 6), sys.version
+
+# verify that we have Python 2 and Python 3 kernelspecs
+from jupyter_client.kernelspec import KernelSpecManager
+ksm = KernelSpecManager()
+specs = ksm.get_all_specs()
+assert sorted(specs) == ['python2', 'python3'], specs.keys()
+
+# verify that we created the kernel env
+import json
+from subprocess import check_output
+envs = json.loads(check_output(['conda', 'env', 'list', '--json']).decode('utf8'))
+assert envs == {'envs': ['/srv/conda/envs/kernel']}, envs
+
+pkgs = json.loads(check_output(['conda', 'list', '-n', 'kernel', '--json']).decode('utf8'))
+pkg_names = [pkg['name'] for pkg in pkgs]
+assert 'ipykernel' in pkg_names, pkg_names
+assert 'numpy' in pkg_names
+for pkg in pkgs:
+    if pkg['name'] == 'python':
+        assert pkg['version'].startswith('2.7.')
+        break
+else:
+    assert False, "python not found in %s" % pkg_names