[conda] create separate frozen envs for Python versions

this will allow us to create different base envs based on Python major.minor versions

repo2docker/buildpacks/conda/environment.frozen.yml

@@ -1,3 +1,5 @@
+# AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY
+# Frozen on 2018-01-03 15:33:55 UTC
 name: r2d
 channels:
 - conda-forge
@@ -11,13 +13,13 @@ dependencies:
 - gmp=6.1.2=0
 - html5lib=1.0.1=py_0
 - ipykernel=4.7.0=py36_0
-- ipython=6.2.1=py36_0
+- ipython=6.2.1=py36_1
 - ipython_genutils=0.2.0=py36_0
 - ipywidgets=6.0.1=py36_0
 - jedi=0.10.2=py36_0
 - jinja2=2.10=py36_0
 - jsonschema=2.6.0=py36_0
-- jupyter_client=5.1.0=py36_0
+- jupyter_client=5.2.0=py36_0
 - jupyter_core=4.4.0=py_0
 - jupyterlab=0.30.6=py36_0
 - jupyterlab_launcher=0.6.0=py36_0
@@ -31,23 +33,23 @@ dependencies:
 - openssl=1.0.2n=0
 - pandoc=2.0.5=0
 - pandocfilters=1.4.1=py36_0
-- pexpect=4.3.0=py36_0
+- pexpect=4.3.1=py36_0
 - pickleshare=0.7.4=py36_0
-- pip=9.0.1=py36_0
+- pip=9.0.1=py36_1
 - prompt_toolkit=1.0.15=py36_0
 - ptyprocess=0.5.2=py36_0
 - pygments=2.2.0=py36_0
-- python=3.6.1=3
+- python=3.6.4=0
 - python-dateutil=2.6.1=py36_0
 - pyzmq=16.0.2=py36_2
-- readline=6.2=0
+- readline=7.0=0
 - setuptools=38.2.4=py36_0
 - simplegeneric=0.8.1=py36_0
 - six=1.11.0=py36_1
-- sqlite=3.13.0=1
+- sqlite=3.20.1=2
 - terminado=0.8.1=py36_0
 - testpath=0.3.1=py36_0
-- tk=8.5.19=2
+- tk=8.6.7=0
 - tornado=4.5.2=py36_0
 - traitlets=4.3.2=py36_0
 - wcwidth=0.1.7=py36_0

repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml

@@ -0,0 +1,73 @@
+# AUTO GENERATED FROM environment.py-2.7.yml, DO NOT MANUALLY MODIFY
+# Frozen on 2018-01-03 15:29:14 UTC
+name: r2d
+channels:
+- conda-forge
+- defaults
+dependencies:
+- backports=1.0=py27_1
+- backports.shutil_get_terminal_size=1.0.0=py_3
+- backports_abc=0.5=py27_0
+- bleach=2.0.0=py27_0
+- ca-certificates=2017.11.5=0
+- certifi=2017.11.5=py27_0
+- configparser=3.5.0=py27_0
+- decorator=4.1.2=py27_0
+- entrypoints=0.2.3=py27_1
+- enum34=1.1.6=py27_1
+- functools32=3.2.3.2=py27_1
+- futures=3.2.0=py27_0
+- gmp=6.1.2=0
+- html5lib=1.0.1=py_0
+- ipykernel=4.7.0=py27_0
+- ipython=5.5.0=py27_0
+- ipython_genutils=0.2.0=py27_0
+- ipywidgets=6.0.1=py27_0
+- jinja2=2.10=py27_0
+- jsonschema=2.6.0=py27_0
+- jupyter_client=5.2.0=py27_0
+- jupyter_core=4.4.0=py_0
+- jupyterlab=0.30.6=py27_0
+- jupyterlab_launcher=0.6.0=py27_0
+- libsodium=1.0.15=1
+- markupsafe=1.0=py27_0
+- mistune=0.8.3=py_0
+- nbconvert=5.3.1=py_1
+- nbformat=4.4.0=py27_0
+- ncurses=5.9=10
+- notebook=5.2.2=py27_1
+- openssl=1.0.2n=0
+- pandoc=2.0.5=0
+- pandocfilters=1.4.1=py27_0
+- pathlib2=2.3.0=py27_0
+- pexpect=4.3.1=py27_0
+- pickleshare=0.7.4=py27_0
+- pip=9.0.1=py27_1
+- prompt_toolkit=1.0.15=py27_0
+- ptyprocess=0.5.2=py27_0
+- pygments=2.2.0=py27_0
+- python=2.7.14=4
+- python-dateutil=2.6.1=py27_0
+- pyzmq=16.0.2=py27_2
+- readline=7.0=0
+- scandir=1.6=py27_0
+- setuptools=38.2.4=py27_0
+- simplegeneric=0.8.1=py27_0
+- singledispatch=3.4.0.3=py27_0
+- six=1.11.0=py27_1
+- sqlite=3.20.1=2
+- ssl_match_hostname=3.5.0.1=py27_1
+- subprocess32=3.2.7=py27_0
+- terminado=0.8.1=py27_0
+- testpath=0.3.1=py27_0
+- tk=8.6.7=0
+- tornado=4.5.2=py27_0
+- traitlets=4.3.2=py27_0
+- wcwidth=0.1.7=py27_0
+- webencodings=0.5=py27_0
+- wheel=0.30.0=py_1
+- widgetsnbextension=2.0.1=py27_0
+- zeromq=4.2.1=1
+- zlib=1.2.11=0
+prefix: /opt/conda/envs/r2d
+

repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml

@@ -0,0 +1,63 @@
+# AUTO GENERATED FROM environment.py-3.5.yml, DO NOT MANUALLY MODIFY
+# Frozen on 2018-01-03 15:31:22 UTC
+name: r2d
+channels:
+- conda-forge
+- defaults
+dependencies:
+- bleach=2.0.0=py35_0
+- ca-certificates=2017.11.5=0
+- certifi=2017.11.5=py35_0
+- decorator=4.1.2=py35_0
+- entrypoints=0.2.3=py35_1
+- gmp=6.1.2=0
+- html5lib=1.0.1=py_0
+- ipykernel=4.7.0=py35_0
+- ipython=6.2.1=py35_1
+- ipython_genutils=0.2.0=py35_0
+- ipywidgets=6.0.1=py35_0
+- jedi=0.10.2=py35_0
+- jinja2=2.10=py35_0
+- jsonschema=2.6.0=py35_0
+- jupyter_client=5.2.0=py35_0
+- jupyter_core=4.4.0=py_0
+- jupyterlab=0.30.6=py35_0
+- jupyterlab_launcher=0.6.0=py35_0
+- libsodium=1.0.15=1
+- markupsafe=1.0=py35_0
+- mistune=0.8.3=py_0
+- nbconvert=5.3.1=py_1
+- nbformat=4.4.0=py35_0
+- ncurses=5.9=10
+- notebook=5.2.2=py35_1
+- openssl=1.0.2n=0
+- pandoc=2.0.5=0
+- pandocfilters=1.4.1=py35_0
+- pexpect=4.3.1=py35_0
+- pickleshare=0.7.4=py35_0
+- pip=9.0.1=py35_1
+- prompt_toolkit=1.0.15=py35_0
+- ptyprocess=0.5.2=py35_0
+- pygments=2.2.0=py35_0
+- python=3.5.4=3
+- python-dateutil=2.6.1=py35_0
+- pyzmq=16.0.2=py35_2
+- readline=7.0=0
+- setuptools=38.2.4=py35_0
+- simplegeneric=0.8.1=py35_0
+- six=1.11.0=py35_1
+- sqlite=3.20.1=2
+- terminado=0.8.1=py35_0
+- testpath=0.3.1=py35_0
+- tk=8.6.7=0
+- tornado=4.5.2=py35_0
+- traitlets=4.3.2=py35_0
+- wcwidth=0.1.7=py35_0
+- webencodings=0.5=py35_0
+- wheel=0.30.0=py_1
+- widgetsnbextension=2.0.1=py35_0
+- xz=5.2.3=0
+- zeromq=4.2.1=1
+- zlib=1.2.11=0
+prefix: /opt/conda/envs/r2d
+

repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml

@@ -0,0 +1,63 @@
+# AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY
+# Frozen on 2018-01-03 15:33:55 UTC
+name: r2d
+channels:
+- conda-forge
+- defaults
+dependencies:
+- bleach=2.0.0=py36_0
+- ca-certificates=2017.11.5=0
+- certifi=2017.11.5=py36_0
+- decorator=4.1.2=py36_0
+- entrypoints=0.2.3=py36_1
+- gmp=6.1.2=0
+- html5lib=1.0.1=py_0
+- ipykernel=4.7.0=py36_0
+- ipython=6.2.1=py36_1
+- ipython_genutils=0.2.0=py36_0
+- ipywidgets=6.0.1=py36_0
+- jedi=0.10.2=py36_0
+- jinja2=2.10=py36_0
+- jsonschema=2.6.0=py36_0
+- jupyter_client=5.2.0=py36_0
+- jupyter_core=4.4.0=py_0
+- jupyterlab=0.30.6=py36_0
+- jupyterlab_launcher=0.6.0=py36_0
+- libsodium=1.0.15=1
+- markupsafe=1.0=py36_0
+- mistune=0.8.3=py_0
+- nbconvert=5.3.1=py_1
+- nbformat=4.4.0=py36_0
+- ncurses=5.9=10
+- notebook=5.2.2=py36_1
+- openssl=1.0.2n=0
+- pandoc=2.0.5=0
+- pandocfilters=1.4.1=py36_0
+- pexpect=4.3.1=py36_0
+- pickleshare=0.7.4=py36_0
+- pip=9.0.1=py36_1
+- prompt_toolkit=1.0.15=py36_0
+- ptyprocess=0.5.2=py36_0
+- pygments=2.2.0=py36_0
+- python=3.6.4=0
+- python-dateutil=2.6.1=py36_0
+- pyzmq=16.0.2=py36_2
+- readline=7.0=0
+- setuptools=38.2.4=py36_0
+- simplegeneric=0.8.1=py36_0
+- six=1.11.0=py36_1
+- sqlite=3.20.1=2
+- terminado=0.8.1=py36_0
+- testpath=0.3.1=py36_0
+- tk=8.6.7=0
+- tornado=4.5.2=py36_0
+- traitlets=4.3.2=py36_0
+- wcwidth=0.1.7=py36_0
+- webencodings=0.5=py36_0
+- wheel=0.30.0=py_1
+- widgetsnbextension=2.0.1=py36_0
+- xz=5.2.3=0
+- zeromq=4.2.1=1
+- zlib=1.2.11=0
+prefix: /opt/conda/envs/r2d
+

repo2docker/buildpacks/conda/environment.yml

@@ -1,5 +1,5 @@
 dependencies:
-  - python==3.6.1
+  - python==3.6.*
   - ipywidgets==6.0.1
   - jupyterlab==0.30.6
   - notebook==5.2.2

repo2docker/buildpacks/conda/freeze.py

@@ -8,6 +8,7 @@ Run in a continuumio/miniconda3 image to ensure portability
 from datetime import datetime
 import os
 import pathlib
+import shutil
 from subprocess import check_call
 
 from ruamel.yaml import YAML
@@ -18,7 +19,10 @@ MINICONDA_VERSION = '4.3.27'
 HERE = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))
 
 ENV_FILE = 'environment.yml'
-FROZEN_FILE = 'environment.frozen.yml'
+FROZEN_FILE = os.path.splitext(ENV_FILE)[0] + '.frozen.yml'
+
+ENV_FILE_T = 'environment.py-{py}.yml'
+FROZEN_FILE_T = os.path.splitext(ENV_FILE_T)[0] + '.frozen.yml'
 
 yaml = YAML(typ='rt')
 
@@ -78,11 +82,40 @@ def freeze(env_file, frozen_file):
             'conda config --add channels conda-forge',
             'conda config --system --set auto_update_conda false',
             f"conda env create -v -f /r2d/{env_file} -n r2d",
-            f"conda env export -n r2d > /r2d/{frozen_file}",
+            f"conda env export -n r2d >> /r2d/{frozen_file}",
         ])
     ])
     fixup(HERE / frozen_file)
 
 
+def set_python(py_env_file, py):
+    """Set the Python version in an env file"""
+    if os.path.exists(py_env_file):
+        # only clobber auto-generated files
+        with open(py_env_file) as f:
+            text = f.read()
+            if text and 'GENERATED' not in text:
+                return
+    with open(ENV_FILE) as f:
+        env = yaml.load(f)
+    for idx, dep in enumerate(env['dependencies']):
+        if dep.split('=')[0] == 'python':
+            env['dependencies'][idx] = f'python={py}.*'
+            break
+    else:
+        raise ValueError(f"python dependency not found in {env['dependencies']}")
+    # update python dependency
+    with open(py_env_file, 'w') as f:
+        f.write(f"# AUTO GENERATED FROM {ENV_FILE}, DO NOT MANUALLY MODIFY\n")
+        f.write(f"# Generated on {datetime.utcnow():%Y-%m-%d %H:%M:%S UTC}\n")
+        yaml.dump(env, f)
+
+
 if __name__ == '__main__':
-    freeze(ENV_FILE, FROZEN_FILE)
+    for py in ('2.7', '3.5', '3.6'):
+        env_file = ENV_FILE_T.format(py=py)
+        set_python(env_file, py)
+        frozen_file = os.path.splitext(env_file)[0] + '.frozen.yml'
+        freeze(env_file, frozen_file)
+    # use last version as default
+    shutil.copy(frozen_file, FROZEN_FILE)