From 5984a08e6670af11855dafada9bc029d92859d11 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 3 Jan 2018 16:37:40 +0100 Subject: [PATCH] [conda] create separate frozen envs for Python versions this will allow us to create different base envs based on Python major.minor versions --- .../buildpacks/conda/environment.frozen.yml | 18 +++-- .../conda/environment.py-2.7.frozen.yml | 73 +++++++++++++++++++ .../conda/environment.py-3.5.frozen.yml | 63 ++++++++++++++++ .../conda/environment.py-3.6.frozen.yml | 63 ++++++++++++++++ repo2docker/buildpacks/conda/environment.yml | 2 +- repo2docker/buildpacks/conda/freeze.py | 39 +++++++++- 6 files changed, 246 insertions(+), 12 deletions(-) create mode 100644 repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml create mode 100644 repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml create mode 100644 repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml diff --git a/repo2docker/buildpacks/conda/environment.frozen.yml b/repo2docker/buildpacks/conda/environment.frozen.yml index f63b4f0f..19338f96 100644 --- a/repo2docker/buildpacks/conda/environment.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.frozen.yml @@ -1,3 +1,5 @@ +# AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY +# Frozen on 2018-01-03 15:33:55 UTC name: r2d channels: - conda-forge @@ -11,13 +13,13 @@ dependencies: - gmp=6.1.2=0 - html5lib=1.0.1=py_0 - ipykernel=4.7.0=py36_0 -- ipython=6.2.1=py36_0 +- ipython=6.2.1=py36_1 - ipython_genutils=0.2.0=py36_0 - ipywidgets=6.0.1=py36_0 - jedi=0.10.2=py36_0 - jinja2=2.10=py36_0 - jsonschema=2.6.0=py36_0 -- jupyter_client=5.1.0=py36_0 +- jupyter_client=5.2.0=py36_0 - jupyter_core=4.4.0=py_0 - jupyterlab=0.30.6=py36_0 - jupyterlab_launcher=0.6.0=py36_0 @@ -31,23 +33,23 @@ dependencies: - openssl=1.0.2n=0 - pandoc=2.0.5=0 - pandocfilters=1.4.1=py36_0 -- pexpect=4.3.0=py36_0 +- pexpect=4.3.1=py36_0 - pickleshare=0.7.4=py36_0 -- pip=9.0.1=py36_0 +- pip=9.0.1=py36_1 - prompt_toolkit=1.0.15=py36_0 - ptyprocess=0.5.2=py36_0 - pygments=2.2.0=py36_0 -- python=3.6.1=3 +- python=3.6.4=0 - python-dateutil=2.6.1=py36_0 - pyzmq=16.0.2=py36_2 -- readline=6.2=0 +- readline=7.0=0 - setuptools=38.2.4=py36_0 - simplegeneric=0.8.1=py36_0 - six=1.11.0=py36_1 -- sqlite=3.13.0=1 +- sqlite=3.20.1=2 - terminado=0.8.1=py36_0 - testpath=0.3.1=py36_0 -- tk=8.5.19=2 +- tk=8.6.7=0 - tornado=4.5.2=py36_0 - traitlets=4.3.2=py36_0 - wcwidth=0.1.7=py36_0 diff --git a/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml b/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml new file mode 100644 index 00000000..3e52dbb0 --- /dev/null +++ b/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml @@ -0,0 +1,73 @@ +# AUTO GENERATED FROM environment.py-2.7.yml, DO NOT MANUALLY MODIFY +# Frozen on 2018-01-03 15:29:14 UTC +name: r2d +channels: +- conda-forge +- defaults +dependencies: +- backports=1.0=py27_1 +- backports.shutil_get_terminal_size=1.0.0=py_3 +- backports_abc=0.5=py27_0 +- bleach=2.0.0=py27_0 +- ca-certificates=2017.11.5=0 +- certifi=2017.11.5=py27_0 +- configparser=3.5.0=py27_0 +- decorator=4.1.2=py27_0 +- entrypoints=0.2.3=py27_1 +- enum34=1.1.6=py27_1 +- functools32=3.2.3.2=py27_1 +- futures=3.2.0=py27_0 +- gmp=6.1.2=0 +- html5lib=1.0.1=py_0 +- ipykernel=4.7.0=py27_0 +- ipython=5.5.0=py27_0 +- ipython_genutils=0.2.0=py27_0 +- ipywidgets=6.0.1=py27_0 +- jinja2=2.10=py27_0 +- jsonschema=2.6.0=py27_0 +- jupyter_client=5.2.0=py27_0 +- jupyter_core=4.4.0=py_0 +- jupyterlab=0.30.6=py27_0 +- jupyterlab_launcher=0.6.0=py27_0 +- libsodium=1.0.15=1 +- markupsafe=1.0=py27_0 +- mistune=0.8.3=py_0 +- nbconvert=5.3.1=py_1 +- nbformat=4.4.0=py27_0 +- ncurses=5.9=10 +- notebook=5.2.2=py27_1 +- openssl=1.0.2n=0 +- pandoc=2.0.5=0 +- pandocfilters=1.4.1=py27_0 +- pathlib2=2.3.0=py27_0 +- pexpect=4.3.1=py27_0 +- pickleshare=0.7.4=py27_0 +- pip=9.0.1=py27_1 +- prompt_toolkit=1.0.15=py27_0 +- ptyprocess=0.5.2=py27_0 +- pygments=2.2.0=py27_0 +- python=2.7.14=4 +- python-dateutil=2.6.1=py27_0 +- pyzmq=16.0.2=py27_2 +- readline=7.0=0 +- scandir=1.6=py27_0 +- setuptools=38.2.4=py27_0 +- simplegeneric=0.8.1=py27_0 +- singledispatch=3.4.0.3=py27_0 +- six=1.11.0=py27_1 +- sqlite=3.20.1=2 +- ssl_match_hostname=3.5.0.1=py27_1 +- subprocess32=3.2.7=py27_0 +- terminado=0.8.1=py27_0 +- testpath=0.3.1=py27_0 +- tk=8.6.7=0 +- tornado=4.5.2=py27_0 +- traitlets=4.3.2=py27_0 +- wcwidth=0.1.7=py27_0 +- webencodings=0.5=py27_0 +- wheel=0.30.0=py_1 +- widgetsnbextension=2.0.1=py27_0 +- zeromq=4.2.1=1 +- zlib=1.2.11=0 +prefix: /opt/conda/envs/r2d + diff --git a/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml b/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml new file mode 100644 index 00000000..6aadce8a --- /dev/null +++ b/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml @@ -0,0 +1,63 @@ +# AUTO GENERATED FROM environment.py-3.5.yml, DO NOT MANUALLY MODIFY +# Frozen on 2018-01-03 15:31:22 UTC +name: r2d +channels: +- conda-forge +- defaults +dependencies: +- bleach=2.0.0=py35_0 +- ca-certificates=2017.11.5=0 +- certifi=2017.11.5=py35_0 +- decorator=4.1.2=py35_0 +- entrypoints=0.2.3=py35_1 +- gmp=6.1.2=0 +- html5lib=1.0.1=py_0 +- ipykernel=4.7.0=py35_0 +- ipython=6.2.1=py35_1 +- ipython_genutils=0.2.0=py35_0 +- ipywidgets=6.0.1=py35_0 +- jedi=0.10.2=py35_0 +- jinja2=2.10=py35_0 +- jsonschema=2.6.0=py35_0 +- jupyter_client=5.2.0=py35_0 +- jupyter_core=4.4.0=py_0 +- jupyterlab=0.30.6=py35_0 +- jupyterlab_launcher=0.6.0=py35_0 +- libsodium=1.0.15=1 +- markupsafe=1.0=py35_0 +- mistune=0.8.3=py_0 +- nbconvert=5.3.1=py_1 +- nbformat=4.4.0=py35_0 +- ncurses=5.9=10 +- notebook=5.2.2=py35_1 +- openssl=1.0.2n=0 +- pandoc=2.0.5=0 +- pandocfilters=1.4.1=py35_0 +- pexpect=4.3.1=py35_0 +- pickleshare=0.7.4=py35_0 +- pip=9.0.1=py35_1 +- prompt_toolkit=1.0.15=py35_0 +- ptyprocess=0.5.2=py35_0 +- pygments=2.2.0=py35_0 +- python=3.5.4=3 +- python-dateutil=2.6.1=py35_0 +- pyzmq=16.0.2=py35_2 +- readline=7.0=0 +- setuptools=38.2.4=py35_0 +- simplegeneric=0.8.1=py35_0 +- six=1.11.0=py35_1 +- sqlite=3.20.1=2 +- terminado=0.8.1=py35_0 +- testpath=0.3.1=py35_0 +- tk=8.6.7=0 +- tornado=4.5.2=py35_0 +- traitlets=4.3.2=py35_0 +- wcwidth=0.1.7=py35_0 +- webencodings=0.5=py35_0 +- wheel=0.30.0=py_1 +- widgetsnbextension=2.0.1=py35_0 +- xz=5.2.3=0 +- zeromq=4.2.1=1 +- zlib=1.2.11=0 +prefix: /opt/conda/envs/r2d + diff --git a/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml b/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml new file mode 100644 index 00000000..19338f96 --- /dev/null +++ b/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml @@ -0,0 +1,63 @@ +# AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY +# Frozen on 2018-01-03 15:33:55 UTC +name: r2d +channels: +- conda-forge +- defaults +dependencies: +- bleach=2.0.0=py36_0 +- ca-certificates=2017.11.5=0 +- certifi=2017.11.5=py36_0 +- decorator=4.1.2=py36_0 +- entrypoints=0.2.3=py36_1 +- gmp=6.1.2=0 +- html5lib=1.0.1=py_0 +- ipykernel=4.7.0=py36_0 +- ipython=6.2.1=py36_1 +- ipython_genutils=0.2.0=py36_0 +- ipywidgets=6.0.1=py36_0 +- jedi=0.10.2=py36_0 +- jinja2=2.10=py36_0 +- jsonschema=2.6.0=py36_0 +- jupyter_client=5.2.0=py36_0 +- jupyter_core=4.4.0=py_0 +- jupyterlab=0.30.6=py36_0 +- jupyterlab_launcher=0.6.0=py36_0 +- libsodium=1.0.15=1 +- markupsafe=1.0=py36_0 +- mistune=0.8.3=py_0 +- nbconvert=5.3.1=py_1 +- nbformat=4.4.0=py36_0 +- ncurses=5.9=10 +- notebook=5.2.2=py36_1 +- openssl=1.0.2n=0 +- pandoc=2.0.5=0 +- pandocfilters=1.4.1=py36_0 +- pexpect=4.3.1=py36_0 +- pickleshare=0.7.4=py36_0 +- pip=9.0.1=py36_1 +- prompt_toolkit=1.0.15=py36_0 +- ptyprocess=0.5.2=py36_0 +- pygments=2.2.0=py36_0 +- python=3.6.4=0 +- python-dateutil=2.6.1=py36_0 +- pyzmq=16.0.2=py36_2 +- readline=7.0=0 +- setuptools=38.2.4=py36_0 +- simplegeneric=0.8.1=py36_0 +- six=1.11.0=py36_1 +- sqlite=3.20.1=2 +- terminado=0.8.1=py36_0 +- testpath=0.3.1=py36_0 +- tk=8.6.7=0 +- tornado=4.5.2=py36_0 +- traitlets=4.3.2=py36_0 +- wcwidth=0.1.7=py36_0 +- webencodings=0.5=py36_0 +- wheel=0.30.0=py_1 +- widgetsnbextension=2.0.1=py36_0 +- xz=5.2.3=0 +- zeromq=4.2.1=1 +- zlib=1.2.11=0 +prefix: /opt/conda/envs/r2d + diff --git a/repo2docker/buildpacks/conda/environment.yml b/repo2docker/buildpacks/conda/environment.yml index fec62aa0..3b8f3b12 100644 --- a/repo2docker/buildpacks/conda/environment.yml +++ b/repo2docker/buildpacks/conda/environment.yml @@ -1,5 +1,5 @@ dependencies: - - python==3.6.1 + - python==3.6.* - ipywidgets==6.0.1 - jupyterlab==0.30.6 - notebook==5.2.2 diff --git a/repo2docker/buildpacks/conda/freeze.py b/repo2docker/buildpacks/conda/freeze.py index 75d503f9..6302120e 100644 --- a/repo2docker/buildpacks/conda/freeze.py +++ b/repo2docker/buildpacks/conda/freeze.py @@ -8,6 +8,7 @@ Run in a continuumio/miniconda3 image to ensure portability from datetime import datetime import os import pathlib +import shutil from subprocess import check_call from ruamel.yaml import YAML @@ -18,7 +19,10 @@ MINICONDA_VERSION = '4.3.27' HERE = pathlib.Path(os.path.dirname(os.path.abspath(__file__))) ENV_FILE = 'environment.yml' -FROZEN_FILE = 'environment.frozen.yml' +FROZEN_FILE = os.path.splitext(ENV_FILE)[0] + '.frozen.yml' + +ENV_FILE_T = 'environment.py-{py}.yml' +FROZEN_FILE_T = os.path.splitext(ENV_FILE_T)[0] + '.frozen.yml' yaml = YAML(typ='rt') @@ -78,11 +82,40 @@ def freeze(env_file, frozen_file): 'conda config --add channels conda-forge', 'conda config --system --set auto_update_conda false', f"conda env create -v -f /r2d/{env_file} -n r2d", - f"conda env export -n r2d > /r2d/{frozen_file}", + f"conda env export -n r2d >> /r2d/{frozen_file}", ]) ]) fixup(HERE / frozen_file) +def set_python(py_env_file, py): + """Set the Python version in an env file""" + if os.path.exists(py_env_file): + # only clobber auto-generated files + with open(py_env_file) as f: + text = f.read() + if text and 'GENERATED' not in text: + return + with open(ENV_FILE) as f: + env = yaml.load(f) + for idx, dep in enumerate(env['dependencies']): + if dep.split('=')[0] == 'python': + env['dependencies'][idx] = f'python={py}.*' + break + else: + raise ValueError(f"python dependency not found in {env['dependencies']}") + # update python dependency + with open(py_env_file, 'w') as f: + f.write(f"# AUTO GENERATED FROM {ENV_FILE}, DO NOT MANUALLY MODIFY\n") + f.write(f"# Generated on {datetime.utcnow():%Y-%m-%d %H:%M:%S UTC}\n") + yaml.dump(env, f) + + if __name__ == '__main__': - freeze(ENV_FILE, FROZEN_FILE) + for py in ('2.7', '3.5', '3.6'): + env_file = ENV_FILE_T.format(py=py) + set_python(env_file, py) + frozen_file = os.path.splitext(env_file)[0] + '.frozen.yml' + freeze(env_file, frozen_file) + # use last version as default + shutil.copy(frozen_file, FROZEN_FILE)