Creates a frozen environment for legacy buildpack.

repo2docker/buildpacks/legacy/__init__.py

@@ -1,8 +1,10 @@
 """
 Generates a variety of Dockerfiles based on an input matrix
 """
+import os
+import shutil
 from textwrap import dedent
-from .docker import DockerBuildPack
+from ..docker import DockerBuildPack
 
 class LegacyBinderDockerBuildPack(DockerBuildPack):
 
@@ -14,13 +16,18 @@ class LegacyBinderDockerBuildPack(DockerBuildPack):
     RUN chown -R main:main /home/main/notebooks
     USER main
     WORKDIR /home/main/notebooks
-    ENV PATH /home/main/anaconda2/envs/python3/bin:$PATH
-    RUN conda install -yq -n python3 notebook==5.0.0 ipykernel==4.6.0 && \
+    # update conda in two steps because the base image
+    # has very old conda that can't upgrade past 4.3
+    RUN conda install -yq conda>=4.3 && \
+        conda install -yq conda==4.4.11 && \
+        conda env update -n python3 -f python3.frozen.yml && \
         conda remove -yq -n python3 nb_conda_kernels && \
-        conda install -yq -n root ipykernel==4.6.0 && \
+        conda env update -n root -f root.frozen.yml && \
         /home/main/anaconda2/envs/python3/bin/ipython kernel install --sys-prefix && \
         /home/main/anaconda2/bin/ipython kernel install --prefix=/home/main/anaconda2/envs/python3 && \
         /home/main/anaconda2/bin/ipython kernel install --sys-prefix
+    RUN rm python3.frozen.yml root.frozen.yml
+    ENV PATH /home/main/anaconda2/envs/python3/bin:$PATH
     ENV JUPYTER_PATH /home/main/anaconda2/share/jupyter:$JUPYTER_PATH
     CMD jupyter notebook --ip 0.0.0.0
     """)
@@ -29,9 +36,22 @@ class LegacyBinderDockerBuildPack(DockerBuildPack):
         with open('Dockerfile') as f:
             return '\n'.join([f.read(), self.legacy_appendix, self.appendix, ''])
 
+    def get_build_script_files(self):
+       return {
+            'legacy/root.frozen.yml': '/tmp/root.frozen.yml',
+            'legacy/python3.frozen.yml': '/tmp/python3.frozen.yml',
+        }
+            
     def build(self, image_spec, memory_limit, build_args):
         with open(self.dockerfile, 'w') as f:
             f.write(self.render())
+        for env in ('root', 'python3'):
+            env_file = env + '.frozen.yml'
+            src_path = os.path.join(
+                os.path.dirname(__file__),
+                env_file,
+            )
+            shutil.copy(src_path, env_file)
         return super().build(image_spec, memory_limit, build_args)
 
     def detect(self):

repo2docker/buildpacks/legacy/freeze.py

@@ -16,25 +16,13 @@ import shutil
 from subprocess import check_call
 import sys
 
-from ruamel.yaml import YAML
-
-
-MINICONDA_VERSION = '4.3.27'
 # need conda ≥ 4.4 to avoid bug adding spurious pip dependencies
-CONDA_VERSION = '4.4.8'
+CONDA_VERSION = '4.4.11'
 
 HERE = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))
 
-ENV_FILE = 'environment.yml'
-FROZEN_FILE = os.path.splitext(ENV_FILE)[0] + '.frozen.yml'
 
-ENV_FILE_T = 'environment.py-{py}.yml'
-FROZEN_FILE_T = os.path.splitext(ENV_FILE_T)[0] + '.frozen.yml'
-
-yaml = YAML(typ='rt')
-
-
-def freeze(env_file, frozen_file):
+def freeze(env_name, env_file, frozen_file):
     """Freeze a conda environment.yml
 
     By running in docker:
@@ -56,54 +44,23 @@ def freeze(env_file, frozen_file):
         '--rm',
         '-v' f"{HERE}:/r2d",
         '-it',
-        f"continuumio/miniconda3:{MINICONDA_VERSION}",
+        f"andrewosh/binder-base",
         "sh", "-c",
         '; '.join([
+            "conda update -yq conda",
             f"conda install -yq conda={CONDA_VERSION}",
-            'conda config --add channels conda-forge',
             'conda config --system --set auto_update_conda false',
-            f"conda env create -v -f /r2d/{env_file} -n r2d",
-            # add conda-forge broken channel as lowest priority in case
-            # any of our frozen packages are marked as broken after freezing
-            'conda config --append channels conda-forge/label/broken',
-            f"conda env export -n r2d >> /r2d/{frozen_file}",
+            f"conda env update -f /r2d/{env_file} -n {env_name}",
+            # exclude conda packages because we don't want to pin them
+            f"conda env export -n {env_name} | grep -v conda >> /r2d/{frozen_file}",
         ])
     ])
 
 
-def set_python(py_env_file, py):
-    """Set the Python version in an env file"""
-    if os.path.exists(py_env_file):
-        # only clobber auto-generated files
-        with open(py_env_file) as f:
-            text = f.read()
-            if text and 'GENERATED' not in text:
-                return
-
-    print(f"Regenerating {py_env_file} from {ENV_FILE}")
-    with open(ENV_FILE) as f:
-        env = yaml.load(f)
-    for idx, dep in enumerate(env['dependencies']):
-        if dep.split('=')[0] == 'python':
-            env['dependencies'][idx] = f'python={py}.*'
-            break
-    else:
-        raise ValueError(f"python dependency not found in {env['dependencies']}")
-    # update python dependency
-    with open(py_env_file, 'w') as f:
-        f.write(f"# AUTO GENERATED FROM {ENV_FILE}, DO NOT MANUALLY MODIFY\n")
-        f.write(f"# Generated on {datetime.utcnow():%Y-%m-%d %H:%M:%S UTC}\n")
-        yaml.dump(env, f)
-
-
 if __name__ == '__main__':
     # allow specifying which Pythons to update on argv
-    pys = sys.argv[1:] or ('2.7', '3.5', '3.6')
-    for py in pys:
-        env_file = ENV_FILE_T.format(py=py)
-        set_python(env_file, py)
+    env_names = sys.argv[1:] or ('root', 'python3')
+    for env_name in env_names:
+        env_file = env_name + ".yml"
         frozen_file = os.path.splitext(env_file)[0] + '.frozen.yml'
-        freeze(env_file, frozen_file)
-
-    # use last version as default
-    shutil.copy(frozen_file, FROZEN_FILE)
+        freeze(env_name, env_file, frozen_file)

repo2docker/buildpacks/legacy/python3.frozen.yml

@@ -0,0 +1,197 @@
+# AUTO GENERATED FROM python3.yml, DO NOT MANUALLY MODIFY
+# Frozen on 2018-03-20 11:41:06 UTC
+name: python3
+channels:
+  - defaults
+dependencies:
+  - _license=1.1=py35_1
+  - _nb_ext_conf=0.3.0=py35_0
+  - alabaster=0.7.9=py35_0
+  - argcomplete=1.0.0=py35_1
+  - astroid=1.4.7=py35_0
+  - astropy=1.2.1=np111py35_0
+  - babel=2.3.4=py35_0
+  - backports=1.0=py35_0
+  - beautifulsoup4=4.5.1=py35_0
+  - bitarray=0.8.1=py35_0
+  - blaze=0.10.1=py35_0
+  - bokeh=0.12.2=py35_0
+  - boto=2.42.0=py35_0
+  - bottleneck=1.1.0=np111py35_0
+  - cairo=1.12.18=6
+  - cffi=1.7.0=py35_0
+  - chest=0.2.3=py35_0
+  - click=6.6=py35_0
+  - cloudpickle=0.2.1=py35_0
+  - clyent=1.2.2=py35_0
+  - colorama=0.3.7=py35_0
+  - configobj=5.0.6=py35_0
+  - contextlib2=0.5.3=py35_0
+  - cryptography=1.5=py35_0
+  - curl=7.49.0=1
+  - cycler=0.10.0=py35_0
+  - cython=0.24.1=py35_0
+  - cytoolz=0.8.0=py35_0
+  - dask=0.11.0=py35_0
+  - datashape=0.5.2=py35_0
+  - dbus=1.10.10=0
+  - decorator=4.0.10=py35_0
+  - dill=0.2.5=py35_0
+  - docutils=0.12=py35_2
+  - dynd-python=0.7.2=py35_0
+  - entrypoints=0.2.2=py35_0
+  - et_xmlfile=1.0.1=py35_0
+  - expat=2.1.0=0
+  - fastcache=1.0.2=py35_1
+  - filelock=2.0.6=py35_0
+  - flask=0.11.1=py35_0
+  - flask-cors=2.1.2=py35_0
+  - fontconfig=2.11.1=6
+  - freetype=2.5.5=1
+  - get_terminal_size=1.0.0=py35_0
+  - gevent=1.1.2=py35_0
+  - glib=2.43.0=1
+  - greenlet=0.4.10=py35_0
+  - gst-plugins-base=1.8.0=0
+  - gstreamer=1.8.0=0
+  - h5py=2.6.0=np111py35_2
+  - harfbuzz=0.9.39=1
+  - hdf5=1.8.17=1
+  - heapdict=1.0.0=py35_1
+  - icu=54.1=0
+  - idna=2.1=py35_0
+  - imagesize=0.7.1=py35_0
+  - ipykernel=4.5.0=py35_0
+  - ipython=5.1.0=py35_0
+  - ipython_genutils=0.1.0=py35_0
+  - ipywidgets=5.2.2=py35_0
+  - itsdangerous=0.24=py35_0
+  - jbig=2.1=0
+  - jdcal=1.2=py35_1
+  - jedi=0.9.0=py35_1
+  - jinja2=2.8=py35_1
+  - jpeg=8d=2
+  - jsonschema=2.5.1=py35_0
+  - jupyter=1.0.0=py35_3
+  - jupyter_client=4.4.0=py35_0
+  - jupyter_console=5.0.0=py35_0
+  - jupyter_core=4.2.0=py35_0
+  - lazy-object-proxy=1.2.1=py35_0
+  - libdynd=0.7.2=0
+  - libffi=3.2.1=0
+  - libgcc=4.8.5=2
+  - libgfortran=3.0.0=1
+  - libpng=1.6.22=0
+  - libsodium=1.0.10=0
+  - libtiff=4.0.6=2
+  - libxcb=1.12=0
+  - libxml2=2.9.2=0
+  - libxslt=1.1.28=0
+  - llvmlite=0.13.0=py35_0
+  - locket=0.2.0=py35_1
+  - lxml=3.6.4=py35_0
+  - markupsafe=0.23=py35_2
+  - matplotlib=1.5.3=np111py35_0
+  - mistune=0.7.3=py35_0
+  - mkl=11.3.3=0
+  - mkl-service=1.1.2=py35_2
+  - mpmath=0.19=py35_1
+  - multipledispatch=0.4.8=py35_0
+  - nbconvert=4.2.0=py35_0
+  - nbformat=4.1.0=py35_0
+  - nbpresent=3.0.2=py35_0
+  - networkx=1.11=py35_0
+  - nltk=3.2.1=py35_0
+  - nose=1.3.7=py35_1
+  - notebook=4.2.3=py35_0
+  - numba=0.28.1=np111py35_0
+  - numexpr=2.6.1=np111py35_0
+  - numpy=1.11.1=py35_0
+  - odo=0.5.0=py35_1
+  - openpyxl=2.3.2=py35_0
+  - openssl=1.0.2j=0
+  - pandas=0.18.1=np111py35_0
+  - partd=0.3.6=py35_0
+  - patchelf=0.9=0
+  - path.py=8.2.1=py35_0
+  - pathlib2=2.1.0=py35_0
+  - patsy=0.4.1=py35_0
+  - pep8=1.7.0=py35_0
+  - pexpect=4.0.1=py35_0
+  - pickleshare=0.7.4=py35_0
+  - pillow=3.3.1=py35_0
+  - pip=8.1.2=py35_0
+  - pixman=0.32.6=0
+  - pkginfo=1.3.2=py35_0
+  - ply=3.9=py35_0
+  - prompt_toolkit=1.0.3=py35_0
+  - psutil=4.3.1=py35_0
+  - ptyprocess=0.5.1=py35_0
+  - py=1.4.31=py35_0
+  - pyasn1=0.1.9=py35_0
+  - pycosat=0.6.1=py35_1
+  - pycparser=2.14=py35_1
+  - pycrypto=2.6.1=py35_4
+  - pycurl=7.43.0=py35_0
+  - pyflakes=1.3.0=py35_0
+  - pygments=2.1.3=py35_0
+  - pylint=1.5.4=py35_1
+  - pyopenssl=16.0.0=py35_0
+  - pyparsing=2.1.4=py35_0
+  - pyqt=5.6.0=py35_0
+  - pytables=3.2.3.1=np111py35_0
+  - pytest=2.9.2=py35_0
+  - python=3.5.2=0
+  - python-dateutil=2.5.3=py35_0
+  - pytz=2016.6.1=py35_0
+  - pyyaml=3.12=py35_0
+  - pyzmq=15.4.0=py35_0
+  - qt=5.6.0=0
+  - qtawesome=0.3.3=py35_0
+  - qtconsole=4.2.1=py35_1
+  - qtpy=1.1.2=py35_0
+  - readline=6.2=2
+  - redis=3.2.0=0
+  - redis-py=2.10.5=py35_0
+  - requests=2.11.1=py35_0
+  - rope=0.9.4=py35_1
+  - scikit-image=0.12.3=np111py35_1
+  - scikit-learn=0.17.1=np111py35_2
+  - scipy=0.18.1=np111py35_0
+  - setuptools=27.2.0=py35_0
+  - simplegeneric=0.8.1=py35_1
+  - singledispatch=3.4.0.3=py35_0
+  - sip=4.18=py35_0
+  - six=1.10.0=py35_0
+  - snowballstemmer=1.2.1=py35_0
+  - sockjs-tornado=1.0.3=py35_0
+  - sphinx=1.4.6=py35_0
+  - spyder=3.0.0=py35_0
+  - sqlalchemy=1.0.13=py35_0
+  - sqlite=3.13.0=0
+  - statsmodels=0.6.1=np111py35_1
+  - sympy=1.0=py35_0
+  - terminado=0.6=py35_0
+  - tk=8.5.18=0
+  - toolz=0.8.0=py35_0
+  - tornado=4.4.1=py35_0
+  - traitlets=4.3.0=py35_0
+  - unicodecsv=0.14.1=py35_0
+  - wcwidth=0.1.7=py35_0
+  - werkzeug=0.11.11=py35_0
+  - wheel=0.29.0=py35_0
+  - widgetsnbextension=1.2.6=py35_0
+  - wrapt=1.10.6=py35_0
+  - xlrd=1.0.0=py35_0
+  - xlsxwriter=0.9.3=py35_0
+  - xlwt=1.1.2=py35_0
+  - xz=5.2.2=0
+  - yaml=0.1.6=0
+  - zeromq=4.1.4=0
+  - zlib=1.2.8=3
+  - pip:
+    - backports.shutil-get-terminal-size==1.0.0
+    - dynd==0.7.3.dev1
+    - rope-py3k==0.9.4.post1
+    - tables==3.2.3.1
+

repo2docker/buildpacks/legacy/python3.yml

@@ -0,0 +1,4 @@
+dependencies:
+  - notebook==5.4.1
+  - ipykernel==4.8.2
+  - tornado<5

repo2docker/buildpacks/legacy/root.frozen.yml

@@ -0,0 +1,183 @@
+# AUTO GENERATED FROM root.yml, DO NOT MANUALLY MODIFY
+# Frozen on 2018-03-20 11:39:45 UTC
+name: root
+channels:
+  - defaults
+dependencies:
+  - alabaster=0.7.7=py27_0
+  - argcomplete=1.0.0=py27_1
+  - astropy=1.1.2=np110py27_0
+  - babel=2.2.0=py27_0
+  - backports_abc=0.4=py27_0
+  - beautifulsoup4=4.4.1=py27_0
+  - bitarray=0.8.1=py27_0
+  - blaze=0.9.1=py27_0
+  - bokeh=0.11.1=py27_0
+  - boto=2.39.0=py27_0
+  - bottleneck=1.0.0=np110py27_0
+  - ca-certificates=2017.08.26=h1d4fec5_0
+  - cairo=1.12.18=6
+  - cdecimal=2.3=py27_0
+  - certifi=2018.1.18=py27_0
+  - cffi=1.5.2=py27_0
+  - chest=0.2.3=py27_0
+  - cloudpickle=0.1.1=py27_0
+  - clyent=1.2.1=py27_0
+  - colorama=0.3.7=py27_0
+  - configobj=5.0.6=py27_0
+  - cryptography=1.4=py27_0
+  - curl=7.45.0=0
+  - cycler=0.10.0=py27_0
+  - cython=0.23.4=py27_0
+  - cytoolz=0.7.5=py27_0
+  - dask=0.8.1=py27_0
+  - datashape=0.5.1=py27_0
+  - decorator=4.0.9=py27_0
+  - dill=0.2.4=py27_0
+  - docutils=0.12=py27_0
+  - dynd-python=0.7.2=py27_0
+  - enum34=1.1.2=py27_0
+  - et_xmlfile=1.0.1=py27_0
+  - fastcache=1.0.2=py27_0
+  - flask=0.10.1=py27_1
+  - flask-cors=2.1.2=py27_0
+  - fontconfig=2.11.1=5
+  - freetype=2.5.5=0
+  - funcsigs=0.4=py27_0
+  - futures=3.0.3=py27_0
+  - gevent=1.1.0=py27_0
+  - greenlet=0.4.9=py27_0
+  - grin=1.2.1=py27_1
+  - h5py=2.5.0=np110py27_4
+  - hdf5=1.8.15.1=2
+  - heapdict=1.0.0=py27_0
+  - idna=2.0=py27_0
+  - ipaddress=1.0.14=py27_0
+  - ipykernel=4.8.2=py27_0
+  - ipython=4.1.2=py27_1
+  - ipython_genutils=0.1.0=py27_0
+  - ipywidgets=4.1.1=py27_0
+  - itsdangerous=0.24=py27_0
+  - jbig=2.1=0
+  - jdcal=1.2=py27_0
+  - jedi=0.9.0=py27_0
+  - jinja2=2.8=py27_0
+  - jpeg=8d=0
+  - jsonschema=2.4.0=py27_0
+  - jupyter=1.0.0=py27_2
+  - jupyter_client=4.2.2=py27_0
+  - jupyter_console=4.1.1=py27_0
+  - jupyter_core=4.1.0=py27_0
+  - libdynd=0.7.2=0
+  - libffi=3.0.13=0
+  - libgcc-ng=7.2.0=hdf63c60_3
+  - libgfortran=3.0=0
+  - libpng=1.6.17=0
+  - libsodium=1.0.3=0
+  - libtiff=4.0.6=1
+  - libxml2=2.9.2=0
+  - libxslt=1.1.28=0
+  - llvmlite=0.9.0=py27_0
+  - locket=0.2.0=py27_0
+  - lxml=3.6.0=py27_0
+  - markupsafe=0.23=py27_0
+  - matplotlib=1.5.1=np110py27_0
+  - mistune=0.7.2=py27_0
+  - mkl=11.3.1=0
+  - mkl-service=1.1.2=py27_0
+  - mpmath=0.19=py27_0
+  - multipledispatch=0.4.8=py27_0
+  - nbconvert=4.1.0=py27_0
+  - nbformat=4.0.1=py27_0
+  - networkx=1.11=py27_0
+  - nltk=3.2=py27_0
+  - nose=1.3.7=py27_0
+  - notebook=4.1.0=py27_1
+  - numba=0.24.0=np110py27_0
+  - numexpr=2.5=np110py27_0
+  - numpy=1.10.4=py27_1
+  - odo=0.4.2=py27_0
+  - openpyxl=2.3.2=py27_0
+  - openssl=1.0.2n=hb7f436b_0
+  - pandas=0.18.0=np110py27_0
+  - partd=0.3.2=py27_1
+  - patchelf=0.8=0
+  - path.py=8.1.2=py27_1
+  - patsy=0.4.0=np110py27_0
+  - pep8=1.7.0=py27_0
+  - pexpect=4.0.1=py27_0
+  - pickleshare=0.5=py27_0
+  - pillow=3.1.1=py27_0
+  - pip=8.1.1=py27_1
+  - pixman=0.32.6=0
+  - ply=3.8=py27_0
+  - psutil=4.1.0=py27_0
+  - ptyprocess=0.5=py27_0
+  - py=1.4.31=py27_0
+  - pyasn1=0.1.9=py27_0
+  - pycairo=1.10.0=py27_0
+  - pycosat=0.6.3=py27ha4109ae_0
+  - pycparser=2.14=py27_0
+  - pycrypto=2.6.1=py27_0
+  - pycurl=7.19.5.3=py27_0
+  - pyflakes=1.1.0=py27_0
+  - pygments=2.1.1=py27_0
+  - pyopenssl=16.2.0=py27_0
+  - pyparsing=2.0.3=py27_0
+  - pyqt=4.11.4=py27_1
+  - pytables=3.2.2=np110py27_1
+  - pytest=2.8.5=py27_0
+  - python=2.7.13=0
+  - python-dateutil=2.5.1=py27_0
+  - pytz=2016.2=py27_0
+  - pyyaml=3.11=py27_1
+  - pyzmq=15.2.0=py27_0
+  - qt=4.8.7=1
+  - qtawesome=0.3.2=py27_0
+  - qtconsole=4.2.0=py27_0
+  - qtpy=1.0=py27_0
+  - readline=6.2=2
+  - redis=2.6.9=0
+  - redis-py=2.10.3=py27_0
+  - requests=2.12.4=py27_0
+  - rope=0.9.4=py27_1
+  - ruamel_yaml=0.11.14=py27_1
+  - scikit-image=0.12.3=np110py27_0
+  - scikit-learn=0.17.1=np110py27_0
+  - scipy=0.17.0=np110py27_2
+  - setuptools=20.3=py27_0
+  - simplegeneric=0.8.1=py27_0
+  - singledispatch=3.4.0.3=py27_0
+  - sip=4.16.9=py27_0
+  - six=1.10.0=py27_0
+  - snowballstemmer=1.2.1=py27_0
+  - sockjs-tornado=1.0.1=py27_0
+  - sphinx=1.3.5=py27_0
+  - sphinx_rtd_theme=0.1.9=py27_0
+  - spyder=2.3.8=py27_1
+  - sqlalchemy=1.0.12=py27_0
+  - sqlite=3.13.0=0
+  - ssl_match_hostname=3.4.0.2=py27_0
+  - statsmodels=0.6.1=np110py27_0
+  - sympy=1.0=py27_0
+  - terminado=0.5=py27_1
+  - tk=8.5.18=0
+  - toolz=0.7.4=py27_0
+  - tornado=4.5.3=py27_0
+  - traitlets=4.2.1=py27_0
+  - unicodecsv=0.14.1=py27_0
+  - util-linux=2.21=0
+  - werkzeug=0.11.4=py27_0
+  - wheel=0.29.0=py27_0
+  - xlrd=0.9.4=py27_0
+  - xlsxwriter=0.8.4=py27_0
+  - xlwt=1.0.0=py27_0
+  - xz=5.0.5=1
+  - yaml=0.1.6=0
+  - zeromq=4.1.3=0
+  - zlib=1.2.8=0
+  - pip:
+    - backports.ssl-match-hostname==3.4.0.2
+    - dynd==0.7.3.dev1
+    - tables==3.2.2
+

repo2docker/buildpacks/legacy/root.yml

@@ -0,0 +1,3 @@
+dependencies:
+  - ipykernel==4.8.2
+  - tornado<5