Merge pull request #156 from yuvipanda/frozen-base

Use a frozen requirements.txt for base install in venv buildpacks

MANIFEST.in

@@ -1,4 +1,4 @@
 include *.md
 include LICENSE
 include setup.cfg
-recursive-include repo2docker/files *
+recursive-include repo2docker/buildpacks *

repo2docker/buildpacks/base.py

@@ -381,7 +381,7 @@ class BuildPack(LoggingConfigurable):
 
         for src in sorted(self.build_script_files):
             src_parts = src.split('/')
-            src_path = os.path.join(os.path.dirname(__file__), '..', 'files', *src_parts)
+            src_path = os.path.join(os.path.dirname(__file__), *src_parts)
             tar.add(src_path, src, filter=_filter_tar)
 
         tar.add('.', 'src/', filter=_filter_tar)

repo2docker/buildpacks/conda/__init__.py

@@ -3,7 +3,7 @@ Generates a variety of Dockerfiles based on an input matrix
 """
 from traitlets import default
 import os
-from .base import BuildPack
+from ..base import BuildPack
 
 
 class CondaBuildPack(BuildPack):

repo2docker/buildpacks/python/__init__.py

@@ -3,7 +3,7 @@ Generates a variety of Dockerfiles based on an input matrix
 """
 from traitlets import default
 import os
-from .base import BuildPack
+from ..base import BuildPack
 
 
 class PythonBuildPack(BuildPack):
@@ -26,6 +26,11 @@ class PythonBuildPack(BuildPack):
         "${VENV_PATH}/bin"
     ]
 
+
+    build_script_files = {
+        'python/requirements.frozen.txt': '/tmp/requirements.frozen.txt',
+    }
+
     build_scripts = [
         (
             "root",
@@ -43,10 +48,7 @@ class PythonBuildPack(BuildPack):
         (
             "${NB_USER}",
             r"""
-            pip install --no-cache-dir \
-                notebook==5.2.2 \
-                ipywidgets==6.0.0 \
-                jupyterlab==0.28 && \
+            pip install --no-cache-dir -r /tmp/requirements.frozen.txt && \
             jupyter nbextension enable --py widgetsnbextension --sys-prefix && \
             jupyter serverextension enable --py jupyterlab --sys-prefix
             """
@@ -92,6 +94,10 @@ class Python2BuildPack(BuildPack):
         'virtualenv'
     }
 
+    build_script_files = {
+        'python/requirements2.frozen.txt': '/tmp/requirements2.frozen.txt',
+    }
+
     env = [
         ('VENV2_PATH', '${APP_BASE}/venv2')
     ]
@@ -117,8 +123,7 @@ class Python2BuildPack(BuildPack):
         (
             "${NB_USER}",
             r"""
-            pip2 install --no-cache-dir \
-                 ipykernel==4.6.1 && \
+            pip2 install --no-cache-dir -r /tmp/requirements2.frozen.txt && \
             python2 -m ipykernel install --prefix=${NB_PYTHON_PREFIX}
             """
         )

repo2docker/buildpacks/python/freeze.bash

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -euo pipefail
+# Freeze requirements.txt into requirements.frozen.txt, pinning all dependent library versions to
+# versions that are resolved at time of freezing.
+# Does the same for requirements2.txt to requirements2.frozen.txt...
+
+# cd to the directory where the freeze script is located
+dirname "$(readlink -f "$0")"
+
+function freeze-requirements {
+    # Freeze a requirements file $2 into a frozen requirements file $3
+    # Requires that a completely empty venv of appropriate version exist in $1
+    VENV_PATH="$1"
+    REQUIREMENTS_FILE="$2"
+    FROZEN_FILE="$3"
+
+    ./${VENV_PATH}/bin/pip install --no-cache-dir -r ${REQUIREMENTS_FILE}
+    echo "# AUTO GENERATED FROM ${REQUIREMENTS_FILE}, DO NOT MANUALLY MODIFY" > ${FROZEN_FILE}
+    echo "# Frozen on $(date -u)" >> ${FROZEN_FILE}
+    ./${VENV_PATH}/bin/pip freeze | sort >> ${FROZEN_FILE}
+}
+
+rm -rf py3venv
+python3 -m venv py3venv
+freeze-requirements py3venv requirements.txt requirements.frozen.txt
+rm -rf py3venv
+
+
+rm -rf py2venv
+virtualenv -p python2 py2venv
+freeze-requirements py2venv requirements2.txt requirements2.frozen.txt
+rm -rf py2venv

repo2docker/buildpacks/python/requirements.frozen.txt

@@ -0,0 +1,41 @@
+# AUTO GENERATED FROM requirements.txt, DO NOT MANUALLY MODIFY
+# Frozen on Thu Nov 30 11:49:16 UTC 2017
+bleach==2.1.1
+decorator==4.1.2
+entrypoints==0.2.3
+html5lib==1.0b10
+ipykernel==4.6.1
+ipython==6.2.1
+ipython-genutils==0.2.0
+ipywidgets==6.0.0
+jedi==0.11.0
+Jinja2==2.10
+jsonschema==2.6.0
+jupyter-client==5.1.0
+jupyter-core==4.4.0
+jupyterlab==0.29.2
+jupyterlab-launcher==0.5.5
+MarkupSafe==1.0
+mistune==0.8.1
+nbconvert==5.3.1
+nbformat==4.4.0
+notebook==5.2.2
+pandocfilters==1.4.2
+parso==0.1.0
+pexpect==4.3.0
+pickleshare==0.7.4
+pkg-resources==0.0.0
+prompt-toolkit==1.0.15
+ptyprocess==0.5.2
+Pygments==2.2.0
+python-dateutil==2.6.1
+pyzmq==16.0.3
+simplegeneric==0.8.1
+six==1.11.0
+terminado==0.8.1
+testpath==0.3.1
+tornado==4.5.2
+traitlets==4.3.2
+wcwidth==0.1.7
+webencodings==0.5.1
+widgetsnbextension==2.0.1

repo2docker/buildpacks/python/requirements.txt

@@ -0,0 +1,3 @@
+notebook==5.2.2
+ipywidgets==6.0.0
+jupyterlab==0.29.2

repo2docker/buildpacks/python/requirements2.frozen.txt

@@ -0,0 +1,28 @@
+# AUTO GENERATED FROM requirements2.txt, DO NOT MANUALLY MODIFY
+# Frozen on Thu Nov 30 11:49:30 UTC 2017
+backports-abc==0.5
+backports.shutil-get-terminal-size==1.0.0
+certifi==2017.11.5
+decorator==4.1.2
+enum34==1.1.6
+ipykernel==4.6.1
+ipython==5.5.0
+ipython-genutils==0.2.0
+jupyter-client==5.1.0
+jupyter-core==4.4.0
+pathlib2==2.3.0
+pexpect==4.3.0
+pickleshare==0.7.4
+pkg-resources==0.0.0
+prompt-toolkit==1.0.15
+ptyprocess==0.5.2
+Pygments==2.2.0
+python-dateutil==2.6.1
+pyzmq==16.0.3
+scandir==1.6
+simplegeneric==0.8.1
+singledispatch==3.4.0.3
+six==1.11.0
+tornado==4.5.2
+traitlets==4.3.2
+wcwidth==0.1.7

repo2docker/buildpacks/python/requirements2.txt

@@ -0,0 +1 @@
+ipykernel==4.6.1