Merge pull request #298 from minrk/always-conda-python

Always install Python with conda

repo2docker/app.py

@@ -29,7 +29,7 @@ from traitlets.config import Application
 from . import __version__
 from .buildpacks import (
     PythonBuildPack, DockerBuildPack, LegacyBinderDockerBuildPack,
-    CondaBuildPack, JuliaBuildPack, Python2BuildPack, BaseImage,
+    CondaBuildPack, JuliaBuildPack, BaseImage,
     RBuildPack
 )
 from .utils import (
@@ -67,7 +67,6 @@ class Repo2Docker(Application):
             DockerBuildPack,
             JuliaBuildPack,
             CondaBuildPack,
-            Python2BuildPack,
             RBuildPack,
             PythonBuildPack,
         ],

repo2docker/buildpacks/__init__.py

@@ -1,5 +1,5 @@
 from .base import BuildPack, BaseImage
-from .python import Python2BuildPack, PythonBuildPack
+from .python import PythonBuildPack
 from .conda import CondaBuildPack
 from .julia import JuliaBuildPack
 from .docker import DockerBuildPack

repo2docker/buildpacks/conda/__init__.py

@@ -25,17 +25,26 @@ class CondaBuildPack(BaseImage):
         the `NB_PYTHON_PREFIX` to the location of the jupyter binary.
 
         """
-        return super().get_env() + [
+        env = super().get_env() + [
             ('CONDA_DIR', '${APP_BASE}/conda'),
             ('NB_PYTHON_PREFIX', '${CONDA_DIR}'),
         ]
+        if self.py2:
+            env.append(('KERNEL_PYTHON_PREFIX', '${CONDA_DIR}/envs/kernel'))
+        else:
+            env.append(('KERNEL_PYTHON_PREFIX', '${NB_PYTHON_PREFIX}'))
+        return env
 
     def get_path(self):
         """Return paths (including conda environment path) to be added to
         the PATH environment variable.
 
         """
-        return super().get_path() + ['${CONDA_DIR}/bin']
+        path = super().get_path()
+        if self.py2:
+            path.insert(0, '${KERNEL_PYTHON_PREFIX}/bin')
+        path.insert(0, '${CONDA_DIR}/bin')
+        return path
 
     def get_build_scripts(self):
         """

repo2docker/buildpacks/python/__init__.py

@@ -1,139 +1,70 @@
 """Generates Dockerfiles based on an input matrix based on Python."""
 import os
 
-from ..base import BaseImage
+from ..conda import CondaBuildPack
 
 
-class PythonBuildPack(BaseImage):
+class PythonBuildPack(CondaBuildPack):
-    """Setup Python 3 for use with a repository."""
+    """Setup Python for use with a repository."""
-    def get_packages(self):
-        """Return a list of the Python 3 core language packages to be installed
-           via apt-get for this BuildPack.
 
-           Note: The packages specified here are for the core Python3 language.
+    @property
-           Third party libraries are specified in other configuration files.
+    def python_version(self):
+        if hasattr(self, '_python_version'):
+            return self._python_version
 
-        """
+        try:
-        return super().get_packages().union({
+            with open(self.binder_path('runtime.txt')) as f:
-            'python3',
+                runtime = f.read().strip()
-            'python3-venv',
+        except FileNotFoundError:
-            'python3-dev',
+            runtime = ''
-        })
 
-    def get_env(self):
+        if not runtime.startswith('python-'):
-        """
+            # not a Python runtime (e.g. R, which subclasses this)
-        Return environment variables to be set.
+            # use the default Python
+            self._python_version = self.major_pythons['3']
+            return self._python_version
 
-        We set `VENV_PATH` to the virtual environment location and
+        py_version_info = runtime.split('-', 1)[1].split('.')
-        the `NB_PYTHON_PREFIX` to the location of the jupyter binary.
+        py_version = ''
-
+        if len(py_version_info) == 1:
-        """
+            py_version = self.major_pythons[py_version_info[0]]
-        return super().get_env() + [
+        else:
-            ("VENV_PATH", "${APP_BASE}/venv"),
+            # get major.minor
-            # Prefix to use for installing kernels and finding jupyter binary
+            py_version = '.'.join(py_version_info[:2])
-            ("NB_PYTHON_PREFIX", "${VENV_PATH}"),
+        self._python_version = py_version
-        ]
+        return self._python_version
-
-    def get_path(self):
-        """Return paths (including virtual environment path) to be added to
-        the PATH environment variable.
-
-        """
-        return super().get_path() + [
-            "${VENV_PATH}/bin"
-        ]
-
-    def get_build_script_files(self):
-        """
-        Dict of files to be copied to the container image for use in building.
-
-        This is copied before the `build_scripts` & `assemble_scripts` are
-        run, so can be executed from either of them.
-
-        It's a dictionary where the key is the source file path in the host
-        system, and the value is the destination file path inside the
-        container image.
-
-        This currently adds a frozen set of Python 3 requirements to the dict
-        of files.
-
-        """
-        files = {
-            'python/requirements.frozen.txt': '/tmp/requirements.frozen.txt',
-        }
-        files.update(super().get_build_script_files())
-        return files
-
-    def get_build_scripts(self):
-        """
-        Return series of build-steps common to all Python 3 repositories.
-
-        All scripts here should be independent of contents of the repository.
-
-        This sets up:
-
-        - a directory for the virtual environment and its ownership by the
-          notebook user
-        - a Python 3 interpreter for the virtual environement
-        - a Python 3 jupyter kernel including a base set of requirements
-        - support for Jupyter widgets
-        - support for JupyterLab
-        - support for nteract
-
-        """
-        return super().get_build_scripts() + [
-            (
-                "root",
-                r"""
-                mkdir -p ${VENV_PATH} && \
-                chown -R ${NB_USER}:${NB_USER} ${VENV_PATH}
-                """
-            ),
-            (
-                "${NB_USER}",
-                r"""
-                python3 -m venv ${VENV_PATH}
-                """
-            ),
-            (
-                "${NB_USER}",
-                r"""
-                pip install --no-cache-dir -r /tmp/requirements.frozen.txt && \
-                jupyter nbextension enable --py widgetsnbextension --sys-prefix && \
-                jupyter serverextension enable --py jupyterlab --sys-prefix && \
-                jupyter serverextension enable nteract_on_jupyter --sys-prefix
-                """
-            )
-        ]
 
     def get_assemble_scripts(self):
         """Return series of build-steps specific to this repository.
         """
         # If we have a runtime.txt & that's set to python-2.7,
-        # we will *not* install requirements.txt but will find &
+        # requirements.txt will be installed in the *kernel* env
-        # install a requirements3.txt file if it exists.
+        # and requirements3.txt (if it exists)
-        # This way, when using python2 venv, requirements.txt will
+        # will be installed in the python 3 notebook server env.
-        # be installed in the python2 venv, and requirements3.txt
-        # will be installed in python3 venv. This is less of a
-        # surprise than requiring python2 to be requirements2.txt tho.
         assemble_scripts = super().get_assemble_scripts()
         setup_py = 'setup.py'
-        try:
+        # KERNEL_PYTHON_PREFIX is the env with the kernel,
-            with open(self.binder_path('runtime.txt')) as f:
+        # whether it's distinct from the notebook or the same.
-                runtime = f.read().strip()
+        pip = '${KERNEL_PYTHON_PREFIX}/bin/pip'
-        except FileNotFoundError:
+        if self.py2:
-            runtime = 'python-3.5'
+            # using python 2 kernel,
-        if runtime == 'python-2.7':
+            # requirements3.txt allows installation in the notebook server env
-            pip = "pip2"
+            nb_requirements_file = self.binder_path('requirements3.txt')
-            requirements_file = self.binder_path('requirements3.txt')
+            if os.path.exists(nb_requirements_file):
-        else:
+                assemble_scripts.append((
-            pip = "pip3"
+                    '${NB_USER}',
+                    '${NB_PYTHON_PREFIX}/bin/pip install --no-cache-dir -r "{}"'.format(nb_requirements_file)
+                ))
+
+        # install requirements.txt in the kernel env
         requirements_file = self.binder_path('requirements.txt')
         if os.path.exists(requirements_file):
             assemble_scripts.append((
                 '${NB_USER}',
-                'pip3 install --no-cache-dir -r "{}"'.format(requirements_file)
+                '{} install --no-cache-dir -r "{}"'.format(pip, requirements_file)
             ))
+
+        # setup.py exists *and* binder dir is not used
         if not os.path.exists('binder') and os.path.exists(setup_py):
             assemble_scripts.append((
                 '${NB_USER}',
@@ -142,7 +73,7 @@ class PythonBuildPack(BaseImage):
         return assemble_scripts
 
     def detect(self):
-        """Check if current repo should be built with the Python 3 Build pack.
+        """Check if current repo should be built with the Python buildpack.
         """
         requirements_txt = self.binder_path('requirements.txt')
         runtime_txt = self.binder_path('runtime.txt')
@@ -151,136 +82,10 @@ class PythonBuildPack(BaseImage):
         if os.path.exists(runtime_txt):
             with open(runtime_txt) as f:
                 runtime = f.read().strip()
-            if runtime.startswith("python-3"):
+            if runtime.startswith("python-"):
                 return True
             else:
                 return False
         if not os.path.exists('binder') and os.path.exists(setup_py):
             return True
         return os.path.exists(requirements_txt)
-
-
-class Python2BuildPack(PythonBuildPack):
-    """Setup Python 2 for use with a repository."""
-    def get_packages(self):
-        """Return a list of the Python 2 core language packages to be installed
-           via apt-get for this BuildPack.
-
-           Note: The packages specified here are for the core Python2 language.
-           Third party libraries are specified in other configuration files.
-
-        """
-        return super().get_packages().union({
-            'python',
-            'python-dev',
-            'virtualenv'
-        })
-
-    def get_env(self):
-        """
-        Return environment variables to be set.
-
-        We set `VENV_PATH` to the virtual environment location containing
-        Python 2.
-
-        """
-        return super().get_env() + [
-            ('VENV2_PATH', '${APP_BASE}/venv2')
-        ]
-
-    def get_path(self):
-        """Return paths (including virtual environment path) to be added to
-        the PATH environment variable.
-
-        """
-        return super().get_path() + [
-            "${VENV2_PATH}/bin"
-        ]
-
-    def get_build_script_files(self):
-        """
-        Dict of files to be copied to the container image for use in building.
-
-        This is copied before the `build_scripts` & `assemble_scripts` are
-        run, so can be executed from either of them.
-
-        It's a dictionary where the key is the source file path in the host
-        system, and the value is the destination file path inside the
-        container image.
-
-        This currently adds a frozen set of Python 2 requirements to the dict
-        of files.
-
-        """
-        files = {
-            'python/requirements2.frozen.txt': '/tmp/requirements2.frozen.txt',
-        }
-        files.update(super().get_build_script_files())
-        return files
-
-    def get_build_scripts(self):
-        """
-        Return series of build-steps common to all Python 2 repositories.
-
-        All scripts here should be independent of contents of the repository.
-
-        This sets up:
-
-        - a directory for the virtual environment and its ownership by the
-          notebook user
-        - a Python 2 interpreter for the virtual environement
-        - a Python 2 jupyter kernel
-
-        """
-        return super().get_build_scripts() + [
-            (
-                "root",
-                r"""
-                mkdir -p ${VENV2_PATH} && \
-                chown -R ${NB_USER}:${NB_USER} ${VENV2_PATH}
-                """
-            ),
-            (
-                "${NB_USER}",
-                r"""
-                virtualenv -p python2 ${VENV2_PATH}
-                """
-            ),
-            (
-                "${NB_USER}",
-                r"""
-                pip2 install --no-cache-dir -r /tmp/requirements2.frozen.txt && \
-                python2 -m ipykernel install --prefix=${NB_PYTHON_PREFIX}
-                """
-            )
-        ]
-
-    def get_assemble_scripts(self):
-        """Return series of build-steps specific to this repository.
-        """
-        requirements_txt = self.binder_path('requirements.txt')
-        assemble_scripts = super().get_assemble_scripts()
-        if os.path.exists(requirements_txt):
-            assemble_scripts.insert(0, (
-                '${NB_USER}',
-                'pip2 install --no-cache-dir -r "{}"'.format(requirements_txt)
-            ))
-        return assemble_scripts
-
-    def detect(self):
-        """Check if current repo should be built with the Python 2 Build pack.
-        """
-        runtime_txt = self.binder_path('runtime.txt')
-
-        if os.path.exists(runtime_txt):
-            with open(runtime_txt) as f:
-                runtime = f.read().strip()
-            if runtime == 'python-2.7':
-                return True
-            elif runtime.startswith('python-2'):
-                raise ValueError(
-                    "Only python-2.7 or python-3.x is supported in "
-                    "runtime.txt, not '{}'".format(runtime_txt))
-            else:
-                return False
-        return False

repo2docker/buildpacks/python/freeze.bash

@@ -1,39 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-# Freeze requirements.txt into requirements.frozen.txt, pinning all dependent library versions to
-# versions that are resolved at time of freezing.
-# Does the same for requirements2.txt to requirements2.frozen.txt...
-
-# cd to the directory where the freeze script is located
-if [[ ! -z "$(which realpath 2>/dev/null)" ]]; then
-    realpath=realpath
-else
-    realpath="readlink -f"
-fi
-
-cd $(dirname "$($realpath "$0")")
-
-
-function freeze-requirements {
-    # Freeze a requirements file $2 into a frozen requirements file $3
-    # Requires that a completely empty venv of appropriate version exist in $1
-    PYTHON_VERSION="$1"
-    REQUIREMENTS_FILE="$2"
-    FROZEN_FILE="$3"
-    if [[ $(echo ${PYTHON_VERSION} | cut -d. -f 1) == "2" ]]; then
-        VENV=virtualenv
-    else
-        VENV=venv
-    fi
-
-    echo "# AUTO GENERATED FROM ${REQUIREMENTS_FILE}, DO NOT MANUALLY MODIFY" > ${FROZEN_FILE}
-    echo "# Frozen on $(date -u)" >> ${FROZEN_FILE}
-    docker run --rm -v $PWD:/python -it python:${PYTHON_VERSION} \
-        sh -c "
-            python -m $VENV /venv
-            /venv/bin/pip install -r /python/${REQUIREMENTS_FILE} &&
-            /venv/bin/pip freeze | sort --ignore-case >> /python/${FROZEN_FILE}"
-}
-
-freeze-requirements 3.5 requirements.txt requirements.frozen.txt
-freeze-requirements 2.7 requirements2.txt requirements2.frozen.txt

repo2docker/buildpacks/python/requirements.frozen.txt

@@ -1,42 +0,0 @@
-# AUTO GENERATED FROM requirements.txt, DO NOT MANUALLY MODIFY
-# Frozen on Mon Apr  2 07:17:48 UTC 2018
-bleach==2.1.3
-decorator==4.2.1
-entrypoints==0.2.3
-html5lib==1.0.1
-ipykernel==4.8.2
-ipython-genutils==0.2.0
-ipython==6.2.1
-ipywidgets==7.1.1
-jedi==0.11.1
-Jinja2==2.10
-jsonschema==2.6.0
-jupyter-client==5.2.3
-jupyter-core==4.4.0
-jupyterlab-launcher==0.10.5
-jupyterlab==0.31.5
-MarkupSafe==1.0
-mistune==0.8.3
-nbconvert==5.3.1
-nbformat==4.4.0
-notebook==5.4.1
-nteract-on-jupyter==1.6.0
-pandocfilters==1.4.2
-parso==0.1.1
-pexpect==4.4.0
-pickleshare==0.7.4
-prompt-toolkit==1.0.15
-ptyprocess==0.5.2
-Pygments==2.2.0
-python-dateutil==2.7.2
-pyzmq==17.0.0
-Send2Trash==1.5.0
-simplegeneric==0.8.1
-six==1.11.0
-terminado==0.8.1
-testpath==0.3.1
-tornado==4.5.3
-traitlets==4.3.2
-wcwidth==0.1.7
-webencodings==0.5.1
-widgetsnbextension==3.1.4

repo2docker/buildpacks/python/requirements.txt

@@ -1,5 +0,0 @@
-notebook==5.4.1
-tornado==4.5.3
-ipywidgets==7.1.1
-jupyterlab==0.31.5
-nteract_on_jupyter==1.6.0

repo2docker/buildpacks/python/requirements2.frozen.txt

@@ -1,27 +0,0 @@
-# AUTO GENERATED FROM requirements2.txt, DO NOT MANUALLY MODIFY
-# Frozen on Mon Apr  2 07:19:00 UTC 2018
-backports-abc==0.5
-backports.shutil-get-terminal-size==1.0.0
-certifi==2018.1.18
-decorator==4.2.1
-enum34==1.1.6
-ipykernel==4.8.2
-ipython-genutils==0.2.0
-ipython==5.5.0
-jupyter-client==5.2.3
-jupyter-core==4.4.0
-pathlib2==2.3.0
-pexpect==4.4.0
-pickleshare==0.7.4
-prompt-toolkit==1.0.15
-ptyprocess==0.5.2
-Pygments==2.2.0
-python-dateutil==2.7.2
-pyzmq==17.0.0
-scandir==1.7
-simplegeneric==0.8.1
-singledispatch==3.4.0.3
-six==1.11.0
-tornado==4.5.3
-traitlets==4.3.2
-wcwidth==0.1.7

repo2docker/buildpacks/python/requirements2.txt

@@ -1,2 +0,0 @@
-ipykernel==4.8.2
-tornado==4.5.3

tests/venv/py2/README.rst

@@ -3,3 +3,7 @@ System - Specifying runtime environments
 
 You can specify runtime environments (such as Python 2 or 3) with a
 ``runtime.txt`` file.
+
+You can specify just the major version,
+such as `python-2` or `python-3`,
+or a minor version, such as `python-3.5`.

tests/venv/py3/README.rst

@@ -1,9 +0,0 @@
-System - Specifying runtime environments
-----------------------------------------
-
-You can specify runtime environments (such as Python 2 or 3) with a
-``runtime.txt`` file.
-
-This is an example that selects Python 3. Currently you can not use
-this to select a sepcific version of Python 3 (e.g. 3.4 vs 3.6). If you
-need this level of control we recommend you use a `environment.yml`.

tests/venv/py35/runtime.txt

@@ -0,0 +1 @@
+python-3.5

tests/venv/py35/verify

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+import sys
+
+print(sys.version_info)
+assert sys.version_info[:2] == (3, 5), sys.version