From b0c67f143cae10a54539a4b9d004728e025c0236 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 10 Apr 2018 15:38:52 +0200 Subject: [PATCH 01/39] add KERNEL_PYTHON_PREFIX for py2 env and put it on PATH --- repo2docker/buildpacks/conda/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/repo2docker/buildpacks/conda/__init__.py b/repo2docker/buildpacks/conda/__init__.py index 4de9638e..e7ec6bf5 100644 --- a/repo2docker/buildpacks/conda/__init__.py +++ b/repo2docker/buildpacks/conda/__init__.py @@ -25,17 +25,24 @@ class CondaBuildPack(BaseImage): the `NB_PYTHON_PREFIX` to the location of the jupyter binary. """ - return super().get_env() + [ + env = super().get_env() + [ ('CONDA_DIR', '${APP_BASE}/conda'), ('NB_PYTHON_PREFIX', '${CONDA_DIR}'), ] + if self.py2: + env.append(('KERNEL_PYTHON_PREFIX', '${CONDA_DIR}/envs/kernel')) + return env def get_path(self): """Return paths (including conda environment path) to be added to the PATH environment variable. """ - return super().get_path() + ['${CONDA_DIR}/bin'] + path = super().get_path() + if self.py2: + path.append('${KERNEL_PYTHON_PREFIX}/bin') + path.append('${CONDA_DIR}/bin') + return path def get_build_scripts(self): """ From 6b15b01fbd712488f99278430bbb00d6b558e819 Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 10 Apr 2018 15:44:03 +0200 Subject: [PATCH 02/39] inherit PythonBuildPack from conda buildpack - sets up base env with conda - installs packages with pip - eliminates duplicate frozen envs for pip - py3/py2 split happens in one class, not separate classes --- repo2docker/app.py | 3 +- repo2docker/buildpacks/__init__.py | 2 +- repo2docker/buildpacks/python/__init__.py | 285 +++--------------- repo2docker/buildpacks/python/freeze.bash | 39 --- .../buildpacks/python/requirements.frozen.txt | 42 --- .../buildpacks/python/requirements.txt | 5 - .../python/requirements2.frozen.txt | 27 -- .../buildpacks/python/requirements2.txt | 2 - 8 files changed, 45 insertions(+), 360 deletions(-) delete mode 100755 repo2docker/buildpacks/python/freeze.bash delete mode 100644 repo2docker/buildpacks/python/requirements.frozen.txt delete mode 100644 repo2docker/buildpacks/python/requirements.txt delete mode 100644 repo2docker/buildpacks/python/requirements2.frozen.txt delete mode 100644 repo2docker/buildpacks/python/requirements2.txt diff --git a/repo2docker/app.py b/repo2docker/app.py index 3e62f5db..40d6fa83 100644 --- a/repo2docker/app.py +++ b/repo2docker/app.py @@ -29,7 +29,7 @@ from traitlets.config import Application from . import __version__ from .buildpacks import ( PythonBuildPack, DockerBuildPack, LegacyBinderDockerBuildPack, - CondaBuildPack, JuliaBuildPack, Python2BuildPack, BaseImage, + CondaBuildPack, JuliaBuildPack, BaseImage, RBuildPack ) from .utils import ( @@ -67,7 +67,6 @@ class Repo2Docker(Application): DockerBuildPack, JuliaBuildPack, CondaBuildPack, - Python2BuildPack, RBuildPack, PythonBuildPack, ], diff --git a/repo2docker/buildpacks/__init__.py b/repo2docker/buildpacks/__init__.py index ae60c05a..5ccc0c0e 100644 --- a/repo2docker/buildpacks/__init__.py +++ b/repo2docker/buildpacks/__init__.py @@ -1,5 +1,5 @@ from .base import BuildPack, BaseImage -from .python import Python2BuildPack, PythonBuildPack +from .python import PythonBuildPack from .conda import CondaBuildPack from .julia import JuliaBuildPack from .docker import DockerBuildPack diff --git a/repo2docker/buildpacks/python/__init__.py b/repo2docker/buildpacks/python/__init__.py index 76a57832..1dcfb4be 100644 --- a/repo2docker/buildpacks/python/__init__.py +++ b/repo2docker/buildpacks/python/__init__.py @@ -1,110 +1,31 @@ """Generates Dockerfiles based on an input matrix based on Python.""" import os -from ..base import BaseImage +from ..conda import CondaBuildPack -class PythonBuildPack(BaseImage): - """Setup Python 3 for use with a repository.""" - def get_packages(self): - """Return a list of the Python 3 core language packages to be installed - via apt-get for this BuildPack. +class PythonBuildPack(CondaBuildPack): + """Setup Python for use with a repository.""" - Note: The packages specified here are for the core Python3 language. - Third party libraries are specified in other configuration files. + @property + def python_version(self): + if hasattr(self, '_python_version'): + return self._python_version - """ - return super().get_packages().union({ - 'python3', - 'python3-venv', - 'python3-dev', - }) - - def get_env(self): - """ - Return environment variables to be set. - - We set `VENV_PATH` to the virtual environment location and - the `NB_PYTHON_PREFIX` to the location of the jupyter binary. - - """ - return super().get_env() + [ - ("VENV_PATH", "${APP_BASE}/venv"), - # Prefix to use for installing kernels and finding jupyter binary - ("NB_PYTHON_PREFIX", "${VENV_PATH}"), - ] - - def get_path(self): - """Return paths (including virtual environment path) to be added to - the PATH environment variable. - - """ - return super().get_path() + [ - "${VENV_PATH}/bin" - ] - - def get_build_script_files(self): - """ - Dict of files to be copied to the container image for use in building. - - This is copied before the `build_scripts` & `assemble_scripts` are - run, so can be executed from either of them. - - It's a dictionary where the key is the source file path in the host - system, and the value is the destination file path inside the - container image. - - This currently adds a frozen set of Python 3 requirements to the dict - of files. - - """ - files = { - 'python/requirements.frozen.txt': '/tmp/requirements.frozen.txt', - } - files.update(super().get_build_script_files()) - return files - - def get_build_scripts(self): - """ - Return series of build-steps common to all Python 3 repositories. - - All scripts here should be independent of contents of the repository. - - This sets up: - - - a directory for the virtual environment and its ownership by the - notebook user - - a Python 3 interpreter for the virtual environement - - a Python 3 jupyter kernel including a base set of requirements - - support for Jupyter widgets - - support for JupyterLab - - support for nteract - - """ - return super().get_build_scripts() + [ - ( - "root", - r""" - mkdir -p ${VENV_PATH} && \ - chown -R ${NB_USER}:${NB_USER} ${VENV_PATH} - """ - ), - ( - "${NB_USER}", - r""" - python3 -m venv ${VENV_PATH} - """ - ), - ( - "${NB_USER}", - r""" - pip install --no-cache-dir -r /tmp/requirements.frozen.txt && \ - jupyter nbextension enable --py widgetsnbextension --sys-prefix && \ - jupyter serverextension enable --py jupyterlab --sys-prefix && \ - jupyter serverextension enable nteract_on_jupyter --sys-prefix - """ - ) - ] + try: + with open(self.binder_path('runtime.txt')) as f: + runtime = f.read().strip() + except FileNotFoundError: + runtime = 'python-' + self.major_pythons['3'] + py_version_info = runtime.split('-', 1)[1].split('.') + py_version = '' + if len(py_version_info) == 1: + py_version = self.major_pythons[py_version_info[0]] + else: + # get major.minor + py_version = '.'.join(py_version_info[:2]) + self._python_version = py_version + return self._python_version def get_assemble_scripts(self): """Return series of build-steps specific to this repository. @@ -112,28 +33,34 @@ class PythonBuildPack(BaseImage): # If we have a runtime.txt & that's set to python-2.7, # we will *not* install requirements.txt but will find & # install a requirements3.txt file if it exists. - # This way, when using python2 venv, requirements.txt will - # be installed in the python2 venv, and requirements3.txt - # will be installed in python3 venv. This is less of a + # This way, when using python2 env, requirements.txt will + # be installed in the python2 env, and requirements3.txt + # will be installed in python3 env. This is less of a # surprise than requiring python2 to be requirements2.txt tho. assemble_scripts = super().get_assemble_scripts() setup_py = 'setup.py' - try: - with open(self.binder_path('runtime.txt')) as f: - runtime = f.read().strip() - except FileNotFoundError: - runtime = 'python-3.5' - if runtime == 'python-2.7': - pip = "pip2" - requirements_file = self.binder_path('requirements3.txt') - else: - pip = "pip3" - requirements_file = self.binder_path('requirements.txt') + pip = 'pip' + if self.py2: + # using python 2 kernel, + # requirements3.txt allows installation in the notebook env + nb_requirements_file = self.binder_path('requirements3.txt') + if os.path.exists(nb_requirements_file): + assemble_scripts.append(( + '${NB_USER}', + 'pip install --no-cache-dir -r "{}"'.format(nb_requirements_file) + )) + # subsequent `pip` calls should run in the kernel env + pip = '${KERNEL_PYTHON_PREFIX}/bin/pip' + + # install requirements.txt in the kernel env + requirements_file = self.binder_path('requirements.txt') if os.path.exists(requirements_file): assemble_scripts.append(( '${NB_USER}', - 'pip3 install --no-cache-dir -r "{}"'.format(requirements_file) + '{} install --no-cache-dir -r "{}"'.format(pip, requirements_file) )) + + # setup.py exists *and* binder dir is not used if not os.path.exists('binder') and os.path.exists(setup_py): assemble_scripts.append(( '${NB_USER}', @@ -151,136 +78,10 @@ class PythonBuildPack(BaseImage): if os.path.exists(runtime_txt): with open(runtime_txt) as f: runtime = f.read().strip() - if runtime.startswith("python-3"): + if runtime.startswith("python-"): return True else: return False if not os.path.exists('binder') and os.path.exists(setup_py): return True return os.path.exists(requirements_txt) - - -class Python2BuildPack(PythonBuildPack): - """Setup Python 2 for use with a repository.""" - def get_packages(self): - """Return a list of the Python 2 core language packages to be installed - via apt-get for this BuildPack. - - Note: The packages specified here are for the core Python2 language. - Third party libraries are specified in other configuration files. - - """ - return super().get_packages().union({ - 'python', - 'python-dev', - 'virtualenv' - }) - - def get_env(self): - """ - Return environment variables to be set. - - We set `VENV_PATH` to the virtual environment location containing - Python 2. - - """ - return super().get_env() + [ - ('VENV2_PATH', '${APP_BASE}/venv2') - ] - - def get_path(self): - """Return paths (including virtual environment path) to be added to - the PATH environment variable. - - """ - return super().get_path() + [ - "${VENV2_PATH}/bin" - ] - - def get_build_script_files(self): - """ - Dict of files to be copied to the container image for use in building. - - This is copied before the `build_scripts` & `assemble_scripts` are - run, so can be executed from either of them. - - It's a dictionary where the key is the source file path in the host - system, and the value is the destination file path inside the - container image. - - This currently adds a frozen set of Python 2 requirements to the dict - of files. - - """ - files = { - 'python/requirements2.frozen.txt': '/tmp/requirements2.frozen.txt', - } - files.update(super().get_build_script_files()) - return files - - def get_build_scripts(self): - """ - Return series of build-steps common to all Python 2 repositories. - - All scripts here should be independent of contents of the repository. - - This sets up: - - - a directory for the virtual environment and its ownership by the - notebook user - - a Python 2 interpreter for the virtual environement - - a Python 2 jupyter kernel - - """ - return super().get_build_scripts() + [ - ( - "root", - r""" - mkdir -p ${VENV2_PATH} && \ - chown -R ${NB_USER}:${NB_USER} ${VENV2_PATH} - """ - ), - ( - "${NB_USER}", - r""" - virtualenv -p python2 ${VENV2_PATH} - """ - ), - ( - "${NB_USER}", - r""" - pip2 install --no-cache-dir -r /tmp/requirements2.frozen.txt && \ - python2 -m ipykernel install --prefix=${NB_PYTHON_PREFIX} - """ - ) - ] - - def get_assemble_scripts(self): - """Return series of build-steps specific to this repository. - """ - requirements_txt = self.binder_path('requirements.txt') - assemble_scripts = super().get_assemble_scripts() - if os.path.exists(requirements_txt): - assemble_scripts.insert(0, ( - '${NB_USER}', - 'pip2 install --no-cache-dir -r "{}"'.format(requirements_txt) - )) - return assemble_scripts - - def detect(self): - """Check if current repo should be built with the Python 2 Build pack. - """ - runtime_txt = self.binder_path('runtime.txt') - - if os.path.exists(runtime_txt): - with open(runtime_txt) as f: - runtime = f.read().strip() - if runtime == 'python-2.7': - return True - elif runtime.startswith('python-2'): - raise ValueError( - "Only python-2.7 or python-3.x is supported in " - "runtime.txt, not '{}'".format(runtime_txt)) - else: - return False - return False diff --git a/repo2docker/buildpacks/python/freeze.bash b/repo2docker/buildpacks/python/freeze.bash deleted file mode 100755 index 54bb30f9..00000000 --- a/repo2docker/buildpacks/python/freeze.bash +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -euo pipefail -# Freeze requirements.txt into requirements.frozen.txt, pinning all dependent library versions to -# versions that are resolved at time of freezing. -# Does the same for requirements2.txt to requirements2.frozen.txt... - -# cd to the directory where the freeze script is located -if [[ ! -z "$(which realpath 2>/dev/null)" ]]; then - realpath=realpath -else - realpath="readlink -f" -fi - -cd $(dirname "$($realpath "$0")") - - -function freeze-requirements { - # Freeze a requirements file $2 into a frozen requirements file $3 - # Requires that a completely empty venv of appropriate version exist in $1 - PYTHON_VERSION="$1" - REQUIREMENTS_FILE="$2" - FROZEN_FILE="$3" - if [[ $(echo ${PYTHON_VERSION} | cut -d. -f 1) == "2" ]]; then - VENV=virtualenv - else - VENV=venv - fi - - echo "# AUTO GENERATED FROM ${REQUIREMENTS_FILE}, DO NOT MANUALLY MODIFY" > ${FROZEN_FILE} - echo "# Frozen on $(date -u)" >> ${FROZEN_FILE} - docker run --rm -v $PWD:/python -it python:${PYTHON_VERSION} \ - sh -c " - python -m $VENV /venv - /venv/bin/pip install -r /python/${REQUIREMENTS_FILE} && - /venv/bin/pip freeze | sort --ignore-case >> /python/${FROZEN_FILE}" -} - -freeze-requirements 3.5 requirements.txt requirements.frozen.txt -freeze-requirements 2.7 requirements2.txt requirements2.frozen.txt diff --git a/repo2docker/buildpacks/python/requirements.frozen.txt b/repo2docker/buildpacks/python/requirements.frozen.txt deleted file mode 100644 index 3ca553fe..00000000 --- a/repo2docker/buildpacks/python/requirements.frozen.txt +++ /dev/null @@ -1,42 +0,0 @@ -# AUTO GENERATED FROM requirements.txt, DO NOT MANUALLY MODIFY -# Frozen on Mon Apr 2 07:17:48 UTC 2018 -bleach==2.1.3 -decorator==4.2.1 -entrypoints==0.2.3 -html5lib==1.0.1 -ipykernel==4.8.2 -ipython-genutils==0.2.0 -ipython==6.2.1 -ipywidgets==7.1.1 -jedi==0.11.1 -Jinja2==2.10 -jsonschema==2.6.0 -jupyter-client==5.2.3 -jupyter-core==4.4.0 -jupyterlab-launcher==0.10.5 -jupyterlab==0.31.5 -MarkupSafe==1.0 -mistune==0.8.3 -nbconvert==5.3.1 -nbformat==4.4.0 -notebook==5.4.1 -nteract-on-jupyter==1.6.0 -pandocfilters==1.4.2 -parso==0.1.1 -pexpect==4.4.0 -pickleshare==0.7.4 -prompt-toolkit==1.0.15 -ptyprocess==0.5.2 -Pygments==2.2.0 -python-dateutil==2.7.2 -pyzmq==17.0.0 -Send2Trash==1.5.0 -simplegeneric==0.8.1 -six==1.11.0 -terminado==0.8.1 -testpath==0.3.1 -tornado==4.5.3 -traitlets==4.3.2 -wcwidth==0.1.7 -webencodings==0.5.1 -widgetsnbextension==3.1.4 diff --git a/repo2docker/buildpacks/python/requirements.txt b/repo2docker/buildpacks/python/requirements.txt deleted file mode 100644 index e0e13a5f..00000000 --- a/repo2docker/buildpacks/python/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -notebook==5.4.1 -tornado==4.5.3 -ipywidgets==7.1.1 -jupyterlab==0.31.5 -nteract_on_jupyter==1.6.0 diff --git a/repo2docker/buildpacks/python/requirements2.frozen.txt b/repo2docker/buildpacks/python/requirements2.frozen.txt deleted file mode 100644 index 670eb05a..00000000 --- a/repo2docker/buildpacks/python/requirements2.frozen.txt +++ /dev/null @@ -1,27 +0,0 @@ -# AUTO GENERATED FROM requirements2.txt, DO NOT MANUALLY MODIFY -# Frozen on Mon Apr 2 07:19:00 UTC 2018 -backports-abc==0.5 -backports.shutil-get-terminal-size==1.0.0 -certifi==2018.1.18 -decorator==4.2.1 -enum34==1.1.6 -ipykernel==4.8.2 -ipython-genutils==0.2.0 -ipython==5.5.0 -jupyter-client==5.2.3 -jupyter-core==4.4.0 -pathlib2==2.3.0 -pexpect==4.4.0 -pickleshare==0.7.4 -prompt-toolkit==1.0.15 -ptyprocess==0.5.2 -Pygments==2.2.0 -python-dateutil==2.7.2 -pyzmq==17.0.0 -scandir==1.7 -simplegeneric==0.8.1 -singledispatch==3.4.0.3 -six==1.11.0 -tornado==4.5.3 -traitlets==4.3.2 -wcwidth==0.1.7 diff --git a/repo2docker/buildpacks/python/requirements2.txt b/repo2docker/buildpacks/python/requirements2.txt deleted file mode 100644 index 7653d953..00000000 --- a/repo2docker/buildpacks/python/requirements2.txt +++ /dev/null @@ -1,2 +0,0 @@ -ipykernel==4.8.2 -tornado==4.5.3 From c70057d8e2d84e6c8a8f0645e6b462789b894bc0 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 16 Apr 2018 10:46:04 +0200 Subject: [PATCH 03/39] verify that Python 3.5 works with venv --- tests/venv/py35/README.rst | 9 +++++++++ tests/venv/py35/runtime.txt | 1 + tests/venv/py35/verify | 5 +++++ 3 files changed, 15 insertions(+) create mode 100644 tests/venv/py35/README.rst create mode 100644 tests/venv/py35/runtime.txt create mode 100755 tests/venv/py35/verify diff --git a/tests/venv/py35/README.rst b/tests/venv/py35/README.rst new file mode 100644 index 00000000..f55f95af --- /dev/null +++ b/tests/venv/py35/README.rst @@ -0,0 +1,9 @@ +System - Specifying runtime environments +---------------------------------------- + +You can specify runtime environments (such as Python 2 or 3) with a +``runtime.txt`` file. + +This is an example that selects Python 3. Currently you can not use +this to select a sepcific version of Python 3 (e.g. 3.4 vs 3.6). If you +need this level of control we recommend you use a `environment.yml`. diff --git a/tests/venv/py35/runtime.txt b/tests/venv/py35/runtime.txt new file mode 100644 index 00000000..5cea6be5 --- /dev/null +++ b/tests/venv/py35/runtime.txt @@ -0,0 +1 @@ +python-3.5 diff --git a/tests/venv/py35/verify b/tests/venv/py35/verify new file mode 100755 index 00000000..4abec309 --- /dev/null +++ b/tests/venv/py35/verify @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 +import sys + +print(sys.version_info) +assert sys.version_info[:2] == (3, 5), sys.version From d6bbed3786e63068a1dcfbf830be3f35995aa7c8 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 16 Apr 2018 10:51:27 +0200 Subject: [PATCH 04/39] kernel python prefix is always defined, even when it's the same as NB_PYTHON_PREFIX --- repo2docker/buildpacks/conda/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/repo2docker/buildpacks/conda/__init__.py b/repo2docker/buildpacks/conda/__init__.py index e7ec6bf5..aba09e47 100644 --- a/repo2docker/buildpacks/conda/__init__.py +++ b/repo2docker/buildpacks/conda/__init__.py @@ -31,6 +31,8 @@ class CondaBuildPack(BaseImage): ] if self.py2: env.append(('KERNEL_PYTHON_PREFIX', '${CONDA_DIR}/envs/kernel')) + else: + env.append(('KERNEL_PYTHON_PREFIX', '${NB_PYTHON_PREFIX}')) return env def get_path(self): From 647d04cbc835498f329c32aa7c92a29ec0f6b7f2 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 16 Apr 2018 10:57:42 +0200 Subject: [PATCH 05/39] Fix detect docstring for Python since it's not just about Python 3 anymore --- repo2docker/buildpacks/python/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo2docker/buildpacks/python/__init__.py b/repo2docker/buildpacks/python/__init__.py index 1dcfb4be..1325ba82 100644 --- a/repo2docker/buildpacks/python/__init__.py +++ b/repo2docker/buildpacks/python/__init__.py @@ -69,7 +69,7 @@ class PythonBuildPack(CondaBuildPack): return assemble_scripts def detect(self): - """Check if current repo should be built with the Python 3 Build pack. + """Check if current repo should be built with the Python buildpack. """ requirements_txt = self.binder_path('requirements.txt') runtime_txt = self.binder_path('runtime.txt') From 593d892e0f2a8b600b5de72b5e9af4a0dca0a5fe Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 16 Apr 2018 16:47:00 +0200 Subject: [PATCH 06/39] readmes in py3 venv tests remove outdated note about not being able to pick a Python 3.x --- tests/venv/py3/README.rst | 5 ++--- tests/venv/py35/README.rst | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/venv/py3/README.rst b/tests/venv/py3/README.rst index f55f95af..904bc951 100644 --- a/tests/venv/py3/README.rst +++ b/tests/venv/py3/README.rst @@ -4,6 +4,5 @@ System - Specifying runtime environments You can specify runtime environments (such as Python 2 or 3) with a ``runtime.txt`` file. -This is an example that selects Python 3. Currently you can not use -this to select a sepcific version of Python 3 (e.g. 3.4 vs 3.6). If you -need this level of control we recommend you use a `environment.yml`. +This is an example that selects Python 3. +If you need this level of control we recommend you use a `environment.yml`. diff --git a/tests/venv/py35/README.rst b/tests/venv/py35/README.rst index f55f95af..816f1398 100644 --- a/tests/venv/py35/README.rst +++ b/tests/venv/py35/README.rst @@ -4,6 +4,5 @@ System - Specifying runtime environments You can specify runtime environments (such as Python 2 or 3) with a ``runtime.txt`` file. -This is an example that selects Python 3. Currently you can not use -this to select a sepcific version of Python 3 (e.g. 3.4 vs 3.6). If you -need this level of control we recommend you use a `environment.yml`. +This is an example that selects Python 3.5. +If you need this level of control we recommend you use a `environment.yml`. From 211a6978c691015dd7ba90fdaaad8c9ab5c31a37 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 16 Apr 2018 16:51:15 +0200 Subject: [PATCH 07/39] clarify comments about requirements3.txt --- repo2docker/buildpacks/python/__init__.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/repo2docker/buildpacks/python/__init__.py b/repo2docker/buildpacks/python/__init__.py index 1325ba82..7ac5808d 100644 --- a/repo2docker/buildpacks/python/__init__.py +++ b/repo2docker/buildpacks/python/__init__.py @@ -31,18 +31,15 @@ class PythonBuildPack(CondaBuildPack): """Return series of build-steps specific to this repository. """ # If we have a runtime.txt & that's set to python-2.7, - # we will *not* install requirements.txt but will find & - # install a requirements3.txt file if it exists. - # This way, when using python2 env, requirements.txt will - # be installed in the python2 env, and requirements3.txt - # will be installed in python3 env. This is less of a - # surprise than requiring python2 to be requirements2.txt tho. + # requirements.txt will be installed in the *kernel* env + # and requirements3.txt (if it exists) + # will be installed in the python 3 notebook server env. assemble_scripts = super().get_assemble_scripts() setup_py = 'setup.py' pip = 'pip' if self.py2: # using python 2 kernel, - # requirements3.txt allows installation in the notebook env + # requirements3.txt allows installation in the notebook server env nb_requirements_file = self.binder_path('requirements3.txt') if os.path.exists(nb_requirements_file): assemble_scripts.append(( From 6d12d2d8076b3027d386f9bb07a0e54e5cf75dd4 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 16 Apr 2018 16:53:34 +0200 Subject: [PATCH 08/39] always use $KERNEL_PYTHON_PREFIX/bin/pip it's always defined and always right for the pip for the kernel env, whether it's separate from the notebook or not. --- repo2docker/buildpacks/python/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/repo2docker/buildpacks/python/__init__.py b/repo2docker/buildpacks/python/__init__.py index 7ac5808d..dad1b046 100644 --- a/repo2docker/buildpacks/python/__init__.py +++ b/repo2docker/buildpacks/python/__init__.py @@ -36,7 +36,9 @@ class PythonBuildPack(CondaBuildPack): # will be installed in the python 3 notebook server env. assemble_scripts = super().get_assemble_scripts() setup_py = 'setup.py' - pip = 'pip' + # KERNEL_PYTHON_PREFIX is the env with the kernel, + # whether it's distinct from the notebook or the same. + pip = '${KERNEL_PYTHON_PREFIX}/bin/pip' if self.py2: # using python 2 kernel, # requirements3.txt allows installation in the notebook server env @@ -44,10 +46,8 @@ class PythonBuildPack(CondaBuildPack): if os.path.exists(nb_requirements_file): assemble_scripts.append(( '${NB_USER}', - 'pip install --no-cache-dir -r "{}"'.format(nb_requirements_file) + '${NB_PYTHON_PREFIX}/bin/pip install --no-cache-dir -r "{}"'.format(nb_requirements_file) )) - # subsequent `pip` calls should run in the kernel env - pip = '${KERNEL_PYTHON_PREFIX}/bin/pip' # install requirements.txt in the kernel env requirements_file = self.binder_path('requirements.txt') From 3b091a5502b6efb7ae7fa33afb365e09766b85f3 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 16 Apr 2018 16:54:12 +0200 Subject: [PATCH 09/39] fix inverted path CONDA_DIR should come first, not last --- repo2docker/buildpacks/conda/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repo2docker/buildpacks/conda/__init__.py b/repo2docker/buildpacks/conda/__init__.py index aba09e47..13ba445c 100644 --- a/repo2docker/buildpacks/conda/__init__.py +++ b/repo2docker/buildpacks/conda/__init__.py @@ -42,8 +42,8 @@ class CondaBuildPack(BaseImage): """ path = super().get_path() if self.py2: - path.append('${KERNEL_PYTHON_PREFIX}/bin') - path.append('${CONDA_DIR}/bin') + path.insert(0, '${KERNEL_PYTHON_PREFIX}/bin') + path.insert(0, '${CONDA_DIR}/bin') return path def get_build_scripts(self): From 69a116a07faaa564faade473fdfcdb744072b36e Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 18 Apr 2018 12:40:04 +0200 Subject: [PATCH 10/39] handle runtime not being Python in Python BuildPack because R buildpack subclasses the Python buildpack, where runtime.txt should be ignored for retrieving the Python version --- repo2docker/buildpacks/python/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/repo2docker/buildpacks/python/__init__.py b/repo2docker/buildpacks/python/__init__.py index dad1b046..8141a30d 100644 --- a/repo2docker/buildpacks/python/__init__.py +++ b/repo2docker/buildpacks/python/__init__.py @@ -16,7 +16,14 @@ class PythonBuildPack(CondaBuildPack): with open(self.binder_path('runtime.txt')) as f: runtime = f.read().strip() except FileNotFoundError: - runtime = 'python-' + self.major_pythons['3'] + runtime = '' + + if not runtime.startswith('python-'): + # not a Python runtime (e.g. R, which subclasses this) + # use the default Python + self._python_version = self.major_pythons['3'] + return self._python_version + py_version_info = runtime.split('-', 1)[1].split('.') py_version = '' if len(py_version_info) == 1: From ce88db7b833fff6f24225e5d9627cdd31bef2b5b Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 26 Apr 2018 10:18:27 +0200 Subject: [PATCH 11/39] bump base image to ubuntu 18.04 --- repo2docker/buildpacks/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index 9691559f..8049b914 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -8,7 +8,7 @@ import logging import docker TEMPLATE = r""" -FROM buildpack-deps:artful +FROM buildpack-deps:bionic # Set up locales properly RUN apt-get update && \ From 5070db1beed257eec509b8487aab9d285f33fa5f Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 26 Apr 2018 10:26:49 +0200 Subject: [PATCH 12/39] remove _nb_ext_conf from legacy env _nb_ext_conf adds configuration that requires nb_conda_kernels, which has been removed, preventing launch --- repo2docker/buildpacks/legacy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo2docker/buildpacks/legacy/__init__.py b/repo2docker/buildpacks/legacy/__init__.py index 5219b2ae..cbc51dd1 100644 --- a/repo2docker/buildpacks/legacy/__init__.py +++ b/repo2docker/buildpacks/legacy/__init__.py @@ -23,7 +23,7 @@ class LegacyBinderDockerBuildPack(DockerBuildPack): RUN conda install -yq conda>=4.3 && \ conda install -yq conda==4.4.11 && \ conda env update -n python3 -f /tmp/python3.frozen.yml && \ - conda remove -yq -n python3 nb_conda_kernels && \ + conda remove -yq -n python3 nb_conda_kernels _nb_ext_conf && \ conda env update -n root -f /tmp/root.frozen.yml && \ /home/main/anaconda2/envs/python3/bin/ipython kernel install --sys-prefix && \ /home/main/anaconda2/bin/ipython kernel install --prefix=/home/main/anaconda2/envs/python3 && \ From 7b58a6281707c3f63190d8038cb0e3e8022532f3 Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 26 Apr 2018 12:46:30 +0200 Subject: [PATCH 13/39] set DEBIAN_FRONTEND=noninteractive avoids possible prompts during setup of apt packages --- repo2docker/buildpacks/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index 8049b914..22aa16da 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -10,6 +10,9 @@ import docker TEMPLATE = r""" FROM buildpack-deps:bionic +# avoid prompts from apt +ENV DEBIAN_FRONTEND=noninteractive + # Set up locales properly RUN apt-get update && \ apt-get install --yes --no-install-recommends locales && \ From 2b2eaa8b69cafe8556000afb5385777f31600f97 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 7 May 2018 15:42:44 +0200 Subject: [PATCH 14/39] remove some duplicate READMEs from tests since we don't need every one included in the docs adds detail to the one that remains --- tests/venv/py2/README.rst | 4 ++++ tests/venv/py3/README.rst | 8 -------- tests/venv/py35/README.rst | 8 -------- 3 files changed, 4 insertions(+), 16 deletions(-) delete mode 100644 tests/venv/py3/README.rst delete mode 100644 tests/venv/py35/README.rst diff --git a/tests/venv/py2/README.rst b/tests/venv/py2/README.rst index 8c125c92..1485f91e 100644 --- a/tests/venv/py2/README.rst +++ b/tests/venv/py2/README.rst @@ -3,3 +3,7 @@ System - Specifying runtime environments You can specify runtime environments (such as Python 2 or 3) with a ``runtime.txt`` file. + +You can specify just the major version, +such as `python-2` or `python-3`, +or a minor version, such as `python-3.5`. diff --git a/tests/venv/py3/README.rst b/tests/venv/py3/README.rst deleted file mode 100644 index 904bc951..00000000 --- a/tests/venv/py3/README.rst +++ /dev/null @@ -1,8 +0,0 @@ -System - Specifying runtime environments ----------------------------------------- - -You can specify runtime environments (such as Python 2 or 3) with a -``runtime.txt`` file. - -This is an example that selects Python 3. -If you need this level of control we recommend you use a `environment.yml`. diff --git a/tests/venv/py35/README.rst b/tests/venv/py35/README.rst deleted file mode 100644 index 816f1398..00000000 --- a/tests/venv/py35/README.rst +++ /dev/null @@ -1,8 +0,0 @@ -System - Specifying runtime environments ----------------------------------------- - -You can specify runtime environments (such as Python 2 or 3) with a -``runtime.txt`` file. - -This is an example that selects Python 3.5. -If you need this level of control we recommend you use a `environment.yml`. From b9f2242876d80e4716cbbf47c7975f0f77f39178 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 8 May 2018 13:32:20 -0700 Subject: [PATCH 15/39] Bump nteract jupyter extension --- .../buildpacks/conda/environment.frozen.yml | 35 ++++++++++--------- .../conda/environment.py-2.7.frozen.yml | 22 ++++++------ .../conda/environment.py-3.5.frozen.yml | 35 ++++++++++--------- .../conda/environment.py-3.6.frozen.yml | 35 ++++++++++--------- repo2docker/buildpacks/conda/environment.yml | 2 +- 5 files changed, 66 insertions(+), 63 deletions(-) diff --git a/repo2docker/buildpacks/conda/environment.frozen.yml b/repo2docker/buildpacks/conda/environment.frozen.yml index f2e31901..85d01107 100644 --- a/repo2docker/buildpacks/conda/environment.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.frozen.yml @@ -1,23 +1,24 @@ # AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY -# Frozen on 2018-04-02 07:27:43 UTC +# Frozen on 2018-05-08 20:22:20 UTC name: r2d channels: - conda-forge - defaults - conda-forge/label/broken dependencies: + - backcall=0.1.0=py_0 - bleach=2.1.3=py_0 - - ca-certificates=2018.1.18=0 - - certifi=2018.1.18=py36_0 - - decorator=4.2.1=py36_0 + - ca-certificates=2018.4.16=0 + - certifi=2018.4.16=py36_0 + - decorator=4.3.0=py_0 - entrypoints=0.2.3=py36_1 - gmp=6.1.2=0 - html5lib=1.0.1=py_0 - ipykernel=4.8.2=py36_0 - - ipython=6.2.1=py36_1 + - ipython=6.3.1=py36_0 - ipython_genutils=0.2.0=py36_0 - ipywidgets=7.1.1=py36_0 - - jedi=0.11.1=py36_0 + - jedi=0.12.0=py36_0 - jinja2=2.10=py36_0 - jsonschema=2.6.0=py36_1 - jupyter_client=5.2.3=py36_0 @@ -26,27 +27,27 @@ dependencies: - jupyterlab_launcher=0.10.5=py36_0 - libsodium=1.0.16=0 - markupsafe=1.0=py36_0 - - mistune=0.8.3=py_0 + - mistune=0.8.3=py36_1 - nbconvert=5.3.1=py_1 - nbformat=4.4.0=py36_0 - ncurses=5.9=10 - notebook=5.4.1=py36_0 - - openssl=1.0.2n=0 - - pandoc=2.1.3=0 - - pandocfilters=1.4.1=py36_0 - - parso=0.1.1=py_0 - - pexpect=4.4.0=py36_0 + - openssl=1.0.2o=0 + - pandoc=2.2=0 + - pandocfilters=1.4.2=py36_0 + - parso=0.2.0=py_0 + - pexpect=4.5.0=py36_0 - pickleshare=0.7.4=py36_0 - pip=9.0.3=py36_0 - prompt_toolkit=1.0.15=py36_0 - ptyprocess=0.5.2=py36_0 - pygments=2.2.0=py36_0 - - python=3.6.5=0 + - python=3.6.5=1 - python-dateutil=2.7.2=py_0 - pyzmq=17.0.0=py36_4 - readline=7.0=0 - send2trash=1.5.0=py_0 - - setuptools=39.0.1=py36_0 + - setuptools=39.1.0=py36_0 - simplegeneric=0.8.1=py36_0 - six=1.11.0=py36_1 - sqlite=3.20.1=2 @@ -57,12 +58,12 @@ dependencies: - traitlets=4.3.2=py36_0 - wcwidth=0.1.7=py36_0 - webencodings=0.5=py36_0 - - wheel=0.30.0=py36_2 - - widgetsnbextension=3.2.0=py36_0 + - wheel=0.31.0=py36_0 + - widgetsnbextension=3.2.1=py36_0 - xz=5.2.3=0 - zeromq=4.2.5=1 - zlib=1.2.11=0 - pip: - - nteract-on-jupyter==1.6.0 + - nteract-on-jupyter==1.7.0 prefix: /opt/conda/envs/r2d diff --git a/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml b/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml index 3068800c..a8af048d 100644 --- a/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.py-2.7.yml, DO NOT MANUALLY MODIFY -# Frozen on 2018-04-02 07:21:15 UTC +# Frozen on 2018-05-08 20:15:09 UTC name: r2d channels: - conda-forge @@ -9,31 +9,31 @@ dependencies: - backports=1.0=py27_1 - backports.shutil_get_terminal_size=1.0.0=py_3 - backports_abc=0.5=py27_0 - - ca-certificates=2018.1.18=0 - - certifi=2018.1.18=py27_0 - - decorator=4.2.1=py27_0 + - ca-certificates=2018.4.16=0 + - certifi=2018.4.16=py27_0 + - decorator=4.3.0=py_0 - enum34=1.1.6=py27_1 - ipykernel=4.8.2=py27_0 - - ipython=5.5.0=py27_0 + - ipython=5.6.0=py27_0 - ipython_genutils=0.2.0=py27_0 - jupyter_client=5.2.3=py27_0 - jupyter_core=4.4.0=py_0 - libsodium=1.0.16=0 - ncurses=5.9=10 - - openssl=1.0.2n=0 - - pathlib2=2.3.0=py27_0 - - pexpect=4.4.0=py27_0 + - openssl=1.0.2o=0 + - pathlib2=2.3.2=py27_0 + - pexpect=4.5.0=py27_0 - pickleshare=0.7.4=py27_0 - pip=9.0.3=py27_0 - prompt_toolkit=1.0.15=py27_0 - ptyprocess=0.5.2=py27_0 - pygments=2.2.0=py27_0 - - python=2.7.14=5 + - python=2.7.15=0 - python-dateutil=2.7.2=py_0 - pyzmq=17.0.0=py27_4 - readline=7.0=0 - scandir=1.7=py27_0 - - setuptools=39.0.1=py27_0 + - setuptools=39.1.0=py27_0 - simplegeneric=0.8.1=py27_0 - singledispatch=3.4.0.3=py27_0 - six=1.11.0=py27_1 @@ -43,7 +43,7 @@ dependencies: - tornado=4.5.3=py27_0 - traitlets=4.3.2=py27_0 - wcwidth=0.1.7=py27_0 - - wheel=0.30.0=py27_2 + - wheel=0.31.0=py27_0 - zeromq=4.2.5=1 - zlib=1.2.11=0 - pip: diff --git a/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml b/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml index 0d8aa8c6..01183a6e 100644 --- a/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml @@ -1,23 +1,24 @@ # AUTO GENERATED FROM environment.py-3.5.yml, DO NOT MANUALLY MODIFY -# Frozen on 2018-04-02 07:23:18 UTC +# Frozen on 2018-05-08 20:18:13 UTC name: r2d channels: - conda-forge - defaults - conda-forge/label/broken dependencies: + - backcall=0.1.0=py_0 - bleach=2.1.3=py_0 - - ca-certificates=2018.1.18=0 - - certifi=2018.1.18=py35_0 - - decorator=4.2.1=py35_0 + - ca-certificates=2018.4.16=0 + - certifi=2018.4.16=py35_0 + - decorator=4.3.0=py_0 - entrypoints=0.2.3=py35_1 - gmp=6.1.2=0 - html5lib=1.0.1=py_0 - ipykernel=4.8.2=py35_0 - - ipython=6.2.1=py35_1 + - ipython=6.3.1=py35_0 - ipython_genutils=0.2.0=py35_0 - ipywidgets=7.1.1=py35_0 - - jedi=0.11.1=py35_0 + - jedi=0.12.0=py35_0 - jinja2=2.10=py35_0 - jsonschema=2.6.0=py35_1 - jupyter_client=5.2.3=py35_0 @@ -26,27 +27,27 @@ dependencies: - jupyterlab_launcher=0.10.5=py35_0 - libsodium=1.0.16=0 - markupsafe=1.0=py35_0 - - mistune=0.8.3=py_0 + - mistune=0.8.3=py35_1 - nbconvert=5.3.1=py_1 - nbformat=4.4.0=py35_0 - ncurses=5.9=10 - notebook=5.4.1=py35_0 - - openssl=1.0.2n=0 - - pandoc=2.1.3=0 - - pandocfilters=1.4.1=py35_0 - - parso=0.1.1=py_0 - - pexpect=4.4.0=py35_0 + - openssl=1.0.2o=0 + - pandoc=2.2=0 + - pandocfilters=1.4.2=py35_0 + - parso=0.2.0=py_0 + - pexpect=4.5.0=py35_0 - pickleshare=0.7.4=py35_0 - pip=9.0.3=py35_0 - prompt_toolkit=1.0.15=py35_0 - ptyprocess=0.5.2=py35_0 - pygments=2.2.0=py35_0 - - python=3.5.5=0 + - python=3.5.5=1 - python-dateutil=2.7.2=py_0 - pyzmq=17.0.0=py35_4 - readline=7.0=0 - send2trash=1.5.0=py_0 - - setuptools=39.0.1=py35_0 + - setuptools=39.1.0=py35_0 - simplegeneric=0.8.1=py35_0 - six=1.11.0=py35_1 - sqlite=3.20.1=2 @@ -57,12 +58,12 @@ dependencies: - traitlets=4.3.2=py35_0 - wcwidth=0.1.7=py35_0 - webencodings=0.5=py35_0 - - wheel=0.30.0=py35_2 - - widgetsnbextension=3.2.0=py35_0 + - wheel=0.31.0=py35_0 + - widgetsnbextension=3.2.1=py35_0 - xz=5.2.3=0 - zeromq=4.2.5=1 - zlib=1.2.11=0 - pip: - - nteract-on-jupyter==1.6.0 + - nteract-on-jupyter==1.7.0 prefix: /opt/conda/envs/r2d diff --git a/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml b/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml index f2e31901..85d01107 100644 --- a/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml @@ -1,23 +1,24 @@ # AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY -# Frozen on 2018-04-02 07:27:43 UTC +# Frozen on 2018-05-08 20:22:20 UTC name: r2d channels: - conda-forge - defaults - conda-forge/label/broken dependencies: + - backcall=0.1.0=py_0 - bleach=2.1.3=py_0 - - ca-certificates=2018.1.18=0 - - certifi=2018.1.18=py36_0 - - decorator=4.2.1=py36_0 + - ca-certificates=2018.4.16=0 + - certifi=2018.4.16=py36_0 + - decorator=4.3.0=py_0 - entrypoints=0.2.3=py36_1 - gmp=6.1.2=0 - html5lib=1.0.1=py_0 - ipykernel=4.8.2=py36_0 - - ipython=6.2.1=py36_1 + - ipython=6.3.1=py36_0 - ipython_genutils=0.2.0=py36_0 - ipywidgets=7.1.1=py36_0 - - jedi=0.11.1=py36_0 + - jedi=0.12.0=py36_0 - jinja2=2.10=py36_0 - jsonschema=2.6.0=py36_1 - jupyter_client=5.2.3=py36_0 @@ -26,27 +27,27 @@ dependencies: - jupyterlab_launcher=0.10.5=py36_0 - libsodium=1.0.16=0 - markupsafe=1.0=py36_0 - - mistune=0.8.3=py_0 + - mistune=0.8.3=py36_1 - nbconvert=5.3.1=py_1 - nbformat=4.4.0=py36_0 - ncurses=5.9=10 - notebook=5.4.1=py36_0 - - openssl=1.0.2n=0 - - pandoc=2.1.3=0 - - pandocfilters=1.4.1=py36_0 - - parso=0.1.1=py_0 - - pexpect=4.4.0=py36_0 + - openssl=1.0.2o=0 + - pandoc=2.2=0 + - pandocfilters=1.4.2=py36_0 + - parso=0.2.0=py_0 + - pexpect=4.5.0=py36_0 - pickleshare=0.7.4=py36_0 - pip=9.0.3=py36_0 - prompt_toolkit=1.0.15=py36_0 - ptyprocess=0.5.2=py36_0 - pygments=2.2.0=py36_0 - - python=3.6.5=0 + - python=3.6.5=1 - python-dateutil=2.7.2=py_0 - pyzmq=17.0.0=py36_4 - readline=7.0=0 - send2trash=1.5.0=py_0 - - setuptools=39.0.1=py36_0 + - setuptools=39.1.0=py36_0 - simplegeneric=0.8.1=py36_0 - six=1.11.0=py36_1 - sqlite=3.20.1=2 @@ -57,12 +58,12 @@ dependencies: - traitlets=4.3.2=py36_0 - wcwidth=0.1.7=py36_0 - webencodings=0.5=py36_0 - - wheel=0.30.0=py36_2 - - widgetsnbextension=3.2.0=py36_0 + - wheel=0.31.0=py36_0 + - widgetsnbextension=3.2.1=py36_0 - xz=5.2.3=0 - zeromq=4.2.5=1 - zlib=1.2.11=0 - pip: - - nteract-on-jupyter==1.6.0 + - nteract-on-jupyter==1.7.0 prefix: /opt/conda/envs/r2d diff --git a/repo2docker/buildpacks/conda/environment.yml b/repo2docker/buildpacks/conda/environment.yml index ee040904..076f6198 100644 --- a/repo2docker/buildpacks/conda/environment.yml +++ b/repo2docker/buildpacks/conda/environment.yml @@ -5,4 +5,4 @@ dependencies: - tornado==4.5.3 - notebook==5.4.1 - pip: - - nteract_on_jupyter==1.6.0 + - nteract_on_jupyter==1.7.0 From 8861382a762a1cefd3c403113e0713957af9921d Mon Sep 17 00:00:00 2001 From: Tim Head Date: Thu, 10 May 2018 21:55:20 +0200 Subject: [PATCH 16/39] Add bug, chat and contrib links to the front page It should be easier to find us to ask questions, report bugs and to get started contributing. --- docs/source/index.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 65883785..9ad490fe 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -5,6 +5,11 @@ jupyter-repo2docker images from source code repositories. See the list below for various ways in which you can use ``repo2docker``. +Please report `Bugs `_, +`ask questions `_ or +`contribute to the project `_. + + Site Contents ------------- .. toctree:: @@ -17,4 +22,3 @@ Site Contents design architecture dev_newbuildpack - From 72292061d2d3f993a39345509ea255df9defd54a Mon Sep 17 00:00:00 2001 From: Tim Head Date: Fri, 11 May 2018 23:13:18 +0200 Subject: [PATCH 17/39] Allow for longer periods of no output during R tests --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f67696de..e017990d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,11 @@ install: script: # cd into tests so CWD being repo2docker does not hide # possible issues with MANIFEST.in - - cd tests && travis_retry pytest --cov repo2docker -v ${REPO_TYPE} + - if [ ${REPO_TYPE} == "r" ]; + then cd tests && travis_wait pytest --cov repo2docker -v ${REPO_TYPE}; + else cd tests && travis_retry pytest --cov repo2docker -v ${REPO_TYPE}; + fi + after_success: - pip install codecov - codecov From c8a31b8baf42c66b9e3f77beb4032d689927626d Mon Sep 17 00:00:00 2001 From: Andrea Borruso Date: Sat, 12 May 2018 08:54:39 +0200 Subject: [PATCH 18/39] To add the user bin folder to the path Here thread https://github.com/jupyterhub/binderhub/issues/557 --- repo2docker/buildpacks/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index 22aa16da..46b18f5f 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -71,6 +71,9 @@ ENV {{item[0]}} {{item[1]}} {% endfor -%} {% endif -%} +# So users can move executables here with postBuild, issue #557 +ENV PATH {{ /home/jovyan/{local}/bin:${PATH}}} + {% if path -%} # Special case PATH ENV PATH {{ ':'.join(path) }}:${PATH} From cd6371c3131202b3d601575b8347bb72c2787256 Mon Sep 17 00:00:00 2001 From: Andrea Borruso Date: Sat, 12 May 2018 10:05:42 +0200 Subject: [PATCH 19/39] Update base.py --- repo2docker/buildpacks/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index 46b18f5f..bb985aa8 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -72,7 +72,7 @@ ENV {{item[0]}} {{item[1]}} {% endif -%} # So users can move executables here with postBuild, issue #557 -ENV PATH {{ /home/jovyan/{local}/bin:${PATH}}} +ENV PATH {{ '/home/jovyan/local/bin':${PATH}}} {% if path -%} # Special case PATH From da3522851fcc03b04ed7137b90a774aa8883f697 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Sat, 12 May 2018 11:32:39 -0700 Subject: [PATCH 20/39] removing extra curly brackets --- repo2docker/buildpacks/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index bb985aa8..94555b68 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -72,7 +72,7 @@ ENV {{item[0]}} {{item[1]}} {% endif -%} # So users can move executables here with postBuild, issue #557 -ENV PATH {{ '/home/jovyan/local/bin':${PATH}}} +ENV PATH '/home/jovyan/local/bin':${PATH} {% if path -%} # Special case PATH From 751531b63259bc517c3837f5ccf804ecaa1bcca7 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Fri, 18 May 2018 18:21:41 -0700 Subject: [PATCH 21/39] Use XDG standard ~/.local for user software installs ~/.local is also where things like pip install --user seem to install things, so we should use that instead of ~/local. --- repo2docker/buildpacks/base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index 94555b68..7951911e 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -71,9 +71,6 @@ ENV {{item[0]}} {{item[1]}} {% endfor -%} {% endif -%} -# So users can move executables here with postBuild, issue #557 -ENV PATH '/home/jovyan/local/bin':${PATH} - {% if path -%} # Special case PATH ENV PATH {{ ':'.join(path) }}:${PATH} @@ -200,7 +197,10 @@ class BuildPack: Just sets the PATH environment variable. Separated out since it is very commonly set by various buildpacks. """ - return [] + # Allow local user installs into ~/.local, which is where the + # XDG desktop standard suggests these should be + # See https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + return ['$HOME/.local/bin'] def get_labels(self): """ From f818fd82beefe7ff75f05f56d7e4d817bb56173e Mon Sep 17 00:00:00 2001 From: Tim Head Date: Wed, 16 May 2018 07:56:43 +0200 Subject: [PATCH 22/39] Add test to check for extra directory on PATH --- tests/venv/usr-bin/verify | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 tests/venv/usr-bin/verify diff --git a/tests/venv/usr-bin/verify b/tests/venv/usr-bin/verify new file mode 100755 index 00000000..27033ec8 --- /dev/null +++ b/tests/venv/usr-bin/verify @@ -0,0 +1,6 @@ +#!/usr/bin/env python +# Verify that ~/local/bin is on the PATH +import os + +PATH = os.getenv("PATH") +assert "/home/jovyan/local/bin" in PATH From 5005c309cc3834a5f5e02c0c077c963153da0d6e Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Fri, 18 May 2018 18:23:54 -0700 Subject: [PATCH 23/39] Modify usr-bin test --- tests/venv/usr-bin/verify | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/venv/usr-bin/verify b/tests/venv/usr-bin/verify index 27033ec8..19642a9f 100755 --- a/tests/venv/usr-bin/verify +++ b/tests/venv/usr-bin/verify @@ -1,6 +1,5 @@ #!/usr/bin/env python -# Verify that ~/local/bin is on the PATH +# Verify that ~/.local/bin is on the PATH import os -PATH = os.getenv("PATH") -assert "/home/jovyan/local/bin" in PATH +assert "$HOME/.local/bin" in os.getenv("PATH") \ No newline at end of file From 2f2fde8a112db3f0c324f5616b99731423ddebc0 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Sat, 19 May 2018 08:02:59 -0700 Subject: [PATCH 24/39] short docs explanation --- docs/source/faq.md | 19 +++++++++++++------ tests/venv/usr-bin/verify | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/source/faq.md b/docs/source/faq.md index 6f5cdb56..560fe561 100644 --- a/docs/source/faq.md +++ b/docs/source/faq.md @@ -9,6 +9,19 @@ If you have a question & have found an answer, send a PR to add it here! Currently the best way to do this is by using a conda ``environment.yml`` file and setting the language to whichever version of Python you like. +## Can I add binary files to the user's PATH? + +Yes! Using a ``postBuild`` file, you can place any files that should be called +from the command line in the folder ``~/.local/``. This folder will be +available in a user's PATH, and can be run from the command line (or as +a subsequent build step.) + +## How do I set environment variables? + +Use the `-e` or `--env` flag for each variable that you want to define. + +For example `jupyter-repo2docker -e VAR1=val1 -e VAR2=val2 ...` + ## Can I use repo2docker to bootstrap my own Dockerfile? No, you can't. @@ -24,9 +37,3 @@ or similar traditional docker command. Check out the [binder-examples](http://github.com/binder-examples/) github organization for example Dockerfiles you can copy & modify for your own use! - -## How do I set environment variables? - -Use the `-e` or `--env` flag for each variable that you want to define. - -For example `jupyter-repo2docker -e VAR1=val1 -e VAR2=val2 ...` diff --git a/tests/venv/usr-bin/verify b/tests/venv/usr-bin/verify index 19642a9f..e79282c1 100755 --- a/tests/venv/usr-bin/verify +++ b/tests/venv/usr-bin/verify @@ -2,4 +2,4 @@ # Verify that ~/.local/bin is on the PATH import os -assert "$HOME/.local/bin" in os.getenv("PATH") \ No newline at end of file +assert "$HOME/.local/bin" in os.getenv("PATH") From 808b6fa20bd3043131124280469c16fc6466505f Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Sat, 19 May 2018 10:06:10 -0700 Subject: [PATCH 25/39] Update faq.md --- docs/source/faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/faq.md b/docs/source/faq.md index 560fe561..053b9f35 100644 --- a/docs/source/faq.md +++ b/docs/source/faq.md @@ -9,7 +9,7 @@ If you have a question & have found an answer, send a PR to add it here! Currently the best way to do this is by using a conda ``environment.yml`` file and setting the language to whichever version of Python you like. -## Can I add binary files to the user's PATH? +## Can I add executable files to the user's PATH? Yes! Using a ``postBuild`` file, you can place any files that should be called from the command line in the folder ``~/.local/``. This folder will be From 91fdfb4c29ab627dda3f139605027c55b4d3dc56 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 23 May 2018 15:22:21 +0200 Subject: [PATCH 26/39] prefer R to conda allows mixing conda and R because R inherits from Python (and thereby conda) --- repo2docker/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo2docker/app.py b/repo2docker/app.py index 40d6fa83..7534f7c2 100644 --- a/repo2docker/app.py +++ b/repo2docker/app.py @@ -66,8 +66,8 @@ class Repo2Docker(Application): LegacyBinderDockerBuildPack, DockerBuildPack, JuliaBuildPack, - CondaBuildPack, RBuildPack, + CondaBuildPack, PythonBuildPack, ], config=True, From 3092365890dfa751dfb11a3665baa7959a9e8c5e Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 23 May 2018 15:52:24 +0200 Subject: [PATCH 27/39] nbrsessionproxy to 0.7 for shiny support! --- repo2docker/buildpacks/r.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repo2docker/buildpacks/r.py b/repo2docker/buildpacks/r.py index a10a4b53..12650c0c 100644 --- a/repo2docker/buildpacks/r.py +++ b/repo2docker/buildpacks/r.py @@ -4,6 +4,7 @@ import datetime from .python import PythonBuildPack + class RBuildPack(PythonBuildPack): """ Setup R for use with a repository @@ -169,7 +170,7 @@ class RBuildPack(PythonBuildPack): "${NB_USER}", # Install nbrsessionproxy r""" - pip install --no-cache-dir nbrsessionproxy==0.6.1 && \ + pip install --no-cache-dir nbrsessionproxy==0.7.0 && \ jupyter serverextension enable nbrsessionproxy --sys-prefix && \ jupyter nbextension install --py nbrsessionproxy --sys-prefix && \ jupyter nbextension enable --py nbrsessionproxy --sys-prefix From 150d1d245e7763eb4f1c166c5e78279e70735999 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 23 May 2018 16:03:07 +0200 Subject: [PATCH 28/39] fix verify script for ~/.local/bin --- tests/venv/usr-bin/verify | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/venv/usr-bin/verify b/tests/venv/usr-bin/verify index e79282c1..4d372157 100755 --- a/tests/venv/usr-bin/verify +++ b/tests/venv/usr-bin/verify @@ -2,4 +2,4 @@ # Verify that ~/.local/bin is on the PATH import os -assert "$HOME/.local/bin" in os.getenv("PATH") +assert os.path.expanduser('~/.local/bin') in os.getenv("PATH"), os.getenv("PATH") From 106087cedb31671f965dd00f31f4fc9e736687ce Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Wed, 23 May 2018 10:34:05 -0700 Subject: [PATCH 29/39] Install shiny-server. --- repo2docker/buildpacks/r.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/repo2docker/buildpacks/r.py b/repo2docker/buildpacks/r.py index a10a4b53..6c7723bd 100644 --- a/repo2docker/buildpacks/r.py +++ b/repo2docker/buildpacks/r.py @@ -128,6 +128,10 @@ class RBuildPack(PythonBuildPack): # This is MD5, because that is what RStudio download page provides! rstudio_checksum = '24cd11f0405d8372b4168fc9956e0386' + # Via https://www.rstudio.com/products/shiny/download-server/ + shiny_url = 'https://download3.rstudio.org/ubuntu-14.04/x86_64/shiny-server-1.5.7.907-amd64.deb' + shiny_checksum = '78371a8361ba0e7fec44edd2b8e425ac' + # Version of MRAN to pull devtools from. devtools_version = '2018-02-01' @@ -155,6 +159,20 @@ class RBuildPack(PythonBuildPack): rstudio_checksum=rstudio_checksum ) ), + ( + "root", + # Install Shiny Server! + r""" + curl --silent --location --fail {url} > {deb} && \ + echo '{checksum} {deb}' | md5sum -c - && \ + dpkg -i {deb} && \ + rm {deb} + """.format( + url=shiny_url, + checksum=shiny_checksum, + deb='/tmp/shiny.deb' + ) + ), ( "root", # Set paths so that RStudio shares libraries with base R From 6deda3cd7abf9ae959617324a82649a01aec3de1 Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Wed, 23 May 2018 11:13:11 -0700 Subject: [PATCH 30/39] Create shiny-server output directories. --- repo2docker/buildpacks/r.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/repo2docker/buildpacks/r.py b/repo2docker/buildpacks/r.py index 6c7723bd..c12d6d98 100644 --- a/repo2docker/buildpacks/r.py +++ b/repo2docker/buildpacks/r.py @@ -226,6 +226,16 @@ class RBuildPack(PythonBuildPack): echo "options(repos = c(CRAN='{mran_url}'), download.file.method = 'libcurl')" > /etc/R/Rprofile.site """.format(mran_url=mran_url) ), + ( + # Not all of these locations are configurable; log_dir is + "root", + r""" + install -o ${NB_USER} -g ${NB_USER} -d /var/log/shiny-server && \ + install -o ${NB_USER} -g ${NB_USER} -d /var/lib/shiny-server && \ + install -o ${NB_USER} -g ${NB_USER} /dev/null /var/log/shiny-server.log && \ + install -o ${NB_USER} -g ${NB_USER} /dev/null /var/run/shiny-server.pid + """ + ), ] installR_path = self.binder_path('install.R') From 709ac4c757d90f4a1b5709dc67da08645e00f5d3 Mon Sep 17 00:00:00 2001 From: Gladys Nalvarte Date: Thu, 24 May 2018 10:04:33 +0200 Subject: [PATCH 31/39] Checks if the environment.yml file is empty and if it contains a dictionary. If the environment.yml is empty, an empty dictionary is instantiated. This will rise an TypeError if the environment.yml file contains a list or other data structure different than a dictionary. --- repo2docker/buildpacks/conda/__init__.py | 6 ++++++ tests/test_env_yml.py | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 tests/test_env_yml.py diff --git a/repo2docker/buildpacks/conda/__init__.py b/repo2docker/buildpacks/conda/__init__.py index 13ba445c..f8d51f7b 100644 --- a/repo2docker/buildpacks/conda/__init__.py +++ b/repo2docker/buildpacks/conda/__init__.py @@ -136,6 +136,12 @@ class CondaBuildPack(BaseImage): py_version = None with open(environment_yml) as f: env = YAML().load(f) + # check if the env file is empty, if so instantiate an empty dictionary. + if env is None: + env = {} + # check if the env file has a dictionary not a list or other data structure. + if not isinstance(env, dict): + raise TypeError("environment.yml should contain a dictionary. Got %r" % env) for dep in env.get('dependencies', []): if not isinstance(dep, str): continue diff --git a/tests/test_env_yml.py b/tests/test_env_yml.py new file mode 100644 index 00000000..0943ca51 --- /dev/null +++ b/tests/test_env_yml.py @@ -0,0 +1,23 @@ +""" +Test if the environment.yml is empty or it constains other data structure than a dictionary +""" +import os +import sys +import pytest +from repo2docker import buildpacks + +#@pytest.mark.xfail(reason="Not yet implemented") +def test_empty_env_yml(tmpdir): + tmpdir.chdir() + p = tmpdir.join("environment.yml") + bp = buildpacks.CondaBuildPack() + with pytest.raises(Exception): + py_ver = bp.python_version() + +def test_no_dict_env_yml(tmpdir): + tmpdir.chdir() + q = tmpdir.join("environment.yml") + q.write("list/n ,string") + bq = buildpacks.CondaBuildPack() + with pytest.raises(Exception): + py_ver = bq.python_version() \ No newline at end of file From 63e459664c697ed1f7ccf7ae060092c0f1f68adb Mon Sep 17 00:00:00 2001 From: Gladys Nalvarte Date: Thu, 24 May 2018 15:29:48 +0200 Subject: [PATCH 32/39] Updated test_env_yml.py file --- tests/test_env_yml.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_env_yml.py b/tests/test_env_yml.py index 0943ca51..d6a49b4f 100644 --- a/tests/test_env_yml.py +++ b/tests/test_env_yml.py @@ -6,18 +6,20 @@ import sys import pytest from repo2docker import buildpacks -#@pytest.mark.xfail(reason="Not yet implemented") + def test_empty_env_yml(tmpdir): tmpdir.chdir() p = tmpdir.join("environment.yml") + p.write("") bp = buildpacks.CondaBuildPack() - with pytest.raises(Exception): - py_ver = bp.python_version() + py_ver = bp.python_version + # If the environment.yml is empty python_version will get an empty string + assert py_ver == '' def test_no_dict_env_yml(tmpdir): tmpdir.chdir() q = tmpdir.join("environment.yml") q.write("list/n ,string") bq = buildpacks.CondaBuildPack() - with pytest.raises(Exception): - py_ver = bq.python_version() \ No newline at end of file + with pytest.raises(TypeError): + py_ver = bq.python_version \ No newline at end of file From 4a6bbae7da92823bb2fefc03cc7268b8bc3e3de5 Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Thu, 24 May 2018 09:48:17 -0700 Subject: [PATCH 33/39] Install shiny library. --- repo2docker/buildpacks/r.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/repo2docker/buildpacks/r.py b/repo2docker/buildpacks/r.py index c12d6d98..568b01b0 100644 --- a/repo2docker/buildpacks/r.py +++ b/repo2docker/buildpacks/r.py @@ -203,6 +203,13 @@ class RBuildPack(PythonBuildPack): """.format( devtools_version=devtools_version, irkernel_version=irkernel_version + ), + ( + "${NB_USER}", + # Install a pinned version of IRKernel and set it up for use! + r""" + R --quiet -e "install.packages('shiny', repos='https://mran.microsoft.com/snapshot/{}', method='libcurl')" + """.format(self.checkpoint_date.isoformat()) ) ) ] From f90758057adf633731a51d75c47848c6f088b321 Mon Sep 17 00:00:00 2001 From: ryanlovett Date: Thu, 24 May 2018 09:50:19 -0700 Subject: [PATCH 34/39] Update comment. --- repo2docker/buildpacks/r.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo2docker/buildpacks/r.py b/repo2docker/buildpacks/r.py index 7265dc1b..b40503d7 100644 --- a/repo2docker/buildpacks/r.py +++ b/repo2docker/buildpacks/r.py @@ -207,7 +207,7 @@ class RBuildPack(PythonBuildPack): ), ( "${NB_USER}", - # Install a pinned version of IRKernel and set it up for use! + # Install shiny library r""" R --quiet -e "install.packages('shiny', repos='https://mran.microsoft.com/snapshot/{}', method='libcurl')" """.format(self.checkpoint_date.isoformat()) From c688b706960c4b5442b32bc9cad1e8d549e03d0d Mon Sep 17 00:00:00 2001 From: Tim Head Date: Fri, 25 May 2018 08:14:14 +0200 Subject: [PATCH 35/39] Fix parentheses in the build scripts for R buildpack --- repo2docker/buildpacks/r.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/repo2docker/buildpacks/r.py b/repo2docker/buildpacks/r.py index b40503d7..ebd5416b 100644 --- a/repo2docker/buildpacks/r.py +++ b/repo2docker/buildpacks/r.py @@ -204,15 +204,17 @@ class RBuildPack(PythonBuildPack): """.format( devtools_version=devtools_version, irkernel_version=irkernel_version + ) ), ( "${NB_USER}", # Install shiny library r""" R --quiet -e "install.packages('shiny', repos='https://mran.microsoft.com/snapshot/{}', method='libcurl')" - """.format(self.checkpoint_date.isoformat()) + """.format( + self.checkpoint_date.isoformat() ) - ) + ), ] def get_assemble_scripts(self): From 43f367c590cac1ecafef4e57dbd85eed297a75a6 Mon Sep 17 00:00:00 2001 From: Gladys Nalvarte Date: Mon, 4 Jun 2018 12:56:20 +0200 Subject: [PATCH 36/39] Update the TypeError message and example in the unit test. --- repo2docker/buildpacks/conda/__init__.py | 2 +- tests/test_env_yml.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/repo2docker/buildpacks/conda/__init__.py b/repo2docker/buildpacks/conda/__init__.py index f8d51f7b..17474bf1 100644 --- a/repo2docker/buildpacks/conda/__init__.py +++ b/repo2docker/buildpacks/conda/__init__.py @@ -141,7 +141,7 @@ class CondaBuildPack(BaseImage): env = {} # check if the env file has a dictionary not a list or other data structure. if not isinstance(env, dict): - raise TypeError("environment.yml should contain a dictionary. Got %r" % env) + raise TypeError("environment.yml should contain a dictionary. Got %r" % type(env)) for dep in env.get('dependencies', []): if not isinstance(dep, str): continue diff --git a/tests/test_env_yml.py b/tests/test_env_yml.py index d6a49b4f..c6d146f6 100644 --- a/tests/test_env_yml.py +++ b/tests/test_env_yml.py @@ -19,7 +19,8 @@ def test_empty_env_yml(tmpdir): def test_no_dict_env_yml(tmpdir): tmpdir.chdir() q = tmpdir.join("environment.yml") - q.write("list/n ,string") + q.write("numpy\n " + "matplotlib\n") bq = buildpacks.CondaBuildPack() with pytest.raises(TypeError): py_ver = bq.python_version \ No newline at end of file From ba6aa6308530eb7d4319a68c419ad0164a014726 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Wed, 6 Jun 2018 19:07:53 +0200 Subject: [PATCH 37/39] Switch to miniconda v4.5.1 --- repo2docker/buildpacks/conda/install-miniconda.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repo2docker/buildpacks/conda/install-miniconda.bash b/repo2docker/buildpacks/conda/install-miniconda.bash index f88433b2..1073afba 100755 --- a/repo2docker/buildpacks/conda/install-miniconda.bash +++ b/repo2docker/buildpacks/conda/install-miniconda.bash @@ -3,7 +3,7 @@ set -ex cd $(dirname $0) -CONDA_VERSION=4.3.30 +CONDA_VERSION=4.5.1 URL="https://repo.continuum.io/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh" INSTALLER_PATH=/tmp/miniconda-installer.sh @@ -12,7 +12,7 @@ chmod +x ${INSTALLER_PATH} # Only MD5 checksums are available for miniconda # Can be obtained from https://repo.continuum.io/miniconda/ -MD5SUM="0b80a152332a4ce5250f3c09589c7a81" +MD5SUM="0c28787e3126238df24c5d4858bd0744" if ! echo "${MD5SUM} ${INSTALLER_PATH}" | md5sum --quiet -c -; then echo "md5sum mismatch for ${INSTALLER_PATH}, exiting!" From 157ce1e1e7f979c528ad91ccac13fe32eaea3774 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Wed, 6 Jun 2018 21:52:41 +0200 Subject: [PATCH 38/39] Add root conda environment to expected output --- tests/conda/simple-py2/verify | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conda/simple-py2/verify b/tests/conda/simple-py2/verify index 38c58a19..57f75c5e 100755 --- a/tests/conda/simple-py2/verify +++ b/tests/conda/simple-py2/verify @@ -13,7 +13,7 @@ assert sorted(specs) == ['python2', 'python3'], specs.keys() import json from subprocess import check_output envs = json.loads(check_output(['conda', 'env', 'list', '--json']).decode('utf8')) -assert envs == {'envs': ['/srv/conda/envs/kernel']}, envs +assert envs == {'envs': ['/srv/conda', '/srv/conda/envs/kernel']}, envs pkgs = json.loads(check_output(['conda', 'list', '-n', 'kernel', '--json']).decode('utf8')) pkg_names = [pkg['name'] for pkg in pkgs] From 3be7099970077e162a83664b60788648a597cdf5 Mon Sep 17 00:00:00 2001 From: Todd Date: Fri, 8 Jun 2018 16:57:51 -0400 Subject: [PATCH 39/39] Include LICENSE file in wheels The license requires that all copies of the software include the license. This makes sure the license is included in the wheels. See the wheel documentation [here](https://wheel.readthedocs.io/en/stable/#including-the-license-in-the-generated-wheel-file) for more information. --- setup.cfg | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index d7de2b16..947f15f2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,5 @@ [wheel] -universal=1 \ No newline at end of file +universal=1 + +[metadata] +license_file = LICENSE