diff --git a/.travis.yml b/.travis.yml index f67696de..e017990d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,11 @@ install: script: # cd into tests so CWD being repo2docker does not hide # possible issues with MANIFEST.in - - cd tests && travis_retry pytest --cov repo2docker -v ${REPO_TYPE} + - if [ ${REPO_TYPE} == "r" ]; + then cd tests && travis_wait pytest --cov repo2docker -v ${REPO_TYPE}; + else cd tests && travis_retry pytest --cov repo2docker -v ${REPO_TYPE}; + fi + after_success: - pip install codecov - codecov diff --git a/docs/source/faq.md b/docs/source/faq.md index 6f5cdb56..053b9f35 100644 --- a/docs/source/faq.md +++ b/docs/source/faq.md @@ -9,6 +9,19 @@ If you have a question & have found an answer, send a PR to add it here! Currently the best way to do this is by using a conda ``environment.yml`` file and setting the language to whichever version of Python you like. +## Can I add executable files to the user's PATH? + +Yes! Using a ``postBuild`` file, you can place any files that should be called +from the command line in the folder ``~/.local/``. This folder will be +available in a user's PATH, and can be run from the command line (or as +a subsequent build step.) + +## How do I set environment variables? + +Use the `-e` or `--env` flag for each variable that you want to define. + +For example `jupyter-repo2docker -e VAR1=val1 -e VAR2=val2 ...` + ## Can I use repo2docker to bootstrap my own Dockerfile? No, you can't. @@ -24,9 +37,3 @@ or similar traditional docker command. Check out the [binder-examples](http://github.com/binder-examples/) github organization for example Dockerfiles you can copy & modify for your own use! - -## How do I set environment variables? - -Use the `-e` or `--env` flag for each variable that you want to define. - -For example `jupyter-repo2docker -e VAR1=val1 -e VAR2=val2 ...` diff --git a/docs/source/index.rst b/docs/source/index.rst index 65883785..9ad490fe 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -5,6 +5,11 @@ jupyter-repo2docker images from source code repositories. See the list below for various ways in which you can use ``repo2docker``. +Please report `Bugs `_, +`ask questions `_ or +`contribute to the project `_. + + Site Contents ------------- .. toctree:: @@ -17,4 +22,3 @@ Site Contents design architecture dev_newbuildpack - diff --git a/repo2docker/app.py b/repo2docker/app.py index 7820f0b4..e7dd4980 100644 --- a/repo2docker/app.py +++ b/repo2docker/app.py @@ -29,7 +29,7 @@ from traitlets.config import Application from . import __version__ from .buildpacks import ( PythonBuildPack, DockerBuildPack, LegacyBinderDockerBuildPack, - CondaBuildPack, JuliaBuildPack, Python2BuildPack, BaseImage, + CondaBuildPack, JuliaBuildPack, BaseImage, RBuildPack ) from .utils import ( @@ -66,9 +66,8 @@ class Repo2Docker(Application): LegacyBinderDockerBuildPack, DockerBuildPack, JuliaBuildPack, - CondaBuildPack, - Python2BuildPack, RBuildPack, + CondaBuildPack, PythonBuildPack, ], config=True, diff --git a/repo2docker/buildpacks/__init__.py b/repo2docker/buildpacks/__init__.py index ae60c05a..5ccc0c0e 100644 --- a/repo2docker/buildpacks/__init__.py +++ b/repo2docker/buildpacks/__init__.py @@ -1,5 +1,5 @@ from .base import BuildPack, BaseImage -from .python import Python2BuildPack, PythonBuildPack +from .python import PythonBuildPack from .conda import CondaBuildPack from .julia import JuliaBuildPack from .docker import DockerBuildPack diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index a595fed5..dee6249a 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -9,7 +9,10 @@ import docker import sys TEMPLATE = r""" -FROM buildpack-deps:artful +FROM buildpack-deps:bionic + +# avoid prompts from apt +ENV DEBIAN_FRONTEND=noninteractive # Set up locales properly RUN apt-get update && \ @@ -198,7 +201,10 @@ class BuildPack: Just sets the PATH environment variable. Separated out since it is very commonly set by various buildpacks. """ - return [] + # Allow local user installs into ~/.local, which is where the + # XDG desktop standard suggests these should be + # See https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + return ['$HOME/.local/bin'] def get_labels(self): """ diff --git a/repo2docker/buildpacks/conda/__init__.py b/repo2docker/buildpacks/conda/__init__.py index 4de9638e..17474bf1 100644 --- a/repo2docker/buildpacks/conda/__init__.py +++ b/repo2docker/buildpacks/conda/__init__.py @@ -25,17 +25,26 @@ class CondaBuildPack(BaseImage): the `NB_PYTHON_PREFIX` to the location of the jupyter binary. """ - return super().get_env() + [ + env = super().get_env() + [ ('CONDA_DIR', '${APP_BASE}/conda'), ('NB_PYTHON_PREFIX', '${CONDA_DIR}'), ] + if self.py2: + env.append(('KERNEL_PYTHON_PREFIX', '${CONDA_DIR}/envs/kernel')) + else: + env.append(('KERNEL_PYTHON_PREFIX', '${NB_PYTHON_PREFIX}')) + return env def get_path(self): """Return paths (including conda environment path) to be added to the PATH environment variable. """ - return super().get_path() + ['${CONDA_DIR}/bin'] + path = super().get_path() + if self.py2: + path.insert(0, '${KERNEL_PYTHON_PREFIX}/bin') + path.insert(0, '${CONDA_DIR}/bin') + return path def get_build_scripts(self): """ @@ -127,6 +136,12 @@ class CondaBuildPack(BaseImage): py_version = None with open(environment_yml) as f: env = YAML().load(f) + # check if the env file is empty, if so instantiate an empty dictionary. + if env is None: + env = {} + # check if the env file has a dictionary not a list or other data structure. + if not isinstance(env, dict): + raise TypeError("environment.yml should contain a dictionary. Got %r" % type(env)) for dep in env.get('dependencies', []): if not isinstance(dep, str): continue diff --git a/repo2docker/buildpacks/conda/environment.frozen.yml b/repo2docker/buildpacks/conda/environment.frozen.yml index f2e31901..85d01107 100644 --- a/repo2docker/buildpacks/conda/environment.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.frozen.yml @@ -1,23 +1,24 @@ # AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY -# Frozen on 2018-04-02 07:27:43 UTC +# Frozen on 2018-05-08 20:22:20 UTC name: r2d channels: - conda-forge - defaults - conda-forge/label/broken dependencies: + - backcall=0.1.0=py_0 - bleach=2.1.3=py_0 - - ca-certificates=2018.1.18=0 - - certifi=2018.1.18=py36_0 - - decorator=4.2.1=py36_0 + - ca-certificates=2018.4.16=0 + - certifi=2018.4.16=py36_0 + - decorator=4.3.0=py_0 - entrypoints=0.2.3=py36_1 - gmp=6.1.2=0 - html5lib=1.0.1=py_0 - ipykernel=4.8.2=py36_0 - - ipython=6.2.1=py36_1 + - ipython=6.3.1=py36_0 - ipython_genutils=0.2.0=py36_0 - ipywidgets=7.1.1=py36_0 - - jedi=0.11.1=py36_0 + - jedi=0.12.0=py36_0 - jinja2=2.10=py36_0 - jsonschema=2.6.0=py36_1 - jupyter_client=5.2.3=py36_0 @@ -26,27 +27,27 @@ dependencies: - jupyterlab_launcher=0.10.5=py36_0 - libsodium=1.0.16=0 - markupsafe=1.0=py36_0 - - mistune=0.8.3=py_0 + - mistune=0.8.3=py36_1 - nbconvert=5.3.1=py_1 - nbformat=4.4.0=py36_0 - ncurses=5.9=10 - notebook=5.4.1=py36_0 - - openssl=1.0.2n=0 - - pandoc=2.1.3=0 - - pandocfilters=1.4.1=py36_0 - - parso=0.1.1=py_0 - - pexpect=4.4.0=py36_0 + - openssl=1.0.2o=0 + - pandoc=2.2=0 + - pandocfilters=1.4.2=py36_0 + - parso=0.2.0=py_0 + - pexpect=4.5.0=py36_0 - pickleshare=0.7.4=py36_0 - pip=9.0.3=py36_0 - prompt_toolkit=1.0.15=py36_0 - ptyprocess=0.5.2=py36_0 - pygments=2.2.0=py36_0 - - python=3.6.5=0 + - python=3.6.5=1 - python-dateutil=2.7.2=py_0 - pyzmq=17.0.0=py36_4 - readline=7.0=0 - send2trash=1.5.0=py_0 - - setuptools=39.0.1=py36_0 + - setuptools=39.1.0=py36_0 - simplegeneric=0.8.1=py36_0 - six=1.11.0=py36_1 - sqlite=3.20.1=2 @@ -57,12 +58,12 @@ dependencies: - traitlets=4.3.2=py36_0 - wcwidth=0.1.7=py36_0 - webencodings=0.5=py36_0 - - wheel=0.30.0=py36_2 - - widgetsnbextension=3.2.0=py36_0 + - wheel=0.31.0=py36_0 + - widgetsnbextension=3.2.1=py36_0 - xz=5.2.3=0 - zeromq=4.2.5=1 - zlib=1.2.11=0 - pip: - - nteract-on-jupyter==1.6.0 + - nteract-on-jupyter==1.7.0 prefix: /opt/conda/envs/r2d diff --git a/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml b/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml index 3068800c..a8af048d 100644 --- a/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.py-2.7.yml, DO NOT MANUALLY MODIFY -# Frozen on 2018-04-02 07:21:15 UTC +# Frozen on 2018-05-08 20:15:09 UTC name: r2d channels: - conda-forge @@ -9,31 +9,31 @@ dependencies: - backports=1.0=py27_1 - backports.shutil_get_terminal_size=1.0.0=py_3 - backports_abc=0.5=py27_0 - - ca-certificates=2018.1.18=0 - - certifi=2018.1.18=py27_0 - - decorator=4.2.1=py27_0 + - ca-certificates=2018.4.16=0 + - certifi=2018.4.16=py27_0 + - decorator=4.3.0=py_0 - enum34=1.1.6=py27_1 - ipykernel=4.8.2=py27_0 - - ipython=5.5.0=py27_0 + - ipython=5.6.0=py27_0 - ipython_genutils=0.2.0=py27_0 - jupyter_client=5.2.3=py27_0 - jupyter_core=4.4.0=py_0 - libsodium=1.0.16=0 - ncurses=5.9=10 - - openssl=1.0.2n=0 - - pathlib2=2.3.0=py27_0 - - pexpect=4.4.0=py27_0 + - openssl=1.0.2o=0 + - pathlib2=2.3.2=py27_0 + - pexpect=4.5.0=py27_0 - pickleshare=0.7.4=py27_0 - pip=9.0.3=py27_0 - prompt_toolkit=1.0.15=py27_0 - ptyprocess=0.5.2=py27_0 - pygments=2.2.0=py27_0 - - python=2.7.14=5 + - python=2.7.15=0 - python-dateutil=2.7.2=py_0 - pyzmq=17.0.0=py27_4 - readline=7.0=0 - scandir=1.7=py27_0 - - setuptools=39.0.1=py27_0 + - setuptools=39.1.0=py27_0 - simplegeneric=0.8.1=py27_0 - singledispatch=3.4.0.3=py27_0 - six=1.11.0=py27_1 @@ -43,7 +43,7 @@ dependencies: - tornado=4.5.3=py27_0 - traitlets=4.3.2=py27_0 - wcwidth=0.1.7=py27_0 - - wheel=0.30.0=py27_2 + - wheel=0.31.0=py27_0 - zeromq=4.2.5=1 - zlib=1.2.11=0 - pip: diff --git a/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml b/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml index 0d8aa8c6..01183a6e 100644 --- a/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml @@ -1,23 +1,24 @@ # AUTO GENERATED FROM environment.py-3.5.yml, DO NOT MANUALLY MODIFY -# Frozen on 2018-04-02 07:23:18 UTC +# Frozen on 2018-05-08 20:18:13 UTC name: r2d channels: - conda-forge - defaults - conda-forge/label/broken dependencies: + - backcall=0.1.0=py_0 - bleach=2.1.3=py_0 - - ca-certificates=2018.1.18=0 - - certifi=2018.1.18=py35_0 - - decorator=4.2.1=py35_0 + - ca-certificates=2018.4.16=0 + - certifi=2018.4.16=py35_0 + - decorator=4.3.0=py_0 - entrypoints=0.2.3=py35_1 - gmp=6.1.2=0 - html5lib=1.0.1=py_0 - ipykernel=4.8.2=py35_0 - - ipython=6.2.1=py35_1 + - ipython=6.3.1=py35_0 - ipython_genutils=0.2.0=py35_0 - ipywidgets=7.1.1=py35_0 - - jedi=0.11.1=py35_0 + - jedi=0.12.0=py35_0 - jinja2=2.10=py35_0 - jsonschema=2.6.0=py35_1 - jupyter_client=5.2.3=py35_0 @@ -26,27 +27,27 @@ dependencies: - jupyterlab_launcher=0.10.5=py35_0 - libsodium=1.0.16=0 - markupsafe=1.0=py35_0 - - mistune=0.8.3=py_0 + - mistune=0.8.3=py35_1 - nbconvert=5.3.1=py_1 - nbformat=4.4.0=py35_0 - ncurses=5.9=10 - notebook=5.4.1=py35_0 - - openssl=1.0.2n=0 - - pandoc=2.1.3=0 - - pandocfilters=1.4.1=py35_0 - - parso=0.1.1=py_0 - - pexpect=4.4.0=py35_0 + - openssl=1.0.2o=0 + - pandoc=2.2=0 + - pandocfilters=1.4.2=py35_0 + - parso=0.2.0=py_0 + - pexpect=4.5.0=py35_0 - pickleshare=0.7.4=py35_0 - pip=9.0.3=py35_0 - prompt_toolkit=1.0.15=py35_0 - ptyprocess=0.5.2=py35_0 - pygments=2.2.0=py35_0 - - python=3.5.5=0 + - python=3.5.5=1 - python-dateutil=2.7.2=py_0 - pyzmq=17.0.0=py35_4 - readline=7.0=0 - send2trash=1.5.0=py_0 - - setuptools=39.0.1=py35_0 + - setuptools=39.1.0=py35_0 - simplegeneric=0.8.1=py35_0 - six=1.11.0=py35_1 - sqlite=3.20.1=2 @@ -57,12 +58,12 @@ dependencies: - traitlets=4.3.2=py35_0 - wcwidth=0.1.7=py35_0 - webencodings=0.5=py35_0 - - wheel=0.30.0=py35_2 - - widgetsnbextension=3.2.0=py35_0 + - wheel=0.31.0=py35_0 + - widgetsnbextension=3.2.1=py35_0 - xz=5.2.3=0 - zeromq=4.2.5=1 - zlib=1.2.11=0 - pip: - - nteract-on-jupyter==1.6.0 + - nteract-on-jupyter==1.7.0 prefix: /opt/conda/envs/r2d diff --git a/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml b/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml index f2e31901..85d01107 100644 --- a/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml @@ -1,23 +1,24 @@ # AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY -# Frozen on 2018-04-02 07:27:43 UTC +# Frozen on 2018-05-08 20:22:20 UTC name: r2d channels: - conda-forge - defaults - conda-forge/label/broken dependencies: + - backcall=0.1.0=py_0 - bleach=2.1.3=py_0 - - ca-certificates=2018.1.18=0 - - certifi=2018.1.18=py36_0 - - decorator=4.2.1=py36_0 + - ca-certificates=2018.4.16=0 + - certifi=2018.4.16=py36_0 + - decorator=4.3.0=py_0 - entrypoints=0.2.3=py36_1 - gmp=6.1.2=0 - html5lib=1.0.1=py_0 - ipykernel=4.8.2=py36_0 - - ipython=6.2.1=py36_1 + - ipython=6.3.1=py36_0 - ipython_genutils=0.2.0=py36_0 - ipywidgets=7.1.1=py36_0 - - jedi=0.11.1=py36_0 + - jedi=0.12.0=py36_0 - jinja2=2.10=py36_0 - jsonschema=2.6.0=py36_1 - jupyter_client=5.2.3=py36_0 @@ -26,27 +27,27 @@ dependencies: - jupyterlab_launcher=0.10.5=py36_0 - libsodium=1.0.16=0 - markupsafe=1.0=py36_0 - - mistune=0.8.3=py_0 + - mistune=0.8.3=py36_1 - nbconvert=5.3.1=py_1 - nbformat=4.4.0=py36_0 - ncurses=5.9=10 - notebook=5.4.1=py36_0 - - openssl=1.0.2n=0 - - pandoc=2.1.3=0 - - pandocfilters=1.4.1=py36_0 - - parso=0.1.1=py_0 - - pexpect=4.4.0=py36_0 + - openssl=1.0.2o=0 + - pandoc=2.2=0 + - pandocfilters=1.4.2=py36_0 + - parso=0.2.0=py_0 + - pexpect=4.5.0=py36_0 - pickleshare=0.7.4=py36_0 - pip=9.0.3=py36_0 - prompt_toolkit=1.0.15=py36_0 - ptyprocess=0.5.2=py36_0 - pygments=2.2.0=py36_0 - - python=3.6.5=0 + - python=3.6.5=1 - python-dateutil=2.7.2=py_0 - pyzmq=17.0.0=py36_4 - readline=7.0=0 - send2trash=1.5.0=py_0 - - setuptools=39.0.1=py36_0 + - setuptools=39.1.0=py36_0 - simplegeneric=0.8.1=py36_0 - six=1.11.0=py36_1 - sqlite=3.20.1=2 @@ -57,12 +58,12 @@ dependencies: - traitlets=4.3.2=py36_0 - wcwidth=0.1.7=py36_0 - webencodings=0.5=py36_0 - - wheel=0.30.0=py36_2 - - widgetsnbextension=3.2.0=py36_0 + - wheel=0.31.0=py36_0 + - widgetsnbextension=3.2.1=py36_0 - xz=5.2.3=0 - zeromq=4.2.5=1 - zlib=1.2.11=0 - pip: - - nteract-on-jupyter==1.6.0 + - nteract-on-jupyter==1.7.0 prefix: /opt/conda/envs/r2d diff --git a/repo2docker/buildpacks/conda/environment.yml b/repo2docker/buildpacks/conda/environment.yml index ee040904..076f6198 100644 --- a/repo2docker/buildpacks/conda/environment.yml +++ b/repo2docker/buildpacks/conda/environment.yml @@ -5,4 +5,4 @@ dependencies: - tornado==4.5.3 - notebook==5.4.1 - pip: - - nteract_on_jupyter==1.6.0 + - nteract_on_jupyter==1.7.0 diff --git a/repo2docker/buildpacks/conda/install-miniconda.bash b/repo2docker/buildpacks/conda/install-miniconda.bash index f88433b2..1073afba 100755 --- a/repo2docker/buildpacks/conda/install-miniconda.bash +++ b/repo2docker/buildpacks/conda/install-miniconda.bash @@ -3,7 +3,7 @@ set -ex cd $(dirname $0) -CONDA_VERSION=4.3.30 +CONDA_VERSION=4.5.1 URL="https://repo.continuum.io/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh" INSTALLER_PATH=/tmp/miniconda-installer.sh @@ -12,7 +12,7 @@ chmod +x ${INSTALLER_PATH} # Only MD5 checksums are available for miniconda # Can be obtained from https://repo.continuum.io/miniconda/ -MD5SUM="0b80a152332a4ce5250f3c09589c7a81" +MD5SUM="0c28787e3126238df24c5d4858bd0744" if ! echo "${MD5SUM} ${INSTALLER_PATH}" | md5sum --quiet -c -; then echo "md5sum mismatch for ${INSTALLER_PATH}, exiting!" diff --git a/repo2docker/buildpacks/legacy/__init__.py b/repo2docker/buildpacks/legacy/__init__.py index 5219b2ae..cbc51dd1 100644 --- a/repo2docker/buildpacks/legacy/__init__.py +++ b/repo2docker/buildpacks/legacy/__init__.py @@ -23,7 +23,7 @@ class LegacyBinderDockerBuildPack(DockerBuildPack): RUN conda install -yq conda>=4.3 && \ conda install -yq conda==4.4.11 && \ conda env update -n python3 -f /tmp/python3.frozen.yml && \ - conda remove -yq -n python3 nb_conda_kernels && \ + conda remove -yq -n python3 nb_conda_kernels _nb_ext_conf && \ conda env update -n root -f /tmp/root.frozen.yml && \ /home/main/anaconda2/envs/python3/bin/ipython kernel install --sys-prefix && \ /home/main/anaconda2/bin/ipython kernel install --prefix=/home/main/anaconda2/envs/python3 && \ diff --git a/repo2docker/buildpacks/python/__init__.py b/repo2docker/buildpacks/python/__init__.py index 76a57832..8141a30d 100644 --- a/repo2docker/buildpacks/python/__init__.py +++ b/repo2docker/buildpacks/python/__init__.py @@ -1,139 +1,70 @@ """Generates Dockerfiles based on an input matrix based on Python.""" import os -from ..base import BaseImage +from ..conda import CondaBuildPack -class PythonBuildPack(BaseImage): - """Setup Python 3 for use with a repository.""" - def get_packages(self): - """Return a list of the Python 3 core language packages to be installed - via apt-get for this BuildPack. +class PythonBuildPack(CondaBuildPack): + """Setup Python for use with a repository.""" - Note: The packages specified here are for the core Python3 language. - Third party libraries are specified in other configuration files. + @property + def python_version(self): + if hasattr(self, '_python_version'): + return self._python_version - """ - return super().get_packages().union({ - 'python3', - 'python3-venv', - 'python3-dev', - }) + try: + with open(self.binder_path('runtime.txt')) as f: + runtime = f.read().strip() + except FileNotFoundError: + runtime = '' - def get_env(self): - """ - Return environment variables to be set. + if not runtime.startswith('python-'): + # not a Python runtime (e.g. R, which subclasses this) + # use the default Python + self._python_version = self.major_pythons['3'] + return self._python_version - We set `VENV_PATH` to the virtual environment location and - the `NB_PYTHON_PREFIX` to the location of the jupyter binary. - - """ - return super().get_env() + [ - ("VENV_PATH", "${APP_BASE}/venv"), - # Prefix to use for installing kernels and finding jupyter binary - ("NB_PYTHON_PREFIX", "${VENV_PATH}"), - ] - - def get_path(self): - """Return paths (including virtual environment path) to be added to - the PATH environment variable. - - """ - return super().get_path() + [ - "${VENV_PATH}/bin" - ] - - def get_build_script_files(self): - """ - Dict of files to be copied to the container image for use in building. - - This is copied before the `build_scripts` & `assemble_scripts` are - run, so can be executed from either of them. - - It's a dictionary where the key is the source file path in the host - system, and the value is the destination file path inside the - container image. - - This currently adds a frozen set of Python 3 requirements to the dict - of files. - - """ - files = { - 'python/requirements.frozen.txt': '/tmp/requirements.frozen.txt', - } - files.update(super().get_build_script_files()) - return files - - def get_build_scripts(self): - """ - Return series of build-steps common to all Python 3 repositories. - - All scripts here should be independent of contents of the repository. - - This sets up: - - - a directory for the virtual environment and its ownership by the - notebook user - - a Python 3 interpreter for the virtual environement - - a Python 3 jupyter kernel including a base set of requirements - - support for Jupyter widgets - - support for JupyterLab - - support for nteract - - """ - return super().get_build_scripts() + [ - ( - "root", - r""" - mkdir -p ${VENV_PATH} && \ - chown -R ${NB_USER}:${NB_USER} ${VENV_PATH} - """ - ), - ( - "${NB_USER}", - r""" - python3 -m venv ${VENV_PATH} - """ - ), - ( - "${NB_USER}", - r""" - pip install --no-cache-dir -r /tmp/requirements.frozen.txt && \ - jupyter nbextension enable --py widgetsnbextension --sys-prefix && \ - jupyter serverextension enable --py jupyterlab --sys-prefix && \ - jupyter serverextension enable nteract_on_jupyter --sys-prefix - """ - ) - ] + py_version_info = runtime.split('-', 1)[1].split('.') + py_version = '' + if len(py_version_info) == 1: + py_version = self.major_pythons[py_version_info[0]] + else: + # get major.minor + py_version = '.'.join(py_version_info[:2]) + self._python_version = py_version + return self._python_version def get_assemble_scripts(self): """Return series of build-steps specific to this repository. """ # If we have a runtime.txt & that's set to python-2.7, - # we will *not* install requirements.txt but will find & - # install a requirements3.txt file if it exists. - # This way, when using python2 venv, requirements.txt will - # be installed in the python2 venv, and requirements3.txt - # will be installed in python3 venv. This is less of a - # surprise than requiring python2 to be requirements2.txt tho. + # requirements.txt will be installed in the *kernel* env + # and requirements3.txt (if it exists) + # will be installed in the python 3 notebook server env. assemble_scripts = super().get_assemble_scripts() setup_py = 'setup.py' - try: - with open(self.binder_path('runtime.txt')) as f: - runtime = f.read().strip() - except FileNotFoundError: - runtime = 'python-3.5' - if runtime == 'python-2.7': - pip = "pip2" - requirements_file = self.binder_path('requirements3.txt') - else: - pip = "pip3" - requirements_file = self.binder_path('requirements.txt') + # KERNEL_PYTHON_PREFIX is the env with the kernel, + # whether it's distinct from the notebook or the same. + pip = '${KERNEL_PYTHON_PREFIX}/bin/pip' + if self.py2: + # using python 2 kernel, + # requirements3.txt allows installation in the notebook server env + nb_requirements_file = self.binder_path('requirements3.txt') + if os.path.exists(nb_requirements_file): + assemble_scripts.append(( + '${NB_USER}', + '${NB_PYTHON_PREFIX}/bin/pip install --no-cache-dir -r "{}"'.format(nb_requirements_file) + )) + + # install requirements.txt in the kernel env + requirements_file = self.binder_path('requirements.txt') if os.path.exists(requirements_file): assemble_scripts.append(( '${NB_USER}', - 'pip3 install --no-cache-dir -r "{}"'.format(requirements_file) + '{} install --no-cache-dir -r "{}"'.format(pip, requirements_file) )) + + # setup.py exists *and* binder dir is not used if not os.path.exists('binder') and os.path.exists(setup_py): assemble_scripts.append(( '${NB_USER}', @@ -142,7 +73,7 @@ class PythonBuildPack(BaseImage): return assemble_scripts def detect(self): - """Check if current repo should be built with the Python 3 Build pack. + """Check if current repo should be built with the Python buildpack. """ requirements_txt = self.binder_path('requirements.txt') runtime_txt = self.binder_path('runtime.txt') @@ -151,136 +82,10 @@ class PythonBuildPack(BaseImage): if os.path.exists(runtime_txt): with open(runtime_txt) as f: runtime = f.read().strip() - if runtime.startswith("python-3"): + if runtime.startswith("python-"): return True else: return False if not os.path.exists('binder') and os.path.exists(setup_py): return True return os.path.exists(requirements_txt) - - -class Python2BuildPack(PythonBuildPack): - """Setup Python 2 for use with a repository.""" - def get_packages(self): - """Return a list of the Python 2 core language packages to be installed - via apt-get for this BuildPack. - - Note: The packages specified here are for the core Python2 language. - Third party libraries are specified in other configuration files. - - """ - return super().get_packages().union({ - 'python', - 'python-dev', - 'virtualenv' - }) - - def get_env(self): - """ - Return environment variables to be set. - - We set `VENV_PATH` to the virtual environment location containing - Python 2. - - """ - return super().get_env() + [ - ('VENV2_PATH', '${APP_BASE}/venv2') - ] - - def get_path(self): - """Return paths (including virtual environment path) to be added to - the PATH environment variable. - - """ - return super().get_path() + [ - "${VENV2_PATH}/bin" - ] - - def get_build_script_files(self): - """ - Dict of files to be copied to the container image for use in building. - - This is copied before the `build_scripts` & `assemble_scripts` are - run, so can be executed from either of them. - - It's a dictionary where the key is the source file path in the host - system, and the value is the destination file path inside the - container image. - - This currently adds a frozen set of Python 2 requirements to the dict - of files. - - """ - files = { - 'python/requirements2.frozen.txt': '/tmp/requirements2.frozen.txt', - } - files.update(super().get_build_script_files()) - return files - - def get_build_scripts(self): - """ - Return series of build-steps common to all Python 2 repositories. - - All scripts here should be independent of contents of the repository. - - This sets up: - - - a directory for the virtual environment and its ownership by the - notebook user - - a Python 2 interpreter for the virtual environement - - a Python 2 jupyter kernel - - """ - return super().get_build_scripts() + [ - ( - "root", - r""" - mkdir -p ${VENV2_PATH} && \ - chown -R ${NB_USER}:${NB_USER} ${VENV2_PATH} - """ - ), - ( - "${NB_USER}", - r""" - virtualenv -p python2 ${VENV2_PATH} - """ - ), - ( - "${NB_USER}", - r""" - pip2 install --no-cache-dir -r /tmp/requirements2.frozen.txt && \ - python2 -m ipykernel install --prefix=${NB_PYTHON_PREFIX} - """ - ) - ] - - def get_assemble_scripts(self): - """Return series of build-steps specific to this repository. - """ - requirements_txt = self.binder_path('requirements.txt') - assemble_scripts = super().get_assemble_scripts() - if os.path.exists(requirements_txt): - assemble_scripts.insert(0, ( - '${NB_USER}', - 'pip2 install --no-cache-dir -r "{}"'.format(requirements_txt) - )) - return assemble_scripts - - def detect(self): - """Check if current repo should be built with the Python 2 Build pack. - """ - runtime_txt = self.binder_path('runtime.txt') - - if os.path.exists(runtime_txt): - with open(runtime_txt) as f: - runtime = f.read().strip() - if runtime == 'python-2.7': - return True - elif runtime.startswith('python-2'): - raise ValueError( - "Only python-2.7 or python-3.x is supported in " - "runtime.txt, not '{}'".format(runtime_txt)) - else: - return False - return False diff --git a/repo2docker/buildpacks/python/freeze.bash b/repo2docker/buildpacks/python/freeze.bash deleted file mode 100755 index 54bb30f9..00000000 --- a/repo2docker/buildpacks/python/freeze.bash +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -euo pipefail -# Freeze requirements.txt into requirements.frozen.txt, pinning all dependent library versions to -# versions that are resolved at time of freezing. -# Does the same for requirements2.txt to requirements2.frozen.txt... - -# cd to the directory where the freeze script is located -if [[ ! -z "$(which realpath 2>/dev/null)" ]]; then - realpath=realpath -else - realpath="readlink -f" -fi - -cd $(dirname "$($realpath "$0")") - - -function freeze-requirements { - # Freeze a requirements file $2 into a frozen requirements file $3 - # Requires that a completely empty venv of appropriate version exist in $1 - PYTHON_VERSION="$1" - REQUIREMENTS_FILE="$2" - FROZEN_FILE="$3" - if [[ $(echo ${PYTHON_VERSION} | cut -d. -f 1) == "2" ]]; then - VENV=virtualenv - else - VENV=venv - fi - - echo "# AUTO GENERATED FROM ${REQUIREMENTS_FILE}, DO NOT MANUALLY MODIFY" > ${FROZEN_FILE} - echo "# Frozen on $(date -u)" >> ${FROZEN_FILE} - docker run --rm -v $PWD:/python -it python:${PYTHON_VERSION} \ - sh -c " - python -m $VENV /venv - /venv/bin/pip install -r /python/${REQUIREMENTS_FILE} && - /venv/bin/pip freeze | sort --ignore-case >> /python/${FROZEN_FILE}" -} - -freeze-requirements 3.5 requirements.txt requirements.frozen.txt -freeze-requirements 2.7 requirements2.txt requirements2.frozen.txt diff --git a/repo2docker/buildpacks/python/requirements.frozen.txt b/repo2docker/buildpacks/python/requirements.frozen.txt deleted file mode 100644 index 3ca553fe..00000000 --- a/repo2docker/buildpacks/python/requirements.frozen.txt +++ /dev/null @@ -1,42 +0,0 @@ -# AUTO GENERATED FROM requirements.txt, DO NOT MANUALLY MODIFY -# Frozen on Mon Apr 2 07:17:48 UTC 2018 -bleach==2.1.3 -decorator==4.2.1 -entrypoints==0.2.3 -html5lib==1.0.1 -ipykernel==4.8.2 -ipython-genutils==0.2.0 -ipython==6.2.1 -ipywidgets==7.1.1 -jedi==0.11.1 -Jinja2==2.10 -jsonschema==2.6.0 -jupyter-client==5.2.3 -jupyter-core==4.4.0 -jupyterlab-launcher==0.10.5 -jupyterlab==0.31.5 -MarkupSafe==1.0 -mistune==0.8.3 -nbconvert==5.3.1 -nbformat==4.4.0 -notebook==5.4.1 -nteract-on-jupyter==1.6.0 -pandocfilters==1.4.2 -parso==0.1.1 -pexpect==4.4.0 -pickleshare==0.7.4 -prompt-toolkit==1.0.15 -ptyprocess==0.5.2 -Pygments==2.2.0 -python-dateutil==2.7.2 -pyzmq==17.0.0 -Send2Trash==1.5.0 -simplegeneric==0.8.1 -six==1.11.0 -terminado==0.8.1 -testpath==0.3.1 -tornado==4.5.3 -traitlets==4.3.2 -wcwidth==0.1.7 -webencodings==0.5.1 -widgetsnbextension==3.1.4 diff --git a/repo2docker/buildpacks/python/requirements.txt b/repo2docker/buildpacks/python/requirements.txt deleted file mode 100644 index e0e13a5f..00000000 --- a/repo2docker/buildpacks/python/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -notebook==5.4.1 -tornado==4.5.3 -ipywidgets==7.1.1 -jupyterlab==0.31.5 -nteract_on_jupyter==1.6.0 diff --git a/repo2docker/buildpacks/python/requirements2.frozen.txt b/repo2docker/buildpacks/python/requirements2.frozen.txt deleted file mode 100644 index 670eb05a..00000000 --- a/repo2docker/buildpacks/python/requirements2.frozen.txt +++ /dev/null @@ -1,27 +0,0 @@ -# AUTO GENERATED FROM requirements2.txt, DO NOT MANUALLY MODIFY -# Frozen on Mon Apr 2 07:19:00 UTC 2018 -backports-abc==0.5 -backports.shutil-get-terminal-size==1.0.0 -certifi==2018.1.18 -decorator==4.2.1 -enum34==1.1.6 -ipykernel==4.8.2 -ipython-genutils==0.2.0 -ipython==5.5.0 -jupyter-client==5.2.3 -jupyter-core==4.4.0 -pathlib2==2.3.0 -pexpect==4.4.0 -pickleshare==0.7.4 -prompt-toolkit==1.0.15 -ptyprocess==0.5.2 -Pygments==2.2.0 -python-dateutil==2.7.2 -pyzmq==17.0.0 -scandir==1.7 -simplegeneric==0.8.1 -singledispatch==3.4.0.3 -six==1.11.0 -tornado==4.5.3 -traitlets==4.3.2 -wcwidth==0.1.7 diff --git a/repo2docker/buildpacks/python/requirements2.txt b/repo2docker/buildpacks/python/requirements2.txt deleted file mode 100644 index 7653d953..00000000 --- a/repo2docker/buildpacks/python/requirements2.txt +++ /dev/null @@ -1,2 +0,0 @@ -ipykernel==4.8.2 -tornado==4.5.3 diff --git a/repo2docker/buildpacks/r.py b/repo2docker/buildpacks/r.py index a10a4b53..ebd5416b 100644 --- a/repo2docker/buildpacks/r.py +++ b/repo2docker/buildpacks/r.py @@ -4,6 +4,7 @@ import datetime from .python import PythonBuildPack + class RBuildPack(PythonBuildPack): """ Setup R for use with a repository @@ -128,6 +129,10 @@ class RBuildPack(PythonBuildPack): # This is MD5, because that is what RStudio download page provides! rstudio_checksum = '24cd11f0405d8372b4168fc9956e0386' + # Via https://www.rstudio.com/products/shiny/download-server/ + shiny_url = 'https://download3.rstudio.org/ubuntu-14.04/x86_64/shiny-server-1.5.7.907-amd64.deb' + shiny_checksum = '78371a8361ba0e7fec44edd2b8e425ac' + # Version of MRAN to pull devtools from. devtools_version = '2018-02-01' @@ -155,6 +160,20 @@ class RBuildPack(PythonBuildPack): rstudio_checksum=rstudio_checksum ) ), + ( + "root", + # Install Shiny Server! + r""" + curl --silent --location --fail {url} > {deb} && \ + echo '{checksum} {deb}' | md5sum -c - && \ + dpkg -i {deb} && \ + rm {deb} + """.format( + url=shiny_url, + checksum=shiny_checksum, + deb='/tmp/shiny.deb' + ) + ), ( "root", # Set paths so that RStudio shares libraries with base R @@ -169,7 +188,7 @@ class RBuildPack(PythonBuildPack): "${NB_USER}", # Install nbrsessionproxy r""" - pip install --no-cache-dir nbrsessionproxy==0.6.1 && \ + pip install --no-cache-dir nbrsessionproxy==0.7.0 && \ jupyter serverextension enable nbrsessionproxy --sys-prefix && \ jupyter nbextension install --py nbrsessionproxy --sys-prefix && \ jupyter nbextension enable --py nbrsessionproxy --sys-prefix @@ -186,7 +205,16 @@ class RBuildPack(PythonBuildPack): devtools_version=devtools_version, irkernel_version=irkernel_version ) - ) + ), + ( + "${NB_USER}", + # Install shiny library + r""" + R --quiet -e "install.packages('shiny', repos='https://mran.microsoft.com/snapshot/{}', method='libcurl')" + """.format( + self.checkpoint_date.isoformat() + ) + ), ] def get_assemble_scripts(self): @@ -208,6 +236,16 @@ class RBuildPack(PythonBuildPack): echo "options(repos = c(CRAN='{mran_url}'), download.file.method = 'libcurl')" > /etc/R/Rprofile.site """.format(mran_url=mran_url) ), + ( + # Not all of these locations are configurable; log_dir is + "root", + r""" + install -o ${NB_USER} -g ${NB_USER} -d /var/log/shiny-server && \ + install -o ${NB_USER} -g ${NB_USER} -d /var/lib/shiny-server && \ + install -o ${NB_USER} -g ${NB_USER} /dev/null /var/log/shiny-server.log && \ + install -o ${NB_USER} -g ${NB_USER} /dev/null /var/run/shiny-server.pid + """ + ), ] installR_path = self.binder_path('install.R') diff --git a/setup.cfg b/setup.cfg index d7de2b16..947f15f2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,5 @@ [wheel] -universal=1 \ No newline at end of file +universal=1 + +[metadata] +license_file = LICENSE diff --git a/tests/conda/simple-py2/verify b/tests/conda/simple-py2/verify index 38c58a19..57f75c5e 100755 --- a/tests/conda/simple-py2/verify +++ b/tests/conda/simple-py2/verify @@ -13,7 +13,7 @@ assert sorted(specs) == ['python2', 'python3'], specs.keys() import json from subprocess import check_output envs = json.loads(check_output(['conda', 'env', 'list', '--json']).decode('utf8')) -assert envs == {'envs': ['/srv/conda/envs/kernel']}, envs +assert envs == {'envs': ['/srv/conda', '/srv/conda/envs/kernel']}, envs pkgs = json.loads(check_output(['conda', 'list', '-n', 'kernel', '--json']).decode('utf8')) pkg_names = [pkg['name'] for pkg in pkgs] diff --git a/tests/test_env_yml.py b/tests/test_env_yml.py new file mode 100644 index 00000000..c6d146f6 --- /dev/null +++ b/tests/test_env_yml.py @@ -0,0 +1,26 @@ +""" +Test if the environment.yml is empty or it constains other data structure than a dictionary +""" +import os +import sys +import pytest +from repo2docker import buildpacks + + +def test_empty_env_yml(tmpdir): + tmpdir.chdir() + p = tmpdir.join("environment.yml") + p.write("") + bp = buildpacks.CondaBuildPack() + py_ver = bp.python_version + # If the environment.yml is empty python_version will get an empty string + assert py_ver == '' + +def test_no_dict_env_yml(tmpdir): + tmpdir.chdir() + q = tmpdir.join("environment.yml") + q.write("numpy\n " + "matplotlib\n") + bq = buildpacks.CondaBuildPack() + with pytest.raises(TypeError): + py_ver = bq.python_version \ No newline at end of file diff --git a/tests/venv/py3/README.rst b/tests/venv/py3/README.rst deleted file mode 100644 index 81961f7b..00000000 --- a/tests/venv/py3/README.rst +++ /dev/null @@ -1,20 +0,0 @@ -System - Specifying runtime environments ----------------------------------------- - -You can specify runtime environments (such as Python 2 or 3) with a -``runtime.txt`` file. To do so, include a line of the following form in -your ``runtime.txt`` file: - -``` -python-N -``` - -Where ``N`` is either ``2`` or ``3``. If ``N==2``, Python 2.7 will be used. -If ``N==3``, Python 3.6 will be used. - -This is an example that selects Python 3. Currently you can not use -this to select a specific version of Python 3 (e.g. 3.4 vs 3.6). If you -need this level of control we recommend you use a `environment.yml`. - -Note that you can also install Python environments using the Anaconda -distribution by using an ``environment.yml`` file. diff --git a/tests/venv/py35/runtime.txt b/tests/venv/py35/runtime.txt new file mode 100644 index 00000000..5cea6be5 --- /dev/null +++ b/tests/venv/py35/runtime.txt @@ -0,0 +1 @@ +python-3.5 diff --git a/tests/venv/py35/verify b/tests/venv/py35/verify new file mode 100755 index 00000000..4abec309 --- /dev/null +++ b/tests/venv/py35/verify @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 +import sys + +print(sys.version_info) +assert sys.version_info[:2] == (3, 5), sys.version diff --git a/tests/venv/usr-bin/verify b/tests/venv/usr-bin/verify new file mode 100755 index 00000000..4d372157 --- /dev/null +++ b/tests/venv/usr-bin/verify @@ -0,0 +1,5 @@ +#!/usr/bin/env python +# Verify that ~/.local/bin is on the PATH +import os + +assert os.path.expanduser('~/.local/bin') in os.getenv("PATH"), os.getenv("PATH")