From 5aaf78a828c84b6db9a2c4b72a8feb80c40082b2 Mon Sep 17 00:00:00 2001 From: Carol Willing Date: Mon, 19 Mar 2018 16:27:00 -0700 Subject: [PATCH 01/10] add docstrings for conda buildpack --- repo2docker/buildpacks/conda/__init__.py | 61 +++++++++++++++++++++--- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/repo2docker/buildpacks/conda/__init__.py b/repo2docker/buildpacks/conda/__init__.py index ef39e9a0..4de9638e 100644 --- a/repo2docker/buildpacks/conda/__init__.py +++ b/repo2docker/buildpacks/conda/__init__.py @@ -1,6 +1,4 @@ -""" -Buildpack for conda environments -""" +"""BuildPack for conda environments""" import os import re @@ -15,16 +13,48 @@ HERE = os.path.dirname(os.path.abspath(__file__)) class CondaBuildPack(BaseImage): + """A conda BuildPack. + + Uses miniconda since it is more lightweight than Anaconda. + + """ def get_env(self): + """Return environment variables to be set. + + We set `CONDA_DIR` to the conda install directory and + the `NB_PYTHON_PREFIX` to the location of the jupyter binary. + + """ return super().get_env() + [ ('CONDA_DIR', '${APP_BASE}/conda'), ('NB_PYTHON_PREFIX', '${CONDA_DIR}'), ] def get_path(self): + """Return paths (including conda environment path) to be added to + the PATH environment variable. + + """ return super().get_path() + ['${CONDA_DIR}/bin'] def get_build_scripts(self): + """ + Return series of build-steps common to all Python 3 repositories. + + All scripts here should be independent of contents of the repository. + + This sets up through `install-miniconda.bash` (found in this directory): + + - a directory for the conda environment and its ownership by the + notebook user + - a Python 3 interpreter for the conda environment + - a Python 3 jupyter kernel + - a frozen base set of requirements, including: + - support for Jupyter widgets + - support for JupyterLab + - support for nteract + + """ return super().get_build_scripts() + [ ( "root", @@ -41,6 +71,20 @@ class CondaBuildPack(BaseImage): } def get_build_script_files(self): + """ + Dict of files to be copied to the container image for use in building. + + This is copied before the `build_scripts` & `assemble_scripts` are + run, so can be executed from either of them. + + It's a dictionary where the key is the source file path in the host + system, and the value is the destination file path inside the + container image. + + This currently adds a frozen set of Python requirements to the dict + of files. + + """ files = { 'conda/install-miniconda.bash': '/tmp/install-miniconda.bash', } @@ -69,10 +113,11 @@ class CondaBuildPack(BaseImage): @property def python_version(self): - """ - Detect the Python version for a given environment.yml + """Detect the Python version for a given `environment.yml` + + Will return 'x.y' if version is found (e.g '3.6'), + or a Falsy empty string '' if not found. - Will return 'x.y' if found, or Falsy '' if not. """ environment_yml = self.binder_path('environment.yml') if not os.path.exists(environment_yml): @@ -109,6 +154,8 @@ class CondaBuildPack(BaseImage): return self.python_version and self.python_version.split('.')[0] == '2' def get_assemble_scripts(self): + """Return series of build-steps specific to this source repository. + """ assembly_scripts = [] environment_yml = self.binder_path('environment.yml') env_name = 'kernel' if self.py2 else 'root' @@ -124,4 +171,6 @@ class CondaBuildPack(BaseImage): return super().get_assemble_scripts() + assembly_scripts def detect(self): + """Check if current repo should be built with the Conda BuildPack. + """ return os.path.exists(self.binder_path('environment.yml')) and super().detect() From e9539c0d47dcdba5ca9aaf90b653ebd35132d303 Mon Sep 17 00:00:00 2001 From: Carol Willing Date: Mon, 19 Mar 2018 16:41:48 -0700 Subject: [PATCH 02/10] add docstrings to docker buildpack --- repo2docker/buildpacks/docker.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/repo2docker/buildpacks/docker.py b/repo2docker/buildpacks/docker.py index 8baa5dfb..8f3aefa0 100644 --- a/repo2docker/buildpacks/docker.py +++ b/repo2docker/buildpacks/docker.py @@ -1,5 +1,4 @@ -""" -Generates a variety of Dockerfiles based on an input matrix +"""Generates a variety of Dockerfiles based on an input matrix """ import os import docker @@ -7,17 +6,21 @@ from .base import BuildPack class DockerBuildPack(BuildPack): + """Docker BuildPack""" dockerfile = "Dockerfile" def detect(self): + """Check if current repo should be built with the Docker BuildPack""" return os.path.exists(self.binder_path('Dockerfile')) def render(self): + """Render the Dockerfile using by reading it from the source repo""" Dockerfile = self.binder_path('Dockerfile') with open(Dockerfile) as f: return f.read() def build(self, image_spec, memory_limit, build_args): + """Build a Docker image based on the Dockerfile in the source repo.""" limits = { # Always disable memory swap for building, since mostly # nothing good can come of that. From a2ebe316e424de4035905cb843410648350e81a4 Mon Sep 17 00:00:00 2001 From: Carol Willing Date: Mon, 19 Mar 2018 17:57:49 -0700 Subject: [PATCH 03/10] Refreeze environments --- repo2docker/buildpacks/conda/environment.frozen.yml | 12 ++++++------ .../buildpacks/conda/environment.py-2.7.frozen.yml | 8 ++++---- .../buildpacks/conda/environment.py-3.5.frozen.yml | 12 ++++++------ .../buildpacks/conda/environment.py-3.6.frozen.yml | 12 ++++++------ repo2docker/buildpacks/conda/environment.yml | 2 +- repo2docker/buildpacks/conda/freeze.py | 2 +- .../buildpacks/python/requirements.frozen.txt | 4 ++-- repo2docker/buildpacks/python/requirements.txt | 2 +- .../buildpacks/python/requirements2.frozen.txt | 2 +- 9 files changed, 28 insertions(+), 28 deletions(-) diff --git a/repo2docker/buildpacks/conda/environment.frozen.yml b/repo2docker/buildpacks/conda/environment.frozen.yml index 458ac45a..b17b9543 100644 --- a/repo2docker/buildpacks/conda/environment.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.frozen.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY -# Frozen on 2018-03-13 17:43:53 UTC +# Frozen on 2018-03-20 00:47:27 UTC name: r2d channels: - conda-forge @@ -30,23 +30,23 @@ dependencies: - nbconvert=5.3.1=py_1 - nbformat=4.4.0=py36_0 - ncurses=5.9=10 - - notebook=5.4.0=py36_0 + - notebook=5.4.1=py36_0 - openssl=1.0.2n=0 - - pandoc=2.1.2=0 + - pandoc=2.1.3=0 - pandocfilters=1.4.1=py36_0 - parso=0.1.1=py_0 - pexpect=4.4.0=py36_0 - pickleshare=0.7.4=py36_0 - - pip=9.0.1=py36_1 + - pip=9.0.2=py36_0 - prompt_toolkit=1.0.15=py36_0 - ptyprocess=0.5.2=py36_0 - pygments=2.2.0=py36_0 - python=3.6.4=0 - - python-dateutil=2.6.1=py36_0 + - python-dateutil=2.7.0=py_0 - pyzmq=17.0.0=py36_3 - readline=7.0=0 - send2trash=1.5.0=py_0 - - setuptools=38.5.2=py36_0 + - setuptools=39.0.1=py36_0 - simplegeneric=0.8.1=py36_0 - six=1.11.0=py36_1 - sqlite=3.20.1=2 diff --git a/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml b/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml index 1b3f297e..a98aa621 100644 --- a/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.py-2.7.yml, DO NOT MANUALLY MODIFY -# Frozen on 2018-03-13 17:38:07 UTC +# Frozen on 2018-03-20 00:42:45 UTC name: r2d channels: - conda-forge @@ -24,16 +24,16 @@ dependencies: - pathlib2=2.3.0=py27_0 - pexpect=4.4.0=py27_0 - pickleshare=0.7.4=py27_0 - - pip=9.0.1=py27_1 + - pip=9.0.2=py27_0 - prompt_toolkit=1.0.15=py27_0 - ptyprocess=0.5.2=py27_0 - pygments=2.2.0=py27_0 - python=2.7.14=4 - - python-dateutil=2.6.1=py27_0 + - python-dateutil=2.7.0=py_0 - pyzmq=17.0.0=py27_3 - readline=7.0=0 - scandir=1.7=py27_0 - - setuptools=38.5.2=py27_0 + - setuptools=39.0.1=py27_0 - simplegeneric=0.8.1=py27_0 - singledispatch=3.4.0.3=py27_0 - six=1.11.0=py27_1 diff --git a/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml b/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml index 66c6383d..3d69e6be 100644 --- a/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.py-3.5.yml, DO NOT MANUALLY MODIFY -# Frozen on 2018-03-13 17:40:14 UTC +# Frozen on 2018-03-20 00:43:53 UTC name: r2d channels: - conda-forge @@ -30,23 +30,23 @@ dependencies: - nbconvert=5.3.1=py_1 - nbformat=4.4.0=py35_0 - ncurses=5.9=10 - - notebook=5.4.0=py35_0 + - notebook=5.4.1=py35_0 - openssl=1.0.2n=0 - - pandoc=2.1.2=0 + - pandoc=2.1.3=0 - pandocfilters=1.4.1=py35_0 - parso=0.1.1=py_0 - pexpect=4.4.0=py35_0 - pickleshare=0.7.4=py35_0 - - pip=9.0.1=py35_1 + - pip=9.0.2=py35_0 - prompt_toolkit=1.0.15=py35_0 - ptyprocess=0.5.2=py35_0 - pygments=2.2.0=py35_0 - python=3.5.5=0 - - python-dateutil=2.6.1=py35_0 + - python-dateutil=2.7.0=py_0 - pyzmq=17.0.0=py35_3 - readline=7.0=0 - send2trash=1.5.0=py_0 - - setuptools=38.5.2=py35_0 + - setuptools=39.0.1=py35_0 - simplegeneric=0.8.1=py35_0 - six=1.11.0=py35_1 - sqlite=3.20.1=2 diff --git a/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml b/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml index 458ac45a..b17b9543 100644 --- a/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY -# Frozen on 2018-03-13 17:43:53 UTC +# Frozen on 2018-03-20 00:47:27 UTC name: r2d channels: - conda-forge @@ -30,23 +30,23 @@ dependencies: - nbconvert=5.3.1=py_1 - nbformat=4.4.0=py36_0 - ncurses=5.9=10 - - notebook=5.4.0=py36_0 + - notebook=5.4.1=py36_0 - openssl=1.0.2n=0 - - pandoc=2.1.2=0 + - pandoc=2.1.3=0 - pandocfilters=1.4.1=py36_0 - parso=0.1.1=py_0 - pexpect=4.4.0=py36_0 - pickleshare=0.7.4=py36_0 - - pip=9.0.1=py36_1 + - pip=9.0.2=py36_0 - prompt_toolkit=1.0.15=py36_0 - ptyprocess=0.5.2=py36_0 - pygments=2.2.0=py36_0 - python=3.6.4=0 - - python-dateutil=2.6.1=py36_0 + - python-dateutil=2.7.0=py_0 - pyzmq=17.0.0=py36_3 - readline=7.0=0 - send2trash=1.5.0=py_0 - - setuptools=38.5.2=py36_0 + - setuptools=39.0.1=py36_0 - simplegeneric=0.8.1=py36_0 - six=1.11.0=py36_1 - sqlite=3.20.1=2 diff --git a/repo2docker/buildpacks/conda/environment.yml b/repo2docker/buildpacks/conda/environment.yml index 804bcb71..26ef24af 100644 --- a/repo2docker/buildpacks/conda/environment.yml +++ b/repo2docker/buildpacks/conda/environment.yml @@ -3,6 +3,6 @@ dependencies: - ipywidgets==7.1.1 - jupyterlab==0.31.5 - tornado==4.5.3 - - notebook==5.4.0 + - notebook==5.4.1 - pip: - nteract_on_jupyter==1.5.0 diff --git a/repo2docker/buildpacks/conda/freeze.py b/repo2docker/buildpacks/conda/freeze.py index 37790781..dc8980ed 100755 --- a/repo2docker/buildpacks/conda/freeze.py +++ b/repo2docker/buildpacks/conda/freeze.py @@ -21,7 +21,7 @@ from ruamel.yaml import YAML MINICONDA_VERSION = '4.3.27' # need conda ≥ 4.4 to avoid bug adding spurious pip dependencies -CONDA_VERSION = '4.4.8' +CONDA_VERSION = '4.4.11' HERE = pathlib.Path(os.path.dirname(os.path.abspath(__file__))) diff --git a/repo2docker/buildpacks/python/requirements.frozen.txt b/repo2docker/buildpacks/python/requirements.frozen.txt index 2444c4be..d884a4a5 100644 --- a/repo2docker/buildpacks/python/requirements.frozen.txt +++ b/repo2docker/buildpacks/python/requirements.frozen.txt @@ -1,5 +1,5 @@ # AUTO GENERATED FROM requirements.txt, DO NOT MANUALLY MODIFY -# Frozen on Thu Mar 15 11:51:16 UTC 2018 +# Frozen on Tue Mar 20 00:18:43 UTC 2018 bleach==2.1.3 decorator==4.2.1 entrypoints==0.2.3 @@ -19,7 +19,7 @@ MarkupSafe==1.0 mistune==0.8.3 nbconvert==5.3.1 nbformat==4.4.0 -notebook==5.4.0 +notebook==5.4.1 nteract-on-jupyter==1.5.0 pandocfilters==1.4.2 parso==0.1.1 diff --git a/repo2docker/buildpacks/python/requirements.txt b/repo2docker/buildpacks/python/requirements.txt index 2812eb71..811d36e4 100644 --- a/repo2docker/buildpacks/python/requirements.txt +++ b/repo2docker/buildpacks/python/requirements.txt @@ -1,4 +1,4 @@ -notebook==5.4.0 +notebook==5.4.1 tornado==4.5.3 ipywidgets==7.1.1 jupyterlab==0.31.5 diff --git a/repo2docker/buildpacks/python/requirements2.frozen.txt b/repo2docker/buildpacks/python/requirements2.frozen.txt index b2f5435a..8d44e4bd 100644 --- a/repo2docker/buildpacks/python/requirements2.frozen.txt +++ b/repo2docker/buildpacks/python/requirements2.frozen.txt @@ -1,5 +1,5 @@ # AUTO GENERATED FROM requirements2.txt, DO NOT MANUALLY MODIFY -# Frozen on Thu Mar 15 11:52:00 UTC 2018 +# Frozen on Tue Mar 20 00:20:49 UTC 2018 backports-abc==0.5 backports.shutil-get-terminal-size==1.0.0 certifi==2018.1.18 From 16a42f52b325b09a1ae96b8a037731aeae4f645b Mon Sep 17 00:00:00 2001 From: Carol Willing Date: Mon, 19 Mar 2018 18:05:53 -0700 Subject: [PATCH 04/10] Edit CONTRIBUTING for freezing dependencies --- CONTRIBUTING.md | 132 +++++++++++++++++++++++++++++------------------- 1 file changed, 79 insertions(+), 53 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8772e2bb..47b08216 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,13 @@ -# Local Development +# Contributing to repo2docker development + +This document covers: + +- Setting up for Local Development +- Running Tests +- Updating and Freezing BuildPack Dependencies +- Creating a Release + +## Setting up for Local Development To develop & test repo2docker locally, you need: @@ -9,7 +18,7 @@ To develop & test repo2docker locally, you need: 5. Your favorite text editor 6. A recent version of [Docker Community Edition](https://www.docker.com/community-edition) -## Clone the repository +### Clone the repository First, you need to get a copy of the repo2docker git repository on your local disk. @@ -21,7 +30,7 @@ git clone https://github.com/jupyter/repo2docker This will clone repo2docker into a directory called `repo2docker`. You can make that your current directory with `cd repo2docker`. -## Set up local virtual environment +### Set up a local virtual environment After cloning the repository (or your fork of the repo), you should set up an isolated environment to install libraries required for running / developing @@ -37,7 +46,7 @@ pip3 install -r dev-requirements.txt This should install all the libraries required for testing & running repo2docker! -## Verify that docker is installed and running +### Verify that docker is installed and running If you do not already have [Docker](https://www.docker.com/), you should be able to download and install it for your operating system using the links from the @@ -74,7 +83,7 @@ Then you are good to go! ## Running tests We have a lot of tests for various cases supported by repo2docker in the `tests/` -subdirectory. These use [py.test](https://docs.pytest.org/). +subdirectory. These use [py.test](https://docs.pytest.org/). You can run all the tests with: @@ -88,10 +97,12 @@ If you want to run a specific test, you can do so with: py.test -s tests/ ``` -# Updating libraries installed for all repos +## Update and Freeze BuildPack Dependencies -For both the `conda` and `virtualenv` (`pip`) base environments, we install specific -pinned versions of all dependencies. We explicitly list the dependencies +### Updating libraries installed for all repos + +For both the `conda` and `virtualenv` (`pip`) base environments in the **Conda BuildPack** and **Python BuildPack**, +we install specific pinned versions of all dependencies. We explicitly list the dependencies we want, then *freeze* them at commit time to explicitly list all the transitive dependencies at current versions. This way, we know that all dependencies will have the exact same version installed at all times. @@ -106,77 +117,92 @@ must follow these steps (with more detailed information in the sections below): See the subsections below for more detailed instructions. -## Virtualenv dependencies +### Virtualenv dependencies -There are two files related to virtualenv dependencies: +1. There are two files related to virtualenv dependencies. Edit as needed. -1. `repo2docker/buildpacks/python/requirements.txt` + - `repo2docker/buildpacks/python/requirements.txt` - Contains list of packages to install in Python3 virtualenvs, - which are the default. **This where all Notebook versions & - notebook extensions (such as JupyterLab / nteract) go**. + Contains list of packages to install in Python3 virtualenvs, + which are the default. **This where all Notebook versions & + notebook extensions (such as JupyterLab / nteract) go**. -2. `repo2docker/buildpacks/python/requirements2.txt` + - `repo2docker/buildpacks/python/requirements2.txt` - Contains list of packages to install in Python2 virtualenvs, which - can be specifically requested by users. **This only needs `IPyKernel` - and kernel related libraries** - Notebook / Notebook Extension need - not be installed here. + Contains list of packages to install in Python2 virtualenvs, which + can be specifically requested by users. **This only needs `IPyKernel` + and kernel related libraries** Notebook / Notebook Extension need + not be installed here. -After you edit either of these files to add a new package / bump version on -an existing package, run `./repo2docker/buildpacks/python/freeze.bash`. +2. After you edit either of these files to add a new package / bump version on + an existing package, run: + + ```bash + ./repo2docker/buildpacks/python/freeze.bash + ``` -This script will resolve dependencies and write them to the respective `.frozen.txt` -files. You will need Python3 and Python2 with virtualenv to run the script. + This script will resolve dependencies and write them to the respective `.frozen.txt` + files. + + Note: If you do not have Python3 and Python2 with virtualenv, the script + will create and build Docker containers to process the frozen files. + +3. All the `.txt` files in `repo2docker/buildpacks/python/` should be committed to git. -All the `.txt` files in `repo2docker/buildpacks/python/` should be committed to git. +4. Make a pull request. -## Conda dependencies +### Conda dependencies -There are two files related to conda dependencies: +1. There are two files related to conda dependencies. Edit as needed. -1. `repo2docker/buildpacks/conda/environment.yml` + - `repo2docker/buildpacks/conda/environment.yml` - Contains list of packages to install in Python3 conda environments, - which are the default. **This is where all Notebook versions & - notebook extensions (such as JupyterLab / nteract) go**. + Contains list of packages to install in Python3 conda environments, + which are the default. **This is where all Notebook versions & + notebook extensions (such as JupyterLab / nteract) go**. -2. `repo2docker/buildpacks/conda/environment.py-2.7.yml` + - `repo2docker/buildpacks/conda/environment.py-2.7.yml` - Contains list of packages to install in Python2 conda environments, which - can be specifically requested by users. **This only needs `IPyKernel` - and kernel related libraries** - Notebook / Notebook Extension need - not be installed here. + Contains list of packages to install in Python2 conda environments, which + can be specifically requested by users. **This only needs `IPyKernel` + and kernel related libraries**. Notebook / Notebook Extension need + not be installed here. -Once you edit either of these files to add a new package / bump version on -an existing package, you should then run `./repo2docker/buildpacks/conda/freeze.py`. -This script will resolve dependencies and write them to the respective `.frozen.yml` -files. You will need `docker` installed to run this script. +2. Once you edit either of these files to add a new package / bump version on + an existing package, you should then run: -After the freeze script finishes, a number of files will have been created. -Commit the following subset of files to git: + ```bash + python ./repo2docker/buildpacks/conda/freeze.py + ``` + + This script will resolve dependencies and write them to the respective `.frozen.yml` + files. You will need `docker` installed to run this script. -``` -repo2docker/buildpacks/conda/environment.yml -repo2docker/buildpacks/conda/environment.frozen.yml -repo2docker/buildpacks/conda/environment.py-2.7.yml -repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml -repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml -repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml -``` +3. After the freeze script finishes, a number of files will have been created. + Commit the following subset of files to git: + ``` + repo2docker/buildpacks/conda/environment.yml + repo2docker/buildpacks/conda/environment.frozen.yml + repo2docker/buildpacks/conda/environment.py-2.7.yml + repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml + repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml + repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml + ``` -## Make a Pull Request +4. Make a pull request. + +### Make a Pull Request Once you've made the commit, please make a Pull Request to the `jupyter/repo2docker` repository, with a description of what versions were bumped / what new packages were added and why. -# Release Process +## Creating a Release We try to make a release of repo2docker every few months if possible. -## Access +## Obtain access credentials To release repo2docker, you will need proper access credentials prior to beginning the process. @@ -186,7 +212,7 @@ To release repo2docker, you will need proper access credentials prior to beginni If you do not have access to any of these, please contact a current maintainer of the project! -## Steps +## Release Process Steps 1. Make a PR bumping version number of repo2docker in the `setup.py` file (like https://github.com/jupyter/repo2docker/pull/221), From 2d0772afb424de5862d5a94d3f10870131c58682 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Tue, 20 Mar 2018 18:14:01 +0000 Subject: [PATCH 05/10] fixing python2 search --- repo2docker/buildpacks/base.py | 2 +- repo2docker/buildpacks/python/__init__.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index 41fea076..1599ff8f 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -170,7 +170,7 @@ class BuildPack: # FIXME: Use npm from nodesource! # Everything seems to depend on npm these days, unfortunately. "npm", - + "unzip", } diff --git a/repo2docker/buildpacks/python/__init__.py b/repo2docker/buildpacks/python/__init__.py index 52437850..0bf648f0 100644 --- a/repo2docker/buildpacks/python/__init__.py +++ b/repo2docker/buildpacks/python/__init__.py @@ -12,7 +12,7 @@ class PythonBuildPack(BaseImage): Note: The packages specified here are for the core Python3 language. Third party libraries are specified in other configuration files. - + """ return super().get_packages().union({ 'python3', @@ -37,7 +37,7 @@ class PythonBuildPack(BaseImage): def get_path(self): """Return paths (including virtual environment path) to be added to the PATH environment variable. - + """ return super().get_path() + [ "${VENV_PATH}/bin" @@ -148,7 +148,7 @@ class Python2BuildPack(PythonBuildPack): Note: The packages specified here are for the core Python2 language. Third party libraries are specified in other configuration files. - + """ return super().get_packages().union({ 'python', @@ -162,7 +162,7 @@ class Python2BuildPack(PythonBuildPack): We set `VENV_PATH` to the virtual environment location containing Python 2. - + """ return super().get_env() + [ ('VENV2_PATH', '${APP_BASE}/venv2') @@ -171,7 +171,7 @@ class Python2BuildPack(PythonBuildPack): def get_path(self): """Return paths (including virtual environment path) to be added to the PATH environment variable. - + """ return super().get_path() + [ "${VENV2_PATH}/bin" @@ -190,7 +190,7 @@ class Python2BuildPack(PythonBuildPack): This currently adds a frozen set of Python 2 requirements to the dict of files. - + """ files = { 'python/requirements2.frozen.txt': '/tmp/requirements2.frozen.txt', @@ -238,10 +238,11 @@ class Python2BuildPack(PythonBuildPack): def get_assemble_scripts(self): """Return series of build-steps specific to this repository. """ + requirements_file = self.binder_path('requirements.txt') return super().get_assemble_scripts() + [ ( '${NB_USER}', - 'pip2 install --no-cache-dir -r requirements.txt' + 'pip2 install --no-cache-dir -r "{}"'.format(requirements_file) ) ] From 572abb3fea6c21a2f27c5d97905543bdca58d3b8 Mon Sep 17 00:00:00 2001 From: Carol Willing Date: Mon, 19 Mar 2018 15:40:50 -0700 Subject: [PATCH 06/10] add docstrings for legacy buildpack --- repo2docker/buildpacks/legacy/__init__.py | 35 ++++++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/repo2docker/buildpacks/legacy/__init__.py b/repo2docker/buildpacks/legacy/__init__.py index fe7c56ca..12d1483e 100644 --- a/repo2docker/buildpacks/legacy/__init__.py +++ b/repo2docker/buildpacks/legacy/__init__.py @@ -1,5 +1,10 @@ -""" -Generates a variety of Dockerfiles based on an input matrix +"""Generates Dockerfiles from the legacy Binder Dockerfiles +based on `andrewosh/binder-base`. + +The Dockerfile is amended to add the contents of the repository +to the image and install a supported version of the notebook +and IPython kernel. + """ import os import shutil @@ -7,7 +12,7 @@ from textwrap import dedent from ..docker import DockerBuildPack class LegacyBinderDockerBuildPack(DockerBuildPack): - + """Legacy build pack for compatibility to first version of Binder.""" dockerfile = '._binder.Dockerfile' legacy_prependix = dedent(r""" @@ -39,6 +44,11 @@ class LegacyBinderDockerBuildPack(DockerBuildPack): """) def render(self): + """Render buildpack into a Dockerfile. + + Render appendix (post-build commands) at the end of the Dockerfile. + + """ segments = [ 'FROM andrewosh/binder-base@sha256:eabde24f4c55174832ed8795faa40cea62fc9e2a4a9f1ee1444f8a2e4f9710ee', self.legacy_prependix, @@ -52,12 +62,27 @@ class LegacyBinderDockerBuildPack(DockerBuildPack): return '\n'.join(segments) def get_build_script_files(self): - return { + """ + Dict of files to be copied to the container image for use in building. + + This is copied before the `build_scripts` & `assemble_scripts` are + run, so can be executed from either of them. + + It's a dictionary where the key is the source file path in the host + system, and the value is the destination file path inside the + container image. + + This currently adds a frozen set of Python requirements to the dict + of files. + + """ + return { 'legacy/root.frozen.yml': '/tmp/root.frozen.yml', 'legacy/python3.frozen.yml': '/tmp/python3.frozen.yml', } def build(self, image_spec, memory_limit, build_args): + """Build a legacy Docker image.""" with open(self.dockerfile, 'w') as f: f.write(self.render()) for env in ('root', 'python3'): @@ -70,6 +95,8 @@ class LegacyBinderDockerBuildPack(DockerBuildPack): return super().build(image_spec, memory_limit, build_args) def detect(self): + """Check if current repo should be built with the Legacy BuildPack. + """ try: with open('Dockerfile', 'r') as f: for line in f: From f4843dfdd3a0e4e6d512e7c136a16ed12aed053d Mon Sep 17 00:00:00 2001 From: Carol Willing Date: Wed, 21 Mar 2018 12:06:50 -0700 Subject: [PATCH 07/10] clarify render order --- repo2docker/buildpacks/legacy/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/repo2docker/buildpacks/legacy/__init__.py b/repo2docker/buildpacks/legacy/__init__.py index 12d1483e..5219b2ae 100644 --- a/repo2docker/buildpacks/legacy/__init__.py +++ b/repo2docker/buildpacks/legacy/__init__.py @@ -46,7 +46,9 @@ class LegacyBinderDockerBuildPack(DockerBuildPack): def render(self): """Render buildpack into a Dockerfile. - Render appendix (post-build commands) at the end of the Dockerfile. + Render legacy image source (andrewosh/binder-base at a specific commit) + and then prependix. Render appendix (post-build commands) at the end of + the Dockerfile. """ segments = [ From 975b34921c637637c79f2792150bca3684eb5230 Mon Sep 17 00:00:00 2001 From: Carol Willing Date: Wed, 21 Mar 2018 12:32:16 -0700 Subject: [PATCH 08/10] Add note about os support and update example repo --- README.md | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 9a6a900a..88b44174 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,10 @@ # jupyter-repo2docker - [![Build Status](https://travis-ci.org/jupyter/repo2docker.svg?branch=master)](https://travis-ci.org/jupyter/repo2docker) [![Documentation Status](https://readthedocs.org/projects/repo2docker/badge/?version=latest)](http://repo2docker.readthedocs.io/en/latest/?badge=latest) -**jupyter-repo2docker** takes as input a repository source, such as a GitHub repo. It then builds, runs, and/or pushes Docker images built from -that source. +**jupyter-repo2docker** takes as input a repository source, such as a GitHub +repo. It then builds, runs, and/or pushes Docker images built from that source. See the [repo2docker documentation](http://repo2docker.readthedocs.io) for more information. @@ -16,15 +15,17 @@ for more information. is recommended. 2. Python 3.4+. +Supported on Linux and macOS. [See documentation note about Windows support.](http://repo2docker.readthedocs.io/en/latest/install.html#note-about-windows-support) + ## Installation -To install from pypi, the python packaging index: +To install from PyPI, the python packaging index, using `pip`: ```bash pip install jupyter-repo2docker ``` -To install from source and start contributing: +To install from source: ```bash git clone https://github.com/jupyter/repo2docker.git @@ -34,29 +35,29 @@ pip install -e . ## Usage -The core feature of repo2docker is to fetch a repo (from github or locally), build a container -image based on the specifications found in the repo & optionally launch a local Jupyter Notebook -you can use to explore it. +The core feature of repo2docker is to fetch a repo (from github or locally), +build a container image based on the specifications found in the repo & +optionally launch a local Jupyter Notebook you can use to explore it. **Note that Docker needs to be running on your machine for this to work.** Example: ```bash -jupyter-repo2docker https://github.com/jakevdp/PythonDataScienceHandbook +jupyter-repo2docker https://github.com/norvig/pytudes ``` -After building (it might take a while!), it should output in your terminal something like: +After building (it might take a while!), it should output in your terminal +something like: - -``` +```bash Copy/paste this URL into your browser when you connect for the first time, to login with a token: http://0.0.0.0:36511/?token=f94f8fabb92e22f5bfab116c382b4707fc2cade56ad1ace0 ``` -If you copy paste that URL into your browser you will see a Jupyter Notebook with the -contents of the repository you had just built! +If you copy paste that URL into your browser you will see a Jupyter Notebook +with the contents of the repository you had just built! For more information on how to use ``repo2docker``, see the [usage guide](http://repo2docker.readthedocs.io/en/latest/usage.html). @@ -64,8 +65,8 @@ For more information on how to use ``repo2docker``, see the ## Repository specifications Repo2Docker looks for configuration files in the source repository to -determine how the Docker image should be built. -It is philosophically similar to [Heroku Build Packs](https://devcenter.heroku.com/articles/buildpacks). +determine how the Docker image should be built. It is philosophically similar +to [Heroku Build Packs](https://devcenter.heroku.com/articles/buildpacks). For a list of the configuration files that ``repo2docker`` can use, see the [usage guide](http://repo2docker.readthedocs.io/en/latest/usage.html). From dcde9b4c54d8b0f79fef4e21b57874ce666eef61 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Wed, 21 Mar 2018 08:23:00 +0100 Subject: [PATCH 09/10] Add codecov configuration to enable PR comments --- .codecov.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .codecov.yml diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 00000000..cc80b2c9 --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,10 @@ +# show coverage in CI status, not as a comment. Never consider it a failure. +comment: off +coverage: + status: + project: + default: + target: 0% + patch: + default: + target: 0% From 678cd985cad413fe2c9c989dbde4e9558c56063e Mon Sep 17 00:00:00 2001 From: Tim Head Date: Thu, 22 Mar 2018 17:27:03 +0100 Subject: [PATCH 10/10] Add a mention of adding tests to CONTRIBUTING.md --- CONTRIBUTING.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 47b08216..3b40cd21 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,7 +83,9 @@ Then you are good to go! ## Running tests We have a lot of tests for various cases supported by repo2docker in the `tests/` -subdirectory. These use [py.test](https://docs.pytest.org/). +subdirectory. If you fix a bug or add new functionality consider adding a new +test to prevent the bug from coming back. These use +[py.test](https://docs.pytest.org/). You can run all the tests with: @@ -136,17 +138,17 @@ See the subsections below for more detailed instructions. 2. After you edit either of these files to add a new package / bump version on an existing package, run: - + ```bash ./repo2docker/buildpacks/python/freeze.bash ``` This script will resolve dependencies and write them to the respective `.frozen.txt` - files. - + files. + Note: If you do not have Python3 and Python2 with virtualenv, the script will create and build Docker containers to process the frozen files. - + 3. All the `.txt` files in `repo2docker/buildpacks/python/` should be committed to git. 4. Make a pull request. @@ -174,7 +176,7 @@ See the subsections below for more detailed instructions. ```bash python ./repo2docker/buildpacks/conda/freeze.py ``` - + This script will resolve dependencies and write them to the respective `.frozen.yml` files. You will need `docker` installed to run this script. @@ -196,7 +198,8 @@ See the subsections below for more detailed instructions. Once you've made the commit, please make a Pull Request to the `jupyter/repo2docker` repository, with a description of what versions were bumped / what new packages were -added and why. +added and why. If you fix a bug or add new functionality consider adding a new +test to prevent the bug from coming back/the feature breaking in the future. ## Creating a Release