From fb335af9b3734ee918ff62086b4e88de5df3e99f Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Thu, 4 Apr 2019 11:32:50 -0700 Subject: [PATCH 01/71] bump miniconda and conda versions --- repo2docker/buildpacks/conda/install-miniconda.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repo2docker/buildpacks/conda/install-miniconda.bash b/repo2docker/buildpacks/conda/install-miniconda.bash index 54fcd55d..b3239cba 100755 --- a/repo2docker/buildpacks/conda/install-miniconda.bash +++ b/repo2docker/buildpacks/conda/install-miniconda.bash @@ -3,8 +3,8 @@ set -ex cd $(dirname $0) -MINICONDA_VERSION=4.5.11 -CONDA_VERSION=4.5.11 +MINICONDA_VERSION=4.5.12 +CONDA_VERSION=4.6.11 URL="https://repo.continuum.io/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh" INSTALLER_PATH=/tmp/miniconda-installer.sh From 7e75f9f9157b1b0f2960871beda8c650ae4f6ad8 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Thu, 4 Apr 2019 11:36:54 -0700 Subject: [PATCH 02/71] update checksum for conda installer --- repo2docker/buildpacks/conda/install-miniconda.bash | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/repo2docker/buildpacks/conda/install-miniconda.bash b/repo2docker/buildpacks/conda/install-miniconda.bash index b3239cba..99af584e 100755 --- a/repo2docker/buildpacks/conda/install-miniconda.bash +++ b/repo2docker/buildpacks/conda/install-miniconda.bash @@ -5,6 +5,10 @@ set -ex cd $(dirname $0) MINICONDA_VERSION=4.5.12 CONDA_VERSION=4.6.11 +# Only MD5 checksums are available for miniconda +# Can be obtained from https://repo.continuum.io/miniconda/ +MD5SUM="4be03f925e992a8eda03758b72a77298" + URL="https://repo.continuum.io/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh" INSTALLER_PATH=/tmp/miniconda-installer.sh @@ -15,10 +19,7 @@ unset HOME wget --quiet $URL -O ${INSTALLER_PATH} chmod +x ${INSTALLER_PATH} -# Only MD5 checksums are available for miniconda -# Can be obtained from https://repo.continuum.io/miniconda/ -MD5SUM="e1045ee415162f944b6aebfe560b8fee" - +# check md5 checksum if ! echo "${MD5SUM} ${INSTALLER_PATH}" | md5sum --quiet -c -; then echo "md5sum mismatch for ${INSTALLER_PATH}, exiting!" exit 1 From a3d4f95de025891cb2b74f346a8bb2d02ec201c3 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Thu, 4 Apr 2019 11:54:33 -0700 Subject: [PATCH 03/71] changelog addition --- docs/source/changelog.rst | 2 ++ repo2docker/buildpacks/conda/install-miniconda.bash | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 59903ce2..1ec6f5aa 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -13,6 +13,8 @@ New features :user:`davidanthoff` - Set JULIA_PROJECT globally, so that every julia instance starts with the julia environment activated in :pr:`612` by :user:`davidanthoff`. +- Update Miniconda version to 4.5.12and Conda version to 4.6.11 in :pr:`637` by + :user:`jhamman` API changes ----------- diff --git a/repo2docker/buildpacks/conda/install-miniconda.bash b/repo2docker/buildpacks/conda/install-miniconda.bash index 99af584e..fa728914 100755 --- a/repo2docker/buildpacks/conda/install-miniconda.bash +++ b/repo2docker/buildpacks/conda/install-miniconda.bash @@ -7,7 +7,7 @@ MINICONDA_VERSION=4.5.12 CONDA_VERSION=4.6.11 # Only MD5 checksums are available for miniconda # Can be obtained from https://repo.continuum.io/miniconda/ -MD5SUM="4be03f925e992a8eda03758b72a77298" +MD5SUM="866ae9dff53ad0874e1d1a60b1ad1ef8" URL="https://repo.continuum.io/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh" INSTALLER_PATH=/tmp/miniconda-installer.sh From bf3d0c133c34c9010ca74e9edbe2b5d061035d94 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Thu, 4 Apr 2019 12:29:34 -0700 Subject: [PATCH 04/71] fallback to 4.6.10 --- docs/source/changelog.rst | 2 +- repo2docker/buildpacks/conda/install-miniconda.bash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 1ec6f5aa..94bbeadd 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -13,7 +13,7 @@ New features :user:`davidanthoff` - Set JULIA_PROJECT globally, so that every julia instance starts with the julia environment activated in :pr:`612` by :user:`davidanthoff`. -- Update Miniconda version to 4.5.12and Conda version to 4.6.11 in :pr:`637` by +- Update Miniconda version to 4.5.12 and Conda version to 4.6.10 in :pr:`637` by :user:`jhamman` API changes diff --git a/repo2docker/buildpacks/conda/install-miniconda.bash b/repo2docker/buildpacks/conda/install-miniconda.bash index fa728914..c7dfd5b4 100755 --- a/repo2docker/buildpacks/conda/install-miniconda.bash +++ b/repo2docker/buildpacks/conda/install-miniconda.bash @@ -4,7 +4,7 @@ set -ex cd $(dirname $0) MINICONDA_VERSION=4.5.12 -CONDA_VERSION=4.6.11 +CONDA_VERSION=4.6.10 # Only MD5 checksums are available for miniconda # Can be obtained from https://repo.continuum.io/miniconda/ MD5SUM="866ae9dff53ad0874e1d1a60b1ad1ef8" From f9620edd8e5af51f1c59f2285a68ffa748095a28 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Thu, 4 Apr 2019 13:35:12 -0700 Subject: [PATCH 05/71] freeze versions --- docs/source/changelog.rst | 2 +- .../buildpacks/conda/environment.frozen.yml | 16 ++++++++-------- .../conda/environment.py-2.7.frozen.yml | 8 ++++---- .../conda/environment.py-3.6.frozen.yml | 14 +++++++------- .../buildpacks/conda/environment.py-3.6.yml | 2 +- .../conda/environment.py-3.7.frozen.yml | 16 ++++++++-------- .../buildpacks/conda/environment.py-3.7.yml | 2 +- repo2docker/buildpacks/conda/freeze.py | 4 ++-- .../buildpacks/conda/install-miniconda.bash | 2 +- 9 files changed, 33 insertions(+), 33 deletions(-) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 94bbeadd..35bc0860 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -13,7 +13,7 @@ New features :user:`davidanthoff` - Set JULIA_PROJECT globally, so that every julia instance starts with the julia environment activated in :pr:`612` by :user:`davidanthoff`. -- Update Miniconda version to 4.5.12 and Conda version to 4.6.10 in :pr:`637` by +- Update Miniconda version to 4.5.12 and Conda version to 4.6.9 in :pr:`637` by :user:`jhamman` API changes diff --git a/repo2docker/buildpacks/conda/environment.frozen.yml b/repo2docker/buildpacks/conda/environment.frozen.yml index eda33f89..1c53b19f 100644 --- a/repo2docker/buildpacks/conda/environment.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.frozen.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.py-3.7.yml, DO NOT MANUALLY MODIFY -# Frozen on 2019-03-23 18:02:15 UTC +# Frozen on 2019-04-04 20:24:39 UTC name: r2d channels: - conda-forge @@ -27,7 +27,9 @@ dependencies: - jupyterlab=0.35.4=py37_0 - jupyterlab_server=0.2.0=py_0 - libffi=3.2.1=he1b5a44_1006 + - libgcc-ng=8.2.0=hdf63c60_1 - libsodium=1.0.16=h14c3975_1001 + - libstdcxx-ng=8.2.0=hdf63c60_1 - markupsafe=1.1.1=py37h14c3975_0 - mistune=0.8.4=py37h14c3975_1000 - nbconvert=5.4.1=py_2 @@ -46,7 +48,7 @@ dependencies: - ptyprocess=0.6.0=py37_1000 - pygments=2.3.1=py_0 - pyrsistent=0.14.11=py37h14c3975_0 - - python=3.7.2=h381d211_0 + - python=3.7.3=h5b0a415_0 - python-dateutil=2.8.0=py_0 - pyzmq=18.0.1=py37h0e1adb2_0 - readline=7.0=hf8c457e_1001 @@ -54,10 +56,10 @@ dependencies: - setuptools=40.8.0=py37_0 - six=1.12.0=py37_1000 - sqlite=3.26.0=h67949de_1001 - - terminado=0.8.1=py37_1001 + - terminado=0.8.2=py37_0 - testpath=0.4.2=py_1001 - - tk=8.6.9=h84994c4_1000 - - tornado=6.0.1=py37h14c3975_0 + - tk=8.6.9=h84994c4_1001 + - tornado=6.0.2=py37h516909a_0 - traitlets=4.3.2=py37_1000 - wcwidth=0.1.7=py_1 - webencodings=0.5.1=py_1 @@ -66,8 +68,6 @@ dependencies: - xz=5.2.4=h14c3975_1001 - zeromq=4.2.5=hf484d3e_1006 - zlib=1.2.11=h14c3975_1004 - - libgcc-ng=8.2.0=hdf63c60_1 - - libstdcxx-ng=8.2.0=hdf63c60_1 - pip: - alembic==1.0.8 - async-generator==1.10 @@ -80,7 +80,7 @@ dependencies: - python-editor==1.0.4 - python-oauth2==1.1.0 - requests==2.21.0 - - sqlalchemy==1.3.1 + - sqlalchemy==1.3.2 - urllib3==1.24.1 prefix: /opt/conda/envs/r2d diff --git a/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml b/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml index e7c3e8e5..bad7a55e 100644 --- a/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.py-2.7.yml, DO NOT MANUALLY MODIFY -# Frozen on 2019-03-23 17:59:14 UTC +# Frozen on 2019-04-04 20:21:17 UTC name: r2d channels: - conda-forge @@ -20,7 +20,9 @@ dependencies: - jupyter_client=5.2.4=py_3 - jupyter_core=4.4.0=py_0 - libffi=3.2.1=he1b5a44_1006 + - libgcc-ng=8.2.0=hdf63c60_1 - libsodium=1.0.16=h14c3975_1001 + - libstdcxx-ng=8.2.0=hdf63c60_1 - ncurses=6.1=hf484d3e_1002 - openssl=1.1.1b=h14c3975_1 - pathlib2=2.3.3=py27_1000 @@ -40,14 +42,12 @@ dependencies: - singledispatch=3.4.0.3=py27_1000 - six=1.12.0=py27_1000 - sqlite=3.26.0=h67949de_1001 - - tk=8.6.9=h84994c4_1000 + - tk=8.6.9=h84994c4_1001 - tornado=5.1.1=py27h14c3975_1000 - traitlets=4.3.2=py27_1000 - wcwidth=0.1.7=py_1 - wheel=0.33.1=py27_0 - zeromq=4.2.5=hf484d3e_1006 - zlib=1.2.11=h14c3975_1004 - - libgcc-ng=8.2.0=hdf63c60_1 - - libstdcxx-ng=8.2.0=hdf63c60_1 prefix: /opt/conda/envs/r2d diff --git a/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml b/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml index b57680a8..18afa8a8 100644 --- a/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY -# Frozen on 2019-03-23 18:00:20 UTC +# Frozen on 2019-04-04 20:22:30 UTC name: r2d channels: - conda-forge @@ -26,7 +26,9 @@ dependencies: - jupyterlab=0.35.4=py36_0 - jupyterlab_server=0.2.0=py_0 - libffi=3.2.1=he1b5a44_1006 + - libgcc-ng=8.2.0=hdf63c60_1 - libsodium=1.0.16=h14c3975_1001 + - libstdcxx-ng=8.2.0=hdf63c60_1 - markupsafe=1.1.1=py36h14c3975_0 - mistune=0.8.4=py36h14c3975_1000 - nbconvert=5.4.1=py_2 @@ -53,10 +55,10 @@ dependencies: - setuptools=40.8.0=py36_0 - six=1.12.0=py36_1000 - sqlite=3.26.0=h67949de_1001 - - terminado=0.8.1=py36_1001 + - terminado=0.8.2=py36_0 - testpath=0.4.2=py_1001 - - tk=8.6.9=h84994c4_1000 - - tornado=6.0.1=py36h14c3975_0 + - tk=8.6.9=h84994c4_1001 + - tornado=6.0.2=py36h516909a_0 - traitlets=4.3.2=py36_1000 - wcwidth=0.1.7=py_1 - webencodings=0.5.1=py_1 @@ -65,8 +67,6 @@ dependencies: - xz=5.2.4=h14c3975_1001 - zeromq=4.2.5=hf484d3e_1006 - zlib=1.2.11=h14c3975_1004 - - libgcc-ng=8.2.0=hdf63c60_1 - - libstdcxx-ng=8.2.0=hdf63c60_1 - pip: - alembic==1.0.8 - async-generator==1.10 @@ -79,7 +79,7 @@ dependencies: - python-editor==1.0.4 - python-oauth2==1.1.0 - requests==2.21.0 - - sqlalchemy==1.3.1 + - sqlalchemy==1.3.2 - urllib3==1.24.1 prefix: /opt/conda/envs/r2d diff --git a/repo2docker/buildpacks/conda/environment.py-3.6.yml b/repo2docker/buildpacks/conda/environment.py-3.6.yml index 5fa071c3..801c5835 100644 --- a/repo2docker/buildpacks/conda/environment.py-3.6.yml +++ b/repo2docker/buildpacks/conda/environment.py-3.6.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.yml, DO NOT MANUALLY MODIFY -# Generated on 2019-03-23 18:00:20 UTC +# Generated on 2019-04-04 20:22:30 UTC dependencies: - python=3.6.* - ipywidgets==7.4.2 diff --git a/repo2docker/buildpacks/conda/environment.py-3.7.frozen.yml b/repo2docker/buildpacks/conda/environment.py-3.7.frozen.yml index eda33f89..1c53b19f 100644 --- a/repo2docker/buildpacks/conda/environment.py-3.7.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.py-3.7.frozen.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.py-3.7.yml, DO NOT MANUALLY MODIFY -# Frozen on 2019-03-23 18:02:15 UTC +# Frozen on 2019-04-04 20:24:39 UTC name: r2d channels: - conda-forge @@ -27,7 +27,9 @@ dependencies: - jupyterlab=0.35.4=py37_0 - jupyterlab_server=0.2.0=py_0 - libffi=3.2.1=he1b5a44_1006 + - libgcc-ng=8.2.0=hdf63c60_1 - libsodium=1.0.16=h14c3975_1001 + - libstdcxx-ng=8.2.0=hdf63c60_1 - markupsafe=1.1.1=py37h14c3975_0 - mistune=0.8.4=py37h14c3975_1000 - nbconvert=5.4.1=py_2 @@ -46,7 +48,7 @@ dependencies: - ptyprocess=0.6.0=py37_1000 - pygments=2.3.1=py_0 - pyrsistent=0.14.11=py37h14c3975_0 - - python=3.7.2=h381d211_0 + - python=3.7.3=h5b0a415_0 - python-dateutil=2.8.0=py_0 - pyzmq=18.0.1=py37h0e1adb2_0 - readline=7.0=hf8c457e_1001 @@ -54,10 +56,10 @@ dependencies: - setuptools=40.8.0=py37_0 - six=1.12.0=py37_1000 - sqlite=3.26.0=h67949de_1001 - - terminado=0.8.1=py37_1001 + - terminado=0.8.2=py37_0 - testpath=0.4.2=py_1001 - - tk=8.6.9=h84994c4_1000 - - tornado=6.0.1=py37h14c3975_0 + - tk=8.6.9=h84994c4_1001 + - tornado=6.0.2=py37h516909a_0 - traitlets=4.3.2=py37_1000 - wcwidth=0.1.7=py_1 - webencodings=0.5.1=py_1 @@ -66,8 +68,6 @@ dependencies: - xz=5.2.4=h14c3975_1001 - zeromq=4.2.5=hf484d3e_1006 - zlib=1.2.11=h14c3975_1004 - - libgcc-ng=8.2.0=hdf63c60_1 - - libstdcxx-ng=8.2.0=hdf63c60_1 - pip: - alembic==1.0.8 - async-generator==1.10 @@ -80,7 +80,7 @@ dependencies: - python-editor==1.0.4 - python-oauth2==1.1.0 - requests==2.21.0 - - sqlalchemy==1.3.1 + - sqlalchemy==1.3.2 - urllib3==1.24.1 prefix: /opt/conda/envs/r2d diff --git a/repo2docker/buildpacks/conda/environment.py-3.7.yml b/repo2docker/buildpacks/conda/environment.py-3.7.yml index 8734c443..73a670fc 100644 --- a/repo2docker/buildpacks/conda/environment.py-3.7.yml +++ b/repo2docker/buildpacks/conda/environment.py-3.7.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.yml, DO NOT MANUALLY MODIFY -# Generated on 2019-03-23 18:02:15 UTC +# Generated on 2019-04-04 20:24:39 UTC dependencies: - python=3.7.* - ipywidgets==7.4.2 diff --git a/repo2docker/buildpacks/conda/freeze.py b/repo2docker/buildpacks/conda/freeze.py index b2d6477f..01a9490b 100755 --- a/repo2docker/buildpacks/conda/freeze.py +++ b/repo2docker/buildpacks/conda/freeze.py @@ -21,8 +21,8 @@ from ruamel.yaml import YAML # Docker image version can be different than conda version, # since miniconda3 docker images seem to lag conda releases. -MINICONDA_DOCKER_VERSION = '4.5.11' -CONDA_VERSION = '4.5.11' +MINICONDA_DOCKER_VERSION = '4.5.12' +CONDA_VERSION = '4.6.9' HERE = pathlib.Path(os.path.dirname(os.path.abspath(__file__))) diff --git a/repo2docker/buildpacks/conda/install-miniconda.bash b/repo2docker/buildpacks/conda/install-miniconda.bash index c7dfd5b4..d848b6ff 100755 --- a/repo2docker/buildpacks/conda/install-miniconda.bash +++ b/repo2docker/buildpacks/conda/install-miniconda.bash @@ -4,7 +4,7 @@ set -ex cd $(dirname $0) MINICONDA_VERSION=4.5.12 -CONDA_VERSION=4.6.10 +CONDA_VERSION=4.6.9 # Only MD5 checksums are available for miniconda # Can be obtained from https://repo.continuum.io/miniconda/ MD5SUM="866ae9dff53ad0874e1d1a60b1ad1ef8" From 843b7a5e30fc63d43805d8843410edee33b4d741 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Tue, 16 Apr 2019 21:45:35 +0200 Subject: [PATCH 06/71] Unpin pip again --- docs/source/changelog.rst | 3 +++ repo2docker/buildpacks/python/__init__.py | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 666c8115..34ede806 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -27,6 +27,9 @@ Bug fixes buildpack in :pr:`633` by :user:`betatim`. - Update to version 5.7.6 of the `notebook` package used in all environments in :pr:`628` by :user:`betatim`. +- Switch to newer version of jupyter-server-proxy to fix websocket handling + in :pr:`646` by :user:`betatim`. +- Update to pip v19.0.3 in :pr:`647` by :user:`betatim`. Version 0.8.0 diff --git a/repo2docker/buildpacks/python/__init__.py b/repo2docker/buildpacks/python/__init__.py index 7ab48e2a..fd81560c 100644 --- a/repo2docker/buildpacks/python/__init__.py +++ b/repo2docker/buildpacks/python/__init__.py @@ -63,7 +63,6 @@ class PythonBuildPack(CondaBuildPack): if os.path.exists(requirements_file): assemble_scripts.append(( '${NB_USER}', - 'pip install "pip<19" && ' + \ '{} install --no-cache-dir -r "{}"'.format(pip, requirements_file) )) From 9599b0a43e78bce543d07183f8d1db2f537c6ec5 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Wed, 24 Apr 2019 19:55:24 +0200 Subject: [PATCH 07/71] Bump nteract-on-jupyter to 2.0.12 and notebook to 5.7.8 --- .../buildpacks/conda/environment.frozen.yml | 40 +++++++++---------- .../conda/environment.py-2.7.frozen.yml | 16 ++++---- .../conda/environment.py-3.5.frozen.yml | 2 +- .../conda/environment.py-3.6.frozen.yml | 38 +++++++++--------- .../buildpacks/conda/environment.py-3.6.yml | 6 +-- .../conda/environment.py-3.7.frozen.yml | 40 +++++++++---------- .../buildpacks/conda/environment.py-3.7.yml | 6 +-- repo2docker/buildpacks/conda/environment.yml | 4 +- 8 files changed, 76 insertions(+), 76 deletions(-) diff --git a/repo2docker/buildpacks/conda/environment.frozen.yml b/repo2docker/buildpacks/conda/environment.frozen.yml index eda33f89..19c0dad6 100644 --- a/repo2docker/buildpacks/conda/environment.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.frozen.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.py-3.7.yml, DO NOT MANUALLY MODIFY -# Frozen on 2019-03-23 18:02:15 UTC +# Frozen on 2019-04-24 18:01:25 UTC name: r2d channels: - conda-forge @@ -20,7 +20,7 @@ dependencies: - ipython_genutils=0.2.0=py_1 - ipywidgets=7.4.2=py_0 - jedi=0.13.3=py37_0 - - jinja2=2.10=py_1 + - jinja2=2.10.1=py_0 - jsonschema=3.0.1=py37_0 - jupyter_client=5.2.4=py_3 - jupyter_core=4.4.0=py_0 @@ -33,54 +33,54 @@ dependencies: - nbconvert=5.4.1=py_2 - nbformat=4.4.0=py_1 - ncurses=6.1=hf484d3e_1002 - - notebook=5.7.6=py37_0 + - notebook=5.7.8=py37_0 - openssl=1.1.1b=h14c3975_1 - - pandoc=2.7.1=0 + - pandoc=2.7.2=0 - pandocfilters=1.4.2=py_1 - - parso=0.3.4=py_0 - - pexpect=4.6.0=py37_1000 + - parso=0.4.0=py_0 + - pexpect=4.7.0=py37_0 - pickleshare=0.7.5=py37_1000 - - pip=19.0.3=py37_0 + - pip=19.1=py37_0 - prometheus_client=0.6.0=py_0 - prompt_toolkit=2.0.9=py_0 - - ptyprocess=0.6.0=py37_1000 + - ptyprocess=0.6.0=py_1001 - pygments=2.3.1=py_0 - pyrsistent=0.14.11=py37h14c3975_0 - - python=3.7.2=h381d211_0 + - python=3.7.3=h5b0a415_0 - python-dateutil=2.8.0=py_0 - - pyzmq=18.0.1=py37h0e1adb2_0 + - pyzmq=18.0.1=py37hc4ba49a_1 - readline=7.0=hf8c457e_1001 - send2trash=1.5.0=py_0 - - setuptools=40.8.0=py37_0 + - setuptools=41.0.1=py37_0 - six=1.12.0=py37_1000 - sqlite=3.26.0=h67949de_1001 - - terminado=0.8.1=py37_1001 + - terminado=0.8.2=py37_0 - testpath=0.4.2=py_1001 - - tk=8.6.9=h84994c4_1000 - - tornado=6.0.1=py37h14c3975_0 + - tk=8.6.9=h84994c4_1001 + - tornado=6.0.2=py37h516909a_0 - traitlets=4.3.2=py37_1000 - wcwidth=0.1.7=py_1 - webencodings=0.5.1=py_1 - wheel=0.33.1=py37_0 - widgetsnbextension=3.4.2=py37_1000 - xz=5.2.4=h14c3975_1001 - - zeromq=4.2.5=hf484d3e_1006 + - zeromq=4.3.1=hf484d3e_1000 - zlib=1.2.11=h14c3975_1004 - libgcc-ng=8.2.0=hdf63c60_1 - libstdcxx-ng=8.2.0=hdf63c60_1 - pip: - - alembic==1.0.8 + - alembic==1.0.9 - async-generator==1.10 - chardet==3.0.4 - idna==2.8 - jupyterhub==0.9.4 - - mako==1.0.8 - - nteract-on-jupyter==2.0.0 + - mako==1.0.9 + - nteract-on-jupyter==2.0.12 - pamela==1.0.0 - python-editor==1.0.4 - python-oauth2==1.1.0 - requests==2.21.0 - - sqlalchemy==1.3.1 - - urllib3==1.24.1 + - sqlalchemy==1.3.3 + - urllib3==1.24.2 prefix: /opt/conda/envs/r2d diff --git a/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml b/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml index e7c3e8e5..1c74fe08 100644 --- a/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.py-2.7.yml, DO NOT MANUALLY MODIFY -# Frozen on 2019-03-23 17:59:14 UTC +# Frozen on 2019-04-24 17:58:08 UTC name: r2d channels: - conda-forge @@ -24,28 +24,28 @@ dependencies: - ncurses=6.1=hf484d3e_1002 - openssl=1.1.1b=h14c3975_1 - pathlib2=2.3.3=py27_1000 - - pexpect=4.6.0=py27_1000 + - pexpect=4.7.0=py27_0 - pickleshare=0.7.5=py27_1000 - - pip=19.0.3=py27_0 + - pip=19.1=py27_0 - prompt_toolkit=1.0.15=py_1 - - ptyprocess=0.6.0=py27_1000 + - ptyprocess=0.6.0=py_1001 - pygments=2.3.1=py_0 - python=2.7.15=h721da81_1008 - python-dateutil=2.8.0=py_0 - - pyzmq=18.0.1=py27h0e1adb2_0 + - pyzmq=18.0.1=py27hc4ba49a_1 - readline=7.0=hf8c457e_1001 - scandir=1.10.0=py27h14c3975_0 - - setuptools=40.8.0=py27_0 + - setuptools=41.0.1=py27_0 - simplegeneric=0.8.1=py_1 - singledispatch=3.4.0.3=py27_1000 - six=1.12.0=py27_1000 - sqlite=3.26.0=h67949de_1001 - - tk=8.6.9=h84994c4_1000 + - tk=8.6.9=h84994c4_1001 - tornado=5.1.1=py27h14c3975_1000 - traitlets=4.3.2=py27_1000 - wcwidth=0.1.7=py_1 - wheel=0.33.1=py27_0 - - zeromq=4.2.5=hf484d3e_1006 + - zeromq=4.3.1=hf484d3e_1000 - zlib=1.2.11=h14c3975_1004 - libgcc-ng=8.2.0=hdf63c60_1 - libstdcxx-ng=8.2.0=hdf63c60_1 diff --git a/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml b/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml index 97d21c50..7aa3aea3 100644 --- a/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml @@ -75,7 +75,7 @@ dependencies: - idna==2.7 - jupyterhub==0.9.4 - mako==1.0.7 - - notebook==5.7.6 + - notebook==5.7.8 - nteract-on-jupyter==1.9.6 - pamela==0.3.0 - python-editor==1.0.3 diff --git a/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml b/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml index b57680a8..745375ae 100644 --- a/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY -# Frozen on 2019-03-23 18:00:20 UTC +# Frozen on 2019-04-24 17:59:15 UTC name: r2d channels: - conda-forge @@ -19,7 +19,7 @@ dependencies: - ipython_genutils=0.2.0=py_1 - ipywidgets=7.4.2=py_0 - jedi=0.13.3=py36_0 - - jinja2=2.10=py_1 + - jinja2=2.10.1=py_0 - jsonschema=3.0.1=py36_0 - jupyter_client=5.2.4=py_3 - jupyter_core=4.4.0=py_0 @@ -32,54 +32,54 @@ dependencies: - nbconvert=5.4.1=py_2 - nbformat=4.4.0=py_1 - ncurses=6.1=hf484d3e_1002 - - notebook=5.7.6=py36_0 + - notebook=5.7.8=py36_0 - openssl=1.1.1b=h14c3975_1 - - pandoc=2.7.1=0 + - pandoc=2.7.2=0 - pandocfilters=1.4.2=py_1 - - parso=0.3.4=py_0 - - pexpect=4.6.0=py36_1000 + - parso=0.4.0=py_0 + - pexpect=4.7.0=py36_0 - pickleshare=0.7.5=py36_1000 - - pip=19.0.3=py36_0 + - pip=19.1=py36_0 - prometheus_client=0.6.0=py_0 - prompt_toolkit=2.0.9=py_0 - - ptyprocess=0.6.0=py36_1000 + - ptyprocess=0.6.0=py_1001 - pygments=2.3.1=py_0 - pyrsistent=0.14.11=py36h14c3975_0 - python=3.6.7=h381d211_1004 - python-dateutil=2.8.0=py_0 - - pyzmq=18.0.1=py36h0e1adb2_0 + - pyzmq=18.0.1=py36hc4ba49a_1 - readline=7.0=hf8c457e_1001 - send2trash=1.5.0=py_0 - - setuptools=40.8.0=py36_0 + - setuptools=41.0.1=py36_0 - six=1.12.0=py36_1000 - sqlite=3.26.0=h67949de_1001 - - terminado=0.8.1=py36_1001 + - terminado=0.8.2=py36_0 - testpath=0.4.2=py_1001 - - tk=8.6.9=h84994c4_1000 - - tornado=6.0.1=py36h14c3975_0 + - tk=8.6.9=h84994c4_1001 + - tornado=6.0.2=py36h516909a_0 - traitlets=4.3.2=py36_1000 - wcwidth=0.1.7=py_1 - webencodings=0.5.1=py_1 - wheel=0.33.1=py36_0 - widgetsnbextension=3.4.2=py36_1000 - xz=5.2.4=h14c3975_1001 - - zeromq=4.2.5=hf484d3e_1006 + - zeromq=4.3.1=hf484d3e_1000 - zlib=1.2.11=h14c3975_1004 - libgcc-ng=8.2.0=hdf63c60_1 - libstdcxx-ng=8.2.0=hdf63c60_1 - pip: - - alembic==1.0.8 + - alembic==1.0.9 - async-generator==1.10 - chardet==3.0.4 - idna==2.8 - jupyterhub==0.9.4 - - mako==1.0.8 - - nteract-on-jupyter==2.0.0 + - mako==1.0.9 + - nteract-on-jupyter==2.0.12 - pamela==1.0.0 - python-editor==1.0.4 - python-oauth2==1.1.0 - requests==2.21.0 - - sqlalchemy==1.3.1 - - urllib3==1.24.1 + - sqlalchemy==1.3.3 + - urllib3==1.24.2 prefix: /opt/conda/envs/r2d diff --git a/repo2docker/buildpacks/conda/environment.py-3.6.yml b/repo2docker/buildpacks/conda/environment.py-3.6.yml index 5fa071c3..5aef274a 100644 --- a/repo2docker/buildpacks/conda/environment.py-3.6.yml +++ b/repo2docker/buildpacks/conda/environment.py-3.6.yml @@ -1,11 +1,11 @@ # AUTO GENERATED FROM environment.yml, DO NOT MANUALLY MODIFY -# Generated on 2019-03-23 18:00:20 UTC +# Generated on 2019-04-24 17:59:15 UTC dependencies: - python=3.6.* - ipywidgets==7.4.2 - jupyterlab==0.35.4 - nbconvert==5.4.1 -- notebook==5.7.6 +- notebook==5.7.8 - pip: - - nteract_on_jupyter==2.0.0 + - nteract_on_jupyter==2.0.12 - jupyterhub==0.9.4 diff --git a/repo2docker/buildpacks/conda/environment.py-3.7.frozen.yml b/repo2docker/buildpacks/conda/environment.py-3.7.frozen.yml index eda33f89..19c0dad6 100644 --- a/repo2docker/buildpacks/conda/environment.py-3.7.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.py-3.7.frozen.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.py-3.7.yml, DO NOT MANUALLY MODIFY -# Frozen on 2019-03-23 18:02:15 UTC +# Frozen on 2019-04-24 18:01:25 UTC name: r2d channels: - conda-forge @@ -20,7 +20,7 @@ dependencies: - ipython_genutils=0.2.0=py_1 - ipywidgets=7.4.2=py_0 - jedi=0.13.3=py37_0 - - jinja2=2.10=py_1 + - jinja2=2.10.1=py_0 - jsonschema=3.0.1=py37_0 - jupyter_client=5.2.4=py_3 - jupyter_core=4.4.0=py_0 @@ -33,54 +33,54 @@ dependencies: - nbconvert=5.4.1=py_2 - nbformat=4.4.0=py_1 - ncurses=6.1=hf484d3e_1002 - - notebook=5.7.6=py37_0 + - notebook=5.7.8=py37_0 - openssl=1.1.1b=h14c3975_1 - - pandoc=2.7.1=0 + - pandoc=2.7.2=0 - pandocfilters=1.4.2=py_1 - - parso=0.3.4=py_0 - - pexpect=4.6.0=py37_1000 + - parso=0.4.0=py_0 + - pexpect=4.7.0=py37_0 - pickleshare=0.7.5=py37_1000 - - pip=19.0.3=py37_0 + - pip=19.1=py37_0 - prometheus_client=0.6.0=py_0 - prompt_toolkit=2.0.9=py_0 - - ptyprocess=0.6.0=py37_1000 + - ptyprocess=0.6.0=py_1001 - pygments=2.3.1=py_0 - pyrsistent=0.14.11=py37h14c3975_0 - - python=3.7.2=h381d211_0 + - python=3.7.3=h5b0a415_0 - python-dateutil=2.8.0=py_0 - - pyzmq=18.0.1=py37h0e1adb2_0 + - pyzmq=18.0.1=py37hc4ba49a_1 - readline=7.0=hf8c457e_1001 - send2trash=1.5.0=py_0 - - setuptools=40.8.0=py37_0 + - setuptools=41.0.1=py37_0 - six=1.12.0=py37_1000 - sqlite=3.26.0=h67949de_1001 - - terminado=0.8.1=py37_1001 + - terminado=0.8.2=py37_0 - testpath=0.4.2=py_1001 - - tk=8.6.9=h84994c4_1000 - - tornado=6.0.1=py37h14c3975_0 + - tk=8.6.9=h84994c4_1001 + - tornado=6.0.2=py37h516909a_0 - traitlets=4.3.2=py37_1000 - wcwidth=0.1.7=py_1 - webencodings=0.5.1=py_1 - wheel=0.33.1=py37_0 - widgetsnbextension=3.4.2=py37_1000 - xz=5.2.4=h14c3975_1001 - - zeromq=4.2.5=hf484d3e_1006 + - zeromq=4.3.1=hf484d3e_1000 - zlib=1.2.11=h14c3975_1004 - libgcc-ng=8.2.0=hdf63c60_1 - libstdcxx-ng=8.2.0=hdf63c60_1 - pip: - - alembic==1.0.8 + - alembic==1.0.9 - async-generator==1.10 - chardet==3.0.4 - idna==2.8 - jupyterhub==0.9.4 - - mako==1.0.8 - - nteract-on-jupyter==2.0.0 + - mako==1.0.9 + - nteract-on-jupyter==2.0.12 - pamela==1.0.0 - python-editor==1.0.4 - python-oauth2==1.1.0 - requests==2.21.0 - - sqlalchemy==1.3.1 - - urllib3==1.24.1 + - sqlalchemy==1.3.3 + - urllib3==1.24.2 prefix: /opt/conda/envs/r2d diff --git a/repo2docker/buildpacks/conda/environment.py-3.7.yml b/repo2docker/buildpacks/conda/environment.py-3.7.yml index 8734c443..5465414c 100644 --- a/repo2docker/buildpacks/conda/environment.py-3.7.yml +++ b/repo2docker/buildpacks/conda/environment.py-3.7.yml @@ -1,11 +1,11 @@ # AUTO GENERATED FROM environment.yml, DO NOT MANUALLY MODIFY -# Generated on 2019-03-23 18:02:15 UTC +# Generated on 2019-04-24 18:01:25 UTC dependencies: - python=3.7.* - ipywidgets==7.4.2 - jupyterlab==0.35.4 - nbconvert==5.4.1 -- notebook==5.7.6 +- notebook==5.7.8 - pip: - - nteract_on_jupyter==2.0.0 + - nteract_on_jupyter==2.0.12 - jupyterhub==0.9.4 diff --git a/repo2docker/buildpacks/conda/environment.yml b/repo2docker/buildpacks/conda/environment.yml index 9fde6701..c0abbfd8 100644 --- a/repo2docker/buildpacks/conda/environment.yml +++ b/repo2docker/buildpacks/conda/environment.yml @@ -3,7 +3,7 @@ dependencies: - ipywidgets==7.4.2 - jupyterlab==0.35.4 - nbconvert==5.4.1 - - notebook==5.7.6 + - notebook==5.7.8 - pip: - - nteract_on_jupyter==2.0.0 + - nteract_on_jupyter==2.0.12 - jupyterhub==0.9.4 From 9678dde2510b266fb3a2035e19376312861329a0 Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 25 Apr 2019 15:44:51 +0200 Subject: [PATCH 08/71] install notebook in its own env leave the conda root env alone adds conda activation to profile.d and ensures that we start with a login shell via the ENTRYPOINT --- repo2docker/buildpacks/base.py | 10 +++++ repo2docker/buildpacks/conda/__init__.py | 12 +++--- .../buildpacks/conda/activate-conda.sh | 11 ++++++ .../buildpacks/conda/install-miniconda.bash | 38 +++++++++---------- repo2docker/buildpacks/repo2docker-entrypoint | 5 +++ 5 files changed, 50 insertions(+), 26 deletions(-) create mode 100755 repo2docker/buildpacks/conda/activate-conda.sh create mode 100755 repo2docker/buildpacks/repo2docker-entrypoint diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index 77c1e2f3..1d62d5e5 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -156,6 +156,9 @@ RUN ./{{ s }} {% if start_script is not none -%} RUN chmod +x "{{ start_script }}" ENTRYPOINT ["{{ start_script }}"] +{% else %} +COPY /repo2docker-entrypoint /usr/local/bin/repo2docker-entrypoint +ENTRYPOINT ["/usr/local/bin/repo2docker-entrypoint"] {% endif -%} # Specify the default command to run @@ -167,6 +170,11 @@ CMD ["jupyter", "notebook", "--ip", "0.0.0.0"] {% endif %} """ +ENTRYPOINT_FILE = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "repo2docker-entrypoint", +) + class BuildPack: """ @@ -493,6 +501,8 @@ class BuildPack: src_path = os.path.join(os.path.dirname(__file__), *src_parts) tar.add(src_path, src, filter=_filter_tar) + tar.add(ENTRYPOINT_FILE, "repo2docker-entrypoint", filter=_filter_tar) + tar.add('.', 'src/', filter=_filter_tar) tar.close() diff --git a/repo2docker/buildpacks/conda/__init__.py b/repo2docker/buildpacks/conda/__init__.py index fca58c5e..c1bbbf1a 100644 --- a/repo2docker/buildpacks/conda/__init__.py +++ b/repo2docker/buildpacks/conda/__init__.py @@ -28,7 +28,7 @@ class CondaBuildPack(BaseImage): """ env = super().get_build_env() + [ ('CONDA_DIR', '${APP_BASE}/conda'), - ('NB_PYTHON_PREFIX', '${CONDA_DIR}'), + ('NB_PYTHON_PREFIX', '${CONDA_DIR}/envs/notebook'), ] if self.py2: env.append(('KERNEL_PYTHON_PREFIX', '${CONDA_DIR}/envs/kernel')) @@ -42,9 +42,10 @@ class CondaBuildPack(BaseImage): """ path = super().get_path() + path.insert(0, '${CONDA_DIR}/bin') if self.py2: path.insert(0, '${KERNEL_PYTHON_PREFIX}/bin') - path.insert(0, '${CONDA_DIR}/bin') + path.insert(0, '${NB_PYTHON_PREFIX}/bin') return path def get_build_scripts(self): @@ -97,6 +98,7 @@ class CondaBuildPack(BaseImage): """ files = { 'conda/install-miniconda.bash': '/tmp/install-miniconda.bash', + 'conda/activate-conda.sh': '/etc/profile.d/activate-conda.sh', } py_version = self.python_version self.log.info("Building conda environment for python=%s" % py_version) @@ -174,16 +176,16 @@ class CondaBuildPack(BaseImage): """ assembly_scripts = [] environment_yml = self.binder_path('environment.yml') - env_name = 'kernel' if self.py2 else 'root' + env_prefix = "${KERNEL_PYTHON_PREFIX}" if self.py2 else "${NB_PYTHON_PREFIX}" if os.path.exists(environment_yml): assembly_scripts.append(( '${NB_USER}', r""" - conda env update -n {0} -f "{1}" && \ + conda env update -p {0} -f "{1}" && \ conda clean -tipsy && \ conda list -n {0} && \ rm -rf /srv/conda/pkgs - """.format(env_name, environment_yml) + """.format(env_prefix, environment_yml) )) return super().get_assemble_scripts() + assembly_scripts diff --git a/repo2docker/buildpacks/conda/activate-conda.sh b/repo2docker/buildpacks/conda/activate-conda.sh new file mode 100755 index 00000000..56c4354b --- /dev/null +++ b/repo2docker/buildpacks/conda/activate-conda.sh @@ -0,0 +1,11 @@ +# enable conda and activate the notebook environment +CONDA_PROFILE="${CONDA_DIR}/etc/profile.d/conda.sh" +test -f $CONDA_PROFILE && . $CONDA_PROFILE +if [[ "${KERNEL_PYTHON_PREFIX}" != "${NB_PYTHON_PREFIX}" ]]; then + # if the kernel is a separate env, stack them + # so both are on PATH + conda activate ${KERNEL_PYTHON_PREFIX} + conda activate --stack ${NB_PYTHON_PREFIX} +else + conda activate ${NB_PYTHON_PREFIX} +fi diff --git a/repo2docker/buildpacks/conda/install-miniconda.bash b/repo2docker/buildpacks/conda/install-miniconda.bash index 54fcd55d..39a8d501 100755 --- a/repo2docker/buildpacks/conda/install-miniconda.bash +++ b/repo2docker/buildpacks/conda/install-miniconda.bash @@ -34,22 +34,21 @@ conda config --system --add channels conda-forge conda config --system --set auto_update_conda false conda config --system --set show_channel_urls true -# install conda itself -conda install -yq conda==${CONDA_VERSION} - -# switch Python in its own step -# since switching Python during an env update can -# prevent pip installation. -# we wouldn't have this issue if we did `conda env create` -# instead of `conda env update` in these cases -conda install -y $(cat /tmp/environment.yml | grep -o '\spython=.*') conda==${CONDA_VERSION} - # bug in conda 4.3.>15 prevents --set update_dependencies echo 'update_dependencies: false' >> ${CONDA_DIR}/.condarc -echo "installing root env:" +# install conda itself +conda install -yq conda==${CONDA_VERSION} + +echo "installing notebook env:" cat /tmp/environment.yml -conda env update -n root -f /tmp/environment.yml +conda env create -p ${NB_PYTHON_PREFIX} -f /tmp/environment.yml + +# empty conda history file, +# which seems to result in some effective pinning of packages in the initial env, +# which we don't intend. +# this file must not be *removed*, however +echo '' > ${NB_PYTHON_PREFIX}/conda-meta/history # enable nteract-on-jupyter, which was installed with pip jupyter serverextension enable nteract_on_jupyter --sys-prefix @@ -59,15 +58,11 @@ if [[ -f /tmp/kernel-environment.yml ]]; then echo "installing kernel env:" cat /tmp/kernel-environment.yml - conda env create -n kernel -f /tmp/kernel-environment.yml - ${CONDA_DIR}/envs/kernel/bin/ipython kernel install --prefix "${CONDA_DIR}" - echo '' > ${CONDA_DIR}/envs/kernel/conda-meta/history + conda env create -p ${KERNEL_PYTHON_PREFIX} -f /tmp/kernel-environment.yml + ${KERNEL_PYTHON_PREFIX}/bin/ipython kernel install --prefix "${CONDA_DIR}" + echo '' > ${KERNEL_PYTHON_PREFIX}/envs/kernel/conda-meta/history + conda list -p ${KERNEL_PYTHON_PREFIX} fi -# empty conda history file, -# which seems to result in some effective pinning of packages in the initial env, -# which we don't intend. -# this file must not be *removed*, however -echo '' > ${CONDA_DIR}/conda-meta/history # Clean things out! conda clean -tipsy @@ -77,4 +72,5 @@ rm ${INSTALLER_PATH} chown -R $NB_USER:$NB_USER ${CONDA_DIR} -conda list +conda list -n root +conda list -p ${NB_PYTHON_PREFIX} diff --git a/repo2docker/buildpacks/repo2docker-entrypoint b/repo2docker/buildpacks/repo2docker-entrypoint new file mode 100755 index 00000000..317ee96f --- /dev/null +++ b/repo2docker/buildpacks/repo2docker-entrypoint @@ -0,0 +1,5 @@ +#!/bin/bash -l +# lightest possible entrypoint that ensures that +# we use a login shell to get a fully configured shell environment +# (e.g. sourcing /etc/profile.d, ~/.bashrc, and friends) +exec "$@" From 15a23711e6f5f44f5520f352eae05c11d3f2802a Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 25 Apr 2019 15:56:26 +0200 Subject: [PATCH 09/71] invoke start script from our entrypoint rather than replacing our entrypoint --- repo2docker/buildpacks/base.py | 8 ++++---- repo2docker/buildpacks/repo2docker-entrypoint | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index 1d62d5e5..86591d6d 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -154,12 +154,12 @@ RUN ./{{ s }} # Add start script {% if start_script is not none -%} -RUN chmod +x "{{ start_script }}" -ENTRYPOINT ["{{ start_script }}"] -{% else %} +ENV R2D_ENTRYPOINT "{{ start_script }}" +{% endif -%} + +# Add entrypoint COPY /repo2docker-entrypoint /usr/local/bin/repo2docker-entrypoint ENTRYPOINT ["/usr/local/bin/repo2docker-entrypoint"] -{% endif -%} # Specify the default command to run CMD ["jupyter", "notebook", "--ip", "0.0.0.0"] diff --git a/repo2docker/buildpacks/repo2docker-entrypoint b/repo2docker/buildpacks/repo2docker-entrypoint index 317ee96f..8d67de46 100755 --- a/repo2docker/buildpacks/repo2docker-entrypoint +++ b/repo2docker/buildpacks/repo2docker-entrypoint @@ -2,4 +2,8 @@ # lightest possible entrypoint that ensures that # we use a login shell to get a fully configured shell environment # (e.g. sourcing /etc/profile.d, ~/.bashrc, and friends) -exec "$@" +if [[ ! -z "${R2D_ENTRYPOINT:-}" ]]; then + exec "$R2D_ENTRYPOINT" "$@" +else + exec "$@" +fi From 36ac03fd9d2c9d895568030e5b3920eca79b88bc Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 25 Apr 2019 15:58:44 +0200 Subject: [PATCH 10/71] env is a prefix now --- repo2docker/buildpacks/conda/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo2docker/buildpacks/conda/__init__.py b/repo2docker/buildpacks/conda/__init__.py index c1bbbf1a..9dec3e99 100644 --- a/repo2docker/buildpacks/conda/__init__.py +++ b/repo2docker/buildpacks/conda/__init__.py @@ -183,7 +183,7 @@ class CondaBuildPack(BaseImage): r""" conda env update -p {0} -f "{1}" && \ conda clean -tipsy && \ - conda list -n {0} && \ + conda list -p {0} && \ rm -rf /srv/conda/pkgs """.format(env_prefix, environment_yml) )) From 5f285b74e829d9202279219a3f1cdd6fae272d91 Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 25 Apr 2019 15:59:04 +0200 Subject: [PATCH 11/71] verify conda activation (and thereby entrypoint) in tests/conda/simple use pytest for better error reporting --- tests/conda/simple/environment.yml | 1 + tests/conda/simple/verify | 8 ++------ tests/conda/simple/verify.py | 12 ++++++++++++ 3 files changed, 15 insertions(+), 6 deletions(-) create mode 100755 tests/conda/simple/verify.py diff --git a/tests/conda/simple/environment.yml b/tests/conda/simple/environment.yml index 6a91d667..c8a54544 100644 --- a/tests/conda/simple/environment.yml +++ b/tests/conda/simple/environment.yml @@ -1,2 +1,3 @@ dependencies: - numpy + - pytest diff --git a/tests/conda/simple/verify b/tests/conda/simple/verify index 54d39472..058dbed0 100755 --- a/tests/conda/simple/verify +++ b/tests/conda/simple/verify @@ -1,6 +1,2 @@ -#!/usr/bin/env python -import sys - -assert sys.version_info[:2] == (3, 7), sys.version - -import numpy +#!/bin/sh +pytest -v ./verify.py diff --git a/tests/conda/simple/verify.py b/tests/conda/simple/verify.py new file mode 100755 index 00000000..6151ae5e --- /dev/null +++ b/tests/conda/simple/verify.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python +import os +import sys + +def test_sys_version(): + assert sys.version_info[:2] == (3, 7) + +def test_numpy(): + import numpy + +def test_conda_activated(): + assert os.environ.get("CONDA_PREFIX") == os.environ["NB_PYTHON_PREFIX"], dict(os.environ) From 51f15ee7bd4de58dddc47e7c2b3d5599acc17a7c Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 25 Apr 2019 16:30:42 +0200 Subject: [PATCH 12/71] update some tests with new expectations - conda is not importable in the frontmost python - sys.executable is in $NB_PYTHON_PREFIX (this was true before, but it's value has changed) --- tests/conda/binder-dir/verify | 6 ++++-- tests/conda/repo-path/verify | 2 +- tests/conda/simple-py2/verify | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/conda/binder-dir/verify b/tests/conda/binder-dir/verify index 002b5b94..64016158 100755 --- a/tests/conda/binder-dir/verify +++ b/tests/conda/binder-dir/verify @@ -1,8 +1,10 @@ #!/usr/bin/env python import sys +from subprocess import check_output assert sys.version_info[:2] == (3, 5), sys.version +out = check_output(['conda', '--version']).decode('utf8').strip() +assert out == 'conda 4.5.11', out + import numpy -import conda -assert conda.__version__ == '4.5.11', conda.__version__ diff --git a/tests/conda/repo-path/verify b/tests/conda/repo-path/verify index 6df74cbd..29559cb6 100755 --- a/tests/conda/repo-path/verify +++ b/tests/conda/repo-path/verify @@ -3,7 +3,7 @@ import sys import os # Python should still be in /srv/conda -assert sys.executable == '/srv/conda/bin/python' +assert sys.executable == os.path.join(os.environ['NB_PYTHON_PREFIX'], 'bin', 'python') # Repo should be in /srv/repo assert os.path.exists('/srv/repo/verify') diff --git a/tests/conda/simple-py2/verify b/tests/conda/simple-py2/verify index a6adf3c3..38c5a5a9 100755 --- a/tests/conda/simple-py2/verify +++ b/tests/conda/simple-py2/verify @@ -13,7 +13,7 @@ assert sorted(specs) == ['python2', 'python3'], specs.keys() import json from subprocess import check_output envs = json.loads(check_output(['conda', 'env', 'list', '--json']).decode('utf8')) -assert envs == {'envs': ['/srv/conda', '/srv/conda/envs/kernel']}, envs +assert envs == {'envs': ['/srv/conda', '/srv/conda/envs/kernel', '/srv/conda/envs/notebook']}, envs pkgs = json.loads(check_output(['conda', 'list', '-n', 'kernel', '--json']).decode('utf8')) pkg_names = [pkg['name'] for pkg in pkgs] From 70d950cc055413f53769d789b86f781826a45383 Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 25 Apr 2019 16:53:50 +0200 Subject: [PATCH 13/71] install python2 kernelspec into notebook env like other things, this was always the right thing to do, but more important now that notebook prefix is not the same as CONDA_DIR --- repo2docker/buildpacks/conda/install-miniconda.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repo2docker/buildpacks/conda/install-miniconda.bash b/repo2docker/buildpacks/conda/install-miniconda.bash index 39a8d501..f50aa653 100755 --- a/repo2docker/buildpacks/conda/install-miniconda.bash +++ b/repo2docker/buildpacks/conda/install-miniconda.bash @@ -59,8 +59,8 @@ if [[ -f /tmp/kernel-environment.yml ]]; then cat /tmp/kernel-environment.yml conda env create -p ${KERNEL_PYTHON_PREFIX} -f /tmp/kernel-environment.yml - ${KERNEL_PYTHON_PREFIX}/bin/ipython kernel install --prefix "${CONDA_DIR}" - echo '' > ${KERNEL_PYTHON_PREFIX}/envs/kernel/conda-meta/history + ${KERNEL_PYTHON_PREFIX}/bin/ipython kernel install --prefix "${NB_PYTHON_PREFIX}" + echo '' > ${KERNEL_PYTHON_PREFIX}/conda-meta/history conda list -p ${KERNEL_PYTHON_PREFIX} fi From 109793bffdbddeac1a6a797fdb54dfe4fd79019f Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 26 Apr 2019 16:09:00 +0200 Subject: [PATCH 14/71] conda 4.6.14 --- docs/source/changelog.rst | 5 ++++- repo2docker/buildpacks/conda/freeze.py | 2 +- repo2docker/buildpacks/conda/install-miniconda.bash | 10 ++++++---- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 27cc4133..71fa6db5 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -13,8 +13,11 @@ New features :user:`davidanthoff` - Set JULIA_PROJECT globally, so that every julia instance starts with the julia environment activated in :pr:`612` by :user:`davidanthoff`. -- Update Miniconda version to 4.5.12 and Conda version to 4.6.9 in :pr:`637` by +- Update Miniconda version to 4.5.12 and Conda version to 4.6.14 in :pr:`637` by :user:`jhamman` +- Install notebook into `notebook` env instead of `root`. + Activate conda environments and shell integration via ENTRYPOINT + in :pr:`651` by :user:`minrk` API changes ----------- diff --git a/repo2docker/buildpacks/conda/freeze.py b/repo2docker/buildpacks/conda/freeze.py index 01a9490b..91bd28c2 100755 --- a/repo2docker/buildpacks/conda/freeze.py +++ b/repo2docker/buildpacks/conda/freeze.py @@ -22,7 +22,7 @@ from ruamel.yaml import YAML # Docker image version can be different than conda version, # since miniconda3 docker images seem to lag conda releases. MINICONDA_DOCKER_VERSION = '4.5.12' -CONDA_VERSION = '4.6.9' +CONDA_VERSION = '4.6.14' HERE = pathlib.Path(os.path.dirname(os.path.abspath(__file__))) diff --git a/repo2docker/buildpacks/conda/install-miniconda.bash b/repo2docker/buildpacks/conda/install-miniconda.bash index 988a8b7a..fcbabace 100755 --- a/repo2docker/buildpacks/conda/install-miniconda.bash +++ b/repo2docker/buildpacks/conda/install-miniconda.bash @@ -3,11 +3,11 @@ set -ex cd $(dirname $0) -MINICONDA_VERSION=4.5.12 -CONDA_VERSION=4.6.9 +MINICONDA_VERSION=4.6.14 +CONDA_VERSION=4.6.14 # Only MD5 checksums are available for miniconda # Can be obtained from https://repo.continuum.io/miniconda/ -MD5SUM="866ae9dff53ad0874e1d1a60b1ad1ef8" +MD5SUM="718259965f234088d785cad1fbd7de03" URL="https://repo.continuum.io/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh" INSTALLER_PATH=/tmp/miniconda-installer.sh @@ -39,7 +39,9 @@ conda config --system --set show_channel_urls true echo 'update_dependencies: false' >> ${CONDA_DIR}/.condarc # install conda itself -conda install -yq conda==${CONDA_VERSION} +if [[ "${CONDA_VERSION}" != "${MINICONDA_VERSION}" ]]; then + conda install -yq conda==${CONDA_VERSION} +fi echo "installing notebook env:" cat /tmp/environment.yml From 9b7618cb79c04321e24d171e1ca12654748061a6 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 26 Apr 2019 16:14:30 +0200 Subject: [PATCH 15/71] refreeze --- repo2docker/buildpacks/conda/environment.frozen.yml | 6 +++--- repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml | 2 +- repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml | 6 +++--- repo2docker/buildpacks/conda/environment.py-3.6.yml | 2 +- repo2docker/buildpacks/conda/environment.py-3.7.frozen.yml | 6 +++--- repo2docker/buildpacks/conda/environment.py-3.7.yml | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/repo2docker/buildpacks/conda/environment.frozen.yml b/repo2docker/buildpacks/conda/environment.frozen.yml index ec1e2f13..2864475d 100644 --- a/repo2docker/buildpacks/conda/environment.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.frozen.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.py-3.7.yml, DO NOT MANUALLY MODIFY -# Frozen on 2019-04-24 18:01:25 UTC +# Frozen on 2019-04-26 14:12:19 UTC name: r2d channels: - conda-forge @@ -16,7 +16,7 @@ dependencies: - defusedxml=0.5.0=py_1 - entrypoints=0.3=py37_1000 - ipykernel=5.1.0=py37h24bf2e0_1002 - - ipython=7.4.0=py37h24bf2e0_0 + - ipython=7.5.0=py37h24bf2e0_0 - ipython_genutils=0.2.0=py_1 - ipywidgets=7.4.2=py_0 - jedi=0.13.3=py37_0 @@ -47,7 +47,7 @@ dependencies: - prompt_toolkit=2.0.9=py_0 - ptyprocess=0.6.0=py_1001 - pygments=2.3.1=py_0 - - pyrsistent=0.14.11=py37h14c3975_0 + - pyrsistent=0.15.1=py37h516909a_0 - python=3.7.3=h5b0a415_0 - python-dateutil=2.8.0=py_0 - pyzmq=18.0.1=py37hc4ba49a_1 diff --git a/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml b/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml index c12efcee..78906321 100644 --- a/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.py-2.7.yml, DO NOT MANUALLY MODIFY -# Frozen on 2019-04-24 17:58:08 UTC +# Frozen on 2019-04-26 14:09:19 UTC name: r2d channels: - conda-forge diff --git a/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml b/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml index e749cad5..c51bb836 100644 --- a/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY -# Frozen on 2019-04-24 17:59:15 UTC +# Frozen on 2019-04-26 14:10:28 UTC name: r2d channels: - conda-forge @@ -15,7 +15,7 @@ dependencies: - defusedxml=0.5.0=py_1 - entrypoints=0.3=py36_1000 - ipykernel=5.1.0=py36h24bf2e0_1002 - - ipython=7.4.0=py36h24bf2e0_0 + - ipython=7.5.0=py36h24bf2e0_0 - ipython_genutils=0.2.0=py_1 - ipywidgets=7.4.2=py_0 - jedi=0.13.3=py36_0 @@ -46,7 +46,7 @@ dependencies: - prompt_toolkit=2.0.9=py_0 - ptyprocess=0.6.0=py_1001 - pygments=2.3.1=py_0 - - pyrsistent=0.14.11=py36h14c3975_0 + - pyrsistent=0.15.1=py36h516909a_0 - python=3.6.7=h381d211_1004 - python-dateutil=2.8.0=py_0 - pyzmq=18.0.1=py36hc4ba49a_1 diff --git a/repo2docker/buildpacks/conda/environment.py-3.6.yml b/repo2docker/buildpacks/conda/environment.py-3.6.yml index 5aef274a..f6aa14b9 100644 --- a/repo2docker/buildpacks/conda/environment.py-3.6.yml +++ b/repo2docker/buildpacks/conda/environment.py-3.6.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.yml, DO NOT MANUALLY MODIFY -# Generated on 2019-04-24 17:59:15 UTC +# Generated on 2019-04-26 14:10:28 UTC dependencies: - python=3.6.* - ipywidgets==7.4.2 diff --git a/repo2docker/buildpacks/conda/environment.py-3.7.frozen.yml b/repo2docker/buildpacks/conda/environment.py-3.7.frozen.yml index ec1e2f13..2864475d 100644 --- a/repo2docker/buildpacks/conda/environment.py-3.7.frozen.yml +++ b/repo2docker/buildpacks/conda/environment.py-3.7.frozen.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.py-3.7.yml, DO NOT MANUALLY MODIFY -# Frozen on 2019-04-24 18:01:25 UTC +# Frozen on 2019-04-26 14:12:19 UTC name: r2d channels: - conda-forge @@ -16,7 +16,7 @@ dependencies: - defusedxml=0.5.0=py_1 - entrypoints=0.3=py37_1000 - ipykernel=5.1.0=py37h24bf2e0_1002 - - ipython=7.4.0=py37h24bf2e0_0 + - ipython=7.5.0=py37h24bf2e0_0 - ipython_genutils=0.2.0=py_1 - ipywidgets=7.4.2=py_0 - jedi=0.13.3=py37_0 @@ -47,7 +47,7 @@ dependencies: - prompt_toolkit=2.0.9=py_0 - ptyprocess=0.6.0=py_1001 - pygments=2.3.1=py_0 - - pyrsistent=0.14.11=py37h14c3975_0 + - pyrsistent=0.15.1=py37h516909a_0 - python=3.7.3=h5b0a415_0 - python-dateutil=2.8.0=py_0 - pyzmq=18.0.1=py37hc4ba49a_1 diff --git a/repo2docker/buildpacks/conda/environment.py-3.7.yml b/repo2docker/buildpacks/conda/environment.py-3.7.yml index 5465414c..73e5e8bf 100644 --- a/repo2docker/buildpacks/conda/environment.py-3.7.yml +++ b/repo2docker/buildpacks/conda/environment.py-3.7.yml @@ -1,5 +1,5 @@ # AUTO GENERATED FROM environment.yml, DO NOT MANUALLY MODIFY -# Generated on 2019-04-24 18:01:25 UTC +# Generated on 2019-04-26 14:12:19 UTC dependencies: - python=3.7.* - ipywidgets==7.4.2 From 186e1bbc5df50bdd5a1d1b15e731e7b0da62406b Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 26 Apr 2019 16:15:40 +0200 Subject: [PATCH 16/71] repo-path verifies conda prefix --- tests/conda/repo-path/verify | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/conda/repo-path/verify b/tests/conda/repo-path/verify index 29559cb6..78a68f37 100755 --- a/tests/conda/repo-path/verify +++ b/tests/conda/repo-path/verify @@ -2,8 +2,10 @@ import sys import os -# Python should still be in /srv/conda -assert sys.executable == os.path.join(os.environ['NB_PYTHON_PREFIX'], 'bin', 'python') +# conda should still be in /srv/conda +# and Python should still be in $NB_PYTHON_PREFIX +assert sys.executable == os.path.join(os.environ['NB_PYTHON_PREFIX'], 'bin', 'python'), sys.executable +assert sys.executable.startswith("/srv/conda/"), sys.executable # Repo should be in /srv/repo assert os.path.exists('/srv/repo/verify') From 2acf0974c310f6965dc20aa8e05280ed85fee5dd Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 26 Apr 2019 16:17:22 +0200 Subject: [PATCH 17/71] miniconda is 4.6.14 --- docs/source/changelog.rst | 2 +- tests/conda/binder-dir/verify | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 71fa6db5..3049d7bf 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -13,7 +13,7 @@ New features :user:`davidanthoff` - Set JULIA_PROJECT globally, so that every julia instance starts with the julia environment activated in :pr:`612` by :user:`davidanthoff`. -- Update Miniconda version to 4.5.12 and Conda version to 4.6.14 in :pr:`637` by +- Update Miniconda version to 4.6.14 and Conda version to 4.6.14 in :pr:`637` by :user:`jhamman` - Install notebook into `notebook` env instead of `root`. Activate conda environments and shell integration via ENTRYPOINT diff --git a/tests/conda/binder-dir/verify b/tests/conda/binder-dir/verify index 64016158..2c3fc1e1 100755 --- a/tests/conda/binder-dir/verify +++ b/tests/conda/binder-dir/verify @@ -5,6 +5,6 @@ from subprocess import check_output assert sys.version_info[:2] == (3, 5), sys.version out = check_output(['conda', '--version']).decode('utf8').strip() -assert out == 'conda 4.5.11', out +assert out == 'conda 4.6.14', out import numpy From 24eb3f923af338aa0b42748d7003195a3d6a4749 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Sat, 27 Apr 2019 10:49:32 +0200 Subject: [PATCH 18/71] Fix handling of memory limit command line argument If a string of only numerals is passed as argument we assume it should be converted to an integer and specifies a size in bytes. --- repo2docker/__main__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/repo2docker/__main__.py b/repo2docker/__main__.py index 62d9d883..1cb4862d 100644 --- a/repo2docker/__main__.py +++ b/repo2docker/__main__.py @@ -299,7 +299,12 @@ def make_r2d(argv=None): r2d.user_name = args.user_name if args.build_memory_limit: - r2d.build_memory_limit = args.build_memory_limit + # if the string only contains numerals we assume it should be an int + # and specifies a size inn bytes + if args.build_memory_limit.isnumeric(): + r2d.build_memory_limit = int(args.build_memory_limit) + else: + r2d.build_memory_limit = args.build_memory_limit if args.environment and not r2d.run: print('To specify environment variables, you also need to run ' From c230c70f4e2d004477b167fc3e85f55d22a9e3ba Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Sat, 27 Apr 2019 14:28:50 -0700 Subject: [PATCH 19/71] support for .binder directory --- docs/source/design.md | 2 +- docs/source/usage.rst | 5 +++-- repo2docker/buildpacks/base.py | 15 +++++++++++---- repo2docker/buildpacks/nix/nix-shell-wrapper | 17 ++++++++++++----- repo2docker/buildpacks/python/__init__.py | 4 ++-- repo2docker/buildpacks/r.py | 4 ++-- 6 files changed, 31 insertions(+), 16 deletions(-) diff --git a/docs/source/design.md b/docs/source/design.md index 1fa40aa0..dff33425 100644 --- a/docs/source/design.md +++ b/docs/source/design.md @@ -7,7 +7,7 @@ The philosophy for the repo2docker buildpacks includes: - using common configuration files for familiar installation and packaging tools - allowing configuration files to be combined to compose more complex setups - specifying default locations for configuration files -(the repository's root directory or .binder directory) +(in the repository's root, `binder` or `.binder` directory) When designing `repo2docker` and adding to it in the future, the diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 3a58f260..87a7d722 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -75,10 +75,11 @@ Where to put configuration files ``repo2docker`` will look for configuration files in either: * A folder named ``binder/`` in the root of the repository. +* A folder named ``.binder/`` in the root of the repository. * The root directory of the repository. -If the folder ``binder/`` is located at the top level of the repository, -only configuration files in the ``binder/`` folder will be considered. +If either the folder ``binder/`` or ``.binder/`` is located at the top level of +the repository, only configuration files in that folder will be considered. Check the complete list of :ref:`configuration files ` supported by ``repo2docker`` to see how to configure the build process. diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index 77c1e2f3..1d211145 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -189,6 +189,7 @@ class BuildPack: self.log = logging.getLogger('repo2docker') self.appendix = '' self.labels = {} + self._binder_dir = None if sys.platform.startswith('win'): self.log.warning("Windows environment detected. Note that Windows " "support is experimental in repo2docker.") @@ -414,12 +415,18 @@ class BuildPack: """ return None + @property + def binder_dir(self): + if self._binder_dir is None: + for dirname in ['binder', '.binder']: + if os.path.exists(dirname): + self._binder_dir = dirname + self._binder_dir = '' + return self._binder_dir + def binder_path(self, path): """Locate a file""" - if os.path.exists('binder'): - return os.path.join('binder', path) - else: - return path + return os.path.join(self.binder_dir, path) def detect(self): return True diff --git a/repo2docker/buildpacks/nix/nix-shell-wrapper b/repo2docker/buildpacks/nix/nix-shell-wrapper index 4de14a7f..143b6330 100644 --- a/repo2docker/buildpacks/nix/nix-shell-wrapper +++ b/repo2docker/buildpacks/nix/nix-shell-wrapper @@ -8,12 +8,19 @@ _term() { trap _term SIGTERM +# find binder sub-directory (if present) +for dir in "./binder" "./.binder"; do + if [ -e $dir ]; then + binder_dir = $dir + fi +done + # if there is a binder/ sub-directory it takes precedence # files outside it are ignored -if [ -e ./binder ]; then - nixpath="./binder/default.nix"; - if [ -f ./binder/start ]; then - chmod u+x ./binder/start +if [ -z $binder_dir ]; then + nixpath="$binder_dir/default.nix"; + if [ -f $binder_dir/start ]; then + chmod u+x $binder_dir/start # Using `$@`` here which is what the internet recommends leads to # errors when the command is something like `jupyter --ip=0.0.0.0 ...` # as nix-shell picks that up as an argument to it instead of the command. @@ -21,7 +28,7 @@ if [ -e ./binder ]; then # for -- to indicate "all arguments after this are for the command, not nix-shell" # but it seems they have stalled/not yet produced an implementation. # So let's use `$*` for now. - nix-shell $nixpath --command "./binder/start $*" & + nix-shell $nixpath --command "$binder_dir/start $*" & else nix-shell $nixpath --command "$*" & fi diff --git a/repo2docker/buildpacks/python/__init__.py b/repo2docker/buildpacks/python/__init__.py index fd81560c..1686ff3c 100644 --- a/repo2docker/buildpacks/python/__init__.py +++ b/repo2docker/buildpacks/python/__init__.py @@ -67,7 +67,7 @@ class PythonBuildPack(CondaBuildPack): )) # setup.py exists *and* binder dir is not used - if not os.path.exists('binder') and os.path.exists(setup_py): + if not self.binder_dir and os.path.exists(setup_py): assemble_scripts.append(( '${NB_USER}', '{} install --no-cache-dir .'.format(pip) @@ -88,6 +88,6 @@ class PythonBuildPack(CondaBuildPack): return True else: return False - if not os.path.exists('binder') and os.path.exists(setup_py): + if not self.binder_dir and os.path.exists(setup_py): return True return os.path.exists(requirements_txt) diff --git a/repo2docker/buildpacks/r.py b/repo2docker/buildpacks/r.py index 8d68d700..f24b6593 100644 --- a/repo2docker/buildpacks/r.py +++ b/repo2docker/buildpacks/r.py @@ -84,7 +84,7 @@ class RBuildPack(PythonBuildPack): return True description_R = 'DESCRIPTION' - if ((not os.path.exists('binder') and os.path.exists(description_R)) + if (not self.binder_dir and os.path.exists(description_R) or 'r' in self.stencila_contexts): if not self.checkpoint_date: # no R snapshot date set through runtime.txt @@ -300,7 +300,7 @@ class RBuildPack(PythonBuildPack): ] description_R = 'DESCRIPTION' - if not os.path.exists('binder') and os.path.exists(description_R): + if not self.binder_dir and os.path.exists(description_R): assemble_scripts += [ ( "${NB_USER}", From 3941a92a7d707f7d9613602492d609cb20b168b4 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Sat, 27 Apr 2019 17:16:14 -0700 Subject: [PATCH 20/71] fix missing () --- repo2docker/buildpacks/r.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo2docker/buildpacks/r.py b/repo2docker/buildpacks/r.py index f24b6593..5719169a 100644 --- a/repo2docker/buildpacks/r.py +++ b/repo2docker/buildpacks/r.py @@ -84,7 +84,7 @@ class RBuildPack(PythonBuildPack): return True description_R = 'DESCRIPTION' - if (not self.binder_dir and os.path.exists(description_R) + if ((not self.binder_dir and os.path.exists(description_R)) or 'r' in self.stencila_contexts): if not self.checkpoint_date: # no R snapshot date set through runtime.txt From e3ee8afa6f081d9be02c795b0679027cc15e9da6 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Sun, 28 Apr 2019 13:02:41 +0200 Subject: [PATCH 21/71] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 26 ++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 20 +++++++++++++++++ .github/ISSUE_TEMPLATE/support-question.md | 14 ++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/support-question.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..809ac848 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,26 @@ +--- +name: Bug report +about: Create a report to help us repair something that is currently broken +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behaviour: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behaviour** +A clear and concise description of what you expected to happen. + +**Desktop (please complete the following information):** + - OS: [e.g. iOS] + - Browser [e.g. chrome, safari] + - Version [e.g. 22] diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..cb41a717 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest a new feature or a big change to repo2docker +title: '' +labels: 'needs: discussion' +assignees: '' + +--- + +Those who maintain this project do so as volunteers and we have no control over their schedule or priorities. + +This means if the feature you are proposing is something you'd like to see added the best way to achieve that is for you to organise the effort required to build it. This could be evangelising for the feature, paying someone, doing it yourself, etc. + +Even so it might never get merged because the trade off between additional happy users and maintenance burden is not favourable. + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. diff --git a/.github/ISSUE_TEMPLATE/support-question.md b/.github/ISSUE_TEMPLATE/support-question.md new file mode 100644 index 00000000..bf10a69a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/support-question.md @@ -0,0 +1,14 @@ +--- +name: Support question +about: Ask a question about using repo2docker +title: '' +labels: '' +assignees: '' + +--- + +🚨 Please do **not** open an issue for support questions. Instead please search for similar issues or post on http://discourse.jupyter.org/c/questions. 🚨 + +More people read the forum than this issue tracker, it is indexed by search engines and easier for others to discover. + +For more details: https://discourse.jupyter.org/t/a-proposal-for-jupyterhub-communications/505 From 991a6826a866d11caae5a77756600e283ab7ae9f Mon Sep 17 00:00:00 2001 From: Tim Head Date: Sun, 28 Apr 2019 18:31:46 +0200 Subject: [PATCH 22/71] Tweak bug report template --- .github/ISSUE_TEMPLATE/bug_report.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 809ac848..2bac7b96 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -20,7 +20,7 @@ Steps to reproduce the behaviour: **Expected behaviour** A clear and concise description of what you expected to happen. -**Desktop (please complete the following information):** - - OS: [e.g. iOS] - - Browser [e.g. chrome, safari] - - Version [e.g. 22] +**Versions (please complete the following information):** + - OS: [e.g. linux, OSX] + - Docker version: `docker version` + - repo2docker version `repo2docker --version` From 1d9a64551ff565580eeb203dbafffb9b614ee257 Mon Sep 17 00:00:00 2001 From: Kirstie Whitaker Date: Sun, 28 Apr 2019 18:12:23 +0100 Subject: [PATCH 23/71] Add html comments to bug report --- .github/ISSUE_TEMPLATE/bug_report.md | 29 +++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 2bac7b96..0dd20516 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -6,21 +6,32 @@ labels: '' assignees: '' --- + -**Describe the bug** -A clear and concise description of what the bug is. +### Bug description + + +#### Expected behaviour + + +#### Actual behaviour + + +### How to reproduce + -**To Reproduce** -Steps to reproduce the behaviour: 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error -**Expected behaviour** -A clear and concise description of what you expected to happen. +### Your personal set up + -**Versions (please complete the following information):** - OS: [e.g. linux, OSX] - - Docker version: `docker version` - - repo2docker version `repo2docker --version` + - Docker version: `docker version` + - repo2docker version `repo2docker --version` + From 6ff13e91623e1903d09397226f59b071e125a6b7 Mon Sep 17 00:00:00 2001 From: Kirstie Whitaker Date: Sun, 28 Apr 2019 18:25:25 +0100 Subject: [PATCH 24/71] Add comments and additional structure to feature request --- .github/ISSUE_TEMPLATE/feature_request.md | 27 +++++++++++++++++------ 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index cb41a717..e2554408 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -6,15 +6,28 @@ labels: 'needs: discussion' assignees: '' --- + -Those who maintain this project do so as volunteers and we have no control over their schedule or priorities. +### Proposed change + -This means if the feature you are proposing is something you'd like to see added the best way to achieve that is for you to organise the effort required to build it. This could be evangelising for the feature, paying someone, doing it yourself, etc. -Even so it might never get merged because the trade off between additional happy users and maintenance burden is not favourable. +### Alternative options + -**Describe the solution you'd like** -A clear and concise description of what you want to happen. -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. +### Who would use this feature? + + + +### How much time will adding it take? + + + +### Who can do this work? + + + +### Thank you maintainers! + + From 1e0eb726cebf06ae16ce1a57c65349696a2ae675 Mon Sep 17 00:00:00 2001 From: Kirstie Whitaker Date: Sun, 28 Apr 2019 18:25:57 +0100 Subject: [PATCH 25/71] Make html comments all on one long line --- .github/ISSUE_TEMPLATE/bug_report.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 0dd20516..2171bd67 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -6,8 +6,7 @@ labels: '' assignees: '' --- - + ### Bug description @@ -27,9 +26,7 @@ issue, but you can delete them once you've read them if you prefer! --> 4. See error ### Your personal set up - + - OS: [e.g. linux, OSX] - Docker version: `docker version` From a27420e29fa548fe7832569fd663d9e0e3c69948 Mon Sep 17 00:00:00 2001 From: Kirstie Whitaker Date: Sun, 28 Apr 2019 19:11:32 +0100 Subject: [PATCH 26/71] Add types of contributions section --- docs/source/contributing/contributing.md | 30 ++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/source/contributing/contributing.md b/docs/source/contributing/contributing.md index b6616075..f6b84bf3 100644 --- a/docs/source/contributing/contributing.md +++ b/docs/source/contributing/contributing.md @@ -1,5 +1,35 @@ # Contributing to repo2docker development +Thank you for thinking about contributing to repo2docker! +This is an open source project that is developed and maintained entirely by volunteers. +*Your contribution* is integral to the future of the project. +THANK YOU! + +## Types of contribution + +There are many ways to contribute to repo2docker: + +* **Update the documentation.** + If you're reading a page or docstring and it doesn't make sense (or doesn't exist!), please let us know by opening a bug report. + It's even more amazing if you can give us a suggested change. +* **Fix bugs or add requested features.** + Have a look through the [issue tracker](https://github.com/jupyter/repo2docker/issues) and see if there are any tagged as ["help wanted"](https://github.com/jupyter/repo2docker/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22). + As the label suggests, we'd love your help! +* **Report a bug.** + If repo2docker isn't doing what you thought it would do then open a [bug report](https://github.com/jupyter/repo2docker/issues/new?template=bug_report.md). + That issue template will ask you a few questions described in more detail below. +* **Suggest a new feature.** + We know that there are lots of ways to extend repo2docker! + If you're interested in adding a feature then please open a [feature request](https://github.com/jupyter/repo2docker/issues/new?template=feature_request.md). + That issue template will ask you a few questions described in detail below. +* **Tell people about repo2docker.** + As we said above, repo2docker is built by and for its community. + If you know anyone who would like to use repo2docker, please tell them about the project! + You could give a talk about it, or run a demonstration. + The sky is the limit :rocket::star2:. + +If you're not sure where to get started, then please come and say hello in our [Gitter channel](https://gitter.im/jupyterhub/binder), or open an discussion thread at the [Jupyter discourse forum](https://discourse.jupyter.org/). + ## Process for making a contribution This outlines the process for getting changes to the repo2docker project merged. From 5a7add79291829a5166da0bb08631720f0aa3dfe Mon Sep 17 00:00:00 2001 From: Kirstie Whitaker Date: Sun, 28 Apr 2019 19:41:33 +0100 Subject: [PATCH 27/71] Update process and guidelines for making contribution --- docs/source/contributing/contributing.md | 67 +++++++++++++----------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/docs/source/contributing/contributing.md b/docs/source/contributing/contributing.md index f6b84bf3..71677c79 100644 --- a/docs/source/contributing/contributing.md +++ b/docs/source/contributing/contributing.md @@ -34,41 +34,48 @@ If you're not sure where to get started, then please come and say hello in our [ This outlines the process for getting changes to the repo2docker project merged. -* If your change is a big change, **open an issue to discuss** - before spending a lot of time writing. Getting consensus with the - community is a great way to save time later. -* Make edits in your fork of the repo2docker repository -* All changes are made by submitting a pull request. Read the next section for - the guidelines for both reviewers and contributors on merging a PR. -* Edit [the changelog](./../../changelog) - by appending your feature / bug fix to the development version. -* Wait for a community member to merge your changes. -* (optional) Deploy a new version of repo2docker to mybinder.org by [following these steps](http://mybinder-sre.readthedocs.io/en/latest/deployment/how.html) +1. Identify the correct issue template: [bug report](https://github.com/jupyter/repo2docker/issues/new?template=bug_report.md) or [feature request](https://github.com/jupyter/repo2docker/issues/new?template=feature_request.md). + **Bug reports** ([examples](https://github.com/jupyter/repo2docker/issues?q=is%3Aissue+is%3Aopen+label%3Abug), [new issue](https://github.com/jupyter/repo2docker/issues/new?template=bug_report.md)) will ask you for a description of the problem, the expected behaviour, the actual behaviour, how to reproduce the problem, and your personal set up. + Bugs can include problems with the documentation, or code not running as expected. + + **Feature requests** ([examples](https://github.com/jupyter/repo2docker/labels/needs%3A%20discussion), [new issue](https://github.com/jupyter/repo2docker/issues/new?template=feature_request.md)) will ask you for the proposed change, any alternatives that you have considered, a description of who would use this feature, and a best-guess of how much work it will take and what skills are required to accomplish. + + Very easy feature requests might be updates to the documentation to clarify steps for new users. + Harder feature requests may be to add new functionality to the project and will need more in depth discussion about who can complete and maintain the work. + +2. Open an issue. + Getting consensus with the community is a great way to save time later. +3. Make edits in [your fork](https://help.github.com/en/articles/fork-a-repo) of the [repo2docker repository](https://github.com/jupyter/repo2docker). +4. Make a [pull request](https://help.github.com/en/articles/about-pull-requests). +Read the [next section](#guidelines-to-getting-a-pull-request-merged) for guidelines for both reviewers and contributors on merging a PR. +5. Edit [the changelog](./../../changelog) by appending your feature / bug fix to the development version. +6. Wait for a community member to merge your changes. +7. (optional) Deploy a new version of repo2docker to mybinder.org by [following these steps](http://mybinder-sre.readthedocs.io/en/latest/deployment/how.html) ## Guidelines to getting a Pull Request merged -These are not hard rules to be enforced by 🚓 but instead guidelines -to help you make a contribution. +These are not hard rules to be enforced by 🚓 but they are suggestions written by the repo2docker maintainers to make getting your contribution completed a smooth process for you and for them. + +* **Create a PR as early as possible**, marking it with `[WIP]` while you work on it. + This avoids duplicated work, lets you get high level feedback on functionality or API changes, and/or helps find collaborators to work with you. +* **Keep your PR focused.** + The best PRs solve one problem. + If you end up changing multiple things, please open separate PRs for the different conceptual changes. +* **Add tests your code.** + PRs will not be merged if Travis is failing. +* **Apply [PEP8](https://www.python.org/dev/peps/pep-0008/)** as much as possible, but not too much. + If in doubt, ask. +* **Use merge commits** instead of merge-by-squashing/-rebasing. + This makes it easier to find all changes since the last deployment `git log --merges --pretty=format:"%h %<(10,trunc)%an %<(15)%ar %s" ..` and your PR easier to review. +* **Make it clear when your PR is ready for review.** + Prefix the title of your pull request (PR) with `[MRG]` if the contribution is complete and should be subjected to a detailed review. +* **Enter your changes into the [changelog](./../../changelog)** in `docs/source/changelog.rst`. +* **Use commit messages to describe _why_ you are proposing the changes you are proposing.** +* **Try to not rush changes** (the definition of rush depends on how big your changes are). + Remember that everyone in the repo2docker team is a volunteer and we can not (nor would we want to) control their time or interests. + Wait patiently for a reviewer to merge the PR. -* prefix the title of your pull request with `[MRG]` if the contribution - is complete and should be subjected to a detailed review; -* create a PR as early as possible, marking it with `[WIP]` while you work on - it (good to avoid duplicated work, get broad review of functionality or API, - or seek collaborators); -* a PR solves one problem (do not mix problems together in one PR) with the - minimal set of changes; -* describe why you are proposing the changes you are proposing; -* try to not rush changes (the definition of rush depends on how big your - changes are); -* Enter your changes into the [changelog](./../../changelog) in `docs/source/changelog.rst`; -* someone else has to merge your PR; -* new code needs to come with a test; -* apply [PEP8](https://www.python.org/dev/peps/pep-0008/) as much - as possible, but not too much; -* no merging if travis is red; -* do use merge commits instead of merge-by-squashing/-rebasing. This makes it - easier to find all changes since the last deployment `git log --merges --pretty=format:"%h %<(10,trunc)%an %<(15)%ar %s" ..` ## Setting up for Local Development From 6c43fb927ac594577b883c47bca0c617e8528eb0 Mon Sep 17 00:00:00 2001 From: Kirstie Whitaker Date: Sun, 28 Apr 2019 19:58:55 +0100 Subject: [PATCH 28/71] Extend the points about feature requests & bug reports --- docs/source/contributing/contributing.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/source/contributing/contributing.md b/docs/source/contributing/contributing.md index 71677c79..a8066534 100644 --- a/docs/source/contributing/contributing.md +++ b/docs/source/contributing/contributing.md @@ -39,11 +39,17 @@ This outlines the process for getting changes to the repo2docker project merged. **Bug reports** ([examples](https://github.com/jupyter/repo2docker/issues?q=is%3Aissue+is%3Aopen+label%3Abug), [new issue](https://github.com/jupyter/repo2docker/issues/new?template=bug_report.md)) will ask you for a description of the problem, the expected behaviour, the actual behaviour, how to reproduce the problem, and your personal set up. Bugs can include problems with the documentation, or code not running as expected. + It is really important that you make it easy for the maintainers to reproduce the problem you're having. + This guide on creating a [minimal, complete and verifiable example](https://stackoverflow.com/help/mcve) is a great place to start. + **Feature requests** ([examples](https://github.com/jupyter/repo2docker/labels/needs%3A%20discussion), [new issue](https://github.com/jupyter/repo2docker/issues/new?template=feature_request.md)) will ask you for the proposed change, any alternatives that you have considered, a description of who would use this feature, and a best-guess of how much work it will take and what skills are required to accomplish. Very easy feature requests might be updates to the documentation to clarify steps for new users. Harder feature requests may be to add new functionality to the project and will need more in depth discussion about who can complete and maintain the work. + Feature requests are a great opportunity for you to advocate for the use case you're suggesting. + They help others understand how much effort it would be to integrate the work,and - if you're successful at convincing them that this effort is worth it - make it more likely that they to choose to work on it with you. + 2. Open an issue. Getting consensus with the community is a great way to save time later. 3. Make edits in [your fork](https://help.github.com/en/articles/fork-a-repo) of the [repo2docker repository](https://github.com/jupyter/repo2docker). @@ -76,8 +82,6 @@ These are not hard rules to be enforced by 🚓 but they are suggestions written Remember that everyone in the repo2docker team is a volunteer and we can not (nor would we want to) control their time or interests. Wait patiently for a reviewer to merge the PR. - - ## Setting up for Local Development To develop & test repo2docker locally, you need: From 13922ee24929b154db71b9135db833afa1442a2e Mon Sep 17 00:00:00 2001 From: Kirstie Whitaker Date: Sun, 28 Apr 2019 20:09:34 +0100 Subject: [PATCH 29/71] Embelish the links in contributing.md --- CONTRIBUTING.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 762e84b2..499e7da9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,7 +1,26 @@ # Contributing to repo2docker development -The repo2docker developer documentation can be found on these pages: +:sparkles: Thank you for thinking about contributing to repo2docker! :sparkles: + +(And thank you particularly for coming to read the guidelines! :heart_eyes:) + +The repo2docker developer documentation is all rendered on our documentation website: [https://repo2docker.readthedocs.io](https://repo2docker.readthedocs.io). +If you're here, you're probably looking for the [Contributing to repo2docker development](https://repo2docker.readthedocs.io/en/latest/contributing/contributing.html) page. + +Please make sure you've read the following sections before opening an issue/pull request: +* [Process for making a contribution](https://repo2docker.readthedocs.io/en/latest/contributing/contributing.html#process-for-making-a-contribution). + * These steps talk you through choosing the right issue template (bug report or feature request) and making a change. +* [Guidelines to getting a Pull Request merged](https://repo2docker.readthedocs.io/en/latest/contributing/contributing.html#guidelines-to-getting-a-pull-request-merged). + * These are tips and tricks to help make your contribution as smooth as possible for you and for the repo2docker maintenance team. + +There are a few other pages to highlight: -* [Contributing to repo2docker](https://repo2docker.readthedocs.io/en/latest/contributing/contributing.html) * [Our roadmap](https://repo2docker.readthedocs.io/en/latest/contributing/roadmap.html) + * We use the roadmap to develop a shared understanding of the project's vision and direction amongst the community of users, contributors, and maintainers. + This is a great place to get a feel for what the maintainers are thinking about for the short, medium and long term future of the project. +* [Design of repo2docker](https://repo2docker.readthedocs.io/en/latest/design.html) + * This page explains some of the design principles behind repo2docker. + Its a really good place to understand _why_ the team have made the decisions that they have along the way! + * We absolutely encourage discussion around refactoring, updating or extending repo2docker, but please make sure that you've understood this page before opening an issue to discuss the change you'd like to propose. * [Common developer tasks and how-tos](https://repo2docker.readthedocs.io/en/latest/contributing/tasks.html) + * Some notes on running tests, buildpack dependencies, creating a release, updating the changelog and keeping the pip files up to date. \ No newline at end of file From 3c0ceaacc21288ad1ec6b48f87ce04c2df575c5f Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Sun, 28 Apr 2019 20:28:09 -0700 Subject: [PATCH 30/71] clarify that only one of binder and .binder can be used at once --- docs/source/usage.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 87a7d722..449af33e 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -72,14 +72,14 @@ specify the ``branch-name`` or ``commit-hash``. For example:: Where to put configuration files ================================ -``repo2docker`` will look for configuration files in either: +``repo2docker`` will look for configuration files in: * A folder named ``binder/`` in the root of the repository. * A folder named ``.binder/`` in the root of the repository. * The root directory of the repository. -If either the folder ``binder/`` or ``.binder/`` is located at the top level of -the repository, only configuration files in that folder will be considered. +`repo2docker` searches for these folders in order (``binder/``, ``.binder/``, +root). Only configuration files in the first identified folder are considered. Check the complete list of :ref:`configuration files ` supported by ``repo2docker`` to see how to configure the build process. From e83b12f5b2bb68a58ed695f763f5ef95d26ea5a7 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Mon, 29 Apr 2019 06:47:51 -0700 Subject: [PATCH 31/71] add missing break --- repo2docker/buildpacks/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index 1d211145..69e19bee 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -419,8 +419,9 @@ class BuildPack: def binder_dir(self): if self._binder_dir is None: for dirname in ['binder', '.binder']: - if os.path.exists(dirname): + if os.path.isdir(dirname): self._binder_dir = dirname + break self._binder_dir = '' return self._binder_dir From 6cd3d1d895a77e2acf28861d27d41e2b1ccc011b Mon Sep 17 00:00:00 2001 From: Kirstie Whitaker Date: Mon, 29 Apr 2019 15:09:42 +0100 Subject: [PATCH 32/71] Update .github/ISSUE_TEMPLATE/feature_request.md --- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index e2554408..cfe1ab5d 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -20,7 +20,7 @@ assignees: '' -### How much time will adding it take? +### How much effort will adding it take? From 7ac4cb4932751107f5fee3e6b422de489eabf10b Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Mon, 29 Apr 2019 08:05:49 -0700 Subject: [PATCH 33/71] error if both binder and .binder dirs exist --- repo2docker/buildpacks/base.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index 69e19bee..c0f0a531 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -189,7 +189,6 @@ class BuildPack: self.log = logging.getLogger('repo2docker') self.appendix = '' self.labels = {} - self._binder_dir = None if sys.platform.startswith('win'): self.log.warning("Windows environment detected. Note that Windows " "support is experimental in repo2docker.") @@ -417,13 +416,20 @@ class BuildPack: @property def binder_dir(self): - if self._binder_dir is None: - for dirname in ['binder', '.binder']: - if os.path.isdir(dirname): - self._binder_dir = dirname - break - self._binder_dir = '' - return self._binder_dir + has_binder = os.path.isdir("binder") + has_dotbinder = os.path.isdir(".binder") + + if has_binder and has_dotbinder: + raise RuntimeError( + "The repository contains both a 'binder' and a '.binder' " + "directory. However they are exclusive.") + + if has_dotbinder: + return ".binder" + elif has_binder: + return "binder" + else: + return "" def binder_path(self, path): """Locate a file""" From f3e004dd19e3c6ddc74c0e4e89c3b1ccef556817 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Mon, 29 Apr 2019 16:39:49 +0100 Subject: [PATCH 34/71] Update CONTRIBUTING.md Co-Authored-By: KirstieJane --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 499e7da9..d750b20f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,7 +20,7 @@ There are a few other pages to highlight: This is a great place to get a feel for what the maintainers are thinking about for the short, medium and long term future of the project. * [Design of repo2docker](https://repo2docker.readthedocs.io/en/latest/design.html) * This page explains some of the design principles behind repo2docker. - Its a really good place to understand _why_ the team have made the decisions that they have along the way! + Its a good place to understand _why_ the team have made the decisions that they have along the way! * We absolutely encourage discussion around refactoring, updating or extending repo2docker, but please make sure that you've understood this page before opening an issue to discuss the change you'd like to propose. * [Common developer tasks and how-tos](https://repo2docker.readthedocs.io/en/latest/contributing/tasks.html) - * Some notes on running tests, buildpack dependencies, creating a release, updating the changelog and keeping the pip files up to date. \ No newline at end of file + * Some notes on running tests, buildpack dependencies, creating a release, updating the changelog and keeping the pip files up to date. From c9a161746de4df5600801de42adc60abc583bafe Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Mon, 29 Apr 2019 16:40:02 +0100 Subject: [PATCH 35/71] Update CONTRIBUTING.md Co-Authored-By: KirstieJane --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d750b20f..1a02fa1a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,7 +17,7 @@ There are a few other pages to highlight: * [Our roadmap](https://repo2docker.readthedocs.io/en/latest/contributing/roadmap.html) * We use the roadmap to develop a shared understanding of the project's vision and direction amongst the community of users, contributors, and maintainers. - This is a great place to get a feel for what the maintainers are thinking about for the short, medium and long term future of the project. + This is a great place to get a feel for what the maintainers are thinking about for the short, medium, and long term future of the project. * [Design of repo2docker](https://repo2docker.readthedocs.io/en/latest/design.html) * This page explains some of the design principles behind repo2docker. Its a good place to understand _why_ the team have made the decisions that they have along the way! From 05273abb38356dec25f829ffa43a8754a16c1eb4 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Mon, 29 Apr 2019 08:52:18 -0700 Subject: [PATCH 36/71] fix bash spacing --- repo2docker/buildpacks/nix/nix-shell-wrapper | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repo2docker/buildpacks/nix/nix-shell-wrapper b/repo2docker/buildpacks/nix/nix-shell-wrapper index 143b6330..7490d78e 100644 --- a/repo2docker/buildpacks/nix/nix-shell-wrapper +++ b/repo2docker/buildpacks/nix/nix-shell-wrapper @@ -9,9 +9,9 @@ _term() { trap _term SIGTERM # find binder sub-directory (if present) -for dir in "./binder" "./.binder"; do +for dir in "./.binder" "./binder"; do if [ -e $dir ]; then - binder_dir = $dir + binder_dir=$dir fi done From f9afea1f29322f29050aa0433f4d20b5cc1adee2 Mon Sep 17 00:00:00 2001 From: Kirstie Whitaker Date: Mon, 29 Apr 2019 17:58:22 +0100 Subject: [PATCH 37/71] Reword comment and remove thank you prompt --- .github/ISSUE_TEMPLATE/feature_request.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index cfe1ab5d..25182fc4 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -17,17 +17,13 @@ assignees: '' ### Who would use this feature? - + ### How much effort will adding it take? - + ### Who can do this work? - -### Thank you maintainers! - - From 34ad355de646dcaf04d6cbdb02ef1db337ea2e18 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Mon, 29 Apr 2019 20:30:07 -0700 Subject: [PATCH 38/71] fix up nix-shell-wrapper for .binder feature --- repo2docker/buildpacks/nix/nix-shell-wrapper | 50 ++++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/repo2docker/buildpacks/nix/nix-shell-wrapper b/repo2docker/buildpacks/nix/nix-shell-wrapper index 7490d78e..e9c8d71a 100644 --- a/repo2docker/buildpacks/nix/nix-shell-wrapper +++ b/repo2docker/buildpacks/nix/nix-shell-wrapper @@ -8,38 +8,38 @@ _term() { trap _term SIGTERM +# if there is a binder/ sub-directory it takes precedence +# files outside it are ignored # find binder sub-directory (if present) -for dir in "./.binder" "./binder"; do +binder_dir="./" +for dir in "./binder" "./.binder" ; do if [ -e $dir ]; then binder_dir=$dir + break fi done -# if there is a binder/ sub-directory it takes precedence -# files outside it are ignored -if [ -z $binder_dir ]; then - nixpath="$binder_dir/default.nix"; - if [ -f $binder_dir/start ]; then - chmod u+x $binder_dir/start - # Using `$@`` here which is what the internet recommends leads to - # errors when the command is something like `jupyter --ip=0.0.0.0 ...` - # as nix-shell picks that up as an argument to it instead of the command. - # There are several issues on the nix repos discussing this and adding support - # for -- to indicate "all arguments after this are for the command, not nix-shell" - # but it seems they have stalled/not yet produced an implementation. - # So let's use `$*` for now. - nix-shell $nixpath --command "$binder_dir/start $*" & - else - nix-shell $nixpath --command "$*" & - fi +# raise error if both binder and .binder are found +if [[ -d "./binder" && -d "./.binder" ]]; then + echo "Error: Found both binder and .binder directories." + exit 1 +fi + +echo "binder_dir is: $binder_dir" + +nixpath="$binder_dir/default.nix"; +if [ -f $binder_dir/start ]; then + chmod u+x $binder_dir/start + # Using `$@`` here which is what the internet recommends leads to + # errors when the command is something like `jupyter --ip=0.0.0.0 ...` + # as nix-shell picks that up as an argument to it instead of the command. + # There are several issues on the nix repos discussing this and adding support + # for -- to indicate "all arguments after this are for the command, not nix-shell" + # but it seems they have stalled/not yet produced an implementation. + # So let's use `$*` for now. + nix-shell $nixpath --command "$binder_dir/start $*" & else - nixpath="./default.nix"; - if [ -f ./start ]; then - chmod u+x ./start - nix-shell $nixpath --command "./start $*" & - else - nix-shell $nixpath --command "$*" & - fi + nix-shell $nixpath --command "$*" & fi PID=$! From e15007d528c9b0e53310f12825e17a837a361b47 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Fri, 26 Apr 2019 18:09:58 +0200 Subject: [PATCH 39/71] Make sure start script is executable --- repo2docker/buildpacks/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index 86591d6d..3e3fe532 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -154,6 +154,7 @@ RUN ./{{ s }} # Add start script {% if start_script is not none -%} +RUN chmod +x "{{ start_script }}" ENV R2D_ENTRYPOINT "{{ start_script }}" {% endif -%} From e2c7de8d19ceb9b20d73826801d7a2e499880920 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Tue, 30 Apr 2019 08:35:15 +0200 Subject: [PATCH 40/71] Update Julia tests for new conda env path --- tests/julia/julia_version-1.0.2/verify | 2 +- tests/julia/julia_version-default/verify | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/julia/julia_version-1.0.2/verify b/tests/julia/julia_version-1.0.2/verify index dcb25826..10fd2866 100755 --- a/tests/julia/julia_version-1.0.2/verify +++ b/tests/julia/julia_version-1.0.2/verify @@ -17,7 +17,7 @@ end # Verify that kernels are not installed in home directory (issue #620) try using IJulia - @assert IJulia.kerneldir() == "/srv/conda/share/jupyter/kernels" + @assert IJulia.kerneldir() == ENV["NB_PYTHON_PREFIX"] * "/share/jupyter/kernels" catch exit(1) end diff --git a/tests/julia/julia_version-default/verify b/tests/julia/julia_version-default/verify index 79feaae1..6a5cd226 100755 --- a/tests/julia/julia_version-default/verify +++ b/tests/julia/julia_version-default/verify @@ -17,7 +17,7 @@ end # Verify that kernels are not installed in home directory (issue #620) try using IJulia - @assert IJulia.kerneldir() == "/srv/conda/share/jupyter/kernels" + @assert IJulia.kerneldir() == ENV["NB_PYTHON_PREFIX"] * "/share/jupyter/kernels" catch exit(1) end From c25e6bc2ae506d2e4dec5fb4babecb37d1a239f2 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Tue, 30 Apr 2019 09:13:32 +0100 Subject: [PATCH 41/71] Update docs/source/contributing/contributing.md Co-Authored-By: KirstieJane --- docs/source/contributing/contributing.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/source/contributing/contributing.md b/docs/source/contributing/contributing.md index a8066534..83603428 100644 --- a/docs/source/contributing/contributing.md +++ b/docs/source/contributing/contributing.md @@ -22,6 +22,12 @@ There are many ways to contribute to repo2docker: We know that there are lots of ways to extend repo2docker! If you're interested in adding a feature then please open a [feature request](https://github.com/jupyter/repo2docker/issues/new?template=feature_request.md). That issue template will ask you a few questions described in detail below. +* **Review someone's Pull Request.** + Whenever somebody proposes changes to the repo2docker codebase, the community reviews + the changes, and provides feedback, edits, and suggestions. Check out the + [open pull requests](https://github.com/jupyter/repo2docker/pulls?q=is%3Apr+is%3Aopen+sort%3Aupdated-desc) + and provide feedback that helps improve the PR and get it merged. Please keep your + feedback positive and constructive! * **Tell people about repo2docker.** As we said above, repo2docker is built by and for its community. If you know anyone who would like to use repo2docker, please tell them about the project! From 288c3fe811ae9b13bc645d1ba3e65f6f069f844f Mon Sep 17 00:00:00 2001 From: Kirstie Whitaker Date: Tue, 30 Apr 2019 09:18:01 +0100 Subject: [PATCH 42/71] Update .github/ISSUE_TEMPLATE/feature_request.md --- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 25182fc4..833ba6e5 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -25,5 +25,5 @@ assignees: '' ### Who can do this work? - + From 34cff4d5d3f57000164c93d667cd5cdcb6d50377 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Tue, 30 Apr 2019 09:20:47 +0100 Subject: [PATCH 43/71] Update docs/source/contributing/contributing.md Co-Authored-By: KirstieJane --- docs/source/contributing/contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/contributing/contributing.md b/docs/source/contributing/contributing.md index 83603428..0b50ea81 100644 --- a/docs/source/contributing/contributing.md +++ b/docs/source/contributing/contributing.md @@ -74,7 +74,7 @@ These are not hard rules to be enforced by 🚓 but they are suggestions written * **Keep your PR focused.** The best PRs solve one problem. If you end up changing multiple things, please open separate PRs for the different conceptual changes. -* **Add tests your code.** +* **Add tests to your code.** PRs will not be merged if Travis is failing. * **Apply [PEP8](https://www.python.org/dev/peps/pep-0008/)** as much as possible, but not too much. If in doubt, ask. From 6344f39d255c81352027a5d59c25edd32f59ca4c Mon Sep 17 00:00:00 2001 From: Kirstie Whitaker Date: Tue, 30 Apr 2019 09:23:34 +0100 Subject: [PATCH 44/71] Update contributing.md --- docs/source/contributing/contributing.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/source/contributing/contributing.md b/docs/source/contributing/contributing.md index 0b50ea81..a317e053 100644 --- a/docs/source/contributing/contributing.md +++ b/docs/source/contributing/contributing.md @@ -63,6 +63,8 @@ This outlines the process for getting changes to the repo2docker project merged. Read the [next section](#guidelines-to-getting-a-pull-request-merged) for guidelines for both reviewers and contributors on merging a PR. 5. Edit [the changelog](./../../changelog) by appending your feature / bug fix to the development version. 6. Wait for a community member to merge your changes. + Remember that **someone else must merge your pull request**. + That goes for new contributors and long term maintainers alike. 7. (optional) Deploy a new version of repo2docker to mybinder.org by [following these steps](http://mybinder-sre.readthedocs.io/en/latest/deployment/how.html) ## Guidelines to getting a Pull Request merged @@ -87,6 +89,7 @@ These are not hard rules to be enforced by 🚓 but they are suggestions written * **Try to not rush changes** (the definition of rush depends on how big your changes are). Remember that everyone in the repo2docker team is a volunteer and we can not (nor would we want to) control their time or interests. Wait patiently for a reviewer to merge the PR. + (Remember that **someone else** must merge your PR, even if you have the admin rights to do so.) ## Setting up for Local Development From 6de4c5ea6bc452b5810afa53402322d668cefbf9 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Tue, 30 Apr 2019 08:35:58 -0700 Subject: [PATCH 45/71] add unit tests for binder dir --- repo2docker/__main__.py | 1 + tests/unit/test_binder_dir.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 tests/unit/test_binder_dir.py diff --git a/repo2docker/__main__.py b/repo2docker/__main__.py index 62d9d883..e5f734a2 100644 --- a/repo2docker/__main__.py +++ b/repo2docker/__main__.py @@ -7,6 +7,7 @@ from .app import Repo2Docker from . import __version__ from .utils import validate_and_generate_port_mapping, is_valid_docker_image_name + def validate_image_name(image_name): """ Validate image_name read by argparse diff --git a/tests/unit/test_binder_dir.py b/tests/unit/test_binder_dir.py new file mode 100644 index 00000000..1dc78fb6 --- /dev/null +++ b/tests/unit/test_binder_dir.py @@ -0,0 +1,31 @@ +import os + +import pytest + +from repo2docker import buildpacks + + +@pytest.mark.parametrize("binder_dir", ['.binder', 'binder']) +def test_binder_dir_property(tmpdir, binder_dir): + tmpdir.chdir() + os.mkdir(binder_dir) + + bp = buildpacks.BuildPack() + assert binder_dir in bp.binder_dir + assert bp.binder_path('foo.yaml') == os.path.join(binder_dir, 'foo.yaml') + + +def test_root_binder_dir(tmpdir): + tmpdir.chdir() + bp = buildpacks.BuildPack() + assert bp.binder_dir == '' + + +def test_exclusive_binder_dir(tmpdir): + tmpdir.chdir() + os.mkdir('./binder') + os.mkdir('./.binder') + + bp = buildpacks.BuildPack() + with pytest.raises(RuntimeError): + _ = bp.binder_dir From 4dd32f3563bebcab75a5877712c07355205d6d40 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Mon, 29 Apr 2019 21:42:29 -0700 Subject: [PATCH 46/71] Make sure ENTRYPOINT is an absolute path Unlike other parts of the generated Dockerfile, the start script is evaluated at run time, rather than at build time. Currently, we assume that the current working directory is the same at runtime as build time for the start script. This doesn't hold true always, and particularly not in JupyterHub environments where ${HOME} is often overlaid with a persistent directory. We change this to always refer to the full path, using the ${REPO_DIR} environment variable. This lets people building JupyterHub images to set REPO_DIR to something like /srv/repo (like hubploy does), and still have a working start script. --- repo2docker/buildpacks/base.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index 265c49c0..8616dbd0 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -668,7 +668,12 @@ class BaseImage(BuildPack): return [] def get_start_script(self): - start = self.binder_path('./start') + start = self.binder_path('start') if os.path.exists(start): - return start + # Return an absolute path to start + # This is important when built container images start with + # a working directory that is different from ${REPO_DIR} + # This isn't a problem with anything else, since start is + # the only path evaluated at container start time rather than build time + return os.path.join('${REPO_DIR}', start) return None From 9ded2b5ee6b878fab24552a025d20c104b5d4fba Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Tue, 30 Apr 2019 10:06:24 -0700 Subject: [PATCH 47/71] add note to changelog --- docs/source/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 3049d7bf..d7dd0e3c 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -18,6 +18,8 @@ New features - Install notebook into `notebook` env instead of `root`. Activate conda environments and shell integration via ENTRYPOINT in :pr:`651` by :user:`minrk` +- Support for `.binder` directory in addition to `binder` directory for location of + configuration files, in :pr:`653` by :user:`jhamman`. API changes ----------- From c0a33b340b96f5fb8bc7baf38d01f46deda2fb25 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 30 Apr 2019 10:46:09 -0700 Subject: [PATCH 48/71] Add test for integral --build-memory-limit --- tests/unit/test_args.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/test_args.py b/tests/unit/test_args.py index a1401468..91ead8bc 100644 --- a/tests/unit/test_args.py +++ b/tests/unit/test_args.py @@ -40,6 +40,17 @@ def test_dry_run(): assert not r2d.run assert not r2d.push + +def test_mem_limit(): + """ + Test various ways of passing --build-memory-limit + """ + r2d = make_r2d(['--build-memory-limit', '1024', '.']) + assert int(r2d.build_memory_limit) == 1024 + + r2d = make_r2d(['--build-memory-limit', '3K', '.']) + assert int(r2d.build_memory_limit) == 1024 * 3 + def test_run_required(): """ Test all the things that should fail if we pass in --no-run From c8cff1bcccad4e5eae060b6bc814879ac77526a4 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 30 Apr 2019 10:46:34 -0700 Subject: [PATCH 49/71] Fix typo in comment --- repo2docker/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo2docker/__main__.py b/repo2docker/__main__.py index 1cb4862d..5911c81a 100644 --- a/repo2docker/__main__.py +++ b/repo2docker/__main__.py @@ -300,7 +300,7 @@ def make_r2d(argv=None): if args.build_memory_limit: # if the string only contains numerals we assume it should be an int - # and specifies a size inn bytes + # and specifies a size in bytes if args.build_memory_limit.isnumeric(): r2d.build_memory_limit = int(args.build_memory_limit) else: From a6911a10a0eeafa2e9d133361b26ff81d9f47bc8 Mon Sep 17 00:00:00 2001 From: Kirstie Whitaker Date: Tue, 30 Apr 2019 20:11:17 +0100 Subject: [PATCH 50/71] Update docs/source/contributing/contributing.md --- docs/source/contributing/contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/contributing/contributing.md b/docs/source/contributing/contributing.md index a317e053..995f8fda 100644 --- a/docs/source/contributing/contributing.md +++ b/docs/source/contributing/contributing.md @@ -69,7 +69,7 @@ Read the [next section](#guidelines-to-getting-a-pull-request-merged) for guidel ## Guidelines to getting a Pull Request merged -These are not hard rules to be enforced by 🚓 but they are suggestions written by the repo2docker maintainers to make getting your contribution completed a smooth process for you and for them. +These are not hard rules to be enforced by 🚓 but they are suggestions written by the repo2docker maintainers to help complete your contribution as smoothly as possible for both you and for them. * **Create a PR as early as possible**, marking it with `[WIP]` while you work on it. This avoids duplicated work, lets you get high level feedback on functionality or API changes, and/or helps find collaborators to work with you. From b20438bd593ecf1a2cf3473272c4ccbf2481cc7d Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Thu, 2 May 2019 15:58:58 -0700 Subject: [PATCH 51/71] specification info --- docs/source/index.rst | 1 + docs/source/specification.rst | 20 ++++++++++++++++++++ docs/source/usage.rst | 3 ++- 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 docs/source/specification.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index f150f352..d5aff875 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -42,6 +42,7 @@ Please report `Bugs `_, :caption: Complete list of configuration files config_files + specification .. toctree:: :maxdepth: 2 diff --git a/docs/source/specification.rst b/docs/source/specification.rst new file mode 100644 index 00000000..2c357c2b --- /dev/null +++ b/docs/source/specification.rst @@ -0,0 +1,20 @@ +.. _specification: + +==================================================== +The Reproducible Execution Environment Specification +==================================================== + +repo2docker scans a repository for particular :ref:`config_files`, such +as ``requirements.txt`` or ``REQUIRE``. The collection of files and their contents +that repo2docker uses is known as the **Reproducible Execution Environment Specification**. + +The goal of the REE Specification is to provide a structure that is clearly-defined and that +can be extended to accomodate more components of a reproducible workflow. + +Currently, the definition of the REE Specification is the following: + +> Any collection of files taken from the :ref:`config_files` +> list, placed either in the root of a folder or in a sub-folder called either ``binder/`` or ``.binder/``. + +In the future, the repo2docker team plans to formalize this specification into a pattern +that can also be followed in other ways, such as by creating a JSON or YAML file. \ No newline at end of file diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 449af33e..c21f7f4b 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -10,7 +10,8 @@ Using ``repo2docker`` order to run ``repo2docker``. For more information on installing ``repo2docker``, see :ref:`install`. -``repo2docker`` is called with a URL/path to a git repository. It then +``repo2docker`` can build a reproducible computational environment for any repository that +follows :ref:`specification`. repo2docker is called with a URL/path to a git repository. It then performs these steps: 1. Inspects the repository for :ref:`configuration files `. These will be used to build From 21c730298e13ccc25f62f8c337497268ebdf69ba Mon Sep 17 00:00:00 2001 From: Wayne's Bioinformatics Code Portal Date: Fri, 3 May 2019 11:14:25 -0400 Subject: [PATCH 52/71] tweak language to make clear `exec "$@"` works originally it was: "The last line must be ``exec "$@"`` equivalent." Seemed open to interpretation as I needed to deduce what the equivalent of that would be here. I think adding 'or' before equivalent shows that will work, but I assume other options exist? Or why note say "The last line must be ``exec "$@"``"? --- docs/source/config_files.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/config_files.rst b/docs/source/config_files.rst index 5af8e02a..22435515 100644 --- a/docs/source/config_files.rst +++ b/docs/source/config_files.rst @@ -179,7 +179,7 @@ A script that can contain simple commands to be run at runtime (as an `ENTRYPOINT `_ to the docker container). If you want this to be a shell script, make sure the first line is ``#!/bin/bash``. The last line must be ``exec "$@"`` -equivalent. +or equivalent. Use this to set environment variables that software installed in your container expects to be set. This script is executed each time your binder is started and From 1c66925090f5f3553ce81754ddcccde1aba87369 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Fri, 3 May 2019 09:33:37 -0700 Subject: [PATCH 53/71] updates per comments --- docs/source/specification.rst | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/docs/source/specification.rst b/docs/source/specification.rst index 2c357c2b..789ccf62 100644 --- a/docs/source/specification.rst +++ b/docs/source/specification.rst @@ -4,17 +4,27 @@ The Reproducible Execution Environment Specification ==================================================== -repo2docker scans a repository for particular :ref:`config_files`, such -as ``requirements.txt`` or ``REQUIRE``. The collection of files and their contents -that repo2docker uses is known as the **Reproducible Execution Environment Specification**. +repo2docker scans a repository for particular :ref:`config-files`, such +as ``requirements.txt`` or ``REQUIRE``. The collection of files, their contents, +and the resulting actions that repo2docker takes is known +as the **Reproducible Execution Environment Specification** (or REES). -The goal of the REE Specification is to provide a structure that is clearly-defined and that -can be extended to accomodate more components of a reproducible workflow. +The goal of the REES is to automate and encourage existing community best practices +for reproducible computational environments. This includes installing +community-standard specification files such as ``requirements.txt`` or ``REQUIRE`` using +standard tools such as ``pip`` or ``conda`` or ``apt``. While repo2docker automates the +creation of the environment, a human should be able to look at a REES-compliant +repository and reproduce the environment using common, clear steps without +repo2docker software. Currently, the definition of the REE Specification is the following: -> Any collection of files taken from the :ref:`config_files` -> list, placed either in the root of a folder or in a sub-folder called either ``binder/`` or ``.binder/``. + Any collection of files taken from the :ref:`config-files` + list, placed either in the root of a folder or in a sub-folder called + either ``binder/`` or ``.binder/``. -In the future, the repo2docker team plans to formalize this specification into a pattern -that can also be followed in other ways, such as by creating a JSON or YAML file. \ No newline at end of file +For example, the REES recognises ``requirements.txt`` as a valid config file. +The file format is as defined by the ``requirements.txt`` standard of the Python +community. A REES-compliant tool will install a Python interpreter (of unspecified version) +and perform the equivalent action of ``pip install -r requirements.txt`` so that the +user can afterwards run python and use the packages installed. From 240efebcf2acd7d307e19fea3fdfbfb3ff9b7716 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Sat, 4 May 2019 12:32:26 +0200 Subject: [PATCH 54/71] Remove the conda package cache as we can't hardlink to it --- repo2docker/buildpacks/conda/install-miniconda.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/repo2docker/buildpacks/conda/install-miniconda.bash b/repo2docker/buildpacks/conda/install-miniconda.bash index fcbabace..aed62ecf 100755 --- a/repo2docker/buildpacks/conda/install-miniconda.bash +++ b/repo2docker/buildpacks/conda/install-miniconda.bash @@ -69,6 +69,7 @@ fi # Clean things out! conda clean -tipsy +rm -rf /srv/conda/pkgs # Remove the big installer so we don't increase docker image size too much rm ${INSTALLER_PATH} From 3a6e4b48513134984690c86ac59d2255bc94d7b0 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Sat, 4 May 2019 13:03:50 +0200 Subject: [PATCH 55/71] Remove pip cache in root's home directory --- repo2docker/buildpacks/conda/install-miniconda.bash | 2 ++ 1 file changed, 2 insertions(+) diff --git a/repo2docker/buildpacks/conda/install-miniconda.bash b/repo2docker/buildpacks/conda/install-miniconda.bash index aed62ecf..eeae52d7 100755 --- a/repo2docker/buildpacks/conda/install-miniconda.bash +++ b/repo2docker/buildpacks/conda/install-miniconda.bash @@ -73,6 +73,8 @@ rm -rf /srv/conda/pkgs # Remove the big installer so we don't increase docker image size too much rm ${INSTALLER_PATH} +# Remove pip cache created as part of installing miniconda +rm -rf /root/.cache chown -R $NB_USER:$NB_USER ${CONDA_DIR} From f86a06d4ba283101aad01b8863e00c0ab2c95162 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Fri, 3 May 2019 15:08:48 -0700 Subject: [PATCH 56/71] tim comments in specification doc --- docs/source/specification.rst | 14 ++++++++------ docs/source/usage.rst | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/source/specification.rst b/docs/source/specification.rst index 789ccf62..a3fef2d3 100644 --- a/docs/source/specification.rst +++ b/docs/source/specification.rst @@ -10,18 +10,20 @@ and the resulting actions that repo2docker takes is known as the **Reproducible Execution Environment Specification** (or REES). The goal of the REES is to automate and encourage existing community best practices -for reproducible computational environments. This includes installing -community-standard specification files such as ``requirements.txt`` or ``REQUIRE`` using -standard tools such as ``pip`` or ``conda`` or ``apt``. While repo2docker automates the +for reproducible computational environments. This includes installing pacakges using +community-standard specification files and their corresponding tools, +such as ``requirements.txt`` (with ``pip``), ``REQUIRE`` (with Julia), or +``apt.txt`` (with ``apt``). While repo2docker automates the creation of the environment, a human should be able to look at a REES-compliant repository and reproduce the environment using common, clear steps without repo2docker software. Currently, the definition of the REE Specification is the following: - Any collection of files taken from the :ref:`config-files` - list, placed either in the root of a folder or in a sub-folder called - either ``binder/`` or ``.binder/``. + Any directory containing zero or more files from the :ref:`config-files` list is a + valid reproducible execution environment as defined by the REES. The + configuration files have to all be placed either in the root of the + directory, in a ``binder/`` sub-directory or a ``.binder/`` sub-directory. For example, the REES recognises ``requirements.txt`` as a valid config file. The file format is as defined by the ``requirements.txt`` standard of the Python diff --git a/docs/source/usage.rst b/docs/source/usage.rst index c21f7f4b..a4360760 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -11,7 +11,7 @@ Using ``repo2docker`` ``repo2docker``, see :ref:`install`. ``repo2docker`` can build a reproducible computational environment for any repository that -follows :ref:`specification`. repo2docker is called with a URL/path to a git repository. It then +follows :ref:`specification`. repo2docker is called with a URL/path to a repository. It then performs these steps: 1. Inspects the repository for :ref:`configuration files `. These will be used to build From 00bbde26bb31fdf9a81121c4b346d68ff21090c1 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Sat, 4 May 2019 15:03:17 -0700 Subject: [PATCH 57/71] release info --- docs/source/contributing/tasks.md | 55 +++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/docs/source/contributing/tasks.md b/docs/source/contributing/tasks.md index e6ceea55..3e05acf8 100644 --- a/docs/source/contributing/tasks.md +++ b/docs/source/contributing/tasks.md @@ -90,7 +90,7 @@ See the subsections below for more detailed instructions. change log (details below) and commit the change log, then update the pull request. - + ### Make a Pull Request Once you've made the commit, please make a Pull Request to the `jupyterhub/repo2docker` @@ -104,19 +104,54 @@ test to prevent the bug from coming back/the feature breaking in the future. We try to make a release of repo2docker every few months if possible. -We follow semantic versioning. +We follow [semantic versioning](https://semver.org/). -Check that the Change log is ready and then tag a new release locally: +A new release will automatically be created when a new git tag is created +and pushed to the repository (using +[Travis CI](https://github.com/jupyter/repo2docker/blob/master/.travis.yml#L52)). + +To create a new release, follow these steps: + +### Confirm that the changelog is ready + +[The changelog](https://github.com/jupyter/repo2docker/blob/master/docs/source/changelog.rst) +should reflect all significant enhancements and fixes to repo2docker and +its documentation. In addition, ensure that the correct version is displayed +at the top, and create a new `dev` section if needed. + +### Create a new tag and push it + +First, tag a new release locally: ```bash -V=0.7.0 git tag -am "release $V" $V +V=0.7.0; git tag -am "release $V" $V +``` + +Then push this change up to the master repository + +``` git push origin --tags ``` -When the travis run completes check that the new release is available on PyPI. +Travis should automatically run the tests and, if they pass, create a +new release on the [repo2docker PyPI](https://pypi.org/project/jupyter-repo2docker/). +Once this has completed, make sure that the new version has been updated. +### Create a new release on the GitHub repository -### Update the change log +Once the new release has been pushed to PyPI, we need to create a new +release on the [GitHub repository releases page](https://github.com/jupyter/repo2docker/releases). Once on that page, follow these steps: + +* Click "Draft a new release" +* Choose a tag version following tag you just created above +* The release name is simply the tag version +* The description is [a link to the Changelog](https://github.com/jupyter/repo2docker/blob/master/docs/source/changelog.rst), + ideally with an anchor to the latest release. +* Finally, click "Publish release" + +That's it! + +## Update the change log To add your change to the change log, find the relevant Feature/Bug fix/API change section for the next release near the top of the file; @@ -156,12 +191,12 @@ should be superseded by either a next release candidate, or the final release for that version (bugfix version 0). -### Keeping the Pipfile and requirements files up to date +## Keeping the Pipfile and requirements files up to date -We now have both a `dev-requirements.txt` and a `Pifile` for repo2docker, as -such it is important to keep these in sync/up-to-date. +We now have both a `dev-requirements.txt` and a `Pifile` for repo2docker, as +such it is important to keep these in sync/up-to-date. -Both files use `pip identifiers` so if you are updating for example the Sphinx version +Both files use `pip identifiers` so if you are updating for example the Sphinx version in the `doc-requirements.txt` (currently `Sphinx = ">=1.4,!=1.5.4"`) you can use the same syntax to update the Pipfile and viceversa. From f51ba7604eda85dcae269d26e9324d457fb767d7 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Sun, 5 May 2019 14:55:58 +0200 Subject: [PATCH 58/71] Use `-f --all` for conda clean up --- repo2docker/buildpacks/conda/__init__.py | 5 ++--- repo2docker/buildpacks/conda/install-miniconda.bash | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/repo2docker/buildpacks/conda/__init__.py b/repo2docker/buildpacks/conda/__init__.py index 9dec3e99..e0a36cfe 100644 --- a/repo2docker/buildpacks/conda/__init__.py +++ b/repo2docker/buildpacks/conda/__init__.py @@ -182,9 +182,8 @@ class CondaBuildPack(BaseImage): '${NB_USER}', r""" conda env update -p {0} -f "{1}" && \ - conda clean -tipsy && \ - conda list -p {0} && \ - rm -rf /srv/conda/pkgs + conda clean --all -f -y && \ + conda list -p {0} """.format(env_prefix, environment_yml) )) return super().get_assemble_scripts() + assembly_scripts diff --git a/repo2docker/buildpacks/conda/install-miniconda.bash b/repo2docker/buildpacks/conda/install-miniconda.bash index eeae52d7..5ee2239d 100755 --- a/repo2docker/buildpacks/conda/install-miniconda.bash +++ b/repo2docker/buildpacks/conda/install-miniconda.bash @@ -68,12 +68,12 @@ if [[ -f /tmp/kernel-environment.yml ]]; then fi # Clean things out! -conda clean -tipsy -rm -rf /srv/conda/pkgs +conda clean --all -f -y # Remove the big installer so we don't increase docker image size too much rm ${INSTALLER_PATH} -# Remove pip cache created as part of installing miniconda + +# Remove the pip cache created as part of installing miniconda rm -rf /root/.cache chown -R $NB_USER:$NB_USER ${CONDA_DIR} From e098d23463693e4e24de5f24a29a91311616e005 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Fri, 3 May 2019 18:30:26 +0200 Subject: [PATCH 59/71] Update change log in preparation for releasing v0.9.0 --- docs/source/changelog.rst | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index d7dd0e3c..d42af7a2 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -2,7 +2,7 @@ Changelog ========= -Version 0.x.x +Version 0.9.0 ============= Release date: TBD @@ -20,6 +20,9 @@ New features in :pr:`651` by :user:`minrk` - Support for `.binder` directory in addition to `binder` directory for location of configuration files, in :pr:`653` by :user:`jhamman`. +- Updated contributor guide and issue templates for bugs, feature requests, + and support questions in :pr:`654` and :pr:`655` by :user:`KirstieJane` and + :user:`betatim`. API changes ----------- @@ -34,9 +37,14 @@ Bug fixes buildpack in :pr:`633` by :user:`betatim`. - Update to version 5.7.6 of the `notebook` package used in all environments in :pr:`628` by :user:`betatim`. +- Update to version 5.7.8 of the `notebook` package and version 2.0.12 of + `nteract-on-jupyter` in :pr:`650` by :user:`betatim`. - Switch to newer version of jupyter-server-proxy to fix websocket handling in :pr:`646` by :user:`betatim`. -- Update to pip v19.0.3 in :pr:`647` by :user:`betatim`. +- Update to pip version 19.0.3 in :pr:`647` by :user:`betatim`. +- Ensure ENTRYPOINT is an absolute path in :pr:`657` by :user:`yuvipanda`. +- Fix handling of `--build-memory-limit` values without a postfix in :pr:`652` + by :user:`betatim`. Version 0.8.0 From 19cb905102d2b87316bcd0ad5aac634c3cef9039 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Sun, 5 May 2019 15:09:59 +0200 Subject: [PATCH 60/71] Mention the invention of the REES in the change log --- docs/source/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index d42af7a2..a7ed0b20 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -23,6 +23,9 @@ New features - Updated contributor guide and issue templates for bugs, feature requests, and support questions in :pr:`654` and :pr:`655` by :user:`KirstieJane` and :user:`betatim`. +- Create a page naming and describing the "Reproducible Execution + Environment Specification" (the specification used by repo2docker) + in :pr:`662` by :user:`choldgraf`. API changes ----------- From a3a4eba89ca4daab0e2a3eecd2e071d2b42e3aa5 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Sun, 5 May 2019 10:28:50 -0700 Subject: [PATCH 61/71] Update docs/source/contributing/tasks.md --- docs/source/contributing/tasks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/contributing/tasks.md b/docs/source/contributing/tasks.md index 3e05acf8..b216f58d 100644 --- a/docs/source/contributing/tasks.md +++ b/docs/source/contributing/tasks.md @@ -143,7 +143,7 @@ Once the new release has been pushed to PyPI, we need to create a new release on the [GitHub repository releases page](https://github.com/jupyter/repo2docker/releases). Once on that page, follow these steps: * Click "Draft a new release" -* Choose a tag version following tag you just created above +* Choose a tag version using the same tag you just created above * The release name is simply the tag version * The description is [a link to the Changelog](https://github.com/jupyter/repo2docker/blob/master/docs/source/changelog.rst), ideally with an anchor to the latest release. From 56c1160d9ad29265f2cf5056c75bb1357157e0f2 Mon Sep 17 00:00:00 2001 From: Chris Holdgraf Date: Sun, 5 May 2019 11:23:12 -0700 Subject: [PATCH 62/71] bumping release date --- docs/source/changelog.rst | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index a7ed0b20..1423d55c 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -2,11 +2,27 @@ Changelog ========= -Version 0.9.0 + +Version x.x.x ============= Release date: TBD +New features +------------ + +API changes +----------- + +Bug fixes +--------- + + +Version 0.9.0 +============= + +Release date: 2019-05-05 + New features ------------ - Support for julia `Project.toml`, `JuliaProject.toml` and `Manifest.toml` files in :pr:`595` by From 5adc4b257e05ae79cdb6643a0ad3217e4aefaf6e Mon Sep 17 00:00:00 2001 From: Tim Head Date: Tue, 7 May 2019 19:12:34 +0200 Subject: [PATCH 63/71] Update base image used for memory limit checks --- tests/memlimit/dockerfile/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/memlimit/dockerfile/Dockerfile b/tests/memlimit/dockerfile/Dockerfile index 0cb02777..9513e5e4 100644 --- a/tests/memlimit/dockerfile/Dockerfile +++ b/tests/memlimit/dockerfile/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:artful +FROM ubuntu:bionic RUN apt-get update && apt-get install --yes python3 From ef2860371a82b804dba9f5b878437ba71689717d Mon Sep 17 00:00:00 2001 From: Tim Head Date: Wed, 8 May 2019 08:11:38 +0200 Subject: [PATCH 64/71] Fix memory limits set for container image builds --- repo2docker/buildpacks/base.py | 16 ++++++++++------ repo2docker/buildpacks/docker.py | 16 ++++++++++------ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index 8616dbd0..dedd915f 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -523,13 +523,17 @@ class BuildPack: tar.close() tarf.seek(0) - limits = { - # Always disable memory swap for building, since mostly - # nothing good can come of that. - 'memswap': -1 - } + # If you work on this bit of code check the corresponding code in + # buildpacks/docker.py where it is duplicated + limits = {} if memory_limit: - limits['memory'] = memory_limit + # We'd like to always disable swap but all we can do is set the + # total amount. This means we only limit it when the caller set + # a memory limit + limits = { + 'memory': memory_limit, + 'memswap': memory_limit + 1 + } build_kwargs = dict( fileobj=tarf, diff --git a/repo2docker/buildpacks/docker.py b/repo2docker/buildpacks/docker.py index 91b58403..a9da371d 100644 --- a/repo2docker/buildpacks/docker.py +++ b/repo2docker/buildpacks/docker.py @@ -21,13 +21,17 @@ class DockerBuildPack(BuildPack): def build(self, client, image_spec, memory_limit, build_args, cache_from, extra_build_kwargs): """Build a Docker image based on the Dockerfile in the source repo.""" - limits = { - # Always disable memory swap for building, since mostly - # nothing good can come of that. - 'memswap': -1 - } + # If you work on this bit of code check the corresponding code in + # buildpacks/base.py where it is duplicated + limits = {} if memory_limit: - limits['memory'] = memory_limit + # We'd like to always disable swap but all we can do is set the + # total amount. This means we onnly limit it when the caller set + # a memory limit + limits = { + 'memory': memory_limit, + 'memswap': memory_limit + 1 + } build_kwargs = dict( path=os.getcwd(), From 9f44075839c175d71213c717e6427c70241132f5 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Wed, 8 May 2019 13:20:10 +0200 Subject: [PATCH 65/71] Fix cache-from tests to pass memory limit as integer --- tests/unit/test_cache_from.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_cache_from.py b/tests/unit/test_cache_from.py index be102a18..270debec 100644 --- a/tests/unit/test_cache_from.py +++ b/tests/unit/test_cache_from.py @@ -10,7 +10,6 @@ from repo2docker.buildpacks import BaseImage, DockerBuildPack, LegacyBinderDocke def test_cache_from_base(tmpdir): - FakeDockerClient = MagicMock() cache_from = [ 'image-1:latest' ] @@ -21,16 +20,14 @@ def test_cache_from_base(tmpdir): # Test base image build pack tmpdir.chdir() - for line in BaseImage().build(fake_client, 'image-2', '1Gi', {}, cache_from, extra_build_kwargs): + for line in BaseImage().build(fake_client, 'image-2', 100, {}, cache_from, extra_build_kwargs): assert line == fake_log_value called_args, called_kwargs = fake_client.build.call_args assert 'cache_from' in called_kwargs assert called_kwargs['cache_from'] == cache_from - def test_cache_from_docker(tmpdir): - FakeDockerClient = MagicMock() cache_from = [ 'image-1:latest' ] @@ -44,7 +41,7 @@ def test_cache_from_docker(tmpdir): with tmpdir.join("Dockerfile").open('w') as f: f.write('FROM scratch\n') - for line in DockerBuildPack().build(fake_client, 'image-2', '1Gi', {}, cache_from, extra_build_kwargs): + for line in DockerBuildPack().build(fake_client, 'image-2', 100, {}, cache_from, extra_build_kwargs): assert line == fake_log_value called_args, called_kwargs = fake_client.build.call_args assert 'cache_from' in called_kwargs @@ -52,7 +49,6 @@ def test_cache_from_docker(tmpdir): def test_cache_from_legacy(tmpdir): - FakeDockerClient = MagicMock() cache_from = [ 'image-1:latest' ] @@ -65,9 +61,8 @@ def test_cache_from_legacy(tmpdir): with tmpdir.join("Dockerfile").open('w') as f: f.write('FROM andrewosh/binder-base\n') - for line in LegacyBinderDockerBuildPack().build(fake_client, 'image-2', '1Gi', {}, cache_from, extra_build_kwargs): + for line in LegacyBinderDockerBuildPack().build(fake_client, 'image-2', 100, {}, cache_from, extra_build_kwargs): assert line == fake_log_value called_args, called_kwargs = fake_client.build.call_args assert 'cache_from' in called_kwargs assert called_kwargs['cache_from'] == cache_from - From 032baf6d0452733fd6b09acb2e73adb07abd9f7f Mon Sep 17 00:00:00 2001 From: Tim Head Date: Wed, 8 May 2019 13:20:31 +0200 Subject: [PATCH 66/71] Add check to `build()` for memory limit type --- repo2docker/buildpacks/base.py | 3 +++ repo2docker/buildpacks/docker.py | 3 +++ tests/unit/test_memlimit.py | 19 +++++++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index dedd915f..ba50356b 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -525,6 +525,9 @@ class BuildPack: # If you work on this bit of code check the corresponding code in # buildpacks/docker.py where it is duplicated + if not isinstance(memory_limit, int): + raise ValueError("The memory limit has to be specified as an" + "integer but is '{}'".format(type(memory_limit))) limits = {} if memory_limit: # We'd like to always disable swap but all we can do is set the diff --git a/repo2docker/buildpacks/docker.py b/repo2docker/buildpacks/docker.py index a9da371d..208807ed 100644 --- a/repo2docker/buildpacks/docker.py +++ b/repo2docker/buildpacks/docker.py @@ -23,6 +23,9 @@ class DockerBuildPack(BuildPack): """Build a Docker image based on the Dockerfile in the source repo.""" # If you work on this bit of code check the corresponding code in # buildpacks/base.py where it is duplicated + if not isinstance(memory_limit, int): + raise ValueError("The memory limit has to be specified as an" + "integer but is '{}'".format(type(memory_limit))) limits = {} if memory_limit: # We'd like to always disable swap but all we can do is set the diff --git a/tests/unit/test_memlimit.py b/tests/unit/test_memlimit.py index f636da7c..1d3f88ba 100644 --- a/tests/unit/test_memlimit.py +++ b/tests/unit/test_memlimit.py @@ -10,9 +10,14 @@ import os import shutil import time +from unittest.mock import MagicMock + +import docker + import pytest from repo2docker.app import Repo2Docker +from repo2docker.buildpacks import BaseImage, DockerBuildPack basedir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -82,3 +87,17 @@ def test_memlimit_same_postbuild(): file_contents.append(f.read()) # Make sure they're all the same assert len(set(file_contents)) == 1 + + +@pytest.mark.parametrize('BuildPack', [BaseImage, DockerBuildPack]) +def test_memlimit_argument_type(BuildPack): + # check that an exception is raised when the memory limit isn't an int + fake_log_value = {'stream': 'fake'} + fake_client = MagicMock(spec=docker.APIClient) + fake_client.build.return_value = iter([fake_log_value]) + + with pytest.raises(ValueError) as exc_info: + for line in BuildPack().build(fake_client, 'image-2', "10Gi", {}, [], {}): + pass + + assert "The memory limit has to be specified as an" in str(exc_info.value) From 2eb4781c2e954f0e19ee3c2ae2c6e170b4c919b2 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Wed, 8 May 2019 18:22:11 +0200 Subject: [PATCH 67/71] Swap limit doesn't have to be bigger than memory limit --- repo2docker/buildpacks/base.py | 2 +- repo2docker/buildpacks/docker.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index ba50356b..e6cb3916 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -535,7 +535,7 @@ class BuildPack: # a memory limit limits = { 'memory': memory_limit, - 'memswap': memory_limit + 1 + 'memswap': memory_limit } build_kwargs = dict( diff --git a/repo2docker/buildpacks/docker.py b/repo2docker/buildpacks/docker.py index 208807ed..3f3c09a4 100644 --- a/repo2docker/buildpacks/docker.py +++ b/repo2docker/buildpacks/docker.py @@ -33,7 +33,7 @@ class DockerBuildPack(BuildPack): # a memory limit limits = { 'memory': memory_limit, - 'memswap': memory_limit + 1 + 'memswap': memory_limit, } build_kwargs = dict( From 74fc378230f8bc138bf0f01661914d2951008093 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Wed, 8 May 2019 09:44:58 -0700 Subject: [PATCH 68/71] Clarify comment around memswap & memory --- repo2docker/buildpacks/base.py | 7 ++++--- repo2docker/buildpacks/docker.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index e6cb3916..a11dd1ed 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -530,9 +530,10 @@ class BuildPack: "integer but is '{}'".format(type(memory_limit))) limits = {} if memory_limit: - # We'd like to always disable swap but all we can do is set the - # total amount. This means we only limit it when the caller set - # a memory limit + # We want to always disable swap. Docker expects `memswap` to + # be total allowable memory, *including* swap - while `memory` + # points to non-swap memory. We set both values to the same so + # we use no swap. limits = { 'memory': memory_limit, 'memswap': memory_limit diff --git a/repo2docker/buildpacks/docker.py b/repo2docker/buildpacks/docker.py index 3f3c09a4..5f437b36 100644 --- a/repo2docker/buildpacks/docker.py +++ b/repo2docker/buildpacks/docker.py @@ -28,9 +28,10 @@ class DockerBuildPack(BuildPack): "integer but is '{}'".format(type(memory_limit))) limits = {} if memory_limit: - # We'd like to always disable swap but all we can do is set the - # total amount. This means we onnly limit it when the caller set - # a memory limit + # We want to always disable swap. Docker expects `memswap` to + # be total allowable memory, *including* swap - while `memory` + # points to non-swap memory. We set both values to the same so + # we use no swap. limits = { 'memory': memory_limit, 'memswap': memory_limit, From 7d948bda8dfab43d1a6ace3da7f4982fb9df46a9 Mon Sep 17 00:00:00 2001 From: "Kacper Kowalik (Xarthisius)" Date: Tue, 7 May 2019 10:33:19 -0500 Subject: [PATCH 69/71] Do not try to build the image with root as the primary user Fixes #267 and #395 --- docs/source/changelog.rst | 2 ++ repo2docker/app.py | 14 ++++++++++++++ tests/unit/test_app.py | 15 ++++++++++++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 1423d55c..2e982ab7 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -16,6 +16,8 @@ API changes Bug fixes --------- +- Prevent building the image as root if --user-id and --user-name are not specified + in :pr:`676` by :user:`Xarthisius`. Version 0.9.0 diff --git a/repo2docker/app.py b/repo2docker/app.py index 7fcd170e..bbf1a39a 100644 --- a/repo2docker/app.py +++ b/repo2docker/app.py @@ -8,6 +8,7 @@ Usage: python -m repo2docker https://github.com/you/your-repo """ import argparse +import errno import json import sys import logging @@ -650,6 +651,19 @@ class Repo2Docker(Application): extra=dict(phase='building')) if not self.dry_run: + if os.geteuid() == 0: + self.log.error( + 'Root as the primary user in the image is not permitted.\n' + ) + self.log.info( + "The uid and the username of the user invoking repo2docker " + "is used to create a mirror account in the image by default. " + "To override that behavior pass --user-id and " + " --user-name to repo2docker.\n" + "Please see repo2docker --help for more details.\n" + ) + sys.exit(errno.EPERM) + build_args = { 'NB_USER': self.user_name, 'NB_UID': str(self.user_id), diff --git a/tests/unit/test_app.py b/tests/unit/test_app.py index ecc78609..9621576f 100644 --- a/tests/unit/test_app.py +++ b/tests/unit/test_app.py @@ -1,3 +1,5 @@ +import errno +import pytest from tempfile import TemporaryDirectory from unittest.mock import patch @@ -101,4 +103,15 @@ def test_run_kwargs(repo_with_content): containers.run.assert_called_once() args, kwargs = containers.run.call_args assert 'somekey' in kwargs - assert kwargs['somekey'] == "somevalue" \ No newline at end of file + assert kwargs['somekey'] == "somevalue" + + +def test_root_not_allowed(): + with TemporaryDirectory() as src, patch('os.geteuid') as geteuid: + geteuid.return_value = 0 + app = Repo2Docker() + argv = [src] + app = make_r2d(argv) + with pytest.raises(SystemExit) as exc: + app.build() + assert exc.code == errno.EPERM From e66f0af23ce07992845139563cc8dd10b3a43019 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Thu, 9 May 2019 13:14:32 +0200 Subject: [PATCH 70/71] Revert "[MRG] Do not try to build the image with root as the primary user." --- docs/source/changelog.rst | 2 -- repo2docker/app.py | 14 -------------- tests/unit/test_app.py | 15 +-------------- 3 files changed, 1 insertion(+), 30 deletions(-) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 2e982ab7..1423d55c 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -16,8 +16,6 @@ API changes Bug fixes --------- -- Prevent building the image as root if --user-id and --user-name are not specified - in :pr:`676` by :user:`Xarthisius`. Version 0.9.0 diff --git a/repo2docker/app.py b/repo2docker/app.py index bbf1a39a..7fcd170e 100644 --- a/repo2docker/app.py +++ b/repo2docker/app.py @@ -8,7 +8,6 @@ Usage: python -m repo2docker https://github.com/you/your-repo """ import argparse -import errno import json import sys import logging @@ -651,19 +650,6 @@ class Repo2Docker(Application): extra=dict(phase='building')) if not self.dry_run: - if os.geteuid() == 0: - self.log.error( - 'Root as the primary user in the image is not permitted.\n' - ) - self.log.info( - "The uid and the username of the user invoking repo2docker " - "is used to create a mirror account in the image by default. " - "To override that behavior pass --user-id and " - " --user-name to repo2docker.\n" - "Please see repo2docker --help for more details.\n" - ) - sys.exit(errno.EPERM) - build_args = { 'NB_USER': self.user_name, 'NB_UID': str(self.user_id), diff --git a/tests/unit/test_app.py b/tests/unit/test_app.py index 9621576f..ecc78609 100644 --- a/tests/unit/test_app.py +++ b/tests/unit/test_app.py @@ -1,5 +1,3 @@ -import errno -import pytest from tempfile import TemporaryDirectory from unittest.mock import patch @@ -103,15 +101,4 @@ def test_run_kwargs(repo_with_content): containers.run.assert_called_once() args, kwargs = containers.run.call_args assert 'somekey' in kwargs - assert kwargs['somekey'] == "somevalue" - - -def test_root_not_allowed(): - with TemporaryDirectory() as src, patch('os.geteuid') as geteuid: - geteuid.return_value = 0 - app = Repo2Docker() - argv = [src] - app = make_r2d(argv) - with pytest.raises(SystemExit) as exc: - app.build() - assert exc.code == errno.EPERM + assert kwargs['somekey'] == "somevalue" \ No newline at end of file From caef6686d12f8d3da30828c32a6f5a04dec187a6 Mon Sep 17 00:00:00 2001 From: "Kacper Kowalik (Xarthisius)" Date: Tue, 7 May 2019 10:33:19 -0500 Subject: [PATCH 71/71] Do not try to build the image with root as the primary user Fixes #267 and #395 --- docs/source/changelog.rst | 2 ++ repo2docker/app.py | 14 ++++++++++++++ tests/unit/test_app.py | 26 +++++++++++++++++++++++++- 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 1423d55c..2e982ab7 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -16,6 +16,8 @@ API changes Bug fixes --------- +- Prevent building the image as root if --user-id and --user-name are not specified + in :pr:`676` by :user:`Xarthisius`. Version 0.9.0 diff --git a/repo2docker/app.py b/repo2docker/app.py index 7fcd170e..d951094e 100644 --- a/repo2docker/app.py +++ b/repo2docker/app.py @@ -8,6 +8,7 @@ Usage: python -m repo2docker https://github.com/you/your-repo """ import argparse +import errno import json import sys import logging @@ -650,6 +651,19 @@ class Repo2Docker(Application): extra=dict(phase='building')) if not self.dry_run: + if self.user_id == 0: + self.log.error( + 'Root as the primary user in the image is not permitted.\n' + ) + self.log.info( + "The uid and the username of the user invoking repo2docker " + "is used to create a mirror account in the image by default. " + "To override that behavior pass --user-id and " + " --user-name to repo2docker.\n" + "Please see repo2docker --help for more details.\n" + ) + sys.exit(errno.EPERM) + build_args = { 'NB_USER': self.user_name, 'NB_UID': str(self.user_id), diff --git a/tests/unit/test_app.py b/tests/unit/test_app.py index ecc78609..4f43d88a 100644 --- a/tests/unit/test_app.py +++ b/tests/unit/test_app.py @@ -1,3 +1,5 @@ +import errno +import pytest from tempfile import TemporaryDirectory from unittest.mock import patch @@ -101,4 +103,26 @@ def test_run_kwargs(repo_with_content): containers.run.assert_called_once() args, kwargs = containers.run.call_args assert 'somekey' in kwargs - assert kwargs['somekey'] == "somevalue" \ No newline at end of file + assert kwargs['somekey'] == "somevalue" + + +def test_root_not_allowed(): + with TemporaryDirectory() as src, patch('os.geteuid') as geteuid: + geteuid.return_value = 0 + argv = [src] + app = make_r2d(argv) + with pytest.raises(SystemExit) as exc: + app.build() + assert exc.code == errno.EPERM + + app = Repo2Docker( + repo=src, + user_id=1000, + user_name='jovyan', + run=False, + ) + app.initialize() + with patch.object(docker.APIClient, 'build') as builds: + builds.return_value = [] + app.build() + builds.assert_called_once()