[MRG] install notebook in its own env (#651)

[MRG] install notebook in its own env

docs/source/changelog.rst

@@ -13,6 +13,11 @@ New features
   :user:`davidanthoff`
 - Set JULIA_PROJECT globally, so that every julia instance starts with the
   julia environment activated in :pr:`612` by :user:`davidanthoff`.
+- Update Miniconda version to 4.6.14 and Conda version to 4.6.14 in :pr:`637` by
+  :user:`jhamman`
+- Install notebook into `notebook` env instead of `root`.
+  Activate conda environments and shell integration via ENTRYPOINT
+  in :pr:`651` by :user:`minrk`
 
 API changes
 -----------

repo2docker/buildpacks/base.py

@@ -155,9 +155,13 @@ RUN ./{{ s }}
 # Add start script
 {% if start_script is not none -%}
 RUN chmod +x "{{ start_script }}"
-ENTRYPOINT ["{{ start_script }}"]
+ENV R2D_ENTRYPOINT "{{ start_script }}"
 {% endif -%}
 
+# Add entrypoint
+COPY /repo2docker-entrypoint /usr/local/bin/repo2docker-entrypoint
+ENTRYPOINT ["/usr/local/bin/repo2docker-entrypoint"]
+
 # Specify the default command to run
 CMD ["jupyter", "notebook", "--ip", "0.0.0.0"]
 
@@ -167,6 +171,11 @@ CMD ["jupyter", "notebook", "--ip", "0.0.0.0"]
 {% endif %}
 """
 
+ENTRYPOINT_FILE = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)),
+    "repo2docker-entrypoint",
+)
+
 
 class BuildPack:
     """
@@ -493,6 +502,8 @@ class BuildPack:
             src_path = os.path.join(os.path.dirname(__file__), *src_parts)
             tar.add(src_path, src, filter=_filter_tar)
 
+        tar.add(ENTRYPOINT_FILE, "repo2docker-entrypoint", filter=_filter_tar)
+
         tar.add('.', 'src/', filter=_filter_tar)
 
         tar.close()

repo2docker/buildpacks/conda/__init__.py

@@ -28,7 +28,7 @@ class CondaBuildPack(BaseImage):
         """
         env = super().get_build_env() + [
             ('CONDA_DIR', '${APP_BASE}/conda'),
-            ('NB_PYTHON_PREFIX', '${CONDA_DIR}'),
+            ('NB_PYTHON_PREFIX', '${CONDA_DIR}/envs/notebook'),
         ]
         if self.py2:
             env.append(('KERNEL_PYTHON_PREFIX', '${CONDA_DIR}/envs/kernel'))
@@ -42,9 +42,10 @@ class CondaBuildPack(BaseImage):
 
         """
         path = super().get_path()
+        path.insert(0, '${CONDA_DIR}/bin')
         if self.py2:
             path.insert(0, '${KERNEL_PYTHON_PREFIX}/bin')
-        path.insert(0, '${CONDA_DIR}/bin')
+        path.insert(0, '${NB_PYTHON_PREFIX}/bin')
         return path
 
     def get_build_scripts(self):
@@ -97,6 +98,7 @@ class CondaBuildPack(BaseImage):
         """
         files = {
             'conda/install-miniconda.bash': '/tmp/install-miniconda.bash',
+            'conda/activate-conda.sh': '/etc/profile.d/activate-conda.sh',
         }
         py_version = self.python_version
         self.log.info("Building conda environment for python=%s" % py_version)
@@ -174,16 +176,16 @@ class CondaBuildPack(BaseImage):
         """
         assembly_scripts = []
         environment_yml = self.binder_path('environment.yml')
-        env_name = 'kernel' if self.py2 else 'root'
+        env_prefix = "${KERNEL_PYTHON_PREFIX}" if self.py2 else "${NB_PYTHON_PREFIX}"
         if os.path.exists(environment_yml):
             assembly_scripts.append((
                 '${NB_USER}',
                 r"""
-                conda env update -n {0} -f "{1}" && \
+                conda env update -p {0} -f "{1}" && \
                 conda clean -tipsy && \
-                conda list -n {0} && \
+                conda list -p {0} && \
                 rm -rf /srv/conda/pkgs
-                """.format(env_name, environment_yml)
+                """.format(env_prefix, environment_yml)
             ))
         return super().get_assemble_scripts() + assembly_scripts
 

repo2docker/buildpacks/conda/activate-conda.sh

@@ -0,0 +1,11 @@
+# enable conda and activate the notebook environment
+CONDA_PROFILE="${CONDA_DIR}/etc/profile.d/conda.sh"
+test -f $CONDA_PROFILE && . $CONDA_PROFILE
+if [[ "${KERNEL_PYTHON_PREFIX}" != "${NB_PYTHON_PREFIX}" ]]; then
+    # if the kernel is a separate env, stack them
+    # so both are on PATH
+    conda activate ${KERNEL_PYTHON_PREFIX}
+    conda activate --stack ${NB_PYTHON_PREFIX}
+else
+    conda activate ${NB_PYTHON_PREFIX}
+fi

repo2docker/buildpacks/conda/environment.frozen.yml

@@ -1,5 +1,5 @@
 # AUTO GENERATED FROM environment.py-3.7.yml, DO NOT MANUALLY MODIFY
-# Frozen on 2019-04-24 18:01:25 UTC
+# Frozen on 2019-04-26 14:12:19 UTC
 name: r2d
 channels:
   - conda-forge
@@ -16,7 +16,7 @@ dependencies:
   - defusedxml=0.5.0=py_1
   - entrypoints=0.3=py37_1000
   - ipykernel=5.1.0=py37h24bf2e0_1002
-  - ipython=7.4.0=py37h24bf2e0_0
+  - ipython=7.5.0=py37h24bf2e0_0
   - ipython_genutils=0.2.0=py_1
   - ipywidgets=7.4.2=py_0
   - jedi=0.13.3=py37_0
@@ -27,7 +27,9 @@ dependencies:
   - jupyterlab=0.35.4=py37_0
   - jupyterlab_server=0.2.0=py_0
   - libffi=3.2.1=he1b5a44_1006
+  - libgcc-ng=8.2.0=hdf63c60_1
   - libsodium=1.0.16=h14c3975_1001
+  - libstdcxx-ng=8.2.0=hdf63c60_1
   - markupsafe=1.1.1=py37h14c3975_0
   - mistune=0.8.4=py37h14c3975_1000
   - nbconvert=5.4.1=py_2
@@ -45,7 +47,7 @@ dependencies:
   - prompt_toolkit=2.0.9=py_0
   - ptyprocess=0.6.0=py_1001
   - pygments=2.3.1=py_0
-  - pyrsistent=0.14.11=py37h14c3975_0
+  - pyrsistent=0.15.1=py37h516909a_0
   - python=3.7.3=h5b0a415_0
   - python-dateutil=2.8.0=py_0
   - pyzmq=18.0.1=py37hc4ba49a_1
@@ -66,8 +68,6 @@ dependencies:
   - xz=5.2.4=h14c3975_1001
   - zeromq=4.3.1=hf484d3e_1000
   - zlib=1.2.11=h14c3975_1004
-  - libgcc-ng=8.2.0=hdf63c60_1
-  - libstdcxx-ng=8.2.0=hdf63c60_1
   - pip:
     - alembic==1.0.9
     - async-generator==1.10

repo2docker/buildpacks/conda/environment.py-2.7.frozen.yml

@@ -1,5 +1,5 @@
 # AUTO GENERATED FROM environment.py-2.7.yml, DO NOT MANUALLY MODIFY
-# Frozen on 2019-04-24 17:58:08 UTC
+# Frozen on 2019-04-26 14:09:19 UTC
 name: r2d
 channels:
   - conda-forge
@@ -20,7 +20,9 @@ dependencies:
   - jupyter_client=5.2.4=py_3
   - jupyter_core=4.4.0=py_0
   - libffi=3.2.1=he1b5a44_1006
+  - libgcc-ng=8.2.0=hdf63c60_1
   - libsodium=1.0.16=h14c3975_1001
+  - libstdcxx-ng=8.2.0=hdf63c60_1
   - ncurses=6.1=hf484d3e_1002
   - openssl=1.1.1b=h14c3975_1
   - pathlib2=2.3.3=py27_1000
@@ -47,7 +49,5 @@ dependencies:
   - wheel=0.33.1=py27_0
   - zeromq=4.3.1=hf484d3e_1000
   - zlib=1.2.11=h14c3975_1004
-  - libgcc-ng=8.2.0=hdf63c60_1
-  - libstdcxx-ng=8.2.0=hdf63c60_1
 prefix: /opt/conda/envs/r2d
 

repo2docker/buildpacks/conda/environment.py-3.6.frozen.yml

@@ -1,5 +1,5 @@
 # AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY
-# Frozen on 2019-04-24 17:59:15 UTC
+# Frozen on 2019-04-26 14:10:28 UTC
 name: r2d
 channels:
   - conda-forge
@@ -15,7 +15,7 @@ dependencies:
   - defusedxml=0.5.0=py_1
   - entrypoints=0.3=py36_1000
   - ipykernel=5.1.0=py36h24bf2e0_1002
-  - ipython=7.4.0=py36h24bf2e0_0
+  - ipython=7.5.0=py36h24bf2e0_0
   - ipython_genutils=0.2.0=py_1
   - ipywidgets=7.4.2=py_0
   - jedi=0.13.3=py36_0
@@ -26,7 +26,9 @@ dependencies:
   - jupyterlab=0.35.4=py36_0
   - jupyterlab_server=0.2.0=py_0
   - libffi=3.2.1=he1b5a44_1006
+  - libgcc-ng=8.2.0=hdf63c60_1
   - libsodium=1.0.16=h14c3975_1001
+  - libstdcxx-ng=8.2.0=hdf63c60_1
   - markupsafe=1.1.1=py36h14c3975_0
   - mistune=0.8.4=py36h14c3975_1000
   - nbconvert=5.4.1=py_2
@@ -44,7 +46,7 @@ dependencies:
   - prompt_toolkit=2.0.9=py_0
   - ptyprocess=0.6.0=py_1001
   - pygments=2.3.1=py_0
-  - pyrsistent=0.14.11=py36h14c3975_0
+  - pyrsistent=0.15.1=py36h516909a_0
   - python=3.6.7=h381d211_1004
   - python-dateutil=2.8.0=py_0
   - pyzmq=18.0.1=py36hc4ba49a_1
@@ -65,8 +67,6 @@ dependencies:
   - xz=5.2.4=h14c3975_1001
   - zeromq=4.3.1=hf484d3e_1000
   - zlib=1.2.11=h14c3975_1004
-  - libgcc-ng=8.2.0=hdf63c60_1
-  - libstdcxx-ng=8.2.0=hdf63c60_1
   - pip:
     - alembic==1.0.9
     - async-generator==1.10

repo2docker/buildpacks/conda/environment.py-3.6.yml

@@ -1,5 +1,5 @@
 # AUTO GENERATED FROM environment.yml, DO NOT MANUALLY MODIFY
-# Generated on 2019-04-24 17:59:15 UTC
+# Generated on 2019-04-26 14:10:28 UTC
 dependencies:
 - python=3.6.*
 - ipywidgets==7.4.2

repo2docker/buildpacks/conda/environment.py-3.7.frozen.yml

@@ -1,5 +1,5 @@
 # AUTO GENERATED FROM environment.py-3.7.yml, DO NOT MANUALLY MODIFY
-# Frozen on 2019-04-24 18:01:25 UTC
+# Frozen on 2019-04-26 14:12:19 UTC
 name: r2d
 channels:
   - conda-forge
@@ -16,7 +16,7 @@ dependencies:
   - defusedxml=0.5.0=py_1
   - entrypoints=0.3=py37_1000
   - ipykernel=5.1.0=py37h24bf2e0_1002
-  - ipython=7.4.0=py37h24bf2e0_0
+  - ipython=7.5.0=py37h24bf2e0_0
   - ipython_genutils=0.2.0=py_1
   - ipywidgets=7.4.2=py_0
   - jedi=0.13.3=py37_0
@@ -27,7 +27,9 @@ dependencies:
   - jupyterlab=0.35.4=py37_0
   - jupyterlab_server=0.2.0=py_0
   - libffi=3.2.1=he1b5a44_1006
+  - libgcc-ng=8.2.0=hdf63c60_1
   - libsodium=1.0.16=h14c3975_1001
+  - libstdcxx-ng=8.2.0=hdf63c60_1
   - markupsafe=1.1.1=py37h14c3975_0
   - mistune=0.8.4=py37h14c3975_1000
   - nbconvert=5.4.1=py_2
@@ -45,7 +47,7 @@ dependencies:
   - prompt_toolkit=2.0.9=py_0
   - ptyprocess=0.6.0=py_1001
   - pygments=2.3.1=py_0
-  - pyrsistent=0.14.11=py37h14c3975_0
+  - pyrsistent=0.15.1=py37h516909a_0
   - python=3.7.3=h5b0a415_0
   - python-dateutil=2.8.0=py_0
   - pyzmq=18.0.1=py37hc4ba49a_1
@@ -66,8 +68,6 @@ dependencies:
   - xz=5.2.4=h14c3975_1001
   - zeromq=4.3.1=hf484d3e_1000
   - zlib=1.2.11=h14c3975_1004
-  - libgcc-ng=8.2.0=hdf63c60_1
-  - libstdcxx-ng=8.2.0=hdf63c60_1
   - pip:
     - alembic==1.0.9
     - async-generator==1.10

repo2docker/buildpacks/conda/environment.py-3.7.yml

@@ -1,5 +1,5 @@
 # AUTO GENERATED FROM environment.yml, DO NOT MANUALLY MODIFY
-# Generated on 2019-04-24 18:01:25 UTC
+# Generated on 2019-04-26 14:12:19 UTC
 dependencies:
 - python=3.7.*
 - ipywidgets==7.4.2

repo2docker/buildpacks/conda/freeze.py

@@ -21,8 +21,8 @@ from ruamel.yaml import YAML
 
 # Docker image version can be different than conda version,
 # since miniconda3 docker images seem to lag conda releases.
-MINICONDA_DOCKER_VERSION = '4.5.11'
-CONDA_VERSION = '4.5.11'
+MINICONDA_DOCKER_VERSION = '4.5.12'
+CONDA_VERSION = '4.6.14'
 
 HERE = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))
 

repo2docker/buildpacks/conda/install-miniconda.bash

@@ -3,8 +3,12 @@
 set -ex
 
 cd $(dirname $0)
-MINICONDA_VERSION=4.5.11
-CONDA_VERSION=4.5.11
+MINICONDA_VERSION=4.6.14
+CONDA_VERSION=4.6.14
+# Only MD5 checksums are available for miniconda
+# Can be obtained from https://repo.continuum.io/miniconda/
+MD5SUM="718259965f234088d785cad1fbd7de03"
+
 URL="https://repo.continuum.io/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh"
 INSTALLER_PATH=/tmp/miniconda-installer.sh
 
@@ -15,10 +19,7 @@ unset HOME
 wget --quiet $URL -O ${INSTALLER_PATH}
 chmod +x ${INSTALLER_PATH}
 
-# Only MD5 checksums are available for miniconda
-# Can be obtained from https://repo.continuum.io/miniconda/
-MD5SUM="e1045ee415162f944b6aebfe560b8fee"
-
+# check md5 checksum
 if ! echo "${MD5SUM}  ${INSTALLER_PATH}" | md5sum  --quiet -c -; then
     echo "md5sum mismatch for ${INSTALLER_PATH}, exiting!"
     exit 1
@@ -34,22 +35,23 @@ conda config --system --add channels conda-forge
 conda config --system --set auto_update_conda false
 conda config --system --set show_channel_urls true
 
-# install conda itself
-conda install -yq conda==${CONDA_VERSION}
-
-# switch Python in its own step
-# since switching Python during an env update can
-# prevent pip installation.
-# we wouldn't have this issue if we did `conda env create`
-# instead of `conda env update` in these cases
-conda install -y $(cat /tmp/environment.yml | grep -o '\spython=.*') conda==${CONDA_VERSION}
-
 # bug in conda 4.3.>15 prevents --set update_dependencies
 echo 'update_dependencies: false' >> ${CONDA_DIR}/.condarc
 
-echo "installing root env:"
+# install conda itself
+if [[ "${CONDA_VERSION}" != "${MINICONDA_VERSION}" ]]; then
+    conda install -yq conda==${CONDA_VERSION}
+fi
+
+echo "installing notebook env:"
 cat /tmp/environment.yml
-conda env update -n root -f /tmp/environment.yml
+conda env create -p ${NB_PYTHON_PREFIX} -f /tmp/environment.yml
+
+# empty conda history file,
+# which seems to result in some effective pinning of packages in the initial env,
+# which we don't intend.
+# this file must not be *removed*, however
+echo '' > ${NB_PYTHON_PREFIX}/conda-meta/history
 
 # enable nteract-on-jupyter, which was installed with pip
 jupyter serverextension enable nteract_on_jupyter --sys-prefix
@@ -59,15 +61,11 @@ if [[ -f /tmp/kernel-environment.yml ]]; then
     echo "installing kernel env:"
     cat /tmp/kernel-environment.yml
 
-    conda env create -n kernel -f /tmp/kernel-environment.yml
-    ${CONDA_DIR}/envs/kernel/bin/ipython kernel install --prefix "${CONDA_DIR}"
-    echo '' > ${CONDA_DIR}/envs/kernel/conda-meta/history
+    conda env create -p ${KERNEL_PYTHON_PREFIX} -f /tmp/kernel-environment.yml
+    ${KERNEL_PYTHON_PREFIX}/bin/ipython kernel install --prefix "${NB_PYTHON_PREFIX}"
+    echo '' > ${KERNEL_PYTHON_PREFIX}/conda-meta/history
+    conda list -p ${KERNEL_PYTHON_PREFIX}
 fi
-# empty conda history file,
-# which seems to result in some effective pinning of packages in the initial env,
-# which we don't intend.
-# this file must not be *removed*, however
-echo '' > ${CONDA_DIR}/conda-meta/history
 
 # Clean things out!
 conda clean -tipsy
@@ -77,4 +75,5 @@ rm ${INSTALLER_PATH}
 
 chown -R $NB_USER:$NB_USER ${CONDA_DIR}
 
-conda list
+conda list -n root
+conda list -p ${NB_PYTHON_PREFIX}

repo2docker/buildpacks/repo2docker-entrypoint

@@ -0,0 +1,9 @@
+#!/bin/bash -l
+# lightest possible entrypoint that ensures that
+# we use a login shell to get a fully configured shell environment
+# (e.g. sourcing /etc/profile.d, ~/.bashrc, and friends)
+if [[ ! -z "${R2D_ENTRYPOINT:-}" ]]; then
+    exec "$R2D_ENTRYPOINT" "$@"
+else
+    exec "$@"
+fi

tests/conda/binder-dir/verify

@@ -1,8 +1,10 @@
 #!/usr/bin/env python
 import sys
+from subprocess import check_output
 
 assert sys.version_info[:2] == (3, 5), sys.version
 
+out = check_output(['conda', '--version']).decode('utf8').strip()
+assert out == 'conda 4.6.14', out
+
 import numpy
-import conda
-assert conda.__version__ == '4.5.11', conda.__version__

tests/conda/repo-path/verify

@@ -2,8 +2,10 @@
 import sys
 import os
 
-# Python should still be in /srv/conda
-assert sys.executable == '/srv/conda/bin/python'
+# conda should still be in /srv/conda
+# and Python should still be in $NB_PYTHON_PREFIX
+assert sys.executable == os.path.join(os.environ['NB_PYTHON_PREFIX'], 'bin', 'python'), sys.executable
+assert sys.executable.startswith("/srv/conda/"), sys.executable
 
 # Repo should be in /srv/repo
 assert os.path.exists('/srv/repo/verify')

tests/conda/simple-py2/verify

@@ -13,7 +13,7 @@ assert sorted(specs) == ['python2', 'python3'], specs.keys()
 import json
 from subprocess import check_output
 envs = json.loads(check_output(['conda', 'env', 'list', '--json']).decode('utf8'))
-assert envs == {'envs': ['/srv/conda', '/srv/conda/envs/kernel']}, envs
+assert envs == {'envs': ['/srv/conda', '/srv/conda/envs/kernel', '/srv/conda/envs/notebook']}, envs
 
 pkgs = json.loads(check_output(['conda', 'list', '-n', 'kernel', '--json']).decode('utf8'))
 pkg_names = [pkg['name'] for pkg in pkgs]

tests/conda/simple/environment.yml

@@ -1,2 +1,3 @@
 dependencies:
   - numpy
+  - pytest

tests/conda/simple/verify

@@ -1,6 +1,2 @@
-#!/usr/bin/env python
-import sys
-
-assert sys.version_info[:2] == (3, 7), sys.version
-
-import numpy
+#!/bin/sh
+pytest -v ./verify.py

tests/conda/simple/verify.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+import os
+import sys
+
+def test_sys_version():
+    assert sys.version_info[:2] == (3, 7)
+
+def test_numpy():
+    import numpy
+
+def test_conda_activated():
+    assert os.environ.get("CONDA_PREFIX") == os.environ["NB_PYTHON_PREFIX"], dict(os.environ)

tests/julia/julia_version-1.0.2/verify

@@ -17,7 +17,7 @@ end
 # Verify that kernels are not installed in home directory (issue #620)
 try
     using IJulia
-    @assert IJulia.kerneldir() == "/srv/conda/share/jupyter/kernels"
+    @assert IJulia.kerneldir() == ENV["NB_PYTHON_PREFIX"] * "/share/jupyter/kernels"
 catch
     exit(1)
 end

tests/julia/julia_version-default/verify

@@ -17,7 +17,7 @@ end
 # Verify that kernels are not installed in home directory (issue #620)
 try
     using IJulia
-    @assert IJulia.kerneldir() == "/srv/conda/share/jupyter/kernels"
+    @assert IJulia.kerneldir() == ENV["NB_PYTHON_PREFIX"] * "/share/jupyter/kernels"
 catch
     exit(1)
 end