diff --git a/repo2docker/buildpacks/conda/__init__.py b/repo2docker/buildpacks/conda/__init__.py index 8325dd72..966d2324 100644 --- a/repo2docker/buildpacks/conda/__init__.py +++ b/repo2docker/buildpacks/conda/__init__.py @@ -23,13 +23,17 @@ class CondaBuildPack(BaseImage): """ - # The kernel environment file, if any. + # The kernel conda environment file, if any. # As an absolute path within the container. _kernel_environment_file = "" + # extra pip requirements.txt for the kernel + _kernel_requirements_file = "" - # The notebook server environment file, if any. + # The notebook server environment file. # As an absolute path within the container. _nb_environment_file = "" + # extra pip requirements.txt for the notebook env + _nb_requirements_file = "" def get_build_env(self): """Return environment variables to be set. @@ -38,18 +42,28 @@ class CondaBuildPack(BaseImage): the `NB_PYTHON_PREFIX` to the location of the jupyter binary. """ + if not self._nb_environment_file: + # get_build_scripts locates requirements/environment files + self.get_build_scripts() + env = super().get_build_env() + [ ("CONDA_DIR", "${APP_BASE}/conda"), ("NB_PYTHON_PREFIX", "${CONDA_DIR}/envs/notebook"), ("NB_ENVIRONMENT_FILE", self._nb_environment_file), ] - if self.py2: + if self._nb_requirements_file: + env.append(("NB_REQUIREMENTS_FILE", self._nb_requirements_file)) + + if self._kernel_environment_file: + # if kernel environment file is separate env.extend( [ ("KERNEL_PYTHON_PREFIX", "${CONDA_DIR}/envs/kernel"), ("KERNEL_ENVIRONMENT_FILE", self._kernel_environment_file), ] ) + if self._kernel_requirements_file: + env.append(("KERNEL_REQUIREMENTS_FILE", self._kernel_requirements_file)) else: env.append(("KERNEL_PYTHON_PREFIX", "${NB_PYTHON_PREFIX}")) return env @@ -95,7 +109,7 @@ class CondaBuildPack(BaseImage): r""" TIMEFORMAT='time: %3R' \ bash -c 'time /tmp/install-miniforge.bash' && \ - rm /tmp/install-miniforge.bash ${NB_ENVIRONMENT_FILE} + rm -rf /tmp/install-miniforge.bash /tmp/env """, ) ] @@ -129,24 +143,37 @@ class CondaBuildPack(BaseImage): # If no version is specified or no matching X.Y version is found, # the default base environment is used. frozen_name = "environment.lock" + pip_frozen_name = "requirements.txt" if py_version: if self.py2: # python 2 goes in a different env files[ "conda/environment.py-2.7.lock" - ] = self._kernel_environment_file = "/tmp/kernel-environment.lock" + ] = self._kernel_environment_file = "/tmp/env/kernel-environment.lock" + # additional pip requirements for kernel env + if os.path.exists(os.path.join(HERE, "requirements.py-2.7.txt")): + files[ + "conda/requirements.py-2.7.txt" + ] = ( + self._kernel_requirements_file + ) = "/tmp/env/kernel-requirements.txt" else: - for ext in [".lock", ".frozen.yml"]: - py_frozen_name = f"environment.py-{py_version}{ext}" - if os.path.exists(os.path.join(HERE, py_frozen_name)): - frozen_name = py_frozen_name - break + py_frozen_name = f"environment.py-{py_version}.lock" + if os.path.exists(os.path.join(HERE, py_frozen_name)): + frozen_name = py_frozen_name + pip_frozen_name = f"requirements.py-{py_version}.txt" if not frozen_name: self.log.warning(f"No frozen env for {py_version}") - _, frozen_ext = os.path.splitext(frozen_name) files[ "conda/" + frozen_name - ] = self._nb_environment_file = f"/tmp/environment{frozen_ext}" + ] = self._nb_environment_file = "/tmp/env/environment.lock" + + # add requirements.txt, if present + if os.path.exists(os.path.join(HERE, pip_frozen_name)): + files[ + "conda/" + pip_frozen_name + ] = self._nb_requirements_file = "/tmp/env/requirements.txt" + files.update(super().get_build_script_files()) return files diff --git a/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml b/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml deleted file mode 100644 index 7aa3aea3..00000000 --- a/repo2docker/buildpacks/conda/environment.py-3.5.frozen.yml +++ /dev/null @@ -1,86 +0,0 @@ -# py3.5 conda packages are no longer being updated as of 12/2018 -# this file is hand-maintained now -# and should only receive security fixes, no new features -name: r2d -channels: - - conda-forge - - defaults - - conda-forge/label/broken -dependencies: - - backcall=0.1.0=py_0 - - bleach=3.0.2=py_0 - - bzip2=1.0.6=h470a237_2 - - ca-certificates=2018.10.15=ha4d7672_0 - - certifi=2018.8.24=py35_1001 - - decorator=4.3.0=py_0 - - defusedxml=0.5.0=py_1 - - entrypoints=0.2.3=py35_2 - - gmp=6.1.2=hfc679d8_0 - - ipykernel=5.1.0=pyh24bf2e0_0 - - ipython=7.0.1=py35h24bf2e0_0 - - ipython_genutils=0.2.0=py_1 - - ipywidgets=7.2.1=py35_1 - - jedi=0.12.1=py35_0 - - jinja2=2.10=py_1 - - jsonschema=2.6.0=py35_2 - - jupyter_client=5.2.3=py_1 - - jupyter_core=4.4.0=py_0 - - jupyterlab=0.34.9=py35_0 - - jupyterlab_launcher=0.13.1=py_2 - - libffi=3.2.1=hfc679d8_5 - - libgcc-ng=7.2.0=hdf63c60_3 - - libsodium=1.0.16=h470a237_1 - - libstdcxx-ng=7.2.0=hdf63c60_3 - - markupsafe=1.0=py35h470a237_1 - - mistune=0.8.3=py35h470a237_2 - - nbconvert=5.4.0=1 - - nbformat=4.4.0=py_1 - - ncurses=6.1=hfc679d8_1 - - openssl=1.0.2p=h470a237_1 - - pandoc=1.19.2=0 - - pandocfilters=1.4.2=py_1 - - parso=0.3.1=py_0 - - pexpect=4.6.0=py35_0 - - pickleshare=0.7.5=py35_0 - - pip=18.0=py35_1001 - - prometheus_client=0.4.2=py_0 - - prompt_toolkit=2.0.7=py_0 - - ptyprocess=0.6.0=py35_1000 - - pygments=2.2.0=py_1 - - python=3.5.5=h5001a0f_2 - - python-dateutil=2.7.5=py_0 - - pyzmq=17.1.2=py35hae99301_0 - - readline=7.0=haf1bffa_1 - - send2trash=1.5.0=py_0 - - setuptools=40.4.3=py35_0 - - simplegeneric=0.8.1=py_1 - - six=1.11.0=py35_1 - - sqlite=3.25.3=hb1c47c0_0 - - terminado=0.8.1=py35_1 - - testpath=0.3.1=py35_1 - - tk=8.6.9=ha92aebf_0 - - tornado=5.1.1=py35h470a237_0 - - traitlets=4.3.2=py35_0 - - wcwidth=0.1.7=py_1 - - webencodings=0.5.1=py_1 - - wheel=0.32.0=py35_1000 - - widgetsnbextension=3.2.1=py35_1 - - xz=5.2.4=h470a237_1 - - zeromq=4.2.5=hfc679d8_6 - - zlib=1.2.11=h470a237_3 - - pip: - - alembic==1.0.3 - - async-generator==1.10 - - chardet==3.0.4 - - idna==2.7 - - jupyterhub==0.9.4 - - mako==1.0.7 - - notebook==5.7.8 - - nteract-on-jupyter==1.9.6 - - pamela==0.3.0 - - python-editor==1.0.3 - - python-oauth2==1.1.0 - - requests==2.20.1 - - sqlalchemy==1.2.14 - - urllib3==1.24.1 -prefix: /opt/conda/envs/r2d diff --git a/repo2docker/buildpacks/conda/environment.py-3.5.lock b/repo2docker/buildpacks/conda/environment.py-3.5.lock new file mode 100644 index 00000000..25297473 --- /dev/null +++ b/repo2docker/buildpacks/conda/environment.py-3.5.lock @@ -0,0 +1,67 @@ +# py3.5 conda packages are no longer being updated as of 12/2018 +# this file is hand-maintained now +# and should only receive security fixes, no new features +@EXPLICIT +https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-main.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2018.10.15-ha4d7672_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/linux-64/libstdcxx-ng-7.2.0-hdf63c60_3.conda +https://repo.anaconda.com/pkgs/main/linux-64/libgcc-ng-7.2.0-hdf63c60_3.conda +https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.6-h470a237_2.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/gmp-6.1.2-hfc679d8_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libffi-3.2.1-hfc679d8_5.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/libsodium-1.0.16-h470a237_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.1-hfc679d8_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/openssl-1.0.2p-h470a237_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.4-h470a237_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.11-h470a237_3.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/pandoc-1.19.2-0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/readline-7.0-haf1bffa_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.9-ha92aebf_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.2.5-hfc679d8_6.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.25.3-hb1c47c0_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/python-3.5.5-h5001a0f_2.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/backcall-0.1.0-py_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/certifi-2018.8.24-py35_1001.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/decorator-4.3.0-py_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.5.0-py_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/entrypoints-0.2.3-py35_2.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/ipython_genutils-0.2.0-py_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/jsonschema-2.6.0-py35_2.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/markupsafe-1.0-py35h470a237_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/mistune-0.8.3-py35h470a237_2.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.4.2-py_1.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/parso-0.3.1-py_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/pickleshare-0.7.5-py35_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/prometheus_client-0.4.2-py_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/ptyprocess-0.6.0-py35_1000.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/pyzmq-17.1.2-py35hae99301_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/send2trash-1.5.0-py_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/simplegeneric-0.8.1-py_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/six-1.11.0-py35_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/testpath-0.3.1-py35_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/tornado-5.1.1-py35h470a237_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.1.7-py_1.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-py_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/jedi-0.12.1-py35_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/pexpect-4.6.0-py35_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.7.5-py_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/setuptools-40.4.3-py35_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/terminado-0.8.1-py35_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/traitlets-4.3.2-py35_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/bleach-3.0.2-py_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/jinja2-2.10-py_1.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/jupyter_core-4.4.0-py_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/pygments-2.2.0-py_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/wheel-0.32.0-py35_1000.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/jupyter_client-5.2.3-py_1.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/nbformat-4.4.0-py_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/pip-18.0-py35_1001.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-2.0.7-py_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/ipython-7.0.1-py35h24bf2e0_0.tar.bz2 +https://conda.anaconda.org/conda-forge/label/broken/noarch/nbconvert-5.4.0-1.tar.bz2 +https://conda.anaconda.org/conda-forge/label/broken/noarch/ipykernel-5.1.0-pyh24bf2e0_0.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/notebook-5.7.0-py35_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/jupyterlab_launcher-0.13.1-py_2.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/widgetsnbextension-3.2.1-py35_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/ipywidgets-7.2.1-py35_1.tar.bz2 +https://conda.anaconda.org/conda-forge/linux-64/jupyterlab-0.34.9-py35_0.tar.bz2 diff --git a/repo2docker/buildpacks/conda/install-miniforge.bash b/repo2docker/buildpacks/conda/install-miniforge.bash index dabdb9e9..3c99b6a0 100755 --- a/repo2docker/buildpacks/conda/install-miniforge.bash +++ b/repo2docker/buildpacks/conda/install-miniforge.bash @@ -47,31 +47,31 @@ time mamba install -y mamba==${MAMBA_VERSION} echo "installing notebook env:" cat "${NB_ENVIRONMENT_FILE}" -if [[ "${NB_ENVIRONMENT_FILE: -5}" == ".lock" ]]; then - create="mamba create" -else - create="mamba env create" +time mamba create -p ${NB_PYTHON_PREFIX} --file "${NB_ENVIRONMENT_FILE}" + +if [[ ! -z "${NB_REQUIREMENTS_FILE:-}" ]]; then + echo "installing pip requirements" + cat "${NB_REQUIREMENTS_FILE}" + ${NB_PYTHON_PREFIX}/bin/python -mpip install --no-cache --no-deps -r "${NB_REQUIREMENTS_FILE}" fi - -time $create -p ${NB_PYTHON_PREFIX} --file "${NB_ENVIRONMENT_FILE}" - # empty conda history file, # which seems to result in some effective pinning of packages in the initial env, # which we don't intend. # this file must not be *removed*, however echo '' > ${NB_PYTHON_PREFIX}/conda-meta/history -if [[ ! -z "${KERNEL_ENVIRONMENT_FILE:-}" && -f "${KERNEL_ENVIRONMENT_FILE}" ]]; then +if [[ ! -z "${KERNEL_ENVIRONMENT_FILE:-}" ]]; then # install kernel env and register kernelspec echo "installing kernel env:" cat "${KERNEL_ENVIRONMENT_FILE}" - if [[ "${KERNEL_ENVIRONMENT_FILE: -5}" == ".lock" ]]; then - create="mamba create" - else - create="mamba env create" + time mamba create -p ${KERNEL_PYTHON_PREFIX} --file "${KERNEL_ENVIRONMENT_FILE}" + + if [[ ! -z "${KERNEL_REQUIREMENTS_FILE:-}" ]]; then + echo "installing pip requirements for kernel" + cat "${KERNEL_REQUIREMENTS_FILE}" + ${KERNEL_PYTHON_PREFIX}/bin/python -mpip install --no-cache --no-deps -r "${KERNEL_REQUIREMENTS_FILE}" fi - time $create -p ${KERNEL_PYTHON_PREFIX} --file "${KERNEL_ENVIRONMENT_FILE}" ${KERNEL_PYTHON_PREFIX}/bin/ipython kernel install --prefix "${NB_PYTHON_PREFIX}" echo '' > ${KERNEL_PYTHON_PREFIX}/conda-meta/history mamba list -p ${KERNEL_PYTHON_PREFIX} diff --git a/repo2docker/buildpacks/conda/requirements.py-3.5.txt b/repo2docker/buildpacks/conda/requirements.py-3.5.txt new file mode 100644 index 00000000..416274ad --- /dev/null +++ b/repo2docker/buildpacks/conda/requirements.py-3.5.txt @@ -0,0 +1,17 @@ +# pip-installed packages for Python 3.5 +# these should only be packages +# whose versions are too new to have builds available for Python 3.5 +alembic==1.0.3 +async-generator==1.10 +chardet==3.0.4 +idna==2.7 +jupyterhub==0.9.4 +mako==1.0.7 +notebook==5.7.8 +nteract-on-jupyter==1.9.6 +pamela==0.3.0 +python-editor==1.0.3 +python-oauth2==1.1.0 +requests==2.20.1 +sqlalchemy==1.2.14 +urllib3==1.24.1