kopia lustrzana https://github.com/jupyterhub/repo2docker
support separate pip requirements.txt in base env
- require base env to be a lockfile, environment.yml no longer allowed - separate `requirements.py-x.y.txt` can be processed post-install, if present - use this for Python 3.5 env, which has some dependencies only from pip (newer versions than the last py35 build on conda-forge)pull/1024/head
Min RK
rodzic
789a3a23c3
commit
03d673b096
|
|
@ -23,13 +23,17 @@ class CondaBuildPack(BaseImage):
|
|||
|
||||
"""
|
||||
|
||||
# The kernel environment file, if any.
|
||||
# The kernel conda environment file, if any.
|
||||
# As an absolute path within the container.
|
||||
_kernel_environment_file = ""
|
||||
# extra pip requirements.txt for the kernel
|
||||
_kernel_requirements_file = ""
|
||||
|
||||
# The notebook server environment file, if any.
|
||||
# The notebook server environment file.
|
||||
# As an absolute path within the container.
|
||||
_nb_environment_file = ""
|
||||
# extra pip requirements.txt for the notebook env
|
||||
_nb_requirements_file = ""
|
||||
|
||||
def get_build_env(self):
|
||||
"""Return environment variables to be set.
|
||||
|
|
@ -38,18 +42,28 @@ class CondaBuildPack(BaseImage):
|
|||
the `NB_PYTHON_PREFIX` to the location of the jupyter binary.
|
||||
|
||||
"""
|
||||
if not self._nb_environment_file:
|
||||
# get_build_scripts locates requirements/environment files
|
||||
self.get_build_scripts()
|
||||
|
||||
env = super().get_build_env() + [
|
||||
("CONDA_DIR", "${APP_BASE}/conda"),
|
||||
("NB_PYTHON_PREFIX", "${CONDA_DIR}/envs/notebook"),
|
||||
("NB_ENVIRONMENT_FILE", self._nb_environment_file),
|
||||
]
|
||||
if self.py2:
|
||||
if self._nb_requirements_file:
|
||||
env.append(("NB_REQUIREMENTS_FILE", self._nb_requirements_file))
|
||||
|
||||
if self._kernel_environment_file:
|
||||
# if kernel environment file is separate
|
||||
env.extend(
|
||||
[
|
||||
("KERNEL_PYTHON_PREFIX", "${CONDA_DIR}/envs/kernel"),
|
||||
("KERNEL_ENVIRONMENT_FILE", self._kernel_environment_file),
|
||||
]
|
||||
)
|
||||
if self._kernel_requirements_file:
|
||||
env.append(("KERNEL_REQUIREMENTS_FILE", self._kernel_requirements_file))
|
||||
else:
|
||||
env.append(("KERNEL_PYTHON_PREFIX", "${NB_PYTHON_PREFIX}"))
|
||||
return env
|
||||
|
|
@ -95,7 +109,7 @@ class CondaBuildPack(BaseImage):
|
|||
r"""
|
||||
TIMEFORMAT='time: %3R' \
|
||||
bash -c 'time /tmp/install-miniforge.bash' && \
|
||||
rm /tmp/install-miniforge.bash ${NB_ENVIRONMENT_FILE}
|
||||
rm -rf /tmp/install-miniforge.bash /tmp/env
|
||||
""",
|
||||
)
|
||||
]
|
||||
|
|
@ -129,24 +143,37 @@ class CondaBuildPack(BaseImage):
|
|||
# If no version is specified or no matching X.Y version is found,
|
||||
# the default base environment is used.
|
||||
frozen_name = "environment.lock"
|
||||
pip_frozen_name = "requirements.txt"
|
||||
if py_version:
|
||||
if self.py2:
|
||||
# python 2 goes in a different env
|
||||
files[
|
||||
"conda/environment.py-2.7.lock"
|
||||
] = self._kernel_environment_file = "/tmp/kernel-environment.lock"
|
||||
] = self._kernel_environment_file = "/tmp/env/kernel-environment.lock"
|
||||
# additional pip requirements for kernel env
|
||||
if os.path.exists(os.path.join(HERE, "requirements.py-2.7.txt")):
|
||||
files[
|
||||
"conda/requirements.py-2.7.txt"
|
||||
] = (
|
||||
self._kernel_requirements_file
|
||||
) = "/tmp/env/kernel-requirements.txt"
|
||||
else:
|
||||
for ext in [".lock", ".frozen.yml"]:
|
||||
py_frozen_name = f"environment.py-{py_version}{ext}"
|
||||
if os.path.exists(os.path.join(HERE, py_frozen_name)):
|
||||
frozen_name = py_frozen_name
|
||||
break
|
||||
py_frozen_name = f"environment.py-{py_version}.lock"
|
||||
if os.path.exists(os.path.join(HERE, py_frozen_name)):
|
||||
frozen_name = py_frozen_name
|
||||
pip_frozen_name = f"requirements.py-{py_version}.txt"
|
||||
if not frozen_name:
|
||||
self.log.warning(f"No frozen env for {py_version}")
|
||||
_, frozen_ext = os.path.splitext(frozen_name)
|
||||
files[
|
||||
"conda/" + frozen_name
|
||||
] = self._nb_environment_file = f"/tmp/environment{frozen_ext}"
|
||||
] = self._nb_environment_file = "/tmp/env/environment.lock"
|
||||
|
||||
# add requirements.txt, if present
|
||||
if os.path.exists(os.path.join(HERE, pip_frozen_name)):
|
||||
files[
|
||||
"conda/" + pip_frozen_name
|
||||
] = self._nb_requirements_file = "/tmp/env/requirements.txt"
|
||||
|
||||
files.update(super().get_build_script_files())
|
||||
return files
|
||||
|
||||
|
|
|
|||
|
|
@ -1,86 +0,0 @@
|
|||
# py3.5 conda packages are no longer being updated as of 12/2018
|
||||
# this file is hand-maintained now
|
||||
# and should only receive security fixes, no new features
|
||||
name: r2d
|
||||
channels:
|
||||
- conda-forge
|
||||
- defaults
|
||||
- conda-forge/label/broken
|
||||
dependencies:
|
||||
- backcall=0.1.0=py_0
|
||||
- bleach=3.0.2=py_0
|
||||
- bzip2=1.0.6=h470a237_2
|
||||
- ca-certificates=2018.10.15=ha4d7672_0
|
||||
- certifi=2018.8.24=py35_1001
|
||||
- decorator=4.3.0=py_0
|
||||
- defusedxml=0.5.0=py_1
|
||||
- entrypoints=0.2.3=py35_2
|
||||
- gmp=6.1.2=hfc679d8_0
|
||||
- ipykernel=5.1.0=pyh24bf2e0_0
|
||||
- ipython=7.0.1=py35h24bf2e0_0
|
||||
- ipython_genutils=0.2.0=py_1
|
||||
- ipywidgets=7.2.1=py35_1
|
||||
- jedi=0.12.1=py35_0
|
||||
- jinja2=2.10=py_1
|
||||
- jsonschema=2.6.0=py35_2
|
||||
- jupyter_client=5.2.3=py_1
|
||||
- jupyter_core=4.4.0=py_0
|
||||
- jupyterlab=0.34.9=py35_0
|
||||
- jupyterlab_launcher=0.13.1=py_2
|
||||
- libffi=3.2.1=hfc679d8_5
|
||||
- libgcc-ng=7.2.0=hdf63c60_3
|
||||
- libsodium=1.0.16=h470a237_1
|
||||
- libstdcxx-ng=7.2.0=hdf63c60_3
|
||||
- markupsafe=1.0=py35h470a237_1
|
||||
- mistune=0.8.3=py35h470a237_2
|
||||
- nbconvert=5.4.0=1
|
||||
- nbformat=4.4.0=py_1
|
||||
- ncurses=6.1=hfc679d8_1
|
||||
- openssl=1.0.2p=h470a237_1
|
||||
- pandoc=1.19.2=0
|
||||
- pandocfilters=1.4.2=py_1
|
||||
- parso=0.3.1=py_0
|
||||
- pexpect=4.6.0=py35_0
|
||||
- pickleshare=0.7.5=py35_0
|
||||
- pip=18.0=py35_1001
|
||||
- prometheus_client=0.4.2=py_0
|
||||
- prompt_toolkit=2.0.7=py_0
|
||||
- ptyprocess=0.6.0=py35_1000
|
||||
- pygments=2.2.0=py_1
|
||||
- python=3.5.5=h5001a0f_2
|
||||
- python-dateutil=2.7.5=py_0
|
||||
- pyzmq=17.1.2=py35hae99301_0
|
||||
- readline=7.0=haf1bffa_1
|
||||
- send2trash=1.5.0=py_0
|
||||
- setuptools=40.4.3=py35_0
|
||||
- simplegeneric=0.8.1=py_1
|
||||
- six=1.11.0=py35_1
|
||||
- sqlite=3.25.3=hb1c47c0_0
|
||||
- terminado=0.8.1=py35_1
|
||||
- testpath=0.3.1=py35_1
|
||||
- tk=8.6.9=ha92aebf_0
|
||||
- tornado=5.1.1=py35h470a237_0
|
||||
- traitlets=4.3.2=py35_0
|
||||
- wcwidth=0.1.7=py_1
|
||||
- webencodings=0.5.1=py_1
|
||||
- wheel=0.32.0=py35_1000
|
||||
- widgetsnbextension=3.2.1=py35_1
|
||||
- xz=5.2.4=h470a237_1
|
||||
- zeromq=4.2.5=hfc679d8_6
|
||||
- zlib=1.2.11=h470a237_3
|
||||
- pip:
|
||||
- alembic==1.0.3
|
||||
- async-generator==1.10
|
||||
- chardet==3.0.4
|
||||
- idna==2.7
|
||||
- jupyterhub==0.9.4
|
||||
- mako==1.0.7
|
||||
- notebook==5.7.8
|
||||
- nteract-on-jupyter==1.9.6
|
||||
- pamela==0.3.0
|
||||
- python-editor==1.0.3
|
||||
- python-oauth2==1.1.0
|
||||
- requests==2.20.1
|
||||
- sqlalchemy==1.2.14
|
||||
- urllib3==1.24.1
|
||||
prefix: /opt/conda/envs/r2d
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
# py3.5 conda packages are no longer being updated as of 12/2018
|
||||
# this file is hand-maintained now
|
||||
# and should only receive security fixes, no new features
|
||||
@EXPLICIT
|
||||
https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-main.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2018.10.15-ha4d7672_0.tar.bz2
|
||||
https://repo.anaconda.com/pkgs/main/linux-64/libstdcxx-ng-7.2.0-hdf63c60_3.conda
|
||||
https://repo.anaconda.com/pkgs/main/linux-64/libgcc-ng-7.2.0-hdf63c60_3.conda
|
||||
https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.6-h470a237_2.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/gmp-6.1.2-hfc679d8_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/libffi-3.2.1-hfc679d8_5.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/libsodium-1.0.16-h470a237_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.1-hfc679d8_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/openssl-1.0.2p-h470a237_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.4-h470a237_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.11-h470a237_3.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/pandoc-1.19.2-0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/readline-7.0-haf1bffa_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.9-ha92aebf_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/zeromq-4.2.5-hfc679d8_6.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/sqlite-3.25.3-hb1c47c0_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/python-3.5.5-h5001a0f_2.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/backcall-0.1.0-py_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/certifi-2018.8.24-py35_1001.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/decorator-4.3.0-py_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/defusedxml-0.5.0-py_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/entrypoints-0.2.3-py35_2.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/ipython_genutils-0.2.0-py_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/jsonschema-2.6.0-py35_2.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/markupsafe-1.0-py35h470a237_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/mistune-0.8.3-py35h470a237_2.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/pandocfilters-1.4.2-py_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/parso-0.3.1-py_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/pickleshare-0.7.5-py35_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/prometheus_client-0.4.2-py_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/ptyprocess-0.6.0-py35_1000.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/pyzmq-17.1.2-py35hae99301_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/send2trash-1.5.0-py_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/simplegeneric-0.8.1-py_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/six-1.11.0-py35_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/testpath-0.3.1-py35_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/tornado-5.1.1-py35h470a237_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/wcwidth-0.1.7-py_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/webencodings-0.5.1-py_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/jedi-0.12.1-py35_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/pexpect-4.6.0-py35_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.7.5-py_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/setuptools-40.4.3-py35_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/terminado-0.8.1-py35_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/traitlets-4.3.2-py35_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/bleach-3.0.2-py_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/jinja2-2.10-py_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/jupyter_core-4.4.0-py_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/pygments-2.2.0-py_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/wheel-0.32.0-py35_1000.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/jupyter_client-5.2.3-py_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/nbformat-4.4.0-py_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/pip-18.0-py35_1001.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-2.0.7-py_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/ipython-7.0.1-py35h24bf2e0_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/label/broken/noarch/nbconvert-5.4.0-1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/label/broken/noarch/ipykernel-5.1.0-pyh24bf2e0_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/notebook-5.7.0-py35_0.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/noarch/jupyterlab_launcher-0.13.1-py_2.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/widgetsnbextension-3.2.1-py35_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/ipywidgets-7.2.1-py35_1.tar.bz2
|
||||
https://conda.anaconda.org/conda-forge/linux-64/jupyterlab-0.34.9-py35_0.tar.bz2
|
||||
|
|
@ -47,31 +47,31 @@ time mamba install -y mamba==${MAMBA_VERSION}
|
|||
echo "installing notebook env:"
|
||||
cat "${NB_ENVIRONMENT_FILE}"
|
||||
|
||||
if [[ "${NB_ENVIRONMENT_FILE: -5}" == ".lock" ]]; then
|
||||
create="mamba create"
|
||||
else
|
||||
create="mamba env create"
|
||||
time mamba create -p ${NB_PYTHON_PREFIX} --file "${NB_ENVIRONMENT_FILE}"
|
||||
|
||||
if [[ ! -z "${NB_REQUIREMENTS_FILE:-}" ]]; then
|
||||
echo "installing pip requirements"
|
||||
cat "${NB_REQUIREMENTS_FILE}"
|
||||
${NB_PYTHON_PREFIX}/bin/python -mpip install --no-cache --no-deps -r "${NB_REQUIREMENTS_FILE}"
|
||||
fi
|
||||
|
||||
time $create -p ${NB_PYTHON_PREFIX} --file "${NB_ENVIRONMENT_FILE}"
|
||||
|
||||
# empty conda history file,
|
||||
# which seems to result in some effective pinning of packages in the initial env,
|
||||
# which we don't intend.
|
||||
# this file must not be *removed*, however
|
||||
echo '' > ${NB_PYTHON_PREFIX}/conda-meta/history
|
||||
|
||||
if [[ ! -z "${KERNEL_ENVIRONMENT_FILE:-}" && -f "${KERNEL_ENVIRONMENT_FILE}" ]]; then
|
||||
if [[ ! -z "${KERNEL_ENVIRONMENT_FILE:-}" ]]; then
|
||||
# install kernel env and register kernelspec
|
||||
echo "installing kernel env:"
|
||||
cat "${KERNEL_ENVIRONMENT_FILE}"
|
||||
if [[ "${KERNEL_ENVIRONMENT_FILE: -5}" == ".lock" ]]; then
|
||||
create="mamba create"
|
||||
else
|
||||
create="mamba env create"
|
||||
time mamba create -p ${KERNEL_PYTHON_PREFIX} --file "${KERNEL_ENVIRONMENT_FILE}"
|
||||
|
||||
if [[ ! -z "${KERNEL_REQUIREMENTS_FILE:-}" ]]; then
|
||||
echo "installing pip requirements for kernel"
|
||||
cat "${KERNEL_REQUIREMENTS_FILE}"
|
||||
${KERNEL_PYTHON_PREFIX}/bin/python -mpip install --no-cache --no-deps -r "${KERNEL_REQUIREMENTS_FILE}"
|
||||
fi
|
||||
|
||||
time $create -p ${KERNEL_PYTHON_PREFIX} --file "${KERNEL_ENVIRONMENT_FILE}"
|
||||
${KERNEL_PYTHON_PREFIX}/bin/ipython kernel install --prefix "${NB_PYTHON_PREFIX}"
|
||||
echo '' > ${KERNEL_PYTHON_PREFIX}/conda-meta/history
|
||||
mamba list -p ${KERNEL_PYTHON_PREFIX}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
# pip-installed packages for Python 3.5
|
||||
# these should only be packages
|
||||
# whose versions are too new to have builds available for Python 3.5
|
||||
alembic==1.0.3
|
||||
async-generator==1.10
|
||||
chardet==3.0.4
|
||||
idna==2.7
|
||||
jupyterhub==0.9.4
|
||||
mako==1.0.7
|
||||
notebook==5.7.8
|
||||
nteract-on-jupyter==1.9.6
|
||||
pamela==0.3.0
|
||||
python-editor==1.0.3
|
||||
python-oauth2==1.1.0
|
||||
requests==2.20.1
|
||||
sqlalchemy==1.2.14
|
||||
urllib3==1.24.1
|
||||
Ładowanie…
Reference in New Issue