kopia lustrzana https://github.com/jupyterhub/repo2docker
Merge pull request #182 from minrk/conda-python-runtime
conda: separate base envs by Python versionpull/183/head
Yuvi Panda
GitHub
commit
839de1ee1b
|
|
@ -1,26 +1,31 @@
|
|||
"""
|
||||
Generates a variety of Dockerfiles based on an input matrix
|
||||
Buildpack for conda environments
|
||||
"""
|
||||
from traitlets import default
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
|
||||
from ruamel.yaml import YAML
|
||||
from traitlets import default, Unicode
|
||||
|
||||
from ..base import BuildPack
|
||||
|
||||
# pattern for parsing conda dependency line
|
||||
PYTHON_REGEX = re.compile(r'python\s*=+\s*([\d\.]*)')
|
||||
# current directory
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
class CondaBuildPack(BuildPack):
|
||||
name = "conda"
|
||||
version = "0.1"
|
||||
env = [
|
||||
('CONDA_DIR', '${APP_BASE}/conda'),
|
||||
('NB_PYTHON_PREFIX', '${CONDA_DIR}')
|
||||
('NB_PYTHON_PREFIX', '${CONDA_DIR}'),
|
||||
]
|
||||
|
||||
path = ['${CONDA_DIR}/bin']
|
||||
|
||||
build_script_files = {
|
||||
'conda/install-miniconda.bash': '/tmp/install-miniconda.bash',
|
||||
'conda/environment.frozen.yml': '/tmp/environment.yml'
|
||||
}
|
||||
|
||||
build_scripts = [
|
||||
(
|
||||
"root",
|
||||
|
|
@ -31,19 +36,90 @@ class CondaBuildPack(BuildPack):
|
|||
)
|
||||
]
|
||||
|
||||
major_pythons = {
|
||||
'2': '2.7',
|
||||
'3': '3.6',
|
||||
}
|
||||
|
||||
@default('build_script_files')
|
||||
def setup_build_script_files(self):
|
||||
files = {
|
||||
'conda/install-miniconda.bash': '/tmp/install-miniconda.bash',
|
||||
}
|
||||
py_version = self.python_version
|
||||
self.log.info("Building conda environment for python=%s" % py_version)
|
||||
# Select the frozen base environment based on Python version.
|
||||
# avoids expensive and possibly conflicting upgrades when changing
|
||||
# major Python versions during upgrade.
|
||||
# If no version is specified or no matching X.Y version is found,
|
||||
# the default base environment is used.
|
||||
frozen_name = 'environment.frozen.yml'
|
||||
if py_version:
|
||||
if self.py2:
|
||||
# python 2 goes in a different env
|
||||
files['conda/environment.py-2.7.frozen.yml'] = '/tmp/kernel-environment.yml'
|
||||
else:
|
||||
py_frozen_name = \
|
||||
'environment.py-{py}.frozen.yml'.format(py=py_version)
|
||||
if os.path.exists(os.path.join(HERE, py_frozen_name)):
|
||||
frozen_name = py_frozen_name
|
||||
else:
|
||||
self.log.warning("No frozen env: %s", py_frozen_name)
|
||||
files['conda/' + frozen_name] = '/tmp/environment.yml'
|
||||
return files
|
||||
|
||||
python_version = Unicode()
|
||||
@default('python_version')
|
||||
def detect_python_version(self):
|
||||
"""Detect the Python version for a given environment.yml
|
||||
|
||||
Will return 'x.y' if found, or Falsy '' if not.
|
||||
"""
|
||||
py_version = None
|
||||
environment_yml = self.binder_path('environment.yml')
|
||||
if not os.path.exists(environment_yml):
|
||||
return ''
|
||||
with open(environment_yml) as f:
|
||||
env = YAML().load(f)
|
||||
for dep in env.get('dependencies', []):
|
||||
if not isinstance(dep, str):
|
||||
continue
|
||||
match = PYTHON_REGEX.match(dep)
|
||||
if not match:
|
||||
continue
|
||||
py_version = match.group(1)
|
||||
break
|
||||
|
||||
# extract major.minor
|
||||
if py_version:
|
||||
if len(py_version) == 1:
|
||||
return self.major_pythons.get(py_version[0])
|
||||
else:
|
||||
# return major.minor
|
||||
return '.'.join(py_version[:2])
|
||||
|
||||
return ''
|
||||
|
||||
@property
|
||||
def py2(self):
|
||||
"""Am I building a Python 2 kernel environment?"""
|
||||
return self.python_version and self.python_version.split('.')[0] == '2'
|
||||
|
||||
@default('assemble_scripts')
|
||||
def setup_assembly(self):
|
||||
assembly_scripts = []
|
||||
environment_yml = self.binder_path('environment.yml')
|
||||
env_name = 'kernel' if self.py2 else 'root'
|
||||
if os.path.exists(environment_yml):
|
||||
assembly_scripts.append((
|
||||
'${NB_USER}',
|
||||
r"""
|
||||
conda env update -v -n root -f "{}" && \
|
||||
conda env update -v -n {} -f "{}" && \
|
||||
conda clean -tipsy
|
||||
""".format(environment_yml)
|
||||
""".format(env_name, environment_yml)
|
||||
))
|
||||
return assembly_scripts
|
||||
|
||||
|
||||
def detect(self):
|
||||
return os.path.exists(self.binder_path('environment.yml')) and super().detect()
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
# AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY
|
||||
# Frozen on 2018-01-03 15:33:55 UTC
|
||||
name: r2d
|
||||
channels:
|
||||
- conda-forge
|
||||
|
|
@ -11,13 +13,13 @@ dependencies:
|
|||
- gmp=6.1.2=0
|
||||
- html5lib=1.0.1=py_0
|
||||
- ipykernel=4.7.0=py36_0
|
||||
- ipython=6.2.1=py36_0
|
||||
- ipython=6.2.1=py36_1
|
||||
- ipython_genutils=0.2.0=py36_0
|
||||
- ipywidgets=6.0.1=py36_0
|
||||
- jedi=0.10.2=py36_0
|
||||
- jinja2=2.10=py36_0
|
||||
- jsonschema=2.6.0=py36_0
|
||||
- jupyter_client=5.1.0=py36_0
|
||||
- jupyter_client=5.2.0=py36_0
|
||||
- jupyter_core=4.4.0=py_0
|
||||
- jupyterlab=0.30.6=py36_0
|
||||
- jupyterlab_launcher=0.6.0=py36_0
|
||||
|
|
@ -31,23 +33,23 @@ dependencies:
|
|||
- openssl=1.0.2n=0
|
||||
- pandoc=2.0.5=0
|
||||
- pandocfilters=1.4.1=py36_0
|
||||
- pexpect=4.3.0=py36_0
|
||||
- pexpect=4.3.1=py36_0
|
||||
- pickleshare=0.7.4=py36_0
|
||||
- pip=9.0.1=py36_0
|
||||
- pip=9.0.1=py36_1
|
||||
- prompt_toolkit=1.0.15=py36_0
|
||||
- ptyprocess=0.5.2=py36_0
|
||||
- pygments=2.2.0=py36_0
|
||||
- python=3.6.1=3
|
||||
- python=3.6.4=0
|
||||
- python-dateutil=2.6.1=py36_0
|
||||
- pyzmq=16.0.2=py36_2
|
||||
- readline=6.2=0
|
||||
- readline=7.0=0
|
||||
- setuptools=38.2.4=py36_0
|
||||
- simplegeneric=0.8.1=py36_0
|
||||
- six=1.11.0=py36_1
|
||||
- sqlite=3.13.0=1
|
||||
- sqlite=3.20.1=2
|
||||
- terminado=0.8.1=py36_0
|
||||
- testpath=0.3.1=py36_0
|
||||
- tk=8.5.19=2
|
||||
- tk=8.6.7=0
|
||||
- tornado=4.5.2=py36_0
|
||||
- traitlets=4.3.2=py36_0
|
||||
- wcwidth=0.1.7=py36_0
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
# AUTO GENERATED FROM environment.py-2.7.yml, DO NOT MANUALLY MODIFY
|
||||
# Frozen on 2018-01-03 16:56:23 UTC
|
||||
name: r2d
|
||||
channels:
|
||||
- conda-forge
|
||||
- defaults
|
||||
dependencies:
|
||||
- backports=1.0=py27_1
|
||||
- backports.shutil_get_terminal_size=1.0.0=py_3
|
||||
- backports_abc=0.5=py27_0
|
||||
- ca-certificates=2017.11.5=0
|
||||
- certifi=2017.11.5=py27_0
|
||||
- decorator=4.1.2=py27_0
|
||||
- enum34=1.1.6=py27_1
|
||||
- ipykernel=4.7.0=py27_0
|
||||
- ipython=5.5.0=py27_0
|
||||
- ipython_genutils=0.2.0=py27_0
|
||||
- jupyter_client=5.2.0=py27_0
|
||||
- jupyter_core=4.4.0=py_0
|
||||
- libsodium=1.0.15=1
|
||||
- ncurses=5.9=10
|
||||
- openssl=1.0.2n=0
|
||||
- pathlib2=2.3.0=py27_0
|
||||
- pexpect=4.3.1=py27_0
|
||||
- pickleshare=0.7.4=py27_0
|
||||
- pip=9.0.1=py27_1
|
||||
- prompt_toolkit=1.0.15=py27_0
|
||||
- ptyprocess=0.5.2=py27_0
|
||||
- pygments=2.2.0=py27_0
|
||||
- python=2.7.14=4
|
||||
- python-dateutil=2.6.1=py27_0
|
||||
- pyzmq=16.0.2=py27_2
|
||||
- readline=7.0=0
|
||||
- scandir=1.6=py27_0
|
||||
- setuptools=38.2.4=py27_0
|
||||
- simplegeneric=0.8.1=py27_0
|
||||
- singledispatch=3.4.0.3=py27_0
|
||||
- six=1.11.0=py27_1
|
||||
- sqlite=3.20.1=2
|
||||
- ssl_match_hostname=3.5.0.1=py27_1
|
||||
- tk=8.6.7=0
|
||||
- tornado=4.5.2=py27_0
|
||||
- traitlets=4.3.2=py27_0
|
||||
- wcwidth=0.1.7=py27_0
|
||||
- wheel=0.30.0=py_1
|
||||
- zeromq=4.2.1=1
|
||||
- zlib=1.2.11=0
|
||||
prefix: /opt/conda/envs/r2d
|
||||
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
dependencies:
|
||||
- python=2.7.*
|
||||
- ipykernel==4.7.0
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
# AUTO GENERATED FROM environment.py-3.5.yml, DO NOT MANUALLY MODIFY
|
||||
# Frozen on 2018-01-03 15:31:22 UTC
|
||||
name: r2d
|
||||
channels:
|
||||
- conda-forge
|
||||
- defaults
|
||||
dependencies:
|
||||
- bleach=2.0.0=py35_0
|
||||
- ca-certificates=2017.11.5=0
|
||||
- certifi=2017.11.5=py35_0
|
||||
- decorator=4.1.2=py35_0
|
||||
- entrypoints=0.2.3=py35_1
|
||||
- gmp=6.1.2=0
|
||||
- html5lib=1.0.1=py_0
|
||||
- ipykernel=4.7.0=py35_0
|
||||
- ipython=6.2.1=py35_1
|
||||
- ipython_genutils=0.2.0=py35_0
|
||||
- ipywidgets=6.0.1=py35_0
|
||||
- jedi=0.10.2=py35_0
|
||||
- jinja2=2.10=py35_0
|
||||
- jsonschema=2.6.0=py35_0
|
||||
- jupyter_client=5.2.0=py35_0
|
||||
- jupyter_core=4.4.0=py_0
|
||||
- jupyterlab=0.30.6=py35_0
|
||||
- jupyterlab_launcher=0.6.0=py35_0
|
||||
- libsodium=1.0.15=1
|
||||
- markupsafe=1.0=py35_0
|
||||
- mistune=0.8.3=py_0
|
||||
- nbconvert=5.3.1=py_1
|
||||
- nbformat=4.4.0=py35_0
|
||||
- ncurses=5.9=10
|
||||
- notebook=5.2.2=py35_1
|
||||
- openssl=1.0.2n=0
|
||||
- pandoc=2.0.5=0
|
||||
- pandocfilters=1.4.1=py35_0
|
||||
- pexpect=4.3.1=py35_0
|
||||
- pickleshare=0.7.4=py35_0
|
||||
- pip=9.0.1=py35_1
|
||||
- prompt_toolkit=1.0.15=py35_0
|
||||
- ptyprocess=0.5.2=py35_0
|
||||
- pygments=2.2.0=py35_0
|
||||
- python=3.5.4=3
|
||||
- python-dateutil=2.6.1=py35_0
|
||||
- pyzmq=16.0.2=py35_2
|
||||
- readline=7.0=0
|
||||
- setuptools=38.2.4=py35_0
|
||||
- simplegeneric=0.8.1=py35_0
|
||||
- six=1.11.0=py35_1
|
||||
- sqlite=3.20.1=2
|
||||
- terminado=0.8.1=py35_0
|
||||
- testpath=0.3.1=py35_0
|
||||
- tk=8.6.7=0
|
||||
- tornado=4.5.2=py35_0
|
||||
- traitlets=4.3.2=py35_0
|
||||
- wcwidth=0.1.7=py35_0
|
||||
- webencodings=0.5=py35_0
|
||||
- wheel=0.30.0=py_1
|
||||
- widgetsnbextension=2.0.1=py35_0
|
||||
- xz=5.2.3=0
|
||||
- zeromq=4.2.1=1
|
||||
- zlib=1.2.11=0
|
||||
prefix: /opt/conda/envs/r2d
|
||||
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
# AUTO GENERATED FROM environment.py-3.6.yml, DO NOT MANUALLY MODIFY
|
||||
# Frozen on 2018-01-03 15:33:55 UTC
|
||||
name: r2d
|
||||
channels:
|
||||
- conda-forge
|
||||
- defaults
|
||||
dependencies:
|
||||
- bleach=2.0.0=py36_0
|
||||
- ca-certificates=2017.11.5=0
|
||||
- certifi=2017.11.5=py36_0
|
||||
- decorator=4.1.2=py36_0
|
||||
- entrypoints=0.2.3=py36_1
|
||||
- gmp=6.1.2=0
|
||||
- html5lib=1.0.1=py_0
|
||||
- ipykernel=4.7.0=py36_0
|
||||
- ipython=6.2.1=py36_1
|
||||
- ipython_genutils=0.2.0=py36_0
|
||||
- ipywidgets=6.0.1=py36_0
|
||||
- jedi=0.10.2=py36_0
|
||||
- jinja2=2.10=py36_0
|
||||
- jsonschema=2.6.0=py36_0
|
||||
- jupyter_client=5.2.0=py36_0
|
||||
- jupyter_core=4.4.0=py_0
|
||||
- jupyterlab=0.30.6=py36_0
|
||||
- jupyterlab_launcher=0.6.0=py36_0
|
||||
- libsodium=1.0.15=1
|
||||
- markupsafe=1.0=py36_0
|
||||
- mistune=0.8.3=py_0
|
||||
- nbconvert=5.3.1=py_1
|
||||
- nbformat=4.4.0=py36_0
|
||||
- ncurses=5.9=10
|
||||
- notebook=5.2.2=py36_1
|
||||
- openssl=1.0.2n=0
|
||||
- pandoc=2.0.5=0
|
||||
- pandocfilters=1.4.1=py36_0
|
||||
- pexpect=4.3.1=py36_0
|
||||
- pickleshare=0.7.4=py36_0
|
||||
- pip=9.0.1=py36_1
|
||||
- prompt_toolkit=1.0.15=py36_0
|
||||
- ptyprocess=0.5.2=py36_0
|
||||
- pygments=2.2.0=py36_0
|
||||
- python=3.6.4=0
|
||||
- python-dateutil=2.6.1=py36_0
|
||||
- pyzmq=16.0.2=py36_2
|
||||
- readline=7.0=0
|
||||
- setuptools=38.2.4=py36_0
|
||||
- simplegeneric=0.8.1=py36_0
|
||||
- six=1.11.0=py36_1
|
||||
- sqlite=3.20.1=2
|
||||
- terminado=0.8.1=py36_0
|
||||
- testpath=0.3.1=py36_0
|
||||
- tk=8.6.7=0
|
||||
- tornado=4.5.2=py36_0
|
||||
- traitlets=4.3.2=py36_0
|
||||
- wcwidth=0.1.7=py36_0
|
||||
- webencodings=0.5=py36_0
|
||||
- wheel=0.30.0=py_1
|
||||
- widgetsnbextension=2.0.1=py36_0
|
||||
- xz=5.2.3=0
|
||||
- zeromq=4.2.1=1
|
||||
- zlib=1.2.11=0
|
||||
prefix: /opt/conda/envs/r2d
|
||||
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
dependencies:
|
||||
- python==3.6.1
|
||||
- python==3.6.*
|
||||
- ipywidgets==6.0.1
|
||||
- jupyterlab==0.30.6
|
||||
- notebook==5.2.2
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ Run in a continuumio/miniconda3 image to ensure portability
|
|||
from datetime import datetime
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
from subprocess import check_call
|
||||
|
||||
from ruamel.yaml import YAML
|
||||
|
|
@ -18,7 +19,10 @@ MINICONDA_VERSION = '4.3.27'
|
|||
HERE = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
ENV_FILE = 'environment.yml'
|
||||
FROZEN_FILE = 'environment.frozen.yml'
|
||||
FROZEN_FILE = os.path.splitext(ENV_FILE)[0] + '.frozen.yml'
|
||||
|
||||
ENV_FILE_T = 'environment.py-{py}.yml'
|
||||
FROZEN_FILE_T = os.path.splitext(ENV_FILE_T)[0] + '.frozen.yml'
|
||||
|
||||
yaml = YAML(typ='rt')
|
||||
|
||||
|
|
@ -78,11 +82,40 @@ def freeze(env_file, frozen_file):
|
|||
'conda config --add channels conda-forge',
|
||||
'conda config --system --set auto_update_conda false',
|
||||
f"conda env create -v -f /r2d/{env_file} -n r2d",
|
||||
f"conda env export -n r2d > /r2d/{frozen_file}",
|
||||
f"conda env export -n r2d >> /r2d/{frozen_file}",
|
||||
])
|
||||
])
|
||||
fixup(HERE / frozen_file)
|
||||
|
||||
|
||||
def set_python(py_env_file, py):
|
||||
"""Set the Python version in an env file"""
|
||||
if os.path.exists(py_env_file):
|
||||
# only clobber auto-generated files
|
||||
with open(py_env_file) as f:
|
||||
text = f.read()
|
||||
if text and 'GENERATED' not in text:
|
||||
return
|
||||
with open(ENV_FILE) as f:
|
||||
env = yaml.load(f)
|
||||
for idx, dep in enumerate(env['dependencies']):
|
||||
if dep.split('=')[0] == 'python':
|
||||
env['dependencies'][idx] = f'python={py}.*'
|
||||
break
|
||||
else:
|
||||
raise ValueError(f"python dependency not found in {env['dependencies']}")
|
||||
# update python dependency
|
||||
with open(py_env_file, 'w') as f:
|
||||
f.write(f"# AUTO GENERATED FROM {ENV_FILE}, DO NOT MANUALLY MODIFY\n")
|
||||
f.write(f"# Generated on {datetime.utcnow():%Y-%m-%d %H:%M:%S UTC}\n")
|
||||
yaml.dump(env, f)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
freeze(ENV_FILE, FROZEN_FILE)
|
||||
for py in ('2.7', '3.5', '3.6'):
|
||||
env_file = ENV_FILE_T.format(py=py)
|
||||
set_python(env_file, py)
|
||||
frozen_file = os.path.splitext(env_file)[0] + '.frozen.yml'
|
||||
freeze(env_file, frozen_file)
|
||||
# use last version as default
|
||||
shutil.copy(frozen_file, FROZEN_FILE)
|
||||
|
|
|
|||
|
|
@ -32,6 +32,12 @@ ${CONDA_DIR}/bin/conda config --system --set show_channel_urls true
|
|||
|
||||
${CONDA_DIR}/bin/conda env update -n root -f /tmp/environment.yml
|
||||
|
||||
if [[ -f /tmp/kernel-environment.yml ]]; then
|
||||
# install kernel env and register kernelspec
|
||||
${CONDA_DIR}/bin/conda env create -n kernel -f /tmp/kernel-environment.yml
|
||||
${CONDA_DIR}/envs/kernel/bin/ipython kernel install --prefix "${CONDA_DIR}"
|
||||
fi
|
||||
|
||||
# Clean things out!
|
||||
${CONDA_DIR}/bin/conda clean -tipsy
|
||||
|
||||
|
|
|
|||
3
setup.py
3
setup.py
|
|
@ -8,7 +8,8 @@ setup(
|
|||
'traitlets',
|
||||
'python-json-logger',
|
||||
'escapism',
|
||||
'jinja2'
|
||||
'jinja2',
|
||||
'ruamel.yaml>=0.15',
|
||||
],
|
||||
python_requires='>=3.4',
|
||||
author='Yuvi Panda',
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
Conda Environment
|
||||
-----------------
|
||||
|
||||
Conda environments files may allow for more complex builds and dependencies. You
|
||||
can specify them in the standard `environment.yml` files.
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
dependencies:
|
||||
- python=2
|
||||
- numpy
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
#!/usr/bin/env python
|
||||
import sys
|
||||
|
||||
assert sys.version_info[:2] == (3, 6), sys.version
|
||||
|
||||
# verify that we have Python 2 and Python 3 kernelspecs
|
||||
from jupyter_client.kernelspec import KernelSpecManager
|
||||
ksm = KernelSpecManager()
|
||||
specs = ksm.get_all_specs()
|
||||
assert sorted(specs) == ['python2', 'python3'], specs.keys()
|
||||
|
||||
# verify that we created the kernel env
|
||||
import json
|
||||
from subprocess import check_output
|
||||
envs = json.loads(check_output(['conda', 'env', 'list', '--json']).decode('utf8'))
|
||||
assert envs == {'envs': ['/srv/conda/envs/kernel']}, envs
|
||||
|
||||
pkgs = json.loads(check_output(['conda', 'list', '-n', 'kernel', '--json']).decode('utf8'))
|
||||
pkg_names = [pkg['name'] for pkg in pkgs]
|
||||
assert 'ipykernel' in pkg_names, pkg_names
|
||||
assert 'numpy' in pkg_names
|
||||
for pkg in pkgs:
|
||||
if pkg['name'] == 'python':
|
||||
assert pkg['version'].startswith('2.7.')
|
||||
break
|
||||
else:
|
||||
assert False, "python not found in %s" % pkg_names
|
||||
Ładowanie…
Reference in New Issue