mirror
/
repo2docker
kopia lustrzana https://github.com/jupyterhub/repo2docker
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge pull request #298 from minrk/always-conda-python 

Always install Python with conda
pull/306/head
Tim Head 2018-05-08 10:33:03 +02:00 zatwierdzone przez GitHub
rodzic 85f69a74df 2b2eaa8b69
commit f73db1121e
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
13 zmienionych plików z 72 dodań i 373 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

3
repo2docker/app.py
Wyświetl plik

@ -29,7 +29,7 @@ from traitlets.config import Application
from . import __version__
from .buildpacks import (
PythonBuildPack, DockerBuildPack, LegacyBinderDockerBuildPack,
CondaBuildPack, JuliaBuildPack, Python2BuildPack, BaseImage,
CondaBuildPack, JuliaBuildPack, BaseImage,
RBuildPack
)
from .utils import (
@ -67,7 +67,6 @@ class Repo2Docker(Application):
DockerBuildPack,
JuliaBuildPack,
CondaBuildPack,
Python2BuildPack,
RBuildPack,
PythonBuildPack,
],

2
repo2docker/buildpacks/__init__.py
Wyświetl plik

@ -1,5 +1,5 @@
from .base import BuildPack, BaseImage
from .python import Python2BuildPack, PythonBuildPack
from .python import PythonBuildPack
from .conda import CondaBuildPack
from .julia import JuliaBuildPack
from .docker import DockerBuildPack

13
repo2docker/buildpacks/conda/__init__.py
Wyświetl plik

@ -25,17 +25,26 @@ class CondaBuildPack(BaseImage):
the `NB_PYTHON_PREFIX` to the location of the jupyter binary.
"""
return super().get_env() + [
env = super().get_env() + [
('CONDA_DIR', '${APP_BASE}/conda'),
('NB_PYTHON_PREFIX', '${CONDA_DIR}'),
]
if self.py2:
env.append(('KERNEL_PYTHON_PREFIX', '${CONDA_DIR}/envs/kernel'))
else:
env.append(('KERNEL_PYTHON_PREFIX', '${NB_PYTHON_PREFIX}'))
return env
def get_path(self):
"""Return paths (including conda environment path) to be added to
the PATH environment variable.
"""
return super().get_path() + ['${CONDA_DIR}/bin']
path = super().get_path()
if self.py2:
path.insert(0, '${KERNEL_PYTHON_PREFIX}/bin')
path.insert(0, '${CONDA_DIR}/bin')
return path
def get_build_scripts(self):
"""

293
repo2docker/buildpacks/python/__init__.py
Wyświetl plik

@ -1,139 +1,70 @@
"""Generates Dockerfiles based on an input matrix based on Python."""
import os
from ..base import BaseImage
from ..conda import CondaBuildPack
class PythonBuildPack(BaseImage):
"""Setup Python 3 for use with a repository."""
def get_packages(self):
"""Return a list of the Python 3 core language packages to be installed
via apt-get for this BuildPack.
class PythonBuildPack(CondaBuildPack):
"""Setup Python for use with a repository."""
Note: The packages specified here are for the core Python3 language.
Third party libraries are specified in other configuration files.
@property
def python_version(self):
if hasattr(self, '_python_version'):
return self._python_version
"""
return super().get_packages().union({
'python3',
'python3-venv',
'python3-dev',
})
try:
with open(self.binder_path('runtime.txt')) as f:
runtime = f.read().strip()
except FileNotFoundError:
runtime = ''
def get_env(self):
"""
Return environment variables to be set.
if not runtime.startswith('python-'):
# not a Python runtime (e.g. R, which subclasses this)
# use the default Python
self._python_version = self.major_pythons['3']
return self._python_version
We set `VENV_PATH` to the virtual environment location and
the `NB_PYTHON_PREFIX` to the location of the jupyter binary.
"""
return super().get_env() + [
("VENV_PATH", "${APP_BASE}/venv"),
# Prefix to use for installing kernels and finding jupyter binary
("NB_PYTHON_PREFIX", "${VENV_PATH}"),
]
def get_path(self):
"""Return paths (including virtual environment path) to be added to
the PATH environment variable.
"""
return super().get_path() + [
"${VENV_PATH}/bin"
]
def get_build_script_files(self):
"""
Dict of files to be copied to the container image for use in building.
This is copied before the `build_scripts` & `assemble_scripts` are
run, so can be executed from either of them.
It's a dictionary where the key is the source file path in the host
system, and the value is the destination file path inside the
container image.
This currently adds a frozen set of Python 3 requirements to the dict
of files.
"""
files = {
'python/requirements.frozen.txt': '/tmp/requirements.frozen.txt',
}
files.update(super().get_build_script_files())
return files
def get_build_scripts(self):
"""
Return series of build-steps common to all Python 3 repositories.
All scripts here should be independent of contents of the repository.
This sets up:
- a directory for the virtual environment and its ownership by the
notebook user
- a Python 3 interpreter for the virtual environement
- a Python 3 jupyter kernel including a base set of requirements
- support for Jupyter widgets
- support for JupyterLab
- support for nteract
"""
return super().get_build_scripts() + [
(
"root",
r"""
mkdir -p ${VENV_PATH} && \
chown -R ${NB_USER}:${NB_USER} ${VENV_PATH}
"""
),
(
"${NB_USER}",
r"""
python3 -m venv ${VENV_PATH}
"""
),
(
"${NB_USER}",
r"""
pip install --no-cache-dir -r /tmp/requirements.frozen.txt && \
jupyter nbextension enable --py widgetsnbextension --sys-prefix && \
jupyter serverextension enable --py jupyterlab --sys-prefix && \
jupyter serverextension enable nteract_on_jupyter --sys-prefix
"""
)
]
py_version_info = runtime.split('-', 1)[1].split('.')
py_version = ''
if len(py_version_info) == 1:
py_version = self.major_pythons[py_version_info[0]]
else:
# get major.minor
py_version = '.'.join(py_version_info[:2])
self._python_version = py_version
return self._python_version
def get_assemble_scripts(self):
"""Return series of build-steps specific to this repository.
"""
# If we have a runtime.txt & that's set to python-2.7,
# we will *not* install requirements.txt but will find &
# install a requirements3.txt file if it exists.
# This way, when using python2 venv, requirements.txt will
# be installed in the python2 venv, and requirements3.txt
# will be installed in python3 venv. This is less of a
# surprise than requiring python2 to be requirements2.txt tho.
# requirements.txt will be installed in the *kernel* env
# and requirements3.txt (if it exists)
# will be installed in the python 3 notebook server env.
assemble_scripts = super().get_assemble_scripts()
setup_py = 'setup.py'
try:
with open(self.binder_path('runtime.txt')) as f:
runtime = f.read().strip()
except FileNotFoundError:
runtime = 'python-3.5'
if runtime == 'python-2.7':
pip = "pip2"
requirements_file = self.binder_path('requirements3.txt')
else:
pip = "pip3"
requirements_file = self.binder_path('requirements.txt')
# KERNEL_PYTHON_PREFIX is the env with the kernel,
# whether it's distinct from the notebook or the same.
pip = '${KERNEL_PYTHON_PREFIX}/bin/pip'
if self.py2:
# using python 2 kernel,
# requirements3.txt allows installation in the notebook server env
nb_requirements_file = self.binder_path('requirements3.txt')
if os.path.exists(nb_requirements_file):
assemble_scripts.append((
'${NB_USER}',
'${NB_PYTHON_PREFIX}/bin/pip install --no-cache-dir -r "{}"'.format(nb_requirements_file)
))
# install requirements.txt in the kernel env
requirements_file = self.binder_path('requirements.txt')
if os.path.exists(requirements_file):
assemble_scripts.append((
'${NB_USER}',
'pip3 install --no-cache-dir -r "{}"'.format(requirements_file)
'{} install --no-cache-dir -r "{}"'.format(pip, requirements_file)
))
# setup.py exists *and* binder dir is not used
if not os.path.exists('binder') and os.path.exists(setup_py):
assemble_scripts.append((
'${NB_USER}',
@ -142,7 +73,7 @@ class PythonBuildPack(BaseImage):
return assemble_scripts
def detect(self):
"""Check if current repo should be built with the Python 3 Build pack.
"""Check if current repo should be built with the Python buildpack.
"""
requirements_txt = self.binder_path('requirements.txt')
runtime_txt = self.binder_path('runtime.txt')
@ -151,136 +82,10 @@ class PythonBuildPack(BaseImage):
if os.path.exists(runtime_txt):
with open(runtime_txt) as f:
runtime = f.read().strip()
if runtime.startswith("python-3"):
if runtime.startswith("python-"):
return True
else:
return False
if not os.path.exists('binder') and os.path.exists(setup_py):
return True
return os.path.exists(requirements_txt)
class Python2BuildPack(PythonBuildPack):
"""Setup Python 2 for use with a repository."""
def get_packages(self):
"""Return a list of the Python 2 core language packages to be installed
via apt-get for this BuildPack.
Note: The packages specified here are for the core Python2 language.
Third party libraries are specified in other configuration files.
"""
return super().get_packages().union({
'python',
'python-dev',
'virtualenv'
})
def get_env(self):
"""
Return environment variables to be set.
We set `VENV_PATH` to the virtual environment location containing
Python 2.
"""
return super().get_env() + [
('VENV2_PATH', '${APP_BASE}/venv2')
]
def get_path(self):
"""Return paths (including virtual environment path) to be added to
the PATH environment variable.
"""
return super().get_path() + [
"${VENV2_PATH}/bin"
]
def get_build_script_files(self):
"""
Dict of files to be copied to the container image for use in building.
This is copied before the `build_scripts` & `assemble_scripts` are
run, so can be executed from either of them.
It's a dictionary where the key is the source file path in the host
system, and the value is the destination file path inside the
container image.
This currently adds a frozen set of Python 2 requirements to the dict
of files.
"""
files = {
'python/requirements2.frozen.txt': '/tmp/requirements2.frozen.txt',
}
files.update(super().get_build_script_files())
return files
def get_build_scripts(self):
"""
Return series of build-steps common to all Python 2 repositories.
All scripts here should be independent of contents of the repository.
This sets up:
- a directory for the virtual environment and its ownership by the
notebook user
- a Python 2 interpreter for the virtual environement
- a Python 2 jupyter kernel
"""
return super().get_build_scripts() + [
(
"root",
r"""
mkdir -p ${VENV2_PATH} && \
chown -R ${NB_USER}:${NB_USER} ${VENV2_PATH}
"""
),
(
"${NB_USER}",
r"""
virtualenv -p python2 ${VENV2_PATH}
"""
),
(
"${NB_USER}",
r"""
pip2 install --no-cache-dir -r /tmp/requirements2.frozen.txt && \
python2 -m ipykernel install --prefix=${NB_PYTHON_PREFIX}
"""
)
]
def get_assemble_scripts(self):
"""Return series of build-steps specific to this repository.
"""
requirements_txt = self.binder_path('requirements.txt')
assemble_scripts = super().get_assemble_scripts()
if os.path.exists(requirements_txt):
assemble_scripts.insert(0, (
'${NB_USER}',
'pip2 install --no-cache-dir -r "{}"'.format(requirements_txt)
))
return assemble_scripts
def detect(self):
"""Check if current repo should be built with the Python 2 Build pack.
"""
runtime_txt = self.binder_path('runtime.txt')
if os.path.exists(runtime_txt):
with open(runtime_txt) as f:
runtime = f.read().strip()
if runtime == 'python-2.7':
return True
elif runtime.startswith('python-2'):
raise ValueError(
"Only python-2.7 or python-3.x is supported in "
"runtime.txt, not '{}'".format(runtime_txt))
else:
return False
return False

39
repo2docker/buildpacks/python/freeze.bash
Wyświetl plik

@ -1,39 +0,0 @@
#!/bin/bash
set -euo pipefail
# Freeze requirements.txt into requirements.frozen.txt, pinning all dependent library versions to
# versions that are resolved at time of freezing.
# Does the same for requirements2.txt to requirements2.frozen.txt...
# cd to the directory where the freeze script is located
if [[ ! -z "$(which realpath 2>/dev/null)" ]]; then
realpath=realpath
else
realpath="readlink -f"
fi
cd $(dirname "$($realpath "$0")")
function freeze-requirements {
# Freeze a requirements file $2 into a frozen requirements file $3
# Requires that a completely empty venv of appropriate version exist in $1
PYTHON_VERSION="$1"
REQUIREMENTS_FILE="$2"
FROZEN_FILE="$3"
if [[ $(echo ${PYTHON_VERSION} | cut -d. -f 1) == "2" ]]; then
VENV=virtualenv
else
VENV=venv
fi
echo "# AUTO GENERATED FROM ${REQUIREMENTS_FILE}, DO NOT MANUALLY MODIFY" > ${FROZEN_FILE}
echo "# Frozen on $(date -u)" >> ${FROZEN_FILE}
docker run --rm -v $PWD:/python -it python:${PYTHON_VERSION} \
sh -c "
python -m $VENV /venv
/venv/bin/pip install -r /python/${REQUIREMENTS_FILE} &&
/venv/bin/pip freeze | sort --ignore-case >> /python/${FROZEN_FILE}"
}
freeze-requirements 3.5 requirements.txt requirements.frozen.txt
freeze-requirements 2.7 requirements2.txt requirements2.frozen.txt

42
repo2docker/buildpacks/python/requirements.frozen.txt
Wyświetl plik

@ -1,42 +0,0 @@
# AUTO GENERATED FROM requirements.txt, DO NOT MANUALLY MODIFY
# Frozen on Mon Apr 2 07:17:48 UTC 2018
bleach==2.1.3
decorator==4.2.1
entrypoints==0.2.3
html5lib==1.0.1
ipykernel==4.8.2
ipython-genutils==0.2.0
ipython==6.2.1
ipywidgets==7.1.1
jedi==0.11.1
Jinja2==2.10
jsonschema==2.6.0
jupyter-client==5.2.3
jupyter-core==4.4.0
jupyterlab-launcher==0.10.5
jupyterlab==0.31.5
MarkupSafe==1.0
mistune==0.8.3
nbconvert==5.3.1
nbformat==4.4.0
notebook==5.4.1
nteract-on-jupyter==1.6.0
pandocfilters==1.4.2
parso==0.1.1
pexpect==4.4.0
pickleshare==0.7.4
prompt-toolkit==1.0.15
ptyprocess==0.5.2
Pygments==2.2.0
python-dateutil==2.7.2
pyzmq==17.0.0
Send2Trash==1.5.0
simplegeneric==0.8.1
six==1.11.0
terminado==0.8.1
testpath==0.3.1
tornado==4.5.3
traitlets==4.3.2
wcwidth==0.1.7
webencodings==0.5.1
widgetsnbextension==3.1.4

5
repo2docker/buildpacks/python/requirements.txt
Wyświetl plik

@ -1,5 +0,0 @@
notebook==5.4.1
tornado==4.5.3
ipywidgets==7.1.1
jupyterlab==0.31.5
nteract_on_jupyter==1.6.0

27
repo2docker/buildpacks/python/requirements2.frozen.txt
Wyświetl plik

@ -1,27 +0,0 @@
# AUTO GENERATED FROM requirements2.txt, DO NOT MANUALLY MODIFY
# Frozen on Mon Apr 2 07:19:00 UTC 2018
backports-abc==0.5
backports.shutil-get-terminal-size==1.0.0
certifi==2018.1.18
decorator==4.2.1
enum34==1.1.6
ipykernel==4.8.2
ipython-genutils==0.2.0
ipython==5.5.0
jupyter-client==5.2.3
jupyter-core==4.4.0
pathlib2==2.3.0
pexpect==4.4.0
pickleshare==0.7.4
prompt-toolkit==1.0.15
ptyprocess==0.5.2
Pygments==2.2.0
python-dateutil==2.7.2
pyzmq==17.0.0
scandir==1.7
simplegeneric==0.8.1
singledispatch==3.4.0.3
six==1.11.0
tornado==4.5.3
traitlets==4.3.2
wcwidth==0.1.7

2
repo2docker/buildpacks/python/requirements2.txt
Wyświetl plik

@ -1,2 +0,0 @@
ipykernel==4.8.2
tornado==4.5.3

4
tests/venv/py2/README.rst
Wyświetl plik

@ -3,3 +3,7 @@ System - Specifying runtime environments
You can specify runtime environments (such as Python 2 or 3) with a
``runtime.txt`` file.
You can specify just the major version,
such as `python-2` or `python-3`,
or a minor version, such as `python-3.5`.

9
tests/venv/py3/README.rst
Wyświetl plik

@ -1,9 +0,0 @@
System - Specifying runtime environments
----------------------------------------
You can specify runtime environments (such as Python 2 or 3) with a
``runtime.txt`` file.
This is an example that selects Python 3. Currently you can not use
this to select a sepcific version of Python 3 (e.g. 3.4 vs 3.6). If you
need this level of control we recommend you use a `environment.yml`.

1
tests/venv/py35/runtime.txt 100644
Wyświetl plik

@ -0,0 +1 @@
python-3.5

5
tests/venv/py35/verify 100755
Wyświetl plik

@ -0,0 +1,5 @@
#!/usr/bin/env python3
import sys
print(sys.version_info)
assert sys.version_info[:2] == (3, 5), sys.version