kopia lustrzana https://github.com/jupyterhub/repo2docker
Merge pull request #207 from yuvipanda/inheritance
Use inheritance to define composed buildpackspull/219/head
Min RK
GitHub
commit
26dac5cc9b
|
|
@ -20,7 +20,7 @@ import pwd
|
|||
|
||||
|
||||
from traitlets.config import Application
|
||||
from traitlets import Unicode, List, default, Tuple, Dict, Int
|
||||
from traitlets import Unicode, List, default, Any, Dict, Int
|
||||
import docker
|
||||
from docker.utils import kwargs_from_env
|
||||
from docker.errors import DockerException
|
||||
|
|
@ -35,15 +35,6 @@ from .utils import execute_cmd, ByteSpecification, maybe_cleanup, is_valid_docke
|
|||
from . import __version__
|
||||
|
||||
|
||||
def compose(buildpacks, parent=None):
|
||||
"""
|
||||
Shortcut to compose many buildpacks together
|
||||
"""
|
||||
image = buildpacks[0](parent=parent)
|
||||
for buildpack in buildpacks[1:]:
|
||||
image = image.compose_with(buildpack(parent=parent))
|
||||
return image
|
||||
|
||||
|
||||
class Repo2Docker(Application):
|
||||
name = 'jupyter-repo2docker'
|
||||
|
|
@ -68,16 +59,12 @@ class Repo2Docker(Application):
|
|||
|
||||
buildpacks = List(
|
||||
[
|
||||
(LegacyBinderDockerBuildPack, ),
|
||||
(DockerBuildPack, ),
|
||||
|
||||
(BaseImage, CondaBuildPack, JuliaBuildPack),
|
||||
(BaseImage, CondaBuildPack),
|
||||
|
||||
(BaseImage, PythonBuildPack, Python2BuildPack, JuliaBuildPack),
|
||||
(BaseImage, PythonBuildPack, JuliaBuildPack),
|
||||
(BaseImage, PythonBuildPack, Python2BuildPack),
|
||||
(BaseImage, PythonBuildPack),
|
||||
LegacyBinderDockerBuildPack(),
|
||||
DockerBuildPack(),
|
||||
JuliaBuildPack(),
|
||||
CondaBuildPack(),
|
||||
Python2BuildPack(),
|
||||
PythonBuildPack()
|
||||
],
|
||||
config=True,
|
||||
help="""
|
||||
|
|
@ -85,8 +72,8 @@ class Repo2Docker(Application):
|
|||
"""
|
||||
)
|
||||
|
||||
default_buildpack = Tuple(
|
||||
(BaseImage, PythonBuildPack),
|
||||
default_buildpack = Any(
|
||||
PythonBuildPack(),
|
||||
config=True,
|
||||
help="""
|
||||
The build pack to use when no buildpacks are found
|
||||
|
|
@ -358,6 +345,7 @@ class Repo2Docker(Application):
|
|||
logHandler = logging.StreamHandler()
|
||||
formatter = jsonlogger.JsonFormatter()
|
||||
logHandler.setFormatter(formatter)
|
||||
self.log = logging.getLogger("repo2docker")
|
||||
self.log.handlers = []
|
||||
self.log.addHandler(logHandler)
|
||||
self.log.setLevel(logging.INFO)
|
||||
|
|
@ -557,10 +545,9 @@ class Repo2Docker(Application):
|
|||
)
|
||||
|
||||
os.chdir(checkout_path)
|
||||
picked_buildpack = compose(self.default_buildpack, parent=self)
|
||||
picked_buildpack = self.default_buildpack
|
||||
|
||||
for bp_spec in self.buildpacks:
|
||||
bp = compose(bp_spec, parent=self)
|
||||
for bp in self.buildpacks:
|
||||
if bp.detect():
|
||||
picked_buildpack = bp
|
||||
break
|
||||
|
|
@ -573,7 +560,7 @@ class Repo2Docker(Application):
|
|||
'NB_USER': self.user_name,
|
||||
'NB_UID': str(self.user_id)
|
||||
}
|
||||
self.log.info('Using %s builder\n', bp.name,
|
||||
self.log.info('Using %s builder\n', bp.__class__.__name__,
|
||||
extra=dict(phase='building'))
|
||||
for l in picked_buildpack.build(self.output_image_spec, self.build_memory_limit, build_args):
|
||||
if 'stream' in l:
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
import textwrap
|
||||
from traitlets.config import LoggingConfigurable
|
||||
from traitlets import Unicode, Set, List, Dict, Tuple, default
|
||||
import jinja2
|
||||
import tarfile
|
||||
import io
|
||||
import os
|
||||
import stat
|
||||
import re
|
||||
import logging
|
||||
import docker
|
||||
|
||||
TEMPLATE = r"""
|
||||
|
|
@ -118,7 +117,7 @@ RUN ./{{ s }}
|
|||
DOC_URL = "http://repo2docker.readthedocs.io/en/latest/samples.html"
|
||||
|
||||
|
||||
class BuildPack(LoggingConfigurable):
|
||||
class BuildPack:
|
||||
"""
|
||||
A composable BuildPack.
|
||||
|
||||
|
|
@ -134,6 +133,10 @@ class BuildPack(LoggingConfigurable):
|
|||
and there are *some* general guarantees of ordering.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.log = logging.getLogger('repo2docker')
|
||||
|
||||
def get_packages(self):
|
||||
"""
|
||||
List of packages that are installed in this BuildPack.
|
||||
|
|
@ -260,49 +263,6 @@ class BuildPack(LoggingConfigurable):
|
|||
"""
|
||||
return []
|
||||
|
||||
name = Unicode(
|
||||
help="""
|
||||
Name of the BuildPack!
|
||||
"""
|
||||
)
|
||||
|
||||
components = Tuple(())
|
||||
|
||||
def compose_with(self, other):
|
||||
"""
|
||||
Compose this BuildPack with another, returning a new one
|
||||
|
||||
Ordering does matter - the properties of the current BuildPack take
|
||||
precedence (wherever that matters) over the properties of other
|
||||
BuildPack. If there are any conflicts, this method is responsible
|
||||
for resolving them.
|
||||
"""
|
||||
result = BuildPack(parent=self)
|
||||
# FIXME: Temporary hack so we can refactor this piece by piece instead of all at once!
|
||||
|
||||
def _merge_dicts(d1, d2):
|
||||
md = {}
|
||||
md.update(d1)
|
||||
md.update(d2)
|
||||
return md
|
||||
|
||||
result.get_packages = lambda: self.get_packages().union(other.get_packages())
|
||||
result.get_base_packages = lambda: self.get_base_packages().union(other.get_base_packages())
|
||||
result.get_path = lambda: self.get_path() + other.get_path()
|
||||
result.get_env = lambda: self.get_env() + other.get_env()
|
||||
result.get_labels = lambda: _merge_dicts(self.get_labels(), other.get_labels())
|
||||
result.get_build_script_files = lambda: _merge_dicts(self.get_build_script_files(), other.get_build_script_files())
|
||||
result.get_build_scripts = lambda: self.get_build_scripts() + other.get_build_scripts()
|
||||
result.get_assemble_scripts = lambda: self.get_assemble_scripts() + other.get_assemble_scripts()
|
||||
result.get_post_build_scripts = lambda: self.get_post_build_scripts() + other.get_post_build_scripts()
|
||||
|
||||
|
||||
result.name = "{}-{}".format(self.name, other.name)
|
||||
|
||||
result.components = ((self, ) + self.components +
|
||||
(other, ) + other.components)
|
||||
return result
|
||||
|
||||
def binder_path(self, path):
|
||||
"""Locate a file"""
|
||||
if os.path.exists('binder'):
|
||||
|
|
@ -311,7 +271,7 @@ class BuildPack(LoggingConfigurable):
|
|||
return path
|
||||
|
||||
def detect(self):
|
||||
return all([p.detect() for p in self.components])
|
||||
return True
|
||||
|
||||
def render(self):
|
||||
"""
|
||||
|
|
@ -407,9 +367,6 @@ class BuildPack(LoggingConfigurable):
|
|||
|
||||
|
||||
class BaseImage(BuildPack):
|
||||
name = "repo2docker"
|
||||
version = "0.1"
|
||||
|
||||
def get_env(self):
|
||||
return [
|
||||
("APP_BASE", "/srv")
|
||||
|
|
|
|||
|
|
@ -6,9 +6,8 @@ import os
|
|||
import re
|
||||
|
||||
from ruamel.yaml import YAML
|
||||
from traitlets import default, Unicode
|
||||
|
||||
from ..base import BuildPack
|
||||
from ..base import BaseImage
|
||||
|
||||
# pattern for parsing conda dependency line
|
||||
PYTHON_REGEX = re.compile(r'python\s*=+\s*([\d\.]*)')
|
||||
|
|
@ -16,20 +15,18 @@ PYTHON_REGEX = re.compile(r'python\s*=+\s*([\d\.]*)')
|
|||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
class CondaBuildPack(BuildPack):
|
||||
name = "conda"
|
||||
version = "0.1"
|
||||
class CondaBuildPack(BaseImage):
|
||||
def get_env(self):
|
||||
return [
|
||||
return super().get_env() + [
|
||||
('CONDA_DIR', '${APP_BASE}/conda'),
|
||||
('NB_PYTHON_PREFIX', '${CONDA_DIR}'),
|
||||
]
|
||||
|
||||
def get_path(self):
|
||||
return ['${CONDA_DIR}/bin']
|
||||
return super().get_path() + ['${CONDA_DIR}/bin']
|
||||
|
||||
def get_build_scripts(self):
|
||||
return [
|
||||
return super().get_build_scripts() + [
|
||||
(
|
||||
"root",
|
||||
r"""
|
||||
|
|
@ -68,6 +65,7 @@ class CondaBuildPack(BuildPack):
|
|||
else:
|
||||
self.log.warning("No frozen env: %s", py_frozen_name)
|
||||
files['conda/' + frozen_name] = '/tmp/environment.yml'
|
||||
files.update(super().get_build_script_files())
|
||||
return files
|
||||
|
||||
@property
|
||||
|
|
@ -123,7 +121,7 @@ class CondaBuildPack(BuildPack):
|
|||
conda clean -tipsy
|
||||
""".format(env_name, environment_yml)
|
||||
))
|
||||
return assembly_scripts
|
||||
return super().get_assemble_scripts() + assembly_scripts
|
||||
|
||||
|
||||
def detect(self):
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ from .base import BuildPack
|
|||
|
||||
|
||||
class DockerBuildPack(BuildPack):
|
||||
name = "Dockerfile"
|
||||
dockerfile = "Dockerfile"
|
||||
|
||||
def detect(self):
|
||||
|
|
|
|||
|
|
@ -1,17 +1,19 @@
|
|||
"""
|
||||
Generates a variety of Dockerfiles based on an input matrix
|
||||
"""
|
||||
from traitlets import default
|
||||
import os
|
||||
from .base import BuildPack
|
||||
from .conda import CondaBuildPack
|
||||
|
||||
|
||||
class JuliaBuildPack(BuildPack):
|
||||
name = "julia"
|
||||
version = "0.1"
|
||||
class JuliaBuildPack(CondaBuildPack):
|
||||
"""
|
||||
Julia + Conda build pack
|
||||
|
||||
Julia does not work with Virtual Envs,
|
||||
see https://github.com/JuliaPy/PyCall.jl/issues/410
|
||||
"""
|
||||
def get_env(self):
|
||||
return [
|
||||
return super().get_env() + [
|
||||
('JULIA_PATH', '${APP_BASE}/julia'),
|
||||
('JULIA_HOME', '${JULIA_PATH}/bin'),
|
||||
('JULIA_PKGDIR', '${JULIA_PATH}/pkg'),
|
||||
|
|
@ -20,10 +22,10 @@ class JuliaBuildPack(BuildPack):
|
|||
]
|
||||
|
||||
def get_path(self):
|
||||
return ['${JULIA_PATH}/bin']
|
||||
return super().get_path() + ['${JULIA_PATH}/bin']
|
||||
|
||||
def get_build_scripts(self):
|
||||
return [
|
||||
return super().get_build_scripts() + [
|
||||
(
|
||||
"root",
|
||||
r"""
|
||||
|
|
@ -51,7 +53,7 @@ class JuliaBuildPack(BuildPack):
|
|||
|
||||
def get_assemble_scripts(self):
|
||||
require = self.binder_path('REQUIRE')
|
||||
return [(
|
||||
return super().get_assemble_scripts() + [(
|
||||
"${NB_USER}",
|
||||
# Pre-compile all libraries if they've opted into it. `using {libraryname}` does the
|
||||
# right thing
|
||||
|
|
@ -67,4 +69,4 @@ class JuliaBuildPack(BuildPack):
|
|||
)]
|
||||
|
||||
def detect(self):
|
||||
return os.path.exists(self.binder_path('REQUIRE')) and super()
|
||||
return os.path.exists(self.binder_path('REQUIRE')) and super().detect()
|
||||
|
|
|
|||
|
|
@ -1,16 +1,14 @@
|
|||
"""
|
||||
Generates a variety of Dockerfiles based on an input matrix
|
||||
"""
|
||||
from traitlets import Unicode
|
||||
from textwrap import dedent
|
||||
from .docker import DockerBuildPack
|
||||
|
||||
class LegacyBinderDockerBuildPack(DockerBuildPack):
|
||||
|
||||
name = 'Legacy Binder Dockerfile'
|
||||
dockerfile = '._binder.Dockerfile'
|
||||
|
||||
dockerfile_appendix = Unicode(dedent(r"""
|
||||
dockerfile_appendix = dedent(r"""
|
||||
USER root
|
||||
COPY . /home/main/notebooks
|
||||
RUN chown -R main:main /home/main/notebooks
|
||||
|
|
@ -25,7 +23,7 @@ class LegacyBinderDockerBuildPack(DockerBuildPack):
|
|||
/home/main/anaconda2/bin/ipython kernel install --sys-prefix
|
||||
ENV JUPYTER_PATH /home/main/anaconda2/share/jupyter:$JUPYTER_PATH
|
||||
CMD jupyter notebook --ip 0.0.0.0
|
||||
"""), config=True)
|
||||
""")
|
||||
|
||||
def render(self):
|
||||
with open('Dockerfile') as f:
|
||||
|
|
|
|||
|
|
@ -1,42 +1,40 @@
|
|||
"""
|
||||
Generates a variety of Dockerfiles based on an input matrix
|
||||
"""
|
||||
from traitlets import default
|
||||
import os
|
||||
from ..base import BuildPack
|
||||
from ..base import BaseImage
|
||||
|
||||
|
||||
class PythonBuildPack(BuildPack):
|
||||
name = "python3.5"
|
||||
version = "0.1"
|
||||
|
||||
class PythonBuildPack(BaseImage):
|
||||
def get_packages(self):
|
||||
return {
|
||||
return super().get_packages().union({
|
||||
'python3',
|
||||
'python3-venv',
|
||||
'python3-dev',
|
||||
}
|
||||
})
|
||||
|
||||
def get_env(self):
|
||||
return [
|
||||
return super().get_env() + [
|
||||
("VENV_PATH", "${APP_BASE}/venv"),
|
||||
# Prefix to use for installing kernels and finding jupyter binary
|
||||
("NB_PYTHON_PREFIX", "${VENV_PATH}"),
|
||||
]
|
||||
|
||||
def get_path(self):
|
||||
return [
|
||||
return super().get_path() + [
|
||||
"${VENV_PATH}/bin"
|
||||
]
|
||||
|
||||
|
||||
def get_build_script_files(self):
|
||||
return {
|
||||
files = {
|
||||
'python/requirements.frozen.txt': '/tmp/requirements.frozen.txt',
|
||||
}
|
||||
files.update(super().get_build_script_files())
|
||||
return files
|
||||
|
||||
def get_build_scripts(self):
|
||||
return [
|
||||
return super().get_build_scripts() + [
|
||||
(
|
||||
"root",
|
||||
r"""
|
||||
|
|
@ -69,6 +67,7 @@ class PythonBuildPack(BuildPack):
|
|||
# be installed in the python2 venv, and requirements3.txt
|
||||
# will be installed in python3 venv. This is less of a
|
||||
# surprise than requiring python2 to be requirements2.txt tho.
|
||||
assemble_scripts = super().get_assemble_scripts()
|
||||
try:
|
||||
with open(self.binder_path('runtime.txt')) as f:
|
||||
runtime = f.read().strip()
|
||||
|
|
@ -79,45 +78,44 @@ class PythonBuildPack(BuildPack):
|
|||
else:
|
||||
requirements_file = self.binder_path('requirements.txt')
|
||||
if os.path.exists(requirements_file):
|
||||
return [(
|
||||
assemble_scripts.append((
|
||||
'${NB_USER}',
|
||||
'pip3 install --no-cache-dir -r "{}"'.format(requirements_file)
|
||||
)]
|
||||
return []
|
||||
))
|
||||
return assemble_scripts
|
||||
|
||||
def detect(self):
|
||||
return os.path.exists('requirements.txt') and super().detect()
|
||||
|
||||
|
||||
class Python2BuildPack(BuildPack):
|
||||
name = "python2.7"
|
||||
version = "0.1"
|
||||
|
||||
class Python2BuildPack(PythonBuildPack):
|
||||
def get_packages(self):
|
||||
return {
|
||||
'python',
|
||||
'python-dev',
|
||||
'virtualenv'
|
||||
}
|
||||
return super().get_packages().union({
|
||||
'python',
|
||||
'python-dev',
|
||||
'virtualenv'
|
||||
})
|
||||
|
||||
|
||||
def get_env(self):
|
||||
return [
|
||||
return super().get_env() + [
|
||||
('VENV2_PATH', '${APP_BASE}/venv2')
|
||||
]
|
||||
|
||||
def get_path(self):
|
||||
return [
|
||||
return super().get_path() + [
|
||||
"${VENV2_PATH}/bin"
|
||||
]
|
||||
|
||||
def get_build_script_files(self):
|
||||
return {
|
||||
files = {
|
||||
'python/requirements2.frozen.txt': '/tmp/requirements2.frozen.txt',
|
||||
}
|
||||
files.update(super().get_build_script_files())
|
||||
return files
|
||||
|
||||
def get_build_scripts(self):
|
||||
return [
|
||||
return super().get_build_scripts() + [
|
||||
(
|
||||
"root",
|
||||
r"""
|
||||
|
|
@ -141,7 +139,7 @@ class Python2BuildPack(BuildPack):
|
|||
]
|
||||
|
||||
def get_assemble_scripts(self):
|
||||
return [
|
||||
return super().get_assemble_scripts() + [
|
||||
(
|
||||
'${NB_USER}',
|
||||
'pip2 install --no-cache-dir -r requirements.txt'
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue