Merge pull request #207 from yuvipanda/inheritance

Use inheritance to define composed buildpacks
pull/219/head
Min RK 2018-02-02 09:36:38 +01:00 zatwierdzone przez GitHub
commit 26dac5cc9b
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
7 zmienionych plików z 68 dodań i 129 usunięć

Wyświetl plik

@ -20,7 +20,7 @@ import pwd
from traitlets.config import Application
from traitlets import Unicode, List, default, Tuple, Dict, Int
from traitlets import Unicode, List, default, Any, Dict, Int
import docker
from docker.utils import kwargs_from_env
from docker.errors import DockerException
@ -35,15 +35,6 @@ from .utils import execute_cmd, ByteSpecification, maybe_cleanup, is_valid_docke
from . import __version__
def compose(buildpacks, parent=None):
"""
Shortcut to compose many buildpacks together
"""
image = buildpacks[0](parent=parent)
for buildpack in buildpacks[1:]:
image = image.compose_with(buildpack(parent=parent))
return image
class Repo2Docker(Application):
name = 'jupyter-repo2docker'
@ -68,16 +59,12 @@ class Repo2Docker(Application):
buildpacks = List(
[
(LegacyBinderDockerBuildPack, ),
(DockerBuildPack, ),
(BaseImage, CondaBuildPack, JuliaBuildPack),
(BaseImage, CondaBuildPack),
(BaseImage, PythonBuildPack, Python2BuildPack, JuliaBuildPack),
(BaseImage, PythonBuildPack, JuliaBuildPack),
(BaseImage, PythonBuildPack, Python2BuildPack),
(BaseImage, PythonBuildPack),
LegacyBinderDockerBuildPack(),
DockerBuildPack(),
JuliaBuildPack(),
CondaBuildPack(),
Python2BuildPack(),
PythonBuildPack()
],
config=True,
help="""
@ -85,8 +72,8 @@ class Repo2Docker(Application):
"""
)
default_buildpack = Tuple(
(BaseImage, PythonBuildPack),
default_buildpack = Any(
PythonBuildPack(),
config=True,
help="""
The build pack to use when no buildpacks are found
@ -358,6 +345,7 @@ class Repo2Docker(Application):
logHandler = logging.StreamHandler()
formatter = jsonlogger.JsonFormatter()
logHandler.setFormatter(formatter)
self.log = logging.getLogger("repo2docker")
self.log.handlers = []
self.log.addHandler(logHandler)
self.log.setLevel(logging.INFO)
@ -557,10 +545,9 @@ class Repo2Docker(Application):
)
os.chdir(checkout_path)
picked_buildpack = compose(self.default_buildpack, parent=self)
picked_buildpack = self.default_buildpack
for bp_spec in self.buildpacks:
bp = compose(bp_spec, parent=self)
for bp in self.buildpacks:
if bp.detect():
picked_buildpack = bp
break
@ -573,7 +560,7 @@ class Repo2Docker(Application):
'NB_USER': self.user_name,
'NB_UID': str(self.user_id)
}
self.log.info('Using %s builder\n', bp.name,
self.log.info('Using %s builder\n', bp.__class__.__name__,
extra=dict(phase='building'))
for l in picked_buildpack.build(self.output_image_spec, self.build_memory_limit, build_args):
if 'stream' in l:

Wyświetl plik

@ -1,12 +1,11 @@
import textwrap
from traitlets.config import LoggingConfigurable
from traitlets import Unicode, Set, List, Dict, Tuple, default
import jinja2
import tarfile
import io
import os
import stat
import re
import logging
import docker
TEMPLATE = r"""
@ -118,7 +117,7 @@ RUN ./{{ s }}
DOC_URL = "http://repo2docker.readthedocs.io/en/latest/samples.html"
class BuildPack(LoggingConfigurable):
class BuildPack:
"""
A composable BuildPack.
@ -134,6 +133,10 @@ class BuildPack(LoggingConfigurable):
and there are *some* general guarantees of ordering.
"""
def __init__(self):
self.log = logging.getLogger('repo2docker')
def get_packages(self):
"""
List of packages that are installed in this BuildPack.
@ -260,49 +263,6 @@ class BuildPack(LoggingConfigurable):
"""
return []
name = Unicode(
help="""
Name of the BuildPack!
"""
)
components = Tuple(())
def compose_with(self, other):
"""
Compose this BuildPack with another, returning a new one
Ordering does matter - the properties of the current BuildPack take
precedence (wherever that matters) over the properties of other
BuildPack. If there are any conflicts, this method is responsible
for resolving them.
"""
result = BuildPack(parent=self)
# FIXME: Temporary hack so we can refactor this piece by piece instead of all at once!
def _merge_dicts(d1, d2):
md = {}
md.update(d1)
md.update(d2)
return md
result.get_packages = lambda: self.get_packages().union(other.get_packages())
result.get_base_packages = lambda: self.get_base_packages().union(other.get_base_packages())
result.get_path = lambda: self.get_path() + other.get_path()
result.get_env = lambda: self.get_env() + other.get_env()
result.get_labels = lambda: _merge_dicts(self.get_labels(), other.get_labels())
result.get_build_script_files = lambda: _merge_dicts(self.get_build_script_files(), other.get_build_script_files())
result.get_build_scripts = lambda: self.get_build_scripts() + other.get_build_scripts()
result.get_assemble_scripts = lambda: self.get_assemble_scripts() + other.get_assemble_scripts()
result.get_post_build_scripts = lambda: self.get_post_build_scripts() + other.get_post_build_scripts()
result.name = "{}-{}".format(self.name, other.name)
result.components = ((self, ) + self.components +
(other, ) + other.components)
return result
def binder_path(self, path):
"""Locate a file"""
if os.path.exists('binder'):
@ -311,7 +271,7 @@ class BuildPack(LoggingConfigurable):
return path
def detect(self):
return all([p.detect() for p in self.components])
return True
def render(self):
"""
@ -407,9 +367,6 @@ class BuildPack(LoggingConfigurable):
class BaseImage(BuildPack):
name = "repo2docker"
version = "0.1"
def get_env(self):
return [
("APP_BASE", "/srv")

Wyświetl plik

@ -6,9 +6,8 @@ import os
import re
from ruamel.yaml import YAML
from traitlets import default, Unicode
from ..base import BuildPack
from ..base import BaseImage
# pattern for parsing conda dependency line
PYTHON_REGEX = re.compile(r'python\s*=+\s*([\d\.]*)')
@ -16,20 +15,18 @@ PYTHON_REGEX = re.compile(r'python\s*=+\s*([\d\.]*)')
HERE = os.path.dirname(os.path.abspath(__file__))
class CondaBuildPack(BuildPack):
name = "conda"
version = "0.1"
class CondaBuildPack(BaseImage):
def get_env(self):
return [
return super().get_env() + [
('CONDA_DIR', '${APP_BASE}/conda'),
('NB_PYTHON_PREFIX', '${CONDA_DIR}'),
]
def get_path(self):
return ['${CONDA_DIR}/bin']
return super().get_path() + ['${CONDA_DIR}/bin']
def get_build_scripts(self):
return [
return super().get_build_scripts() + [
(
"root",
r"""
@ -68,6 +65,7 @@ class CondaBuildPack(BuildPack):
else:
self.log.warning("No frozen env: %s", py_frozen_name)
files['conda/' + frozen_name] = '/tmp/environment.yml'
files.update(super().get_build_script_files())
return files
@property
@ -123,7 +121,7 @@ class CondaBuildPack(BuildPack):
conda clean -tipsy
""".format(env_name, environment_yml)
))
return assembly_scripts
return super().get_assemble_scripts() + assembly_scripts
def detect(self):

Wyświetl plik

@ -7,7 +7,6 @@ from .base import BuildPack
class DockerBuildPack(BuildPack):
name = "Dockerfile"
dockerfile = "Dockerfile"
def detect(self):

Wyświetl plik

@ -1,17 +1,19 @@
"""
Generates a variety of Dockerfiles based on an input matrix
"""
from traitlets import default
import os
from .base import BuildPack
from .conda import CondaBuildPack
class JuliaBuildPack(BuildPack):
name = "julia"
version = "0.1"
class JuliaBuildPack(CondaBuildPack):
"""
Julia + Conda build pack
Julia does not work with Virtual Envs,
see https://github.com/JuliaPy/PyCall.jl/issues/410
"""
def get_env(self):
return [
return super().get_env() + [
('JULIA_PATH', '${APP_BASE}/julia'),
('JULIA_HOME', '${JULIA_PATH}/bin'),
('JULIA_PKGDIR', '${JULIA_PATH}/pkg'),
@ -20,10 +22,10 @@ class JuliaBuildPack(BuildPack):
]
def get_path(self):
return ['${JULIA_PATH}/bin']
return super().get_path() + ['${JULIA_PATH}/bin']
def get_build_scripts(self):
return [
return super().get_build_scripts() + [
(
"root",
r"""
@ -51,7 +53,7 @@ class JuliaBuildPack(BuildPack):
def get_assemble_scripts(self):
require = self.binder_path('REQUIRE')
return [(
return super().get_assemble_scripts() + [(
"${NB_USER}",
# Pre-compile all libraries if they've opted into it. `using {libraryname}` does the
# right thing
@ -67,4 +69,4 @@ class JuliaBuildPack(BuildPack):
)]
def detect(self):
return os.path.exists(self.binder_path('REQUIRE')) and super()
return os.path.exists(self.binder_path('REQUIRE')) and super().detect()

Wyświetl plik

@ -1,16 +1,14 @@
"""
Generates a variety of Dockerfiles based on an input matrix
"""
from traitlets import Unicode
from textwrap import dedent
from .docker import DockerBuildPack
class LegacyBinderDockerBuildPack(DockerBuildPack):
name = 'Legacy Binder Dockerfile'
dockerfile = '._binder.Dockerfile'
dockerfile_appendix = Unicode(dedent(r"""
dockerfile_appendix = dedent(r"""
USER root
COPY . /home/main/notebooks
RUN chown -R main:main /home/main/notebooks
@ -25,7 +23,7 @@ class LegacyBinderDockerBuildPack(DockerBuildPack):
/home/main/anaconda2/bin/ipython kernel install --sys-prefix
ENV JUPYTER_PATH /home/main/anaconda2/share/jupyter:$JUPYTER_PATH
CMD jupyter notebook --ip 0.0.0.0
"""), config=True)
""")
def render(self):
with open('Dockerfile') as f:

Wyświetl plik

@ -1,42 +1,40 @@
"""
Generates a variety of Dockerfiles based on an input matrix
"""
from traitlets import default
import os
from ..base import BuildPack
from ..base import BaseImage
class PythonBuildPack(BuildPack):
name = "python3.5"
version = "0.1"
class PythonBuildPack(BaseImage):
def get_packages(self):
return {
return super().get_packages().union({
'python3',
'python3-venv',
'python3-dev',
}
})
def get_env(self):
return [
return super().get_env() + [
("VENV_PATH", "${APP_BASE}/venv"),
# Prefix to use for installing kernels and finding jupyter binary
("NB_PYTHON_PREFIX", "${VENV_PATH}"),
]
def get_path(self):
return [
return super().get_path() + [
"${VENV_PATH}/bin"
]
def get_build_script_files(self):
return {
files = {
'python/requirements.frozen.txt': '/tmp/requirements.frozen.txt',
}
files.update(super().get_build_script_files())
return files
def get_build_scripts(self):
return [
return super().get_build_scripts() + [
(
"root",
r"""
@ -69,6 +67,7 @@ class PythonBuildPack(BuildPack):
# be installed in the python2 venv, and requirements3.txt
# will be installed in python3 venv. This is less of a
# surprise than requiring python2 to be requirements2.txt tho.
assemble_scripts = super().get_assemble_scripts()
try:
with open(self.binder_path('runtime.txt')) as f:
runtime = f.read().strip()
@ -79,45 +78,44 @@ class PythonBuildPack(BuildPack):
else:
requirements_file = self.binder_path('requirements.txt')
if os.path.exists(requirements_file):
return [(
assemble_scripts.append((
'${NB_USER}',
'pip3 install --no-cache-dir -r "{}"'.format(requirements_file)
)]
return []
))
return assemble_scripts
def detect(self):
return os.path.exists('requirements.txt') and super().detect()
class Python2BuildPack(BuildPack):
name = "python2.7"
version = "0.1"
class Python2BuildPack(PythonBuildPack):
def get_packages(self):
return {
'python',
'python-dev',
'virtualenv'
}
return super().get_packages().union({
'python',
'python-dev',
'virtualenv'
})
def get_env(self):
return [
return super().get_env() + [
('VENV2_PATH', '${APP_BASE}/venv2')
]
def get_path(self):
return [
return super().get_path() + [
"${VENV2_PATH}/bin"
]
def get_build_script_files(self):
return {
files = {
'python/requirements2.frozen.txt': '/tmp/requirements2.frozen.txt',
}
files.update(super().get_build_script_files())
return files
def get_build_scripts(self):
return [
return super().get_build_scripts() + [
(
"root",
r"""
@ -141,7 +139,7 @@ class Python2BuildPack(BuildPack):
]
def get_assemble_scripts(self):
return [
return super().get_assemble_scripts() + [
(
'${NB_USER}',
'pip2 install --no-cache-dir -r requirements.txt'