2017-05-23 19:29:27 +00:00
|
|
|
"""repo2docker: convert git repositories into jupyter-suitable docker images
|
|
|
|
|
2017-10-23 22:39:01 +00:00
|
|
|
Images produced by repo2docker can be used with Jupyter notebooks standalone
|
|
|
|
or with BinderHub.
|
2017-05-23 19:29:27 +00:00
|
|
|
|
|
|
|
Usage:
|
|
|
|
|
|
|
|
python -m repo2docker https://github.com/you/your-repo
|
|
|
|
"""
|
2024-02-05 20:28:56 +00:00
|
|
|
|
2022-10-31 22:32:14 +00:00
|
|
|
import getpass
|
2017-05-09 08:37:19 +00:00
|
|
|
import json
|
2017-05-16 01:54:51 +00:00
|
|
|
import logging
|
2018-02-05 17:25:12 +00:00
|
|
|
import os
|
2018-12-11 09:54:06 +00:00
|
|
|
import shutil
|
2022-10-31 22:32:14 +00:00
|
|
|
import sys
|
2018-02-05 17:25:12 +00:00
|
|
|
import tempfile
|
|
|
|
import time
|
2023-01-02 12:11:34 +00:00
|
|
|
import warnings
|
2018-07-20 09:45:01 +00:00
|
|
|
from urllib.parse import urlparse
|
2021-02-17 10:59:28 +00:00
|
|
|
|
2022-10-31 22:32:14 +00:00
|
|
|
import entrypoints
|
2018-02-05 17:25:12 +00:00
|
|
|
import escapism
|
|
|
|
from pythonjsonlogger import jsonlogger
|
2022-11-08 15:27:45 +00:00
|
|
|
from traitlets import Any, Bool, Dict, Int, List, Unicode, default, observe
|
2018-02-05 17:25:12 +00:00
|
|
|
from traitlets.config import Application
|
2017-05-09 08:37:19 +00:00
|
|
|
|
2022-10-31 22:32:14 +00:00
|
|
|
from . import __version__, contentproviders
|
2017-11-30 07:20:24 +00:00
|
|
|
from .buildpacks import (
|
2019-05-12 13:51:28 +00:00
|
|
|
CondaBuildPack,
|
|
|
|
DockerBuildPack,
|
|
|
|
JuliaProjectTomlBuildPack,
|
|
|
|
JuliaRequireBuildPack,
|
|
|
|
LegacyBinderDockerBuildPack,
|
|
|
|
NixBuildPack,
|
|
|
|
PipfileBuildPack,
|
|
|
|
PythonBuildPack,
|
|
|
|
RBuildPack,
|
2017-05-25 22:15:00 +00:00
|
|
|
)
|
2021-02-17 10:59:28 +00:00
|
|
|
from .engine import BuildError, ContainerEngineException, ImageLoadError
|
2023-01-02 12:11:34 +00:00
|
|
|
from .utils import ByteSpecification, R2dState, chdir, get_platform
|
2017-07-04 17:28:23 +00:00
|
|
|
|
|
|
|
|
2017-05-22 23:22:36 +00:00
|
|
|
class Repo2Docker(Application):
|
2018-02-05 23:15:49 +00:00
|
|
|
"""An application for converting git repositories to docker images"""
|
2019-05-31 09:10:17 +00:00
|
|
|
|
|
|
|
name = "jupyter-repo2docker"
|
2017-05-23 19:29:27 +00:00
|
|
|
version = __version__
|
|
|
|
description = __doc__
|
2022-01-27 13:01:45 +00:00
|
|
|
# disable aliases/flags because we don't use the traitlets for CLI parsing
|
|
|
|
# other than --Class.trait=value
|
|
|
|
aliases = {}
|
|
|
|
flags = {}
|
2017-05-24 00:56:03 +00:00
|
|
|
|
2019-05-31 09:10:17 +00:00
|
|
|
@default("log_level")
|
2017-05-24 21:11:37 +00:00
|
|
|
def _default_log_level(self):
|
2018-02-07 01:25:44 +00:00
|
|
|
"""The application's default log level"""
|
2017-05-24 21:11:37 +00:00
|
|
|
return logging.INFO
|
2017-05-09 08:37:19 +00:00
|
|
|
|
|
|
|
git_workdir = Unicode(
|
2017-11-01 20:15:27 +00:00
|
|
|
None,
|
2017-05-23 03:10:59 +00:00
|
|
|
config=True,
|
2017-11-09 15:41:00 +00:00
|
|
|
allow_none=True,
|
2017-05-23 03:10:59 +00:00
|
|
|
help="""
|
2018-02-05 23:15:49 +00:00
|
|
|
Working directory to use for check out of git repositories.
|
2017-05-23 03:10:59 +00:00
|
|
|
|
2017-11-01 20:15:27 +00:00
|
|
|
The default is to use the system's temporary directory. Should be
|
|
|
|
somewhere ephemeral, such as /tmp.
|
2019-05-31 09:10:17 +00:00
|
|
|
""",
|
2017-05-09 08:37:19 +00:00
|
|
|
)
|
|
|
|
|
2018-09-12 21:43:36 +00:00
|
|
|
subdir = Unicode(
|
2019-05-31 09:10:17 +00:00
|
|
|
"",
|
2018-09-12 21:43:36 +00:00
|
|
|
config=True,
|
|
|
|
help="""
|
|
|
|
Subdirectory of the git repository to examine.
|
|
|
|
|
|
|
|
Defaults to ''.
|
2019-05-31 09:10:17 +00:00
|
|
|
""",
|
2018-09-12 21:43:36 +00:00
|
|
|
)
|
|
|
|
|
2018-12-05 19:01:23 +00:00
|
|
|
cache_from = List(
|
2018-11-29 01:03:41 +00:00
|
|
|
[],
|
|
|
|
config=True,
|
|
|
|
help="""
|
|
|
|
List of images to try & re-use cached image layers from.
|
|
|
|
|
|
|
|
Docker only tries to re-use image layers from images built locally,
|
|
|
|
not pulled from a registry. We can ask it to explicitly re-use layers
|
|
|
|
from non-locally built images by through the 'cache_from' parameter.
|
2019-05-31 09:10:17 +00:00
|
|
|
""",
|
2018-11-29 01:03:41 +00:00
|
|
|
)
|
|
|
|
|
2017-05-09 08:37:19 +00:00
|
|
|
buildpacks = List(
|
2017-07-04 17:28:23 +00:00
|
|
|
[
|
2018-02-09 12:14:34 +00:00
|
|
|
LegacyBinderDockerBuildPack,
|
|
|
|
DockerBuildPack,
|
2019-02-26 16:52:44 +00:00
|
|
|
JuliaProjectTomlBuildPack,
|
|
|
|
JuliaRequireBuildPack,
|
2018-09-17 16:00:23 +00:00
|
|
|
NixBuildPack,
|
2018-02-09 12:14:34 +00:00
|
|
|
RBuildPack,
|
2018-05-23 13:22:21 +00:00
|
|
|
CondaBuildPack,
|
2019-05-12 13:51:28 +00:00
|
|
|
PipfileBuildPack,
|
2018-02-09 12:14:34 +00:00
|
|
|
PythonBuildPack,
|
2017-07-04 17:28:23 +00:00
|
|
|
],
|
2017-05-23 03:10:59 +00:00
|
|
|
config=True,
|
|
|
|
help="""
|
2018-02-05 23:15:49 +00:00
|
|
|
Ordered list of BuildPacks to try when building a git repository.
|
2019-05-31 09:10:17 +00:00
|
|
|
""",
|
2017-05-09 08:37:19 +00:00
|
|
|
)
|
|
|
|
|
2019-02-15 13:57:58 +00:00
|
|
|
extra_build_kwargs = Dict(
|
|
|
|
{},
|
|
|
|
help="""
|
|
|
|
extra kwargs to limit CPU quota when building a docker image.
|
|
|
|
Dictionary that allows the user to set the desired runtime flag
|
|
|
|
to configure the amount of access to CPU resources your container has.
|
|
|
|
Reference https://docs.docker.com/config/containers/resource_constraints/#cpu
|
|
|
|
""",
|
2019-05-31 09:10:17 +00:00
|
|
|
config=True,
|
2019-02-15 13:57:58 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
extra_run_kwargs = Dict(
|
|
|
|
{},
|
|
|
|
help="""
|
|
|
|
extra kwargs to limit CPU quota when running a docker image.
|
|
|
|
Dictionary that allows the user to set the desired runtime flag
|
|
|
|
to configure the amount of access to CPU resources your container has.
|
|
|
|
Reference https://docs.docker.com/config/containers/resource_constraints/#cpu
|
|
|
|
""",
|
2019-05-31 09:10:17 +00:00
|
|
|
config=True,
|
2019-02-15 13:57:58 +00:00
|
|
|
)
|
|
|
|
|
2018-02-01 11:19:06 +00:00
|
|
|
default_buildpack = Any(
|
2018-02-09 12:14:34 +00:00
|
|
|
PythonBuildPack,
|
2017-07-30 00:14:49 +00:00
|
|
|
config=True,
|
|
|
|
help="""
|
2018-02-05 23:15:49 +00:00
|
|
|
The default build pack to use when no other buildpacks are found.
|
2019-05-31 09:10:17 +00:00
|
|
|
""",
|
2017-07-30 00:14:49 +00:00
|
|
|
)
|
|
|
|
|
2018-02-18 12:41:22 +00:00
|
|
|
# Git is our content provider of last resort. This is to maintain the
|
|
|
|
# old behaviour when git and local directories were the only supported
|
|
|
|
# content providers. We can detect local directories from the path, but
|
|
|
|
# detecting if something will successfully `git clone` is very hard if all
|
|
|
|
# you can do is look at the path/URL to it.
|
|
|
|
content_providers = List(
|
2019-09-08 09:53:42 +00:00
|
|
|
[
|
|
|
|
contentproviders.Local,
|
|
|
|
contentproviders.Zenodo,
|
|
|
|
contentproviders.Figshare,
|
2019-07-10 22:44:03 +00:00
|
|
|
contentproviders.Dataverse,
|
2019-09-24 19:37:50 +00:00
|
|
|
contentproviders.Hydroshare,
|
2020-11-24 17:30:08 +00:00
|
|
|
contentproviders.Swhid,
|
2024-02-20 06:37:21 +00:00
|
|
|
contentproviders.CKAN,
|
2020-09-07 08:06:58 +00:00
|
|
|
contentproviders.Mercurial,
|
2019-09-08 09:53:42 +00:00
|
|
|
contentproviders.Git,
|
|
|
|
],
|
2018-02-18 12:41:22 +00:00
|
|
|
config=True,
|
|
|
|
help="""
|
2018-10-16 05:48:34 +00:00
|
|
|
Ordered list by priority of ContentProviders to try in turn to fetch
|
|
|
|
the contents specified by the user.
|
2019-05-31 09:10:17 +00:00
|
|
|
""",
|
2018-02-18 12:41:22 +00:00
|
|
|
)
|
|
|
|
|
2017-12-01 01:14:42 +00:00
|
|
|
build_memory_limit = ByteSpecification(
|
|
|
|
0,
|
|
|
|
help="""
|
|
|
|
Total memory that can be used by the docker image building process.
|
|
|
|
|
|
|
|
Set to 0 for no limits.
|
|
|
|
""",
|
2019-05-31 09:10:17 +00:00
|
|
|
config=True,
|
2017-12-01 01:14:42 +00:00
|
|
|
)
|
|
|
|
|
2017-12-19 19:02:27 +00:00
|
|
|
volumes = Dict(
|
|
|
|
{},
|
|
|
|
help="""
|
|
|
|
Volumes to mount when running the container.
|
|
|
|
|
2018-02-05 23:15:49 +00:00
|
|
|
Only used when running, not during build process!
|
2017-12-19 19:02:27 +00:00
|
|
|
|
2018-02-05 23:15:49 +00:00
|
|
|
Use a key-value pair, with the key being the volume source &
|
|
|
|
value being the destination volume.
|
2018-04-23 21:30:39 +00:00
|
|
|
|
|
|
|
Both source and destination can be relative. Source is resolved
|
2018-02-05 23:15:49 +00:00
|
|
|
relative to the current working directory on the host, and
|
2018-04-23 21:30:39 +00:00
|
|
|
destination is resolved relative to the working directory of the
|
2018-02-05 23:15:49 +00:00
|
|
|
image - ($HOME by default)
|
2017-12-19 19:02:27 +00:00
|
|
|
""",
|
2019-05-31 09:10:17 +00:00
|
|
|
config=True,
|
2017-12-19 19:02:27 +00:00
|
|
|
)
|
|
|
|
|
2017-12-19 20:46:22 +00:00
|
|
|
user_id = Int(
|
|
|
|
help="""
|
|
|
|
UID of the user to create inside the built image.
|
|
|
|
|
|
|
|
Should be a uid that is not currently used by anything in the image.
|
2017-12-21 22:01:22 +00:00
|
|
|
Defaults to uid of currently running user, since that is the most
|
|
|
|
common case when running r2d manually.
|
2017-12-19 20:46:22 +00:00
|
|
|
|
|
|
|
Might not affect Dockerfile builds.
|
|
|
|
""",
|
2019-05-31 09:10:17 +00:00
|
|
|
config=True,
|
2017-12-19 20:46:22 +00:00
|
|
|
)
|
|
|
|
|
2019-05-31 09:10:17 +00:00
|
|
|
@default("user_id")
|
2017-12-21 22:01:22 +00:00
|
|
|
def _user_id_default(self):
|
|
|
|
"""
|
|
|
|
Default user_id to current running user.
|
|
|
|
"""
|
|
|
|
return os.geteuid()
|
|
|
|
|
2017-12-19 20:46:22 +00:00
|
|
|
user_name = Unicode(
|
2019-05-31 09:10:17 +00:00
|
|
|
"jovyan",
|
2017-12-19 20:46:22 +00:00
|
|
|
help="""
|
|
|
|
Username of the user to create inside the built image.
|
|
|
|
|
2018-02-05 23:15:49 +00:00
|
|
|
Should be a username that is not currently used by anything in the
|
|
|
|
image, and should conform to the restrictions on user names for Linux.
|
2017-12-21 22:01:22 +00:00
|
|
|
|
|
|
|
Defaults to username of currently running user, since that is the most
|
2018-02-05 23:15:49 +00:00
|
|
|
common case when running repo2docker manually.
|
2017-12-19 20:46:22 +00:00
|
|
|
""",
|
2019-05-31 09:10:17 +00:00
|
|
|
config=True,
|
2017-12-19 20:46:22 +00:00
|
|
|
)
|
|
|
|
|
2019-05-31 09:10:17 +00:00
|
|
|
@default("user_name")
|
2017-12-21 22:01:22 +00:00
|
|
|
def _user_name_default(self):
|
|
|
|
"""
|
|
|
|
Default user_name to current running user.
|
|
|
|
"""
|
2019-05-21 06:32:53 +00:00
|
|
|
return getpass.getuser()
|
2017-12-21 22:01:22 +00:00
|
|
|
|
2018-02-09 10:54:55 +00:00
|
|
|
appendix = Unicode(
|
|
|
|
config=True,
|
|
|
|
help="""
|
|
|
|
Appendix of Dockerfile commands to run at the end of the build.
|
|
|
|
|
|
|
|
Can be used to customize the resulting image after all
|
|
|
|
standard build steps finish.
|
2019-05-31 09:10:17 +00:00
|
|
|
""",
|
2018-02-09 10:54:55 +00:00
|
|
|
)
|
|
|
|
|
2021-11-23 19:37:37 +00:00
|
|
|
labels = Dict(
|
|
|
|
{},
|
|
|
|
help="""
|
|
|
|
Extra labels to set on the final image.
|
|
|
|
|
|
|
|
Each Label is a key-value pair, with the key being the name of the label
|
|
|
|
and the value its value.
|
|
|
|
""",
|
|
|
|
config=True,
|
|
|
|
)
|
|
|
|
|
2023-01-02 12:11:34 +00:00
|
|
|
platform = Unicode(
|
|
|
|
config=True,
|
|
|
|
help="""
|
|
|
|
Platform to build for, linux/amd64 (recommended) or linux/arm64 (experimental).
|
|
|
|
""",
|
|
|
|
)
|
|
|
|
|
|
|
|
@default("platform")
|
|
|
|
def _platform_default(self):
|
|
|
|
"""
|
|
|
|
Default platform
|
|
|
|
"""
|
|
|
|
p = get_platform()
|
|
|
|
if p == "linux/arm64":
|
|
|
|
warnings.warn(
|
|
|
|
"Building for linux/arm64 is experimental. "
|
|
|
|
"To use the recommended platform set --Repo2Docker.platform=linux/amd64. "
|
|
|
|
"To silence this warning set --Repo2Docker.platform=linux/arm64."
|
|
|
|
)
|
|
|
|
return p
|
|
|
|
|
2021-11-24 18:50:29 +00:00
|
|
|
extra_build_args = Dict(
|
|
|
|
{},
|
|
|
|
help="""
|
|
|
|
Extra build args to pass to the image build process.
|
|
|
|
This is pretty much only useful for custom Dockerfile based builds.
|
|
|
|
""",
|
|
|
|
config=True,
|
|
|
|
)
|
|
|
|
|
2018-12-11 17:11:01 +00:00
|
|
|
json_logs = Bool(
|
|
|
|
False,
|
|
|
|
help="""
|
|
|
|
Log output in structured JSON format.
|
|
|
|
|
|
|
|
Useful when stdout is consumed by other tools
|
|
|
|
""",
|
2019-05-31 09:10:17 +00:00
|
|
|
config=True,
|
2018-12-11 17:11:01 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
repo = Unicode(
|
|
|
|
".",
|
|
|
|
help="""
|
|
|
|
Specification of repository to build image for.
|
|
|
|
|
|
|
|
Could be local path or git URL.
|
|
|
|
""",
|
2019-05-31 09:10:17 +00:00
|
|
|
config=True,
|
2018-12-11 17:11:01 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
ref = Unicode(
|
|
|
|
None,
|
|
|
|
help="""
|
|
|
|
Git ref that should be built.
|
|
|
|
|
|
|
|
If repo is a git repository, this ref is checked out
|
|
|
|
in a local clone before repository is built.
|
|
|
|
""",
|
|
|
|
config=True,
|
2019-05-31 09:10:17 +00:00
|
|
|
allow_none=True,
|
2018-12-11 17:11:01 +00:00
|
|
|
)
|
|
|
|
|
2020-11-24 17:30:08 +00:00
|
|
|
swh_token = Unicode(
|
|
|
|
None,
|
|
|
|
help="""
|
|
|
|
Token to use authenticated SWH API access.
|
|
|
|
|
|
|
|
If unset, default to unauthenticated (limited) usage of the Software
|
|
|
|
Heritage API.
|
|
|
|
""",
|
|
|
|
config=True,
|
|
|
|
allow_none=True,
|
|
|
|
)
|
|
|
|
|
2018-12-11 17:11:01 +00:00
|
|
|
cleanup_checkout = Bool(
|
2022-11-08 15:27:45 +00:00
|
|
|
True,
|
2018-12-11 17:11:01 +00:00
|
|
|
help="""
|
|
|
|
Delete source repository after building is done.
|
|
|
|
|
|
|
|
Useful when repo2docker is doing the git cloning
|
|
|
|
""",
|
2019-05-31 09:10:17 +00:00
|
|
|
config=True,
|
2018-12-11 17:11:01 +00:00
|
|
|
)
|
|
|
|
|
2022-11-08 15:27:45 +00:00
|
|
|
@default("cleanup_checkout")
|
|
|
|
def _defaut_cleanup_checkout(self):
|
|
|
|
# if the source exists locally we don't want to delete it at the end
|
|
|
|
# FIXME: Find a better way to figure out if repo is 'local'. Push this into ContentProvider?
|
|
|
|
return not os.path.exists(self.repo)
|
|
|
|
|
2018-12-11 17:11:01 +00:00
|
|
|
output_image_spec = Unicode(
|
|
|
|
"",
|
|
|
|
help="""
|
|
|
|
Docker Image name:tag to tag the built image with.
|
|
|
|
|
|
|
|
Required parameter.
|
|
|
|
""",
|
2019-05-31 09:10:17 +00:00
|
|
|
config=True,
|
2018-12-11 17:11:01 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
push = Bool(
|
|
|
|
False,
|
|
|
|
help="""
|
|
|
|
Set to true to push docker image after building
|
|
|
|
""",
|
2019-05-31 09:10:17 +00:00
|
|
|
config=True,
|
2018-12-11 17:11:01 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
run = Bool(
|
2022-11-02 13:51:58 +00:00
|
|
|
True,
|
2018-12-11 17:11:01 +00:00
|
|
|
help="""
|
|
|
|
Run docker image after building
|
|
|
|
""",
|
2019-05-31 09:10:17 +00:00
|
|
|
config=True,
|
2018-12-11 17:11:01 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
# FIXME: Refactor class to be able to do --no-build without needing
|
|
|
|
# deep support for it inside other code
|
2018-12-12 00:06:53 +00:00
|
|
|
dry_run = Bool(
|
|
|
|
False,
|
2018-12-11 17:11:01 +00:00
|
|
|
help="""
|
2018-12-12 00:06:53 +00:00
|
|
|
Do not actually build the docker image, just simulate it.
|
2018-12-11 17:11:01 +00:00
|
|
|
""",
|
2019-05-31 09:10:17 +00:00
|
|
|
config=True,
|
2018-12-11 17:11:01 +00:00
|
|
|
)
|
|
|
|
|
2022-11-08 15:27:45 +00:00
|
|
|
@observe("dry_run")
|
|
|
|
def _dry_run_changed(self, change):
|
|
|
|
if change.new:
|
|
|
|
# dry_run forces run and push to be False
|
|
|
|
self.push = self.run = False
|
|
|
|
|
2018-12-11 17:11:01 +00:00
|
|
|
# FIXME: Refactor classes to separate build & run steps
|
|
|
|
run_cmd = List(
|
|
|
|
[],
|
|
|
|
help="""
|
|
|
|
Command to run when running the container
|
|
|
|
|
|
|
|
When left empty, a jupyter notebook is run.
|
|
|
|
""",
|
2019-05-31 09:10:17 +00:00
|
|
|
config=True,
|
2018-12-11 17:11:01 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
all_ports = Bool(
|
|
|
|
False,
|
|
|
|
help="""
|
|
|
|
Publish all declared ports from container whiel running.
|
|
|
|
|
|
|
|
Equivalent to -P option to docker run
|
|
|
|
""",
|
2019-05-31 09:10:17 +00:00
|
|
|
config=True,
|
2018-12-11 17:11:01 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
ports = Dict(
|
|
|
|
{},
|
|
|
|
help="""
|
|
|
|
Port mappings to establish when running the container.
|
|
|
|
|
|
|
|
Equivalent to -p {key}:{value} options to docker run.
|
|
|
|
{key} refers to port inside container, and {value}
|
|
|
|
refers to port / host:port in the host
|
|
|
|
""",
|
2019-05-31 09:10:17 +00:00
|
|
|
config=True,
|
2018-12-11 17:11:01 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
environment = List(
|
|
|
|
[],
|
|
|
|
help="""
|
|
|
|
Environment variables to set when running the built image.
|
|
|
|
|
|
|
|
Each item must be a string formatted as KEY=VALUE
|
|
|
|
""",
|
2019-05-31 09:10:17 +00:00
|
|
|
config=True,
|
2018-12-11 17:11:01 +00:00
|
|
|
)
|
|
|
|
|
2018-12-18 19:21:19 +00:00
|
|
|
target_repo_dir = Unicode(
|
2019-05-31 09:10:17 +00:00
|
|
|
"",
|
2018-12-17 23:32:17 +00:00
|
|
|
help="""
|
2019-06-27 07:58:29 +00:00
|
|
|
Path inside the image where contents of the repositories are copied to,
|
|
|
|
and where all the build operations (such as postBuild) happen.
|
2018-12-17 23:32:17 +00:00
|
|
|
|
2018-12-17 23:50:30 +00:00
|
|
|
Defaults to ${HOME} if not set
|
2018-12-17 23:32:17 +00:00
|
|
|
""",
|
2019-05-31 09:10:17 +00:00
|
|
|
config=True,
|
2018-12-17 23:32:17 +00:00
|
|
|
)
|
|
|
|
|
2020-02-13 11:46:29 +00:00
|
|
|
engine = Unicode(
|
|
|
|
"docker",
|
|
|
|
config=True,
|
|
|
|
help="""
|
|
|
|
Name of the container engine.
|
|
|
|
|
|
|
|
Defaults to 'docker'.
|
|
|
|
""",
|
|
|
|
)
|
|
|
|
|
2022-06-25 01:13:53 +00:00
|
|
|
base_image = Unicode(
|
2023-06-13 07:30:54 +00:00
|
|
|
"docker.io/library/buildpack-deps:jammy",
|
2022-06-25 01:13:53 +00:00
|
|
|
config=True,
|
|
|
|
help="""
|
|
|
|
Base image to use when building docker images.
|
|
|
|
|
2022-07-26 18:53:33 +00:00
|
|
|
Only images that match the following criteria are supported:
|
|
|
|
- Ubuntu based distributions, minimum 18.04
|
|
|
|
- Contains set of base packages installed with the buildpack-deps
|
|
|
|
image family: https://hub.docker.com/_/buildpack-deps
|
|
|
|
|
|
|
|
Other images *may* work, but are not officially supported.
|
2022-06-25 01:13:53 +00:00
|
|
|
""",
|
|
|
|
)
|
|
|
|
|
2020-02-13 11:46:29 +00:00
|
|
|
def get_engine(self):
|
|
|
|
"""Return an instance of the container engine.
|
|
|
|
|
|
|
|
Currently no arguments are passed to the engine constructor.
|
|
|
|
"""
|
|
|
|
engines = entrypoints.get_group_named("repo2docker.engines")
|
|
|
|
try:
|
|
|
|
entry = engines[self.engine]
|
|
|
|
except KeyError:
|
|
|
|
raise ContainerEngineException(
|
2022-10-23 17:18:37 +00:00
|
|
|
f"Container engine '{self.engine}' not found. Available engines: {','.join(engines.keys())}"
|
2020-02-13 11:46:29 +00:00
|
|
|
)
|
|
|
|
engine_class = entry.load()
|
2020-02-14 13:37:31 +00:00
|
|
|
return engine_class(parent=self)
|
2020-02-13 11:46:29 +00:00
|
|
|
|
2017-05-23 03:26:27 +00:00
|
|
|
def fetch(self, url, ref, checkout_path):
|
2018-11-16 21:24:31 +00:00
|
|
|
"""Fetch the contents of `url` and place it in `checkout_path`.
|
|
|
|
|
|
|
|
The `ref` parameter specifies what "version" of the contents should be
|
|
|
|
fetched. In the case of a git repository `ref` is the SHA-1 of a commit.
|
2018-10-16 07:17:24 +00:00
|
|
|
|
|
|
|
Iterate through possible content providers until a valid provider,
|
|
|
|
based on URL, is found.
|
|
|
|
"""
|
2018-02-18 12:41:22 +00:00
|
|
|
picked_content_provider = None
|
2018-10-16 07:17:24 +00:00
|
|
|
for ContentProvider in self.content_providers:
|
|
|
|
cp = ContentProvider()
|
2018-02-18 12:41:22 +00:00
|
|
|
spec = cp.detect(url, ref=ref)
|
|
|
|
if spec is not None:
|
|
|
|
picked_content_provider = cp
|
2022-10-23 17:18:37 +00:00
|
|
|
self.log.info(f"Picked {cp.__class__.__name__} content provider.\n")
|
2018-02-18 12:41:22 +00:00
|
|
|
break
|
|
|
|
|
|
|
|
if picked_content_provider is None:
|
2022-10-23 17:18:37 +00:00
|
|
|
self.log.error(f"No matching content provider found for {url}.")
|
2018-02-18 12:41:22 +00:00
|
|
|
|
2020-11-24 17:30:08 +00:00
|
|
|
swh_token = self.config.get("swh_token", self.swh_token)
|
|
|
|
if swh_token and isinstance(picked_content_provider, contentproviders.Swhid):
|
|
|
|
picked_content_provider.set_auth_token(swh_token)
|
|
|
|
|
2018-02-18 12:41:22 +00:00
|
|
|
for log_line in picked_content_provider.fetch(
|
2019-05-31 09:10:17 +00:00
|
|
|
spec, checkout_path, yield_output=self.json_logs
|
|
|
|
):
|
2022-10-03 21:49:22 +00:00
|
|
|
self.log.info(log_line, extra=dict(phase=R2dState.FETCHING))
|
2017-10-25 06:26:47 +00:00
|
|
|
|
2018-12-17 08:18:30 +00:00
|
|
|
if not self.output_image_spec:
|
2020-11-24 17:19:42 +00:00
|
|
|
image_spec = "r2d" + self.repo
|
2018-12-17 08:18:30 +00:00
|
|
|
# if we are building from a subdirectory include that in the
|
|
|
|
# image name so we can tell builds from different sub-directories
|
|
|
|
# apart.
|
|
|
|
if self.subdir:
|
2020-11-24 17:19:42 +00:00
|
|
|
image_spec += self.subdir
|
2018-12-17 08:18:30 +00:00
|
|
|
if picked_content_provider.content_id is not None:
|
2020-11-24 17:19:42 +00:00
|
|
|
image_spec += picked_content_provider.content_id
|
2018-12-17 08:18:30 +00:00
|
|
|
else:
|
2020-11-24 17:19:42 +00:00
|
|
|
image_spec += str(int(time.time()))
|
|
|
|
self.output_image_spec = escapism.escape(
|
|
|
|
image_spec, escape_char="-"
|
|
|
|
).lower()
|
2017-07-29 06:46:04 +00:00
|
|
|
|
2017-10-17 12:04:16 +00:00
|
|
|
def json_excepthook(self, etype, evalue, traceback):
|
|
|
|
"""Called on an uncaught exception when using json logging
|
|
|
|
|
|
|
|
Avoids non-JSON output on errors when using --json-logs
|
|
|
|
"""
|
2019-05-31 09:10:17 +00:00
|
|
|
self.log.error(
|
2022-10-23 17:18:37 +00:00
|
|
|
f"Error during build: {evalue}",
|
2019-05-31 09:10:17 +00:00
|
|
|
exc_info=(etype, evalue, traceback),
|
2022-10-03 21:49:22 +00:00
|
|
|
extra=dict(phase=R2dState.FAILED),
|
2019-05-31 09:10:17 +00:00
|
|
|
)
|
2017-10-17 12:04:16 +00:00
|
|
|
|
2022-01-26 09:15:38 +00:00
|
|
|
def initialize(self, *args, **kwargs):
|
2018-02-05 23:15:49 +00:00
|
|
|
"""Init repo2docker configuration before start"""
|
2018-12-11 17:11:01 +00:00
|
|
|
# FIXME: Remove this function, move it to setters / traitlet reactors
|
2022-08-22 13:11:28 +00:00
|
|
|
self.log = logging.getLogger("repo2docker")
|
|
|
|
self.log.setLevel(self.log_level)
|
|
|
|
logHandler = logging.StreamHandler()
|
|
|
|
self.log.handlers = []
|
|
|
|
self.log.addHandler(logHandler)
|
2018-12-11 17:11:01 +00:00
|
|
|
if self.json_logs:
|
2017-10-17 12:04:16 +00:00
|
|
|
# register JSON excepthook to avoid non-JSON output on errors
|
|
|
|
sys.excepthook = self.json_excepthook
|
2017-05-24 21:11:37 +00:00
|
|
|
# Need to reset existing handlers, or we repeat messages
|
|
|
|
formatter = jsonlogger.JsonFormatter()
|
|
|
|
logHandler.setFormatter(formatter)
|
|
|
|
else:
|
|
|
|
# due to json logger stuff above,
|
|
|
|
# our log messages include carriage returns, newlines, etc.
|
|
|
|
# remove the additional newline from the stream handler
|
2019-05-31 09:10:17 +00:00
|
|
|
self.log.handlers[0].terminator = ""
|
2017-07-29 03:06:54 +00:00
|
|
|
# We don't want a [Repo2Docker] on all messages
|
2019-05-31 09:10:17 +00:00
|
|
|
self.log.handlers[0].formatter = logging.Formatter(fmt="%(message)s")
|
2017-05-24 21:11:37 +00:00
|
|
|
|
2018-12-12 00:06:53 +00:00
|
|
|
if self.dry_run and (self.run or self.push):
|
2018-12-17 12:11:45 +00:00
|
|
|
raise ValueError("Cannot push or run image if we are not building it")
|
2018-11-29 00:13:20 +00:00
|
|
|
|
2018-12-11 17:11:01 +00:00
|
|
|
if self.volumes and not self.run:
|
2018-12-17 12:11:45 +00:00
|
|
|
raise ValueError("Cannot mount volumes if container is not run")
|
2018-01-07 15:43:03 +00:00
|
|
|
|
2017-05-23 05:55:17 +00:00
|
|
|
def push_image(self):
|
2018-02-05 23:15:49 +00:00
|
|
|
"""Push docker image to registry"""
|
2020-02-13 11:46:29 +00:00
|
|
|
client = self.get_engine()
|
2017-11-13 07:26:58 +00:00
|
|
|
# Build a progress setup for each layer, and only emit per-layer
|
|
|
|
# info every 1.5s
|
include full docker progress data in push events
Most useful is probably progressDetail.progress which is the docker rendered
Example event:
```json
"layers": {
"25b373f5f7f1": {
"status": "Waiting",
"progressDetail": {},
"id": "25b373f5f7f1"
},
"e9f2b5eca21e": {
"status": "Pushing",
"progressDetail": {
"current": 747589632,
"total": 758965327
},
"progress": "[=================================================> ] 747.6MB/759MB",
"id": "e9f2b5eca21e"
},
"7753d7e0913b": {
"status": "Pushed",
"progressDetail": {},
"id": "7753d7e0913b"
},
"ed03b06eb165": {
"status": "Pushed",
"progressDetail": {},
"id": "ed03b06eb165"
},
"01cb88e6c1af": {
"status": "Pushed",
"progressDetail": {},
"id": "01cb88e6c1af"
},
"adda4de99b3d": {
"status": "Mounted from library/buildpack-deps",
"progressDetail": {},
"id": "adda4de99b3d"
},
"3a034154b7b6": {
"status": "Mounted from library/buildpack-deps",
"progressDetail": {},
"id": "3a034154b7b6"
}
```
2019-07-02 12:57:34 +00:00
|
|
|
progress_layers = {}
|
2017-05-23 05:55:17 +00:00
|
|
|
layers = {}
|
|
|
|
last_emit_time = time.time()
|
2020-02-14 16:20:55 +00:00
|
|
|
for chunk in client.push(self.output_image_spec):
|
|
|
|
if client.string_output:
|
2022-10-03 21:49:22 +00:00
|
|
|
self.log.info(chunk, extra=dict(phase=R2dState.PUSHING))
|
2020-02-14 16:20:55 +00:00
|
|
|
continue
|
|
|
|
# else this is Docker output
|
|
|
|
|
2019-07-09 14:16:49 +00:00
|
|
|
# each chunk can be one or more lines of json events
|
|
|
|
# split lines here in case multiple are delivered at once
|
include full docker progress data in push events
Most useful is probably progressDetail.progress which is the docker rendered
Example event:
```json
"layers": {
"25b373f5f7f1": {
"status": "Waiting",
"progressDetail": {},
"id": "25b373f5f7f1"
},
"e9f2b5eca21e": {
"status": "Pushing",
"progressDetail": {
"current": 747589632,
"total": 758965327
},
"progress": "[=================================================> ] 747.6MB/759MB",
"id": "e9f2b5eca21e"
},
"7753d7e0913b": {
"status": "Pushed",
"progressDetail": {},
"id": "7753d7e0913b"
},
"ed03b06eb165": {
"status": "Pushed",
"progressDetail": {},
"id": "ed03b06eb165"
},
"01cb88e6c1af": {
"status": "Pushed",
"progressDetail": {},
"id": "01cb88e6c1af"
},
"adda4de99b3d": {
"status": "Mounted from library/buildpack-deps",
"progressDetail": {},
"id": "adda4de99b3d"
},
"3a034154b7b6": {
"status": "Mounted from library/buildpack-deps",
"progressDetail": {},
"id": "3a034154b7b6"
}
```
2019-07-02 12:57:34 +00:00
|
|
|
for line in chunk.splitlines():
|
|
|
|
line = line.decode("utf-8", errors="replace")
|
|
|
|
try:
|
|
|
|
progress = json.loads(line)
|
|
|
|
except Exception as e:
|
|
|
|
self.log.warning("Not a JSON progress line: %r", line)
|
|
|
|
continue
|
|
|
|
if "error" in progress:
|
2022-10-03 21:49:22 +00:00
|
|
|
self.log.error(progress["error"], extra=dict(phase=R2dState.FAILED))
|
2020-02-11 18:30:18 +00:00
|
|
|
raise ImageLoadError(progress["error"])
|
include full docker progress data in push events
Most useful is probably progressDetail.progress which is the docker rendered
Example event:
```json
"layers": {
"25b373f5f7f1": {
"status": "Waiting",
"progressDetail": {},
"id": "25b373f5f7f1"
},
"e9f2b5eca21e": {
"status": "Pushing",
"progressDetail": {
"current": 747589632,
"total": 758965327
},
"progress": "[=================================================> ] 747.6MB/759MB",
"id": "e9f2b5eca21e"
},
"7753d7e0913b": {
"status": "Pushed",
"progressDetail": {},
"id": "7753d7e0913b"
},
"ed03b06eb165": {
"status": "Pushed",
"progressDetail": {},
"id": "ed03b06eb165"
},
"01cb88e6c1af": {
"status": "Pushed",
"progressDetail": {},
"id": "01cb88e6c1af"
},
"adda4de99b3d": {
"status": "Mounted from library/buildpack-deps",
"progressDetail": {},
"id": "adda4de99b3d"
},
"3a034154b7b6": {
"status": "Mounted from library/buildpack-deps",
"progressDetail": {},
"id": "3a034154b7b6"
}
```
2019-07-02 12:57:34 +00:00
|
|
|
if "id" not in progress:
|
|
|
|
continue
|
|
|
|
# deprecated truncated-progress data
|
|
|
|
if "progressDetail" in progress and progress["progressDetail"]:
|
|
|
|
progress_layers[progress["id"]] = progress["progressDetail"]
|
|
|
|
else:
|
|
|
|
progress_layers[progress["id"]] = progress["status"]
|
|
|
|
# include full progress data for each layer in 'layers' data
|
|
|
|
layers[progress["id"]] = progress
|
|
|
|
if time.time() - last_emit_time > 1.5:
|
|
|
|
self.log.info(
|
2019-07-09 14:16:49 +00:00
|
|
|
"Pushing image\n",
|
|
|
|
extra=dict(
|
2022-10-03 21:49:22 +00:00
|
|
|
progress=progress_layers,
|
|
|
|
layers=layers,
|
|
|
|
phase=R2dState.PUSHING,
|
2019-07-09 14:16:49 +00:00
|
|
|
),
|
include full docker progress data in push events
Most useful is probably progressDetail.progress which is the docker rendered
Example event:
```json
"layers": {
"25b373f5f7f1": {
"status": "Waiting",
"progressDetail": {},
"id": "25b373f5f7f1"
},
"e9f2b5eca21e": {
"status": "Pushing",
"progressDetail": {
"current": 747589632,
"total": 758965327
},
"progress": "[=================================================> ] 747.6MB/759MB",
"id": "e9f2b5eca21e"
},
"7753d7e0913b": {
"status": "Pushed",
"progressDetail": {},
"id": "7753d7e0913b"
},
"ed03b06eb165": {
"status": "Pushed",
"progressDetail": {},
"id": "ed03b06eb165"
},
"01cb88e6c1af": {
"status": "Pushed",
"progressDetail": {},
"id": "01cb88e6c1af"
},
"adda4de99b3d": {
"status": "Mounted from library/buildpack-deps",
"progressDetail": {},
"id": "adda4de99b3d"
},
"3a034154b7b6": {
"status": "Mounted from library/buildpack-deps",
"progressDetail": {},
"id": "3a034154b7b6"
}
```
2019-07-02 12:57:34 +00:00
|
|
|
)
|
|
|
|
last_emit_time = time.time()
|
2019-05-31 09:10:17 +00:00
|
|
|
self.log.info(
|
2022-10-23 16:09:11 +00:00
|
|
|
f"Successfully pushed {self.output_image_spec}",
|
2022-10-03 21:49:22 +00:00
|
|
|
extra=dict(phase=R2dState.PUSHING),
|
2019-05-31 09:10:17 +00:00
|
|
|
)
|
2017-05-23 05:55:17 +00:00
|
|
|
|
|
|
|
def run_image(self):
|
2018-06-15 11:08:20 +00:00
|
|
|
"""Run docker container from built image
|
|
|
|
|
|
|
|
and wait for it to finish.
|
|
|
|
"""
|
|
|
|
container = self.start_container()
|
|
|
|
self.wait_for_container(container)
|
|
|
|
|
|
|
|
def start_container(self):
|
|
|
|
"""Start docker container from built image
|
|
|
|
|
|
|
|
Returns running container
|
|
|
|
"""
|
2020-02-13 11:46:29 +00:00
|
|
|
client = self.get_engine()
|
2018-07-31 09:28:20 +00:00
|
|
|
|
2019-05-31 09:10:17 +00:00
|
|
|
docker_host = os.environ.get("DOCKER_HOST")
|
2018-07-31 09:28:20 +00:00
|
|
|
if docker_host:
|
|
|
|
host_name = urlparse(docker_host).hostname
|
|
|
|
else:
|
2019-05-31 09:10:17 +00:00
|
|
|
host_name = "127.0.0.1"
|
2018-07-31 09:28:20 +00:00
|
|
|
self.hostname = host_name
|
|
|
|
|
2017-07-29 06:46:04 +00:00
|
|
|
if not self.run_cmd:
|
2020-06-26 10:38:25 +00:00
|
|
|
if len(self.ports) == 1:
|
|
|
|
# single port mapping specified
|
|
|
|
# retrieve container and host port from dict
|
|
|
|
# {'8888/tcp': ('hostname', 'port')}
|
|
|
|
# or
|
|
|
|
# {'8888/tcp': 'port'}
|
|
|
|
container_port_proto, host_port = next(iter(self.ports.items()))
|
|
|
|
if isinstance(host_port, tuple):
|
|
|
|
# (hostname, port) tuple or string port
|
|
|
|
host_name, host_port = host_port
|
|
|
|
self.hostname = host_name
|
|
|
|
host_port = int(host_port)
|
|
|
|
container_port = int(container_port_proto.split("/", 1)[0])
|
|
|
|
else:
|
|
|
|
# no port specified, pick a random one
|
|
|
|
container_port = host_port = str(self._get_free_port())
|
2022-10-31 10:43:36 +00:00
|
|
|
self.ports = {f"{container_port}/tcp": host_port}
|
2020-06-26 10:38:25 +00:00
|
|
|
self.port = host_port
|
2018-07-31 09:28:20 +00:00
|
|
|
# To use the option --NotebookApp.custom_display_url
|
2018-08-01 12:21:15 +00:00
|
|
|
# make sure the base-notebook image is updated:
|
2018-07-31 09:28:20 +00:00
|
|
|
# docker pull jupyter/base-notebook
|
|
|
|
run_cmd = [
|
2019-05-31 09:10:17 +00:00
|
|
|
"jupyter",
|
|
|
|
"notebook",
|
2022-10-23 17:18:37 +00:00
|
|
|
"--ip=0.0.0.0",
|
|
|
|
f"--port={container_port}",
|
|
|
|
f"--NotebookApp.custom_display_url=http://{host_name}:{host_port}",
|
2021-04-09 08:43:12 +00:00
|
|
|
"--NotebookApp.default_url=/lab",
|
2018-07-31 09:28:20 +00:00
|
|
|
]
|
2017-07-29 06:46:04 +00:00
|
|
|
else:
|
2017-12-25 02:03:17 +00:00
|
|
|
# run_cmd given by user, if port is also given then pass it on
|
2017-07-29 06:46:04 +00:00
|
|
|
run_cmd = self.run_cmd
|
2018-07-20 09:45:01 +00:00
|
|
|
|
2017-12-19 19:32:59 +00:00
|
|
|
container_volumes = {}
|
|
|
|
if self.volumes:
|
2020-02-11 18:30:18 +00:00
|
|
|
image = client.inspect_image(self.output_image_spec)
|
2020-02-14 22:00:34 +00:00
|
|
|
image_workdir = image.config["WorkingDir"]
|
2017-12-19 19:32:59 +00:00
|
|
|
|
|
|
|
for k, v in self.volumes.items():
|
|
|
|
container_volumes[os.path.abspath(k)] = {
|
2019-05-31 09:10:17 +00:00
|
|
|
"bind": v if v.startswith("/") else os.path.join(image_workdir, v),
|
|
|
|
"mode": "rw",
|
2017-12-19 19:32:59 +00:00
|
|
|
}
|
|
|
|
|
2019-02-15 13:57:58 +00:00
|
|
|
run_kwargs = dict(
|
2017-12-25 02:03:17 +00:00
|
|
|
publish_all_ports=self.all_ports,
|
2020-06-26 10:38:25 +00:00
|
|
|
ports=self.ports,
|
2017-12-19 19:02:27 +00:00
|
|
|
command=run_cmd,
|
2018-01-07 15:43:03 +00:00
|
|
|
volumes=container_volumes,
|
2019-05-31 09:10:17 +00:00
|
|
|
environment=self.environment,
|
2017-05-23 05:55:17 +00:00
|
|
|
)
|
2019-02-15 13:57:58 +00:00
|
|
|
|
|
|
|
run_kwargs.update(self.extra_run_kwargs)
|
|
|
|
|
2020-02-11 18:30:18 +00:00
|
|
|
container = client.run(self.output_image_spec, **run_kwargs)
|
2019-02-15 13:57:58 +00:00
|
|
|
|
2019-05-31 09:10:17 +00:00
|
|
|
while container.status == "created":
|
2017-05-23 06:57:40 +00:00
|
|
|
time.sleep(0.5)
|
|
|
|
container.reload()
|
|
|
|
|
2018-06-15 11:08:20 +00:00
|
|
|
return container
|
|
|
|
|
|
|
|
def wait_for_container(self, container):
|
|
|
|
"""Wait for a container to finish
|
|
|
|
|
|
|
|
Displaying logs while it's running
|
|
|
|
"""
|
|
|
|
|
2021-02-17 10:59:28 +00:00
|
|
|
last_timestamp = None
|
2017-05-23 06:00:37 +00:00
|
|
|
try:
|
2021-02-17 10:59:28 +00:00
|
|
|
for line in container.logs(stream=True, timestamps=True):
|
|
|
|
line = line.decode("utf-8")
|
|
|
|
last_timestamp, line = line.split(" ", maxsplit=1)
|
2022-10-03 21:49:22 +00:00
|
|
|
self.log.info(line, extra=dict(phase=R2dState.RUNNING))
|
2021-01-28 08:10:41 +00:00
|
|
|
|
2017-05-23 06:00:37 +00:00
|
|
|
finally:
|
2017-07-30 00:36:07 +00:00
|
|
|
container.reload()
|
2019-05-31 09:10:17 +00:00
|
|
|
if container.status == "running":
|
2022-10-03 21:49:22 +00:00
|
|
|
self.log.info(
|
|
|
|
"Stopping container...\n", extra=dict(phase=R2dState.RUNNING)
|
|
|
|
)
|
2017-07-29 06:46:04 +00:00
|
|
|
container.kill()
|
2020-02-11 19:01:18 +00:00
|
|
|
exit_code = container.exitcode
|
2021-01-28 08:10:41 +00:00
|
|
|
|
|
|
|
container.wait()
|
|
|
|
|
|
|
|
self.log.info(
|
2022-10-03 21:49:22 +00:00
|
|
|
"Container finished running.\n".upper(),
|
|
|
|
extra=dict(phase=R2dState.RUNNING),
|
2021-01-28 08:10:41 +00:00
|
|
|
)
|
|
|
|
# are there more logs? Let's send them back too
|
2021-02-17 10:59:28 +00:00
|
|
|
late_logs = container.logs(since=last_timestamp).decode("utf-8")
|
2021-01-28 08:10:41 +00:00
|
|
|
for line in late_logs.split("\n"):
|
2022-10-03 21:49:22 +00:00
|
|
|
self.log.debug(line + "\n", extra=dict(phase=R2dState.RUNNING))
|
2021-01-28 08:10:41 +00:00
|
|
|
|
2017-05-23 06:00:37 +00:00
|
|
|
container.remove()
|
2018-02-09 12:52:01 +00:00
|
|
|
if exit_code:
|
|
|
|
sys.exit(exit_code)
|
2017-05-23 05:55:17 +00:00
|
|
|
|
2017-05-24 01:08:53 +00:00
|
|
|
def _get_free_port(self):
|
|
|
|
"""
|
|
|
|
Hacky method to get a free random port on local host
|
|
|
|
"""
|
|
|
|
import socket
|
2019-05-31 09:10:17 +00:00
|
|
|
|
2017-05-24 01:08:53 +00:00
|
|
|
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
2017-11-13 07:26:58 +00:00
|
|
|
s.bind(("", 0))
|
2017-05-24 01:08:53 +00:00
|
|
|
port = s.getsockname()[1]
|
|
|
|
s.close()
|
|
|
|
return port
|
|
|
|
|
2018-12-17 08:18:30 +00:00
|
|
|
def find_image(self):
|
|
|
|
# if this is a dry run it is Ok for dockerd to be unreachable so we
|
|
|
|
# always return False for dry runs.
|
|
|
|
if self.dry_run:
|
|
|
|
return False
|
|
|
|
# check if we already have an image for this content
|
2020-02-13 11:46:29 +00:00
|
|
|
client = self.get_engine()
|
2018-12-17 08:18:30 +00:00
|
|
|
for image in client.images():
|
2020-02-14 13:15:04 +00:00
|
|
|
for tag in image.tags:
|
|
|
|
if tag == self.output_image_spec + ":latest":
|
|
|
|
return True
|
2018-12-17 08:18:30 +00:00
|
|
|
return False
|
|
|
|
|
2018-12-12 00:06:53 +00:00
|
|
|
def build(self):
|
|
|
|
"""
|
|
|
|
Build docker image
|
|
|
|
"""
|
|
|
|
# Check if r2d can connect to docker daemon
|
|
|
|
if not self.dry_run:
|
2018-01-10 02:39:07 +00:00
|
|
|
try:
|
2020-02-13 11:46:29 +00:00
|
|
|
docker_client = self.get_engine()
|
2020-02-11 18:30:18 +00:00
|
|
|
except ContainerEngineException as e:
|
2022-10-23 17:18:37 +00:00
|
|
|
self.log.error(f"\nContainer engine initialization error: {e}\n")
|
2019-09-07 10:07:34 +00:00
|
|
|
self.exit(1)
|
2019-09-07 07:22:00 +00:00
|
|
|
|
2018-10-16 07:17:24 +00:00
|
|
|
# If the source to be executed is a directory, continue using the
|
|
|
|
# directory. In the case of a local directory, it is used as both the
|
|
|
|
# source and target. Reusing a local directory seems better than
|
|
|
|
# making a copy of it as it might contain large files that would be
|
|
|
|
# expensive to copy.
|
2018-10-11 20:06:38 +00:00
|
|
|
if os.path.isdir(self.repo):
|
2022-11-08 15:27:45 +00:00
|
|
|
# never cleanup when we are working in a local repo
|
|
|
|
self.cleanup_checkout = False
|
2017-07-29 21:17:32 +00:00
|
|
|
checkout_path = self.repo
|
|
|
|
else:
|
2017-11-01 20:15:27 +00:00
|
|
|
if self.git_workdir is None:
|
2019-05-31 09:10:17 +00:00
|
|
|
checkout_path = tempfile.mkdtemp(prefix="repo2docker")
|
2017-11-01 20:15:27 +00:00
|
|
|
else:
|
|
|
|
checkout_path = self.git_workdir
|
|
|
|
|
2018-12-11 07:12:42 +00:00
|
|
|
try:
|
2018-10-11 20:06:38 +00:00
|
|
|
self.fetch(self.repo, self.ref, checkout_path)
|
2017-11-01 20:15:27 +00:00
|
|
|
|
2018-12-17 08:18:30 +00:00
|
|
|
if self.find_image():
|
2019-05-31 09:10:17 +00:00
|
|
|
self.log.info(
|
2022-10-23 17:18:37 +00:00
|
|
|
f"Reusing existing image ({self.output_image_spec}), not building."
|
2019-05-31 09:10:17 +00:00
|
|
|
)
|
2018-12-17 08:18:30 +00:00
|
|
|
# no need to build, so skip to the end by `return`ing here
|
|
|
|
# this will still execute the finally clause and let's us
|
|
|
|
# avoid having to indent the build code by an extra level
|
|
|
|
return
|
|
|
|
|
2018-12-05 19:51:22 +00:00
|
|
|
if self.subdir:
|
|
|
|
checkout_path = os.path.join(checkout_path, self.subdir)
|
|
|
|
if not os.path.isdir(checkout_path):
|
2019-05-31 09:10:17 +00:00
|
|
|
self.log.error(
|
2022-10-23 17:18:37 +00:00
|
|
|
f"Subdirectory {self.subdir} does not exist",
|
2022-10-03 21:49:22 +00:00
|
|
|
extra=dict(phase=R2dState.FAILED),
|
2019-05-31 09:10:17 +00:00
|
|
|
)
|
2022-10-23 16:09:11 +00:00
|
|
|
raise FileNotFoundError(f"Could not find {checkout_path}")
|
2018-12-05 19:51:22 +00:00
|
|
|
|
|
|
|
with chdir(checkout_path):
|
|
|
|
for BP in self.buildpacks:
|
2022-06-25 01:13:53 +00:00
|
|
|
bp = BP(base_image=self.base_image)
|
2018-12-05 19:51:22 +00:00
|
|
|
if bp.detect():
|
|
|
|
picked_buildpack = bp
|
|
|
|
break
|
|
|
|
else:
|
2022-07-23 15:48:56 +00:00
|
|
|
picked_buildpack = self.default_buildpack(
|
|
|
|
base_image=self.base_image
|
|
|
|
)
|
2018-12-05 19:51:22 +00:00
|
|
|
|
2023-01-02 12:11:34 +00:00
|
|
|
picked_buildpack.platform = self.platform
|
2018-12-05 19:51:22 +00:00
|
|
|
picked_buildpack.appendix = self.appendix
|
2018-12-13 23:12:09 +00:00
|
|
|
# Add metadata labels
|
2019-05-31 09:10:17 +00:00
|
|
|
picked_buildpack.labels["repo2docker.version"] = self.version
|
|
|
|
repo_label = "local" if os.path.isdir(self.repo) else self.repo
|
|
|
|
picked_buildpack.labels["repo2docker.repo"] = repo_label
|
|
|
|
picked_buildpack.labels["repo2docker.ref"] = self.ref
|
2018-12-05 19:51:22 +00:00
|
|
|
|
2021-11-23 19:37:37 +00:00
|
|
|
picked_buildpack.labels.update(self.labels)
|
|
|
|
|
2022-02-13 07:20:08 +00:00
|
|
|
build_args = {
|
|
|
|
"NB_USER": self.user_name,
|
|
|
|
"NB_UID": str(self.user_id),
|
|
|
|
}
|
|
|
|
if self.target_repo_dir:
|
|
|
|
build_args["REPO_DIR"] = self.target_repo_dir
|
|
|
|
build_args.update(self.extra_build_args)
|
|
|
|
|
2019-09-07 12:25:07 +00:00
|
|
|
if self.dry_run:
|
2022-02-13 07:20:08 +00:00
|
|
|
print(picked_buildpack.render(build_args))
|
2019-09-07 12:25:07 +00:00
|
|
|
else:
|
2019-09-07 12:28:14 +00:00
|
|
|
self.log.debug(
|
2022-02-13 07:20:08 +00:00
|
|
|
picked_buildpack.render(build_args),
|
2022-10-03 21:49:22 +00:00
|
|
|
extra=dict(phase=R2dState.BUILDING),
|
2019-09-07 12:28:14 +00:00
|
|
|
)
|
2019-05-07 15:33:19 +00:00
|
|
|
if self.user_id == 0:
|
2019-06-28 16:46:01 +00:00
|
|
|
raise ValueError(
|
|
|
|
"Root as the primary user in the image is not permitted."
|
2019-05-07 15:33:19 +00:00
|
|
|
)
|
|
|
|
|
2019-05-31 09:10:17 +00:00
|
|
|
self.log.info(
|
2022-10-23 17:18:37 +00:00
|
|
|
f"Using {bp.__class__.__name__} builder\n",
|
2022-10-03 21:49:22 +00:00
|
|
|
extra=dict(phase=R2dState.BUILDING),
|
2019-05-31 09:10:17 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
for l in picked_buildpack.build(
|
|
|
|
docker_client,
|
|
|
|
self.output_image_spec,
|
|
|
|
self.build_memory_limit,
|
|
|
|
build_args,
|
|
|
|
self.cache_from,
|
|
|
|
self.extra_build_kwargs,
|
2023-01-02 12:11:34 +00:00
|
|
|
platform=self.platform,
|
2019-05-31 09:10:17 +00:00
|
|
|
):
|
2020-02-14 16:20:55 +00:00
|
|
|
if docker_client.string_output:
|
2022-10-03 21:49:22 +00:00
|
|
|
self.log.info(l, extra=dict(phase=R2dState.BUILDING))
|
2020-02-14 16:20:55 +00:00
|
|
|
# else this is Docker output
|
|
|
|
elif "stream" in l:
|
2022-10-03 21:49:22 +00:00
|
|
|
self.log.info(
|
|
|
|
l["stream"], extra=dict(phase=R2dState.BUILDING)
|
|
|
|
)
|
2019-05-31 09:10:17 +00:00
|
|
|
elif "error" in l:
|
2022-10-03 21:49:22 +00:00
|
|
|
self.log.info(l["error"], extra=dict(phase=R2dState.FAILED))
|
2020-02-14 13:37:59 +00:00
|
|
|
raise BuildError(l["error"])
|
2019-05-31 09:10:17 +00:00
|
|
|
elif "status" in l:
|
|
|
|
self.log.info(
|
2022-10-03 21:49:22 +00:00
|
|
|
"Fetching base image...\r",
|
|
|
|
extra=dict(phase=R2dState.BUILDING),
|
2019-05-31 09:10:17 +00:00
|
|
|
)
|
2018-12-05 19:51:22 +00:00
|
|
|
else:
|
2022-10-03 21:49:22 +00:00
|
|
|
self.log.info(
|
|
|
|
json.dumps(l), extra=dict(phase=R2dState.BUILDING)
|
|
|
|
)
|
2018-12-17 08:18:30 +00:00
|
|
|
|
2018-12-11 07:12:42 +00:00
|
|
|
finally:
|
2018-12-17 08:18:30 +00:00
|
|
|
# Cleanup checkout if necessary
|
2022-11-08 15:27:45 +00:00
|
|
|
# never cleanup when checking out a local repo
|
2018-12-11 07:12:42 +00:00
|
|
|
if self.cleanup_checkout:
|
|
|
|
shutil.rmtree(checkout_path, ignore_errors=True)
|
2017-05-23 05:55:17 +00:00
|
|
|
|
2018-12-11 20:36:06 +00:00
|
|
|
def start(self):
|
|
|
|
self.build()
|
|
|
|
|
2017-05-23 05:17:02 +00:00
|
|
|
if self.push:
|
2017-05-23 05:55:17 +00:00
|
|
|
self.push_image()
|
2017-05-23 05:17:02 +00:00
|
|
|
|
2017-05-23 05:55:17 +00:00
|
|
|
if self.run:
|
|
|
|
self.run_image()
|