2017-05-19 07:07:39 +00:00
|
|
|
import sys
|
2017-05-09 08:37:19 +00:00
|
|
|
import json
|
|
|
|
import os
|
2017-05-16 01:54:51 +00:00
|
|
|
import time
|
|
|
|
import logging
|
2017-05-22 21:41:52 +00:00
|
|
|
import uuid
|
|
|
|
import shutil
|
2017-05-16 01:54:51 +00:00
|
|
|
from pythonjsonlogger import jsonlogger
|
2017-05-23 05:16:30 +00:00
|
|
|
import escapism
|
2017-05-09 08:37:19 +00:00
|
|
|
|
|
|
|
|
2017-05-22 21:41:52 +00:00
|
|
|
from traitlets.config import Application, LoggingConfigurable
|
|
|
|
from traitlets import Type, Bool, Unicode, Dict, List
|
2017-05-09 08:37:19 +00:00
|
|
|
import docker
|
2017-05-22 17:29:48 +00:00
|
|
|
from docker.utils import kwargs_from_env
|
2017-05-09 08:37:19 +00:00
|
|
|
|
|
|
|
import subprocess
|
|
|
|
|
2017-05-22 16:47:08 +00:00
|
|
|
from .detectors import BuildPack, PythonBuildPack, DockerBuildPack, CondaBuildPack
|
2017-05-16 01:54:51 +00:00
|
|
|
from .utils import execute_cmd
|
|
|
|
|
2017-05-09 08:37:19 +00:00
|
|
|
|
2017-05-22 23:22:36 +00:00
|
|
|
class Repo2Docker(Application):
|
2017-05-09 08:37:19 +00:00
|
|
|
config_file = Unicode(
|
2017-05-22 23:22:36 +00:00
|
|
|
'repo2docker_config.py',
|
2017-05-23 03:28:28 +00:00
|
|
|
config=True,
|
|
|
|
help="""
|
|
|
|
Path to read traitlets configuration file from.
|
|
|
|
"""
|
2017-05-09 08:37:19 +00:00
|
|
|
)
|
|
|
|
|
2017-05-23 03:28:38 +00:00
|
|
|
repo = Unicode(
|
2017-05-23 05:16:46 +00:00
|
|
|
os.getcwd(),
|
2017-05-09 08:37:19 +00:00
|
|
|
allow_none=True,
|
2017-05-23 03:10:59 +00:00
|
|
|
config=True,
|
|
|
|
help="""
|
2017-05-23 03:28:38 +00:00
|
|
|
The git repository to clone.
|
|
|
|
|
|
|
|
Could be a https URL, or a file path.
|
2017-05-23 03:10:59 +00:00
|
|
|
"""
|
2017-05-09 08:37:19 +00:00
|
|
|
)
|
|
|
|
|
2017-05-23 03:28:38 +00:00
|
|
|
ref = Unicode(
|
2017-05-19 07:07:39 +00:00
|
|
|
'master',
|
|
|
|
allow_none=True,
|
2017-05-23 03:10:59 +00:00
|
|
|
config=True,
|
|
|
|
help="""
|
|
|
|
The git ref in the git repository to build.
|
|
|
|
|
|
|
|
Can be a tag, ref or branch.
|
|
|
|
"""
|
2017-05-19 07:07:39 +00:00
|
|
|
)
|
|
|
|
|
2017-05-09 08:37:19 +00:00
|
|
|
output_image_spec = Unicode(
|
|
|
|
None,
|
|
|
|
allow_none=True,
|
2017-05-23 03:10:59 +00:00
|
|
|
config=True,
|
|
|
|
help="""
|
|
|
|
The spec of the image to build.
|
|
|
|
|
|
|
|
Should be the same as the value passed to `-t` param of docker build.
|
|
|
|
"""
|
2017-05-09 08:37:19 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
git_workdir = Unicode(
|
2017-05-23 03:10:59 +00:00
|
|
|
"/tmp",
|
|
|
|
config=True,
|
|
|
|
help="""
|
|
|
|
The directory to use to check out git repositories into.
|
|
|
|
|
|
|
|
Should be somewhere ephemeral, such as /tmp
|
|
|
|
"""
|
2017-05-09 08:37:19 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
buildpacks = List(
|
2017-05-23 03:26:47 +00:00
|
|
|
Type(BuildPack),
|
2017-05-22 16:47:08 +00:00
|
|
|
[DockerBuildPack, CondaBuildPack, PythonBuildPack],
|
2017-05-23 03:10:59 +00:00
|
|
|
config=True,
|
|
|
|
help="""
|
|
|
|
Ordered list of BuildPacks to try to use to build a git repository.
|
|
|
|
"""
|
2017-05-09 08:37:19 +00:00
|
|
|
)
|
|
|
|
|
2017-05-22 21:41:52 +00:00
|
|
|
cleanup_checkout = Bool(
|
|
|
|
True,
|
|
|
|
config=True,
|
|
|
|
help="""
|
|
|
|
Set to True to clean up the checked out directory after building is done.
|
|
|
|
|
|
|
|
Will only clean up after a successful build - failed builds will still leave their
|
|
|
|
checkouts intact.
|
|
|
|
"""
|
|
|
|
)
|
|
|
|
|
2017-05-09 08:37:19 +00:00
|
|
|
aliases = Dict({
|
2017-05-23 03:28:38 +00:00
|
|
|
'repo': 'Repo2Docker.repo',
|
|
|
|
'ref': 'Repo2Docker.ref',
|
2017-05-23 03:13:20 +00:00
|
|
|
'image': 'Repo2Docker.output_image_spec',
|
2017-05-22 23:22:36 +00:00
|
|
|
'clean': 'Repo2Docker.cleanup_checkout',
|
|
|
|
'f': 'Repo2Docker.config_file',
|
2017-05-09 08:37:19 +00:00
|
|
|
})
|
|
|
|
|
|
|
|
|
2017-05-23 03:26:27 +00:00
|
|
|
def fetch(self, url, ref, checkout_path):
|
2017-05-19 07:07:39 +00:00
|
|
|
try:
|
2017-05-23 03:26:27 +00:00
|
|
|
for line in execute_cmd(['git', 'clone', url, checkout_path]):
|
2017-05-19 07:07:39 +00:00
|
|
|
self.log.info(line, extra=dict(phase='fetching'))
|
|
|
|
except subprocess.CalledProcessError:
|
|
|
|
self.log.error('Failed to clone repository!', extra=dict(phase='failed'))
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
|
|
try:
|
2017-05-23 03:26:27 +00:00
|
|
|
for line in execute_cmd(['git', 'reset', '--hard', ref], checkout_path):
|
2017-05-19 07:07:39 +00:00
|
|
|
self.log.info(line, extra=dict(phase='fetching'))
|
|
|
|
except subprocess.CalledProcessError:
|
|
|
|
self.log.error('Failed to check out ref %s', ref, extra=dict(phase='failed'))
|
|
|
|
sys.exit(1)
|
2017-05-09 08:37:19 +00:00
|
|
|
|
|
|
|
def initialize(self, *args, **kwargs):
|
|
|
|
super().initialize(*args, **kwargs)
|
2017-05-16 01:54:51 +00:00
|
|
|
logHandler = logging.StreamHandler()
|
|
|
|
formatter = jsonlogger.JsonFormatter()
|
|
|
|
logHandler.setFormatter(formatter)
|
|
|
|
# Need to reset existing handlers, or we repeat messages
|
|
|
|
self.log.handlers = []
|
|
|
|
self.log.addHandler(logHandler)
|
|
|
|
self.log.setLevel(logging.INFO)
|
2017-05-09 08:37:19 +00:00
|
|
|
self.load_config_file(self.config_file)
|
|
|
|
|
2017-05-23 05:16:30 +00:00
|
|
|
if self.output_image_spec is None:
|
|
|
|
# Attempt to set a sane default!
|
|
|
|
# HACK: Provide something more descriptive?
|
|
|
|
self.output_image_spec = escapism.escape(self.repo).lower() + ':' + self.ref.lower()
|
|
|
|
|
|
|
|
|
2017-05-09 08:37:19 +00:00
|
|
|
def run(self):
|
2017-05-10 03:45:45 +00:00
|
|
|
# HACK: Try to just pull this and see if that works.
|
|
|
|
# if it does, then just bail.
|
|
|
|
# WHAT WE REALLY WANT IS TO NOT DO ANY WORK IF THE IMAGE EXISTS
|
2017-05-22 17:29:48 +00:00
|
|
|
client = docker.APIClient(version='auto', **kwargs_from_env())
|
2017-05-10 03:45:45 +00:00
|
|
|
|
2017-05-16 01:54:51 +00:00
|
|
|
repo, tag = self.output_image_spec.split(':')
|
2017-05-22 17:35:13 +00:00
|
|
|
try:
|
|
|
|
for line in client.pull(
|
|
|
|
repository=repo,
|
|
|
|
tag=tag,
|
|
|
|
stream=True,
|
|
|
|
):
|
|
|
|
progress = json.loads(line.decode('utf-8'))
|
|
|
|
if 'error' in progress:
|
2017-05-22 18:52:53 +00:00
|
|
|
# pull failed, proceed to build
|
2017-05-22 17:35:13 +00:00
|
|
|
break
|
|
|
|
else:
|
2017-05-22 18:52:53 +00:00
|
|
|
# image exists, nothing to build
|
2017-05-22 17:35:13 +00:00
|
|
|
return
|
|
|
|
except docker.errors.ImageNotFound:
|
2017-05-22 18:52:53 +00:00
|
|
|
# image not found, proceed to build
|
2017-05-22 17:35:13 +00:00
|
|
|
pass
|
2017-05-10 03:45:45 +00:00
|
|
|
|
2017-05-22 21:41:52 +00:00
|
|
|
|
2017-05-23 03:26:27 +00:00
|
|
|
checkout_path = os.path.join(self.git_workdir, str(uuid.uuid4()))
|
2017-05-09 08:37:19 +00:00
|
|
|
self.fetch(
|
2017-05-23 03:28:38 +00:00
|
|
|
self.repo,
|
|
|
|
self.ref,
|
2017-05-23 03:26:27 +00:00
|
|
|
checkout_path
|
2017-05-09 08:37:19 +00:00
|
|
|
)
|
|
|
|
for bp_class in self.buildpacks:
|
|
|
|
bp = bp_class()
|
2017-05-23 03:26:27 +00:00
|
|
|
if bp.detect(checkout_path):
|
2017-05-16 01:54:51 +00:00
|
|
|
self.log.info('Using %s builder', bp.name, extra=dict(phase='building'))
|
2017-05-23 03:28:38 +00:00
|
|
|
bp.build(checkout_path, self.ref, self.output_image_spec)
|
2017-05-09 20:39:03 +00:00
|
|
|
break
|
|
|
|
else:
|
2017-05-19 07:07:39 +00:00
|
|
|
self.log.error('Could not figure out how to build this repository! Tell us?', extra=dict(phase='failed'))
|
|
|
|
sys.exit(1)
|
2017-05-09 20:39:03 +00:00
|
|
|
|
2017-05-16 01:54:51 +00:00
|
|
|
# Build a progress setup for each layer, and only emit per-layer info every 1.5s
|
|
|
|
layers = {}
|
|
|
|
last_emit_time = time.time()
|
2017-05-10 03:45:45 +00:00
|
|
|
for line in client.push(self.output_image_spec, stream=True):
|
2017-05-09 08:37:19 +00:00
|
|
|
progress = json.loads(line.decode('utf-8'))
|
2017-05-16 01:54:51 +00:00
|
|
|
if 'id' not in progress:
|
|
|
|
continue
|
|
|
|
if 'progressDetail' in progress and progress['progressDetail']:
|
|
|
|
layers[progress['id']] = progress['progressDetail']
|
|
|
|
else:
|
|
|
|
layers[progress['id']] = progress['status']
|
|
|
|
if time.time() - last_emit_time > 1.5:
|
|
|
|
self.log.info('Pushing image', extra=dict(progress=layers, phase='pushing'))
|
|
|
|
last_emit_time = time.time()
|
|
|
|
|
2017-05-22 21:53:44 +00:00
|
|
|
if self.cleanup_checkout:
|
2017-05-23 03:26:27 +00:00
|
|
|
shutil.rmtree(checkout_path)
|
2017-05-22 21:41:52 +00:00
|
|
|
|
2017-05-09 08:37:19 +00:00
|
|
|
|