kopia lustrzana https://github.com/jupyterhub/repo2docker
Merge pull request #478 from yuvipanda/cache-from
Allow specifying images to reuse cache frompull/495/head
commit
694e728ffd
|
@ -73,6 +73,18 @@ class Repo2Docker(Application):
|
|||
"""
|
||||
)
|
||||
|
||||
cache_from = List(
|
||||
[],
|
||||
config=True,
|
||||
help="""
|
||||
List of images to try & re-use cached image layers from.
|
||||
|
||||
Docker only tries to re-use image layers from images built locally,
|
||||
not pulled from a registry. We can ask it to explicitly re-use layers
|
||||
from non-locally built images by through the 'cache_from' parameter.
|
||||
"""
|
||||
)
|
||||
|
||||
buildpacks = List(
|
||||
[
|
||||
LegacyBinderDockerBuildPack,
|
||||
|
@ -398,6 +410,13 @@ class Repo2Docker(Application):
|
|||
help='Print the repo2docker version and exit.'
|
||||
)
|
||||
|
||||
argparser.add_argument(
|
||||
'--cache-from',
|
||||
action='append',
|
||||
default=[],
|
||||
help=self.traits()['cache_from'].help
|
||||
)
|
||||
|
||||
return argparser
|
||||
|
||||
def json_excepthook(self, etype, evalue, traceback):
|
||||
|
@ -542,6 +561,9 @@ class Repo2Docker(Application):
|
|||
if args.subdir:
|
||||
self.subdir = args.subdir
|
||||
|
||||
if args.cache_from:
|
||||
self.cache_from = args.cache_from
|
||||
|
||||
self.environment = args.environment
|
||||
|
||||
def push_image(self):
|
||||
|
@ -675,13 +697,11 @@ class Repo2Docker(Application):
|
|||
return port
|
||||
|
||||
def start(self):
|
||||
"""Start execution of repo2docker"""
|
||||
# Check if r2d can connect to docker daemon
|
||||
"""Start execution of repo2docker""" # Check if r2d can connect to docker daemon
|
||||
if self.build:
|
||||
try:
|
||||
client = docker.APIClient(version='auto',
|
||||
**kwargs_from_env())
|
||||
del client
|
||||
api_client = docker.APIClient(version='auto',
|
||||
**kwargs_from_env())
|
||||
except DockerException as e:
|
||||
print("Docker client initialization error. Check if docker is"
|
||||
" running on the host.")
|
||||
|
@ -737,8 +757,8 @@ class Repo2Docker(Application):
|
|||
self.log.info('Using %s builder\n', bp.__class__.__name__,
|
||||
extra=dict(phase='building'))
|
||||
|
||||
for l in picked_buildpack.build(self.output_image_spec,
|
||||
self.build_memory_limit, build_args):
|
||||
for l in picked_buildpack.build(api_client, self.output_image_spec,
|
||||
self.build_memory_limit, build_args, self.cache_from):
|
||||
if 'stream' in l:
|
||||
self.log.info(l['stream'],
|
||||
extra=dict(phase='building'))
|
||||
|
|
|
@ -449,7 +449,7 @@ class BuildPack:
|
|||
appendix=self.appendix,
|
||||
)
|
||||
|
||||
def build(self, image_spec, memory_limit, build_args):
|
||||
def build(self, client, image_spec, memory_limit, build_args, cache_from):
|
||||
tarf = io.BytesIO()
|
||||
tar = tarfile.open(fileobj=tarf, mode='w')
|
||||
dockerfile_tarinfo = tarfile.TarInfo("Dockerfile")
|
||||
|
@ -489,8 +489,6 @@ class BuildPack:
|
|||
}
|
||||
if memory_limit:
|
||||
limits['memory'] = memory_limit
|
||||
client = docker.APIClient(version='auto',
|
||||
**docker.utils.kwargs_from_env())
|
||||
for line in client.build(
|
||||
fileobj=tarf,
|
||||
tag=image_spec,
|
||||
|
@ -499,7 +497,8 @@ class BuildPack:
|
|||
decode=True,
|
||||
forcerm=True,
|
||||
rm=True,
|
||||
container_limits=limits
|
||||
container_limits=limits,
|
||||
cache_from=cache_from
|
||||
):
|
||||
yield line
|
||||
|
||||
|
|
|
@ -19,7 +19,7 @@ class DockerBuildPack(BuildPack):
|
|||
with open(Dockerfile) as f:
|
||||
return f.read()
|
||||
|
||||
def build(self, image_spec, memory_limit, build_args):
|
||||
def build(self, client, image_spec, memory_limit, build_args, cache_from):
|
||||
"""Build a Docker image based on the Dockerfile in the source repo."""
|
||||
limits = {
|
||||
# Always disable memory swap for building, since mostly
|
||||
|
@ -28,7 +28,6 @@ class DockerBuildPack(BuildPack):
|
|||
}
|
||||
if memory_limit:
|
||||
limits['memory'] = memory_limit
|
||||
client = docker.APIClient(version='auto', **docker.utils.kwargs_from_env())
|
||||
for line in client.build(
|
||||
path=os.getcwd(),
|
||||
dockerfile=self.binder_path(self.dockerfile),
|
||||
|
@ -37,6 +36,7 @@ class DockerBuildPack(BuildPack):
|
|||
decode=True,
|
||||
forcerm=True,
|
||||
rm=True,
|
||||
container_limits=limits
|
||||
container_limits=limits,
|
||||
cache_from=cache_from
|
||||
):
|
||||
yield line
|
||||
|
|
|
@ -83,7 +83,7 @@ class LegacyBinderDockerBuildPack(DockerBuildPack):
|
|||
'legacy/python3.frozen.yml': '/tmp/python3.frozen.yml',
|
||||
}
|
||||
|
||||
def build(self, image_spec, memory_limit, build_args):
|
||||
def build(self, client, image_spec, memory_limit, build_args, cache_from):
|
||||
"""Build a legacy Docker image."""
|
||||
with open(self.dockerfile, 'w') as f:
|
||||
f.write(self.render())
|
||||
|
@ -94,7 +94,7 @@ class LegacyBinderDockerBuildPack(DockerBuildPack):
|
|||
env_file,
|
||||
)
|
||||
shutil.copy(src_path, env_file)
|
||||
return super().build(image_spec, memory_limit, build_args)
|
||||
return super().build(client, image_spec, memory_limit, build_args, cache_from)
|
||||
|
||||
def detect(self):
|
||||
"""Check if current repo should be built with the Legacy BuildPack.
|
||||
|
|
|
@ -0,0 +1,82 @@
|
|||
"""
|
||||
Test that --cache-from is passed in to docker API properly.
|
||||
"""
|
||||
import os
|
||||
import docker
|
||||
from unittest.mock import MagicMock, patch
|
||||
from repo2docker.buildpacks import BaseImage, DockerBuildPack, LegacyBinderDockerBuildPack
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
def test_cache_from_base(monkeypatch):
|
||||
FakeDockerClient = MagicMock()
|
||||
cache_from = [
|
||||
'image-1:latest'
|
||||
]
|
||||
fake_log_value = {'stream': 'fake'}
|
||||
fake_client = MagicMock(spec=docker.APIClient)
|
||||
fake_client.build.return_value = iter([fake_log_value])
|
||||
|
||||
with TemporaryDirectory() as d:
|
||||
# Test base image build pack
|
||||
monkeypatch.chdir(d)
|
||||
for line in BaseImage().build(fake_client, 'image-2', '1Gi', {}, cache_from):
|
||||
assert line == fake_log_value
|
||||
called_args, called_kwargs = fake_client.build.call_args
|
||||
assert 'cache_from' in called_kwargs
|
||||
assert called_kwargs['cache_from'] == cache_from
|
||||
|
||||
|
||||
|
||||
def test_cache_from_docker(monkeypatch):
|
||||
FakeDockerClient = MagicMock()
|
||||
cache_from = [
|
||||
'image-1:latest'
|
||||
]
|
||||
fake_log_value = {'stream': 'fake'}
|
||||
fake_client = MagicMock(spec=docker.APIClient)
|
||||
fake_client.build.return_value = iter([fake_log_value])
|
||||
|
||||
with TemporaryDirectory() as d:
|
||||
# Test docker image
|
||||
with open(os.path.join(d, 'Dockerfile'), 'w') as f:
|
||||
f.write('FROM scratch\n')
|
||||
|
||||
for line in DockerBuildPack().build(fake_client, 'image-2', '1Gi', {}, cache_from):
|
||||
assert line == fake_log_value
|
||||
called_args, called_kwargs = fake_client.build.call_args
|
||||
assert 'cache_from' in called_kwargs
|
||||
assert called_kwargs['cache_from'] == cache_from
|
||||
|
||||
# Test legacy docker image
|
||||
with open(os.path.join(d, 'Dockerfile'), 'w') as f:
|
||||
f.write('FROM andrewosh/binder-base\n')
|
||||
|
||||
for line in LegacyBinderDockerBuildPack().build(fake_client, 'image-2', '1Gi', {}, cache_from):
|
||||
print(line)
|
||||
assert line == fake_log_value
|
||||
called_args, called_kwargs = fake_client.build.call_args
|
||||
assert 'cache_from' in called_kwargs
|
||||
assert called_kwargs['cache_from'] == cache_from
|
||||
|
||||
|
||||
def test_cache_from_legacy(monkeypatch):
|
||||
FakeDockerClient = MagicMock()
|
||||
cache_from = [
|
||||
'image-1:latest'
|
||||
]
|
||||
fake_log_value = {'stream': 'fake'}
|
||||
fake_client = MagicMock(spec=docker.APIClient)
|
||||
fake_client.build.return_value = iter([fake_log_value])
|
||||
|
||||
with TemporaryDirectory() as d:
|
||||
# Test legacy docker image
|
||||
with open(os.path.join(d, 'Dockerfile'), 'w') as f:
|
||||
f.write('FROM andrewosh/binder-base\n')
|
||||
|
||||
for line in LegacyBinderDockerBuildPack().build(fake_client, 'image-2', '1Gi', {}, cache_from):
|
||||
assert line == fake_log_value
|
||||
called_args, called_kwargs = fake_client.build.call_args
|
||||
assert 'cache_from' in called_kwargs
|
||||
assert called_kwargs['cache_from'] == cache_from
|
||||
|
||||
|
Ładowanie…
Reference in New Issue