diff --git a/repo2docker/app.py b/repo2docker/app.py index 6d37cb7c..bc6357f6 100644 --- a/repo2docker/app.py +++ b/repo2docker/app.py @@ -73,6 +73,18 @@ class Repo2Docker(Application): """ ) + cache_from = List( + [], + config=True, + help=""" + List of images to try & re-use cached image layers from. + + Docker only tries to re-use image layers from images built locally, + not pulled from a registry. We can ask it to explicitly re-use layers + from non-locally built images by through the 'cache_from' parameter. + """ + ) + buildpacks = List( [ LegacyBinderDockerBuildPack, @@ -398,6 +410,13 @@ class Repo2Docker(Application): help='Print the repo2docker version and exit.' ) + argparser.add_argument( + '--cache-from', + action='append', + default=[], + help=self.traits()['cache_from'].help + ) + return argparser def json_excepthook(self, etype, evalue, traceback): @@ -542,6 +561,9 @@ class Repo2Docker(Application): if args.subdir: self.subdir = args.subdir + if args.cache_from: + self.cache_from = args.cache_from + self.environment = args.environment def push_image(self): @@ -675,13 +697,11 @@ class Repo2Docker(Application): return port def start(self): - """Start execution of repo2docker""" - # Check if r2d can connect to docker daemon + """Start execution of repo2docker""" # Check if r2d can connect to docker daemon if self.build: try: - client = docker.APIClient(version='auto', - **kwargs_from_env()) - del client + api_client = docker.APIClient(version='auto', + **kwargs_from_env()) except DockerException as e: print("Docker client initialization error. Check if docker is" " running on the host.") @@ -737,8 +757,8 @@ class Repo2Docker(Application): self.log.info('Using %s builder\n', bp.__class__.__name__, extra=dict(phase='building')) - for l in picked_buildpack.build(self.output_image_spec, - self.build_memory_limit, build_args): + for l in picked_buildpack.build(api_client, self.output_image_spec, + self.build_memory_limit, build_args, self.cache_from): if 'stream' in l: self.log.info(l['stream'], extra=dict(phase='building')) diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index 2576d481..5125c420 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -449,7 +449,7 @@ class BuildPack: appendix=self.appendix, ) - def build(self, image_spec, memory_limit, build_args): + def build(self, client, image_spec, memory_limit, build_args, cache_from): tarf = io.BytesIO() tar = tarfile.open(fileobj=tarf, mode='w') dockerfile_tarinfo = tarfile.TarInfo("Dockerfile") @@ -489,8 +489,6 @@ class BuildPack: } if memory_limit: limits['memory'] = memory_limit - client = docker.APIClient(version='auto', - **docker.utils.kwargs_from_env()) for line in client.build( fileobj=tarf, tag=image_spec, @@ -499,7 +497,8 @@ class BuildPack: decode=True, forcerm=True, rm=True, - container_limits=limits + container_limits=limits, + cache_from=cache_from ): yield line diff --git a/repo2docker/buildpacks/docker.py b/repo2docker/buildpacks/docker.py index 8f3aefa0..5d4d83d2 100644 --- a/repo2docker/buildpacks/docker.py +++ b/repo2docker/buildpacks/docker.py @@ -19,7 +19,7 @@ class DockerBuildPack(BuildPack): with open(Dockerfile) as f: return f.read() - def build(self, image_spec, memory_limit, build_args): + def build(self, client, image_spec, memory_limit, build_args, cache_from): """Build a Docker image based on the Dockerfile in the source repo.""" limits = { # Always disable memory swap for building, since mostly @@ -28,7 +28,6 @@ class DockerBuildPack(BuildPack): } if memory_limit: limits['memory'] = memory_limit - client = docker.APIClient(version='auto', **docker.utils.kwargs_from_env()) for line in client.build( path=os.getcwd(), dockerfile=self.binder_path(self.dockerfile), @@ -37,6 +36,7 @@ class DockerBuildPack(BuildPack): decode=True, forcerm=True, rm=True, - container_limits=limits + container_limits=limits, + cache_from=cache_from ): yield line diff --git a/repo2docker/buildpacks/legacy/__init__.py b/repo2docker/buildpacks/legacy/__init__.py index cbc51dd1..217130c3 100644 --- a/repo2docker/buildpacks/legacy/__init__.py +++ b/repo2docker/buildpacks/legacy/__init__.py @@ -83,7 +83,7 @@ class LegacyBinderDockerBuildPack(DockerBuildPack): 'legacy/python3.frozen.yml': '/tmp/python3.frozen.yml', } - def build(self, image_spec, memory_limit, build_args): + def build(self, client, image_spec, memory_limit, build_args, cache_from): """Build a legacy Docker image.""" with open(self.dockerfile, 'w') as f: f.write(self.render()) @@ -94,7 +94,7 @@ class LegacyBinderDockerBuildPack(DockerBuildPack): env_file, ) shutil.copy(src_path, env_file) - return super().build(image_spec, memory_limit, build_args) + return super().build(client, image_spec, memory_limit, build_args, cache_from) def detect(self): """Check if current repo should be built with the Legacy BuildPack. diff --git a/tests/test_cache_from.py b/tests/test_cache_from.py new file mode 100644 index 00000000..6b7c9640 --- /dev/null +++ b/tests/test_cache_from.py @@ -0,0 +1,82 @@ +""" +Test that --cache-from is passed in to docker API properly. +""" +import os +import docker +from unittest.mock import MagicMock, patch +from repo2docker.buildpacks import BaseImage, DockerBuildPack, LegacyBinderDockerBuildPack +from tempfile import TemporaryDirectory + +def test_cache_from_base(monkeypatch): + FakeDockerClient = MagicMock() + cache_from = [ + 'image-1:latest' + ] + fake_log_value = {'stream': 'fake'} + fake_client = MagicMock(spec=docker.APIClient) + fake_client.build.return_value = iter([fake_log_value]) + + with TemporaryDirectory() as d: + # Test base image build pack + monkeypatch.chdir(d) + for line in BaseImage().build(fake_client, 'image-2', '1Gi', {}, cache_from): + assert line == fake_log_value + called_args, called_kwargs = fake_client.build.call_args + assert 'cache_from' in called_kwargs + assert called_kwargs['cache_from'] == cache_from + + + +def test_cache_from_docker(monkeypatch): + FakeDockerClient = MagicMock() + cache_from = [ + 'image-1:latest' + ] + fake_log_value = {'stream': 'fake'} + fake_client = MagicMock(spec=docker.APIClient) + fake_client.build.return_value = iter([fake_log_value]) + + with TemporaryDirectory() as d: + # Test docker image + with open(os.path.join(d, 'Dockerfile'), 'w') as f: + f.write('FROM scratch\n') + + for line in DockerBuildPack().build(fake_client, 'image-2', '1Gi', {}, cache_from): + assert line == fake_log_value + called_args, called_kwargs = fake_client.build.call_args + assert 'cache_from' in called_kwargs + assert called_kwargs['cache_from'] == cache_from + + # Test legacy docker image + with open(os.path.join(d, 'Dockerfile'), 'w') as f: + f.write('FROM andrewosh/binder-base\n') + + for line in LegacyBinderDockerBuildPack().build(fake_client, 'image-2', '1Gi', {}, cache_from): + print(line) + assert line == fake_log_value + called_args, called_kwargs = fake_client.build.call_args + assert 'cache_from' in called_kwargs + assert called_kwargs['cache_from'] == cache_from + + +def test_cache_from_legacy(monkeypatch): + FakeDockerClient = MagicMock() + cache_from = [ + 'image-1:latest' + ] + fake_log_value = {'stream': 'fake'} + fake_client = MagicMock(spec=docker.APIClient) + fake_client.build.return_value = iter([fake_log_value]) + + with TemporaryDirectory() as d: + # Test legacy docker image + with open(os.path.join(d, 'Dockerfile'), 'w') as f: + f.write('FROM andrewosh/binder-base\n') + + for line in LegacyBinderDockerBuildPack().build(fake_client, 'image-2', '1Gi', {}, cache_from): + assert line == fake_log_value + called_args, called_kwargs = fake_client.build.call_args + assert 'cache_from' in called_kwargs + assert called_kwargs['cache_from'] == cache_from + +