Merge pull request #478 from yuvipanda/cache-from

Allow specifying images to reuse cache from

repo2docker/app.py

@@ -73,6 +73,18 @@ class Repo2Docker(Application):
         """
     )
 
+    cache_from = List(
+        [],
+        config=True,
+        help="""
+        List of images to try & re-use cached image layers from.
+
+        Docker only tries to re-use image layers from images built locally,
+        not pulled from a registry. We can ask it to explicitly re-use layers
+        from non-locally built images by through the 'cache_from' parameter.
+        """
+    )
+
     buildpacks = List(
         [
             LegacyBinderDockerBuildPack,
@@ -398,6 +410,13 @@ class Repo2Docker(Application):
             help='Print the repo2docker version and exit.'
         )
 
+        argparser.add_argument(
+            '--cache-from',
+            action='append',
+            default=[],
+            help=self.traits()['cache_from'].help
+        )
+
         return argparser
 
     def json_excepthook(self, etype, evalue, traceback):
@@ -542,6 +561,9 @@ class Repo2Docker(Application):
         if args.subdir:
             self.subdir = args.subdir
 
+        if args.cache_from:
+            self.cache_from = args.cache_from
+
         self.environment = args.environment
 
     def push_image(self):
@@ -675,13 +697,11 @@ class Repo2Docker(Application):
         return port
 
     def start(self):
-        """Start execution of repo2docker"""
+        """Start execution of repo2docker""" # Check if r2d can connect to docker daemon
-        # Check if r2d can connect to docker daemon
         if self.build:
             try:
-                client = docker.APIClient(version='auto',
+                api_client = docker.APIClient(version='auto',
                                               **kwargs_from_env())
-                del client
             except DockerException as e:
                 print("Docker client initialization error. Check if docker is"
                       " running on the host.")
@@ -737,8 +757,8 @@ class Repo2Docker(Application):
                     self.log.info('Using %s builder\n', bp.__class__.__name__,
                                   extra=dict(phase='building'))
 
-                    for l in picked_buildpack.build(self.output_image_spec,
+                    for l in picked_buildpack.build(api_client, self.output_image_spec,
-                        self.build_memory_limit, build_args):
+                        self.build_memory_limit, build_args, self.cache_from):
                         if 'stream' in l:
                             self.log.info(l['stream'],
                                           extra=dict(phase='building'))

repo2docker/buildpacks/base.py

@@ -449,7 +449,7 @@ class BuildPack:
             appendix=self.appendix,
         )
 
-    def build(self, image_spec, memory_limit, build_args):
+    def build(self, client, image_spec, memory_limit, build_args, cache_from):
         tarf = io.BytesIO()
         tar = tarfile.open(fileobj=tarf, mode='w')
         dockerfile_tarinfo = tarfile.TarInfo("Dockerfile")
@@ -489,8 +489,6 @@ class BuildPack:
         }
         if memory_limit:
             limits['memory'] = memory_limit
-        client = docker.APIClient(version='auto',
-                                  **docker.utils.kwargs_from_env())
         for line in client.build(
                 fileobj=tarf,
                 tag=image_spec,
@@ -499,7 +497,8 @@ class BuildPack:
                 decode=True,
                 forcerm=True,
                 rm=True,
-                container_limits=limits
+                container_limits=limits,
+                cache_from=cache_from
         ):
             yield line
 

repo2docker/buildpacks/docker.py

@@ -19,7 +19,7 @@ class DockerBuildPack(BuildPack):
         with open(Dockerfile) as f:
             return f.read()
 
-    def build(self, image_spec, memory_limit, build_args):
+    def build(self, client, image_spec, memory_limit, build_args, cache_from):
         """Build a Docker image based on the Dockerfile in the source repo."""
         limits = {
             # Always disable memory swap for building, since mostly
@@ -28,7 +28,6 @@ class DockerBuildPack(BuildPack):
         }
         if memory_limit:
             limits['memory'] = memory_limit
-        client = docker.APIClient(version='auto', **docker.utils.kwargs_from_env())
         for line in client.build(
                 path=os.getcwd(),
                 dockerfile=self.binder_path(self.dockerfile),
@@ -37,6 +36,7 @@ class DockerBuildPack(BuildPack):
                 decode=True,
                 forcerm=True,
                 rm=True,
-                container_limits=limits
+                container_limits=limits,
+                cache_from=cache_from
         ):
             yield line

repo2docker/buildpacks/legacy/__init__.py

@@ -83,7 +83,7 @@ class LegacyBinderDockerBuildPack(DockerBuildPack):
             'legacy/python3.frozen.yml': '/tmp/python3.frozen.yml',
         }
             
-    def build(self, image_spec, memory_limit, build_args):
+    def build(self, client, image_spec, memory_limit, build_args, cache_from):
         """Build a legacy Docker image."""
         with open(self.dockerfile, 'w') as f:
             f.write(self.render())
@@ -94,7 +94,7 @@ class LegacyBinderDockerBuildPack(DockerBuildPack):
                 env_file,
             )
             shutil.copy(src_path, env_file)
-        return super().build(image_spec, memory_limit, build_args)
+        return super().build(client, image_spec, memory_limit, build_args, cache_from)
 
     def detect(self):
         """Check if current repo should be built with the Legacy BuildPack.

tests/test_cache_from.py

@@ -0,0 +1,82 @@
+"""
+Test that --cache-from is passed in to docker API properly.
+"""
+import os
+import docker
+from unittest.mock import MagicMock, patch
+from repo2docker.buildpacks import BaseImage, DockerBuildPack, LegacyBinderDockerBuildPack
+from tempfile import TemporaryDirectory
+
+def test_cache_from_base(monkeypatch):
+    FakeDockerClient = MagicMock()
+    cache_from = [
+        'image-1:latest'
+    ]
+    fake_log_value = {'stream': 'fake'}
+    fake_client = MagicMock(spec=docker.APIClient)
+    fake_client.build.return_value = iter([fake_log_value])
+
+    with TemporaryDirectory() as d:
+        # Test base image build pack
+        monkeypatch.chdir(d)
+        for line in BaseImage().build(fake_client, 'image-2', '1Gi', {}, cache_from):
+            assert line == fake_log_value
+        called_args, called_kwargs = fake_client.build.call_args
+        assert 'cache_from' in called_kwargs
+        assert called_kwargs['cache_from'] == cache_from
+
+
+
+def test_cache_from_docker(monkeypatch):
+    FakeDockerClient = MagicMock()
+    cache_from = [
+        'image-1:latest'
+    ]
+    fake_log_value = {'stream': 'fake'}
+    fake_client = MagicMock(spec=docker.APIClient)
+    fake_client.build.return_value = iter([fake_log_value])
+
+    with TemporaryDirectory() as d:
+        # Test docker image
+        with open(os.path.join(d, 'Dockerfile'), 'w') as f:
+            f.write('FROM scratch\n')
+
+        for line in DockerBuildPack().build(fake_client, 'image-2', '1Gi', {}, cache_from):
+            assert line == fake_log_value
+        called_args, called_kwargs = fake_client.build.call_args
+        assert 'cache_from' in called_kwargs
+        assert called_kwargs['cache_from'] == cache_from
+
+        # Test legacy docker image
+        with open(os.path.join(d, 'Dockerfile'), 'w') as f:
+            f.write('FROM andrewosh/binder-base\n')
+
+        for line in LegacyBinderDockerBuildPack().build(fake_client, 'image-2', '1Gi', {}, cache_from):
+            print(line)
+            assert line == fake_log_value
+        called_args, called_kwargs = fake_client.build.call_args
+        assert 'cache_from' in called_kwargs
+        assert called_kwargs['cache_from'] == cache_from
+
+
+def test_cache_from_legacy(monkeypatch):
+    FakeDockerClient = MagicMock()
+    cache_from = [
+        'image-1:latest'
+    ]
+    fake_log_value = {'stream': 'fake'}
+    fake_client = MagicMock(spec=docker.APIClient)
+    fake_client.build.return_value = iter([fake_log_value])
+
+    with TemporaryDirectory() as d:
+        # Test legacy docker image
+        with open(os.path.join(d, 'Dockerfile'), 'w') as f:
+            f.write('FROM andrewosh/binder-base\n')
+
+        for line in LegacyBinderDockerBuildPack().build(fake_client, 'image-2', '1Gi', {}, cache_from):
+            assert line == fake_log_value
+        called_args, called_kwargs = fake_client.build.call_args
+        assert 'cache_from' in called_kwargs
+        assert called_kwargs['cache_from'] == cache_from
+
+