Merge ecdcf9e53a into d2f467cef6

repo2docker/__main__.py

@@ -2,6 +2,7 @@ import argparse
 import logging
 import os
 import sys
+from pathlib import Path
 
 from . import __version__
 from .app import Repo2Docker
@@ -282,6 +283,22 @@ def get_argparser():
         help=Repo2Docker.engine.help,
     )
 
+    argparser.add_argument(
+        "--extra-ignore-file",
+        dest="extra_ignore_file",
+        type=Path,
+        help=Repo2Docker.extra_ignore_file.help,
+    )
+
+    argparser.add_argument(
+        "--ignore-file-strategy",
+        dest="ignore_file_strategy",
+        type=str,
+        choices=Repo2Docker.ignore_file_strategy.values,
+        default=Repo2Docker.ignore_file_strategy.default_value,
+        help=Repo2Docker.ignore_file_strategy.help,
+    )
+
     return argparser
 
 
@@ -464,6 +481,15 @@ def make_r2d(argv=None):
     if args.target_repo_dir:
         r2d.target_repo_dir = args.target_repo_dir
 
+    if args.extra_ignore_file is not None:
+        if not args.extra_ignore_file.exists():
+            print(f"The ignore file {args.extra_ignore_file} does not exist")
+            sys.exit(1)
+        r2d.extra_ignore_file = str(args.extra_ignore_file.resolve())
+
+    if args.ignore_file_strategy is not None:
+        r2d.ignore_file_strategy = args.ignore_file_strategy
+
     return r2d
 
 

repo2docker/app.py

@@ -22,13 +22,14 @@ from urllib.parse import urlparse
 import entrypoints
 import escapism
 from pythonjsonlogger import jsonlogger
-from traitlets import Any, Bool, Dict, Int, List, Unicode, default, observe
+from traitlets import Any, Bool, Dict, Enum, Int, List, Unicode, default, observe
 from traitlets.config import Application
 
 from . import __version__, contentproviders
 from .buildpacks import (
     CondaBuildPack,
     DockerBuildPack,
+    ExcludesStrategy,
     JuliaProjectTomlBuildPack,
     JuliaRequireBuildPack,
     LegacyBinderDockerBuildPack,
@@ -463,6 +464,32 @@ class Repo2Docker(Application):
         """,
     )
 
+    extra_ignore_file = Unicode(
+        "",
+        config=True,
+        help="""
+        Path to an additional .dockerignore or .containerignore file to be applied
+        when building an image.
+
+        Depending on the strategy selected the content of the file will replace,
+        be merged or be ignored.
+        """,
+    )
+
+    ignore_file_strategy = Enum(
+        ExcludesStrategy.values(),
+        config=True,
+        default_value=ExcludesStrategy.THEIRS.value,
+        help="""
+        Strategy to use if an extra ignore file is passed:
+        - merge means that the content of the extra ignore file will be merged
+          with the ignore file contained in the repository (if any)
+        - ours means that the extra ignore file content will be used in any case
+        - theirs means that if there is an ignore file in the repository, the
+          extra ignore file will not be used.
+        """,
+    )
+
     def get_engine(self):
         """Return an instance of the container engine.
 
@@ -861,6 +888,10 @@ class Repo2Docker(Application):
                         self.cache_from,
                         self.extra_build_kwargs,
                         platform=self.platform,
+                        extra_ignore_file=self.extra_ignore_file,
+                        ignore_file_strategy=ExcludesStrategy(
+                            self.ignore_file_strategy
+                        ),
                     ):
                         if docker_client.string_output:
                             self.log.info(l, extra=dict(phase=R2dState.BUILDING))

repo2docker/buildpacks/__init__.py

@@ -1,4 +1,4 @@
-from .base import BaseImage, BuildPack
+from .base import BaseImage, BuildPack, ExcludesStrategy
 from .conda import CondaBuildPack
 from .docker import DockerBuildPack
 from .julia import JuliaProjectTomlBuildPack, JuliaRequireBuildPack

repo2docker/buildpacks/base.py

@@ -7,6 +7,7 @@ import string
 import sys
 import tarfile
 import textwrap
+from enum import Enum
 from functools import lru_cache
 
 import escapism
@@ -205,6 +206,16 @@ HERE = os.path.dirname(os.path.abspath(__file__))
 DEFAULT_NB_UID = 1000
 
 
+class ExcludesStrategy(Enum):
+    THEIRS = "theirs"
+    OURS = "ours"
+    MERGE = "merge"
+
+    @classmethod
+    def values(cls):
+        return [item.value for item in cls]
+
+
 class BuildPack:
     """
     A composable BuildPack.
@@ -582,6 +593,8 @@ class BuildPack:
         cache_from,
         extra_build_kwargs,
         platform=None,
+        extra_ignore_file=None,
+        ignore_file_strategy=ExcludesStrategy.THEIRS,
     ):
         tarf = io.BytesIO()
         tar = tarfile.open(fileobj=tarf, mode="w")
@@ -609,24 +622,35 @@ class BuildPack:
         for fname in ("repo2docker-entrypoint", "python3-login"):
             tar.add(os.path.join(HERE, fname), fname, filter=_filter_tar)
 
-        exclude = []
+        def _read_excludes(filepath):
+            with open(filepath) as ignore_file:
+                cleaned_lines = [
+                    line.strip() for line in ignore_file.read().splitlines()
+                ]
+                return [line for line in cleaned_lines if line != "" and line[0] != "#"]
 
+        extra_excludes = []
+        if extra_ignore_file:
+            extra_excludes = _read_excludes(extra_ignore_file)
+
+        excludes = []
         for ignore_file_name in [".dockerignore", ".containerignore"]:
             ignore_file_name = self.binder_path(ignore_file_name)
             if os.path.exists(ignore_file_name):
-                with open(ignore_file_name) as ignore_file:
-                    cleaned_lines = [
-                        line.strip() for line in ignore_file.read().splitlines()
-                    ]
-                    exclude.extend(
-                        [
-                            line
-                            for line in cleaned_lines
-                            if line != "" and line[0] != "#"
-                        ]
-                    )
+                excludes.extend(_read_excludes(ignore_file_name))
 
-        files_to_add = exclude_paths(".", exclude)
+        if extra_ignore_file is not None:
+            if ignore_file_strategy == ExcludesStrategy.OURS:
+                excludes = extra_excludes
+            elif ignore_file_strategy == ExcludesStrategy.MERGE:
+                excludes.extend(extra_excludes)
+            else:
+                # ignore means that if an ignore file exist, its content is used
+                # otherwise, the extra exclude
+                if not excludes:
+                    excludes = extra_excludes
+
+        files_to_add = exclude_paths(".", excludes)
 
         if files_to_add:
             for item in files_to_add:

repo2docker/buildpacks/docker.py

@@ -5,7 +5,7 @@ import os
 
 import docker
 
-from .base import BuildPack
+from .base import BuildPack, ExcludesStrategy
 
 
 class DockerBuildPack(BuildPack):
@@ -32,6 +32,8 @@ class DockerBuildPack(BuildPack):
         cache_from,
         extra_build_kwargs,
         platform=None,
+        extra_ignore_file=None,
+        ignore_file_strategy=ExcludesStrategy.THEIRS,
     ):
         """Build a Docker image based on the Dockerfile in the source repo."""
         # If you work on this bit of code check the corresponding code in

tests/conda/ignore-file

@@ -0,0 +1,2 @@
+# Docker compatible ignore file
+from-extra-ignore

tests/conda/py311-extra-ignore-file-merge/.dockerignore

@@ -0,0 +1 @@
+from-dockerignore

tests/conda/py311-extra-ignore-file-merge/environment.yml

@@ -0,0 +1,2 @@
+dependencies:
+  - python=3.11

tests/conda/py311-extra-ignore-file-merge/from-dockerignore

@@ -0,0 +1 @@
+Must be ignored from .dockerignore file

tests/conda/py311-extra-ignore-file-merge/from-extra-ignore

@@ -0,0 +1 @@
+Must be ignored from extra ignore file

tests/conda/py311-extra-ignore-file-merge/test-extra-args.yaml

@@ -0,0 +1,5 @@
+# This file is respected by repo2docker's test suite, but not repo2docker
+# itself. It is used solely to help us test repo2docker's command line flags.
+#
+- --extra-ignore-file=tests/conda/ignore-file
+- --ignore-file-strategy=merge

tests/conda/py311-extra-ignore-file-merge/verify

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+import pathlib
+
+assert not pathlib.Path("from-dockerignore").exists()
+assert not pathlib.Path("from-extra-ignore").exists()

tests/conda/py311-extra-ignore-file-ours/.dockerignore

@@ -0,0 +1 @@
+from-dockerignore

tests/conda/py311-extra-ignore-file-ours/environment.yml

@@ -0,0 +1,2 @@
+dependencies:
+  - python=3.11

tests/conda/py311-extra-ignore-file-ours/from-dockerignore

@@ -0,0 +1 @@
+Must not be ignored because of ours strategy and extra ignore file does not contain it.

tests/conda/py311-extra-ignore-file-ours/from-extra-ignore

@@ -0,0 +1 @@
+Must be ignored

tests/conda/py311-extra-ignore-file-ours/test-extra-args.yaml

@@ -0,0 +1,5 @@
+# This file is respected by repo2docker's test suite, but not repo2docker
+# itself. It is used solely to help us test repo2docker's command line flags.
+#
+- --extra-ignore-file=tests/conda/ignore-file
+- --ignore-file-strategy=ours

tests/conda/py311-extra-ignore-file-ours/verify

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+import pathlib
+
+assert pathlib.Path("from-dockerignore").exists()
+assert not pathlib.Path("from-extra-ignore").exists()

tests/conda/py311-extra-ignore-file-theirs-without-file/environment.yml

@@ -0,0 +1,2 @@
+dependencies:
+  - python=3.11

tests/conda/py311-extra-ignore-file-theirs-without-file/from-dockerignore

@@ -0,0 +1 @@
+No docker ignore so should still appear

tests/conda/py311-extra-ignore-file-theirs-without-file/from-extra-ignore

@@ -0,0 +1 @@
+Must be ignored because of extra ignore file

tests/conda/py311-extra-ignore-file-theirs-without-file/test-extra-args.yaml

@@ -0,0 +1,5 @@
+# This file is respected by repo2docker's test suite, but not repo2docker
+# itself. It is used solely to help us test repo2docker's command line flags.
+#
+- --extra-ignore-file=tests/conda/ignore-file
+- --ignore-file-strategy=theirs

tests/conda/py311-extra-ignore-file-theirs-without-file/verify

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+import pathlib
+
+assert pathlib.Path("from-dockerignore").exists()
+assert not pathlib.Path("from-extra-ignore").exists()

tests/conda/py311-extra-ignore-file-theirs/.dockerignore

@@ -0,0 +1 @@
+from-dockerignore

tests/conda/py311-extra-ignore-file-theirs/environment.yml

@@ -0,0 +1,2 @@
+dependencies:
+  - python=3.11

tests/conda/py311-extra-ignore-file-theirs/from-dockerignore

@@ -0,0 +1 @@
+Must be ignored from .dockerignore file

tests/conda/py311-extra-ignore-file-theirs/from-extra-ignore

@@ -0,0 +1 @@
+Shall be present due to strategy being theirs and this file does not appear in .dockerignore

tests/conda/py311-extra-ignore-file-theirs/test-extra-args.yaml

@@ -0,0 +1,5 @@
+# This file is respected by repo2docker's test suite, but not repo2docker
+# itself. It is used solely to help us test repo2docker's command line flags.
+#
+- --extra-ignore-file=tests/conda/ignore-file
+- --ignore-file-strategy=theirs

tests/conda/py311-extra-ignore-file-theirs/verify

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+import pathlib
+
+assert not pathlib.Path("from-dockerignore").exists()
+assert pathlib.Path("from-extra-ignore").exists()

tests/unit/test_args.py

@@ -129,3 +129,8 @@ def test_config_priority(tmp_path, trait, arg, default):
     assert getattr(r2d, trait) == "config"
     r2d = make_r2d(["--config", config_file, arg, "cli", "."])
     assert getattr(r2d, trait) == "cli"
+
+
+def test_non_existing_exclude_file():
+    with pytest.raises(SystemExit):
+        make_r2d(["--extra-ignore-file", "does-not-exist"])

tests/unit/test_argumentvalidation.py

@@ -247,3 +247,28 @@ def test_docker_no_build_success(temp_cwd):
     args_list = ["--no-build", "--no-run"]
 
     assert validate_arguments(builddir, args_list, disable_dockerd=True)
+
+
+@pytest.mark.parametrize(
+    "strategy, is_valid",
+    [
+        ("theirs", True),
+        ("ours", True),
+        ("merge", True),
+        ("invalid", False),
+    ],
+)
+def test_ignore_file_strategy(temp_cwd, strategy, is_valid):
+    """ """
+
+    args_list = ["--no-build", "--no-run", "--ignore-file-strategy", strategy]
+
+    assert (
+        validate_arguments(
+            builddir,
+            args_list,
+            "--ignore-file-strategy: invalid choice: 'invalid' (choose from 'theirs', 'ours', 'merge')",
+            disable_dockerd=True,
+        )
+        == is_valid
+    )