From 8591960f0c34a444ec20e8b360fafe57d20792ab Mon Sep 17 00:00:00 2001 From: Samuel Gaist Date: Wed, 20 Dec 2023 17:44:42 +0100 Subject: [PATCH] feat: implement parameters to allow use of an extra ignore file In supplement to the support for the .dockerignore and .containerignore files, these two new parameters (extra-ignore-file and ingore-file-strategy) allow to modify how the ignore list is managed. This allows, for example in the case of BinderHub, the administrator to have a default set of files or folders that get ignored if the repository does not contain such any ignore file. The following strategies are available: - ours - theirs - merge The first forces the use of the file passed in parameters The second uses the file from the repository if it exists The last puts both together --- repo2docker/__main__.py | 26 ++++++++++ repo2docker/app.py | 31 +++++++++++- repo2docker/buildpacks/__init__.py | 2 +- repo2docker/buildpacks/base.py | 50 ++++++++++++++----- tests/conda/ignore-file | 2 + .../.dockerignore | 1 + .../environment.yml | 2 + .../from-dockerignore | 1 + .../from-extra-ignore | 1 + .../test-extra-args.yaml | 5 ++ .../py311-extra-ignore-file-merge/verify | 6 +++ .../.dockerignore | 1 + .../environment.yml | 2 + .../from-dockerignore | 1 + .../from-extra-ignore | 1 + .../test-extra-args.yaml | 5 ++ .../conda/py311-extra-ignore-file-ours/verify | 6 +++ .../environment.yml | 2 + .../from-dockerignore | 1 + .../from-extra-ignore | 1 + .../test-extra-args.yaml | 5 ++ .../verify | 6 +++ .../.dockerignore | 1 + .../environment.yml | 2 + .../from-dockerignore | 1 + .../from-extra-ignore | 1 + .../test-extra-args.yaml | 5 ++ .../py311-extra-ignore-file-theirs/verify | 6 +++ tests/unit/test_args.py | 5 ++ tests/unit/test_argumentvalidation.py | 25 ++++++++++ 30 files changed, 189 insertions(+), 15 deletions(-) create mode 100644 tests/conda/ignore-file create mode 100644 tests/conda/py311-extra-ignore-file-merge/.dockerignore create mode 100644 tests/conda/py311-extra-ignore-file-merge/environment.yml create mode 100644 tests/conda/py311-extra-ignore-file-merge/from-dockerignore create mode 100644 tests/conda/py311-extra-ignore-file-merge/from-extra-ignore create mode 100644 tests/conda/py311-extra-ignore-file-merge/test-extra-args.yaml create mode 100755 tests/conda/py311-extra-ignore-file-merge/verify create mode 100644 tests/conda/py311-extra-ignore-file-ours/.dockerignore create mode 100644 tests/conda/py311-extra-ignore-file-ours/environment.yml create mode 100644 tests/conda/py311-extra-ignore-file-ours/from-dockerignore create mode 100644 tests/conda/py311-extra-ignore-file-ours/from-extra-ignore create mode 100644 tests/conda/py311-extra-ignore-file-ours/test-extra-args.yaml create mode 100755 tests/conda/py311-extra-ignore-file-ours/verify create mode 100644 tests/conda/py311-extra-ignore-file-theirs-without-file/environment.yml create mode 100644 tests/conda/py311-extra-ignore-file-theirs-without-file/from-dockerignore create mode 100644 tests/conda/py311-extra-ignore-file-theirs-without-file/from-extra-ignore create mode 100644 tests/conda/py311-extra-ignore-file-theirs-without-file/test-extra-args.yaml create mode 100755 tests/conda/py311-extra-ignore-file-theirs-without-file/verify create mode 100644 tests/conda/py311-extra-ignore-file-theirs/.dockerignore create mode 100644 tests/conda/py311-extra-ignore-file-theirs/environment.yml create mode 100644 tests/conda/py311-extra-ignore-file-theirs/from-dockerignore create mode 100644 tests/conda/py311-extra-ignore-file-theirs/from-extra-ignore create mode 100644 tests/conda/py311-extra-ignore-file-theirs/test-extra-args.yaml create mode 100755 tests/conda/py311-extra-ignore-file-theirs/verify diff --git a/repo2docker/__main__.py b/repo2docker/__main__.py index a61ef918..0db47f20 100644 --- a/repo2docker/__main__.py +++ b/repo2docker/__main__.py @@ -2,6 +2,7 @@ import argparse import logging import os import sys +from pathlib import Path from . import __version__ from .app import Repo2Docker @@ -282,6 +283,22 @@ def get_argparser(): help=Repo2Docker.engine.help, ) + argparser.add_argument( + "--extra-ignore-file", + dest="extra_ignore_file", + type=Path, + help=Repo2Docker.extra_ignore_file.help, + ) + + argparser.add_argument( + "--ignore-file-strategy", + dest="ignore_file_strategy", + type=str, + choices=Repo2Docker.ignore_file_strategy.values, + default=Repo2Docker.ignore_file_strategy.default_value, + help=Repo2Docker.ignore_file_strategy.help, + ) + return argparser @@ -464,6 +481,15 @@ def make_r2d(argv=None): if args.target_repo_dir: r2d.target_repo_dir = args.target_repo_dir + if args.extra_ignore_file is not None: + if not args.extra_ignore_file.exists(): + print(f"The ignore file {args.extra_ignore_file} does not exist") + sys.exit(1) + r2d.extra_ignore_file = str(args.extra_ignore_file.resolve()) + + if args.ignore_file_strategy is not None: + r2d.ignore_file_strategy = args.ignore_file_strategy + return r2d diff --git a/repo2docker/app.py b/repo2docker/app.py index 63643da2..d5879656 100755 --- a/repo2docker/app.py +++ b/repo2docker/app.py @@ -21,13 +21,14 @@ from urllib.parse import urlparse import entrypoints import escapism from pythonjsonlogger import jsonlogger -from traitlets import Any, Bool, Dict, Int, List, Unicode, default, observe +from traitlets import Any, Bool, Dict, Enum, Int, List, Unicode, default, observe from traitlets.config import Application from . import __version__, contentproviders from .buildpacks import ( CondaBuildPack, DockerBuildPack, + ExcludesStrategy, JuliaProjectTomlBuildPack, JuliaRequireBuildPack, LegacyBinderDockerBuildPack, @@ -462,6 +463,32 @@ class Repo2Docker(Application): """, ) + extra_ignore_file = Unicode( + "", + config=True, + help=""" + Path to an additional .dockerignore or .containerignore file to be applied + when building an image. + + Depending on the strategy selected the content of the file will replace, + be merged or be ignored. + """, + ) + + ignore_file_strategy = Enum( + ExcludesStrategy.values(), + config=True, + default_value=ExcludesStrategy.theirs, + help=""" + Strategy to use if an extra ignore file is passed: + - merge means that the content of the extra ignore file will be merged + with the ignore file contained in the repository (if any) + - ours means that the extra ignore file content will be used in any case + - theirs means that if there is an ignore file in the repository, the + extra ignore file will not be used. + """, + ) + def get_engine(self): """Return an instance of the container engine. @@ -860,6 +887,8 @@ class Repo2Docker(Application): self.cache_from, self.extra_build_kwargs, platform=self.platform, + extra_ignore_file=self.extra_ignore_file, + ignore_file_strategy=self.ignore_file_strategy, ): if docker_client.string_output: self.log.info(l, extra=dict(phase=R2dState.BUILDING)) diff --git a/repo2docker/buildpacks/__init__.py b/repo2docker/buildpacks/__init__.py index 7568000b..cb22c97e 100644 --- a/repo2docker/buildpacks/__init__.py +++ b/repo2docker/buildpacks/__init__.py @@ -1,4 +1,4 @@ -from .base import BaseImage, BuildPack +from .base import BaseImage, BuildPack, ExcludesStrategy from .conda import CondaBuildPack from .docker import DockerBuildPack from .julia import JuliaProjectTomlBuildPack, JuliaRequireBuildPack diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index f8dcf758..540a9159 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -7,6 +7,7 @@ import string import sys import tarfile import textwrap +from enum import StrEnum, auto from functools import lru_cache import escapism @@ -205,6 +206,16 @@ HERE = os.path.dirname(os.path.abspath(__file__)) DEFAULT_NB_UID = 1000 +class ExcludesStrategy(StrEnum): + theirs = auto() + ours = auto() + merge = auto() + + @classmethod + def values(cls): + return [item.value for item in cls] + + class BuildPack: """ A composable BuildPack. @@ -582,6 +593,8 @@ class BuildPack: cache_from, extra_build_kwargs, platform=None, + extra_ignore_file=None, + ignore_file_strategy=ExcludesStrategy.theirs, ): tarf = io.BytesIO() tar = tarfile.open(fileobj=tarf, mode="w") @@ -609,24 +622,35 @@ class BuildPack: for fname in ("repo2docker-entrypoint", "python3-login"): tar.add(os.path.join(HERE, fname), fname, filter=_filter_tar) - exclude = [] + def _read_excludes(filepath): + with open(filepath) as ignore_file: + cleaned_lines = [ + line.strip() for line in ignore_file.read().splitlines() + ] + return [line for line in cleaned_lines if line != "" and line[0] != "#"] + extra_excludes = [] + if extra_ignore_file: + extra_excludes = _read_excludes(extra_ignore_file) + + excludes = [] for ignore_file_name in [".dockerignore", ".containerignore"]: ignore_file_name = self.binder_path(ignore_file_name) if os.path.exists(ignore_file_name): - with open(ignore_file_name) as ignore_file: - cleaned_lines = [ - line.strip() for line in ignore_file.read().splitlines() - ] - exclude.extend( - [ - line - for line in cleaned_lines - if line != "" and line[0] != "#" - ] - ) + excludes.extend(_read_excludes(ignore_file_name)) - files_to_add = exclude_paths(".", exclude) + if extra_ignore_file is not None: + if ignore_file_strategy == ExcludesStrategy.ours: + excludes = extra_excludes + elif ignore_file_strategy == ExcludesStrategy.merge: + excludes.extend(extra_excludes) + else: + # ignore means that if an ignore file exist, its content is used + # otherwise, the extra exclude + if not excludes: + excludes = extra_excludes + + files_to_add = exclude_paths(".", excludes) if files_to_add: for item in files_to_add: diff --git a/tests/conda/ignore-file b/tests/conda/ignore-file new file mode 100644 index 00000000..add093a0 --- /dev/null +++ b/tests/conda/ignore-file @@ -0,0 +1,2 @@ +# Docker compatible ignore file +from-extra-ignore diff --git a/tests/conda/py311-extra-ignore-file-merge/.dockerignore b/tests/conda/py311-extra-ignore-file-merge/.dockerignore new file mode 100644 index 00000000..e8e3e35f --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-merge/.dockerignore @@ -0,0 +1 @@ +from-dockerignore diff --git a/tests/conda/py311-extra-ignore-file-merge/environment.yml b/tests/conda/py311-extra-ignore-file-merge/environment.yml new file mode 100644 index 00000000..c62c1b26 --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-merge/environment.yml @@ -0,0 +1,2 @@ +dependencies: + - python=3.11 diff --git a/tests/conda/py311-extra-ignore-file-merge/from-dockerignore b/tests/conda/py311-extra-ignore-file-merge/from-dockerignore new file mode 100644 index 00000000..00c279c8 --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-merge/from-dockerignore @@ -0,0 +1 @@ +Must be ignored from .dockerignore file diff --git a/tests/conda/py311-extra-ignore-file-merge/from-extra-ignore b/tests/conda/py311-extra-ignore-file-merge/from-extra-ignore new file mode 100644 index 00000000..2a6716fc --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-merge/from-extra-ignore @@ -0,0 +1 @@ +Must be ignored from extra ignore file diff --git a/tests/conda/py311-extra-ignore-file-merge/test-extra-args.yaml b/tests/conda/py311-extra-ignore-file-merge/test-extra-args.yaml new file mode 100644 index 00000000..5e69a801 --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-merge/test-extra-args.yaml @@ -0,0 +1,5 @@ +# This file is respected by repo2docker's test suite, but not repo2docker +# itself. It is used solely to help us test repo2docker's command line flags. +# +- --extra-ignore-file=ignore-file +- --ignore-file-strategy=merge diff --git a/tests/conda/py311-extra-ignore-file-merge/verify b/tests/conda/py311-extra-ignore-file-merge/verify new file mode 100755 index 00000000..1d0b9bab --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-merge/verify @@ -0,0 +1,6 @@ +#!/usr/bin/env python + +import pathlib + +assert not pathlib.Path("from-dockerignore").exists() +assert not pathlib.Path("from-extra-ignore").exists() diff --git a/tests/conda/py311-extra-ignore-file-ours/.dockerignore b/tests/conda/py311-extra-ignore-file-ours/.dockerignore new file mode 100644 index 00000000..e8e3e35f --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-ours/.dockerignore @@ -0,0 +1 @@ +from-dockerignore diff --git a/tests/conda/py311-extra-ignore-file-ours/environment.yml b/tests/conda/py311-extra-ignore-file-ours/environment.yml new file mode 100644 index 00000000..c62c1b26 --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-ours/environment.yml @@ -0,0 +1,2 @@ +dependencies: + - python=3.11 diff --git a/tests/conda/py311-extra-ignore-file-ours/from-dockerignore b/tests/conda/py311-extra-ignore-file-ours/from-dockerignore new file mode 100644 index 00000000..05989774 --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-ours/from-dockerignore @@ -0,0 +1 @@ +Must not be ignored because of ours strategy and extra ignore file does not contain it. diff --git a/tests/conda/py311-extra-ignore-file-ours/from-extra-ignore b/tests/conda/py311-extra-ignore-file-ours/from-extra-ignore new file mode 100644 index 00000000..1a65827c --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-ours/from-extra-ignore @@ -0,0 +1 @@ +Must be ignored diff --git a/tests/conda/py311-extra-ignore-file-ours/test-extra-args.yaml b/tests/conda/py311-extra-ignore-file-ours/test-extra-args.yaml new file mode 100644 index 00000000..b89a2d30 --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-ours/test-extra-args.yaml @@ -0,0 +1,5 @@ +# This file is respected by repo2docker's test suite, but not repo2docker +# itself. It is used solely to help us test repo2docker's command line flags. +# +- --extra-ignore-file=ignore-file +- --ignore-file-strategy=ours diff --git a/tests/conda/py311-extra-ignore-file-ours/verify b/tests/conda/py311-extra-ignore-file-ours/verify new file mode 100755 index 00000000..754aa728 --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-ours/verify @@ -0,0 +1,6 @@ +#!/usr/bin/env python + +import pathlib + +assert pathlib.Path("from-dockerignore").exists() +assert not pathlib.Path("from-extra-ignore").exists() diff --git a/tests/conda/py311-extra-ignore-file-theirs-without-file/environment.yml b/tests/conda/py311-extra-ignore-file-theirs-without-file/environment.yml new file mode 100644 index 00000000..c62c1b26 --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-theirs-without-file/environment.yml @@ -0,0 +1,2 @@ +dependencies: + - python=3.11 diff --git a/tests/conda/py311-extra-ignore-file-theirs-without-file/from-dockerignore b/tests/conda/py311-extra-ignore-file-theirs-without-file/from-dockerignore new file mode 100644 index 00000000..69effab9 --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-theirs-without-file/from-dockerignore @@ -0,0 +1 @@ +No docker ignore so should still appear diff --git a/tests/conda/py311-extra-ignore-file-theirs-without-file/from-extra-ignore b/tests/conda/py311-extra-ignore-file-theirs-without-file/from-extra-ignore new file mode 100644 index 00000000..5b98049c --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-theirs-without-file/from-extra-ignore @@ -0,0 +1 @@ +Must be ignored because of extra ignore file diff --git a/tests/conda/py311-extra-ignore-file-theirs-without-file/test-extra-args.yaml b/tests/conda/py311-extra-ignore-file-theirs-without-file/test-extra-args.yaml new file mode 100644 index 00000000..a8c00a6b --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-theirs-without-file/test-extra-args.yaml @@ -0,0 +1,5 @@ +# This file is respected by repo2docker's test suite, but not repo2docker +# itself. It is used solely to help us test repo2docker's command line flags. +# +- --extra-ignore-file=ignore-file +- --ignore-file-strategy=theirs diff --git a/tests/conda/py311-extra-ignore-file-theirs-without-file/verify b/tests/conda/py311-extra-ignore-file-theirs-without-file/verify new file mode 100755 index 00000000..754aa728 --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-theirs-without-file/verify @@ -0,0 +1,6 @@ +#!/usr/bin/env python + +import pathlib + +assert pathlib.Path("from-dockerignore").exists() +assert not pathlib.Path("from-extra-ignore").exists() diff --git a/tests/conda/py311-extra-ignore-file-theirs/.dockerignore b/tests/conda/py311-extra-ignore-file-theirs/.dockerignore new file mode 100644 index 00000000..e8e3e35f --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-theirs/.dockerignore @@ -0,0 +1 @@ +from-dockerignore diff --git a/tests/conda/py311-extra-ignore-file-theirs/environment.yml b/tests/conda/py311-extra-ignore-file-theirs/environment.yml new file mode 100644 index 00000000..c62c1b26 --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-theirs/environment.yml @@ -0,0 +1,2 @@ +dependencies: + - python=3.11 diff --git a/tests/conda/py311-extra-ignore-file-theirs/from-dockerignore b/tests/conda/py311-extra-ignore-file-theirs/from-dockerignore new file mode 100644 index 00000000..00c279c8 --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-theirs/from-dockerignore @@ -0,0 +1 @@ +Must be ignored from .dockerignore file diff --git a/tests/conda/py311-extra-ignore-file-theirs/from-extra-ignore b/tests/conda/py311-extra-ignore-file-theirs/from-extra-ignore new file mode 100644 index 00000000..149f0734 --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-theirs/from-extra-ignore @@ -0,0 +1 @@ +Shall be present due to strategy being theirs and this file does not appear in .dockerignore diff --git a/tests/conda/py311-extra-ignore-file-theirs/test-extra-args.yaml b/tests/conda/py311-extra-ignore-file-theirs/test-extra-args.yaml new file mode 100644 index 00000000..a8c00a6b --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-theirs/test-extra-args.yaml @@ -0,0 +1,5 @@ +# This file is respected by repo2docker's test suite, but not repo2docker +# itself. It is used solely to help us test repo2docker's command line flags. +# +- --extra-ignore-file=ignore-file +- --ignore-file-strategy=theirs diff --git a/tests/conda/py311-extra-ignore-file-theirs/verify b/tests/conda/py311-extra-ignore-file-theirs/verify new file mode 100755 index 00000000..e6a7764e --- /dev/null +++ b/tests/conda/py311-extra-ignore-file-theirs/verify @@ -0,0 +1,6 @@ +#!/usr/bin/env python + +import pathlib + +assert not pathlib.Path("from-dockerignore").exists() +assert pathlib.Path("from-extra-ignore").exists() diff --git a/tests/unit/test_args.py b/tests/unit/test_args.py index 53913d94..3a859ec4 100644 --- a/tests/unit/test_args.py +++ b/tests/unit/test_args.py @@ -128,3 +128,8 @@ def test_config_priority(tmp_path, trait, arg, default): assert getattr(r2d, trait) == "config" r2d = make_r2d(["--config", config_file, arg, "cli", "."]) assert getattr(r2d, trait) == "cli" + + +def test_non_existing_exclude_file(): + with pytest.raises(SystemExit): + make_r2d(["--extra-ignore-file", "does-not-exist"]) diff --git a/tests/unit/test_argumentvalidation.py b/tests/unit/test_argumentvalidation.py index 78b96464..f70669b3 100644 --- a/tests/unit/test_argumentvalidation.py +++ b/tests/unit/test_argumentvalidation.py @@ -247,3 +247,28 @@ def test_docker_no_build_success(temp_cwd): args_list = ["--no-build", "--no-run"] assert validate_arguments(builddir, args_list, disable_dockerd=True) + + +@pytest.mark.parametrize( + "strategy, is_valid", + [ + ("theirs", True), + ("ours", True), + ("merge", True), + ("invalid", False), + ], +) +def test_ignore_file_strategy(temp_cwd, strategy, is_valid): + """ """ + + args_list = ["--no-build", "--no-run", "--ignore-file-strategy", strategy] + + assert ( + validate_arguments( + builddir, + args_list, + "--ignore-file-strategy: invalid choice: 'invalid' (choose from 'theirs', 'ours', 'merge')", + disable_dockerd=True, + ) + == is_valid + )