From 1e1d6ac20fb192962011fcab04741b16353fcd8d Mon Sep 17 00:00:00 2001 From: Min RK Date: Thu, 23 Mar 2023 15:58:27 +0100 Subject: [PATCH] memoize all getters explicitly rather than automatically --- repo2docker/buildpacks/base.py | 31 ++++++++++++------- repo2docker/buildpacks/conda/__init__.py | 10 ++++++ repo2docker/buildpacks/julia/julia_project.py | 8 ++++- repo2docker/buildpacks/julia/julia_require.py | 6 ++++ repo2docker/buildpacks/nix/__init__.py | 9 +++++- repo2docker/buildpacks/pipfile/__init__.py | 4 +++ repo2docker/buildpacks/python/__init__.py | 4 +++ repo2docker/buildpacks/r.py | 13 ++++++++ 8 files changed, 71 insertions(+), 14 deletions(-) diff --git a/repo2docker/buildpacks/base.py b/repo2docker/buildpacks/base.py index 60f33edd..9afabf7e 100644 --- a/repo2docker/buildpacks/base.py +++ b/repo2docker/buildpacks/base.py @@ -231,19 +231,8 @@ class BuildPack: "support is experimental in repo2docker." ) self.platform = "" - self._memoize() - - def _memoize(self): - """Memoize `get_foo` methods with lru_cache() - - Avoids duplicate log statements when calling what should be idempotent methods more than once - """ - for method_name in dir(self): - if method_name.startswith("get_"): - method = getattr(self, method_name) - if callable(method): - setattr(self, method_name, lru_cache()(method)) + @lru_cache def get_packages(self): """ List of packages that are installed in this BuildPack. @@ -253,6 +242,7 @@ class BuildPack: """ return set() + @lru_cache def get_base_packages(self): """ Base set of apt packages that are installed for all images. @@ -269,6 +259,7 @@ class BuildPack: "unzip", } + @lru_cache def get_build_env(self): """ Ordered list of environment variables to be set for this image. @@ -284,6 +275,7 @@ class BuildPack: """ return [] + @lru_cache def get_env(self): """ Ordered list of environment variables to be set for this image. @@ -298,6 +290,7 @@ class BuildPack: """ return [] + @lru_cache def get_path(self): """ Ordered list of file system paths to look for executables in. @@ -307,12 +300,14 @@ class BuildPack: """ return [] + @lru_cache def get_labels(self): """ Docker labels to set on the built image. """ return self.labels + @lru_cache def get_build_script_files(self): """ Dict of files to be copied to the container image for use in building. @@ -337,6 +332,7 @@ class BuildPack: f"Found a stencila manifest.xml at {root}. Stencila is no longer supported." ) + @lru_cache def get_build_scripts(self): """ Ordered list of shell script snippets to build the base image. @@ -358,6 +354,7 @@ class BuildPack: return [] + @lru_cache def get_preassemble_script_files(self): """ Dict of files to be copied to the container image for use in preassembly. @@ -371,6 +368,7 @@ class BuildPack: """ return {} + @lru_cache def get_preassemble_scripts(self): """ Ordered list of shell snippets to build an image for this repository. @@ -387,6 +385,7 @@ class BuildPack: """ return [] + @lru_cache def get_assemble_scripts(self): """ Ordered list of shell script snippets to build the repo into the image. @@ -413,6 +412,7 @@ class BuildPack: """ return [] + @lru_cache def get_post_build_scripts(self): """ An ordered list of executable scripts to execute after build. @@ -425,6 +425,7 @@ class BuildPack: """ return [] + @lru_cache def get_start_script(self): """ The path to a script to be executed at container start up. @@ -637,12 +638,14 @@ class BuildPack: class BaseImage(BuildPack): + @lru_cache def get_build_env(self): """Return env directives required for build""" return [ ("APP_BASE", "/srv"), ] + @lru_cache def get_env(self): """Return env directives to be set after build""" return [] @@ -650,6 +653,7 @@ class BaseImage(BuildPack): def detect(self): return True + @lru_cache def get_preassemble_scripts(self): scripts = [] try: @@ -689,16 +693,19 @@ class BaseImage(BuildPack): return scripts + @lru_cache def get_assemble_scripts(self): """Return directives to run after the entire repository has been added to the image""" return [] + @lru_cache def get_post_build_scripts(self): post_build = self.binder_path("postBuild") if os.path.exists(post_build): return [post_build] return [] + @lru_cache def get_start_script(self): start = self.binder_path("start") if os.path.exists(start): diff --git a/repo2docker/buildpacks/conda/__init__.py b/repo2docker/buildpacks/conda/__init__.py index 29682c7e..7dc24481 100644 --- a/repo2docker/buildpacks/conda/__init__.py +++ b/repo2docker/buildpacks/conda/__init__.py @@ -3,6 +3,7 @@ import os import re import warnings from collections.abc import Mapping +from functools import lru_cache from ruamel.yaml import YAML @@ -45,6 +46,7 @@ class CondaBuildPack(BaseImage): return "linux-aarch64" raise ValueError(f"Unknown platform {self.platform}") + @lru_cache def get_build_env(self): """Return environment variables to be set. @@ -88,11 +90,13 @@ class CondaBuildPack(BaseImage): env.append(("KERNEL_PYTHON_PREFIX", "${NB_PYTHON_PREFIX}")) return env + @lru_cache def get_env(self): """Make kernel env the default for `conda install`""" env = super().get_env() + [("CONDA_DEFAULT_ENV", "${KERNEL_PYTHON_PREFIX}")] return env + @lru_cache def get_path(self): """Return paths (including conda environment path) to be added to the PATH environment variable. @@ -107,6 +111,7 @@ class CondaBuildPack(BaseImage): path.append("${NPM_DIR}/bin") return path + @lru_cache def get_build_scripts(self): """ Return series of build-steps common to all Python 3 repositories. @@ -145,6 +150,7 @@ class CondaBuildPack(BaseImage): major_pythons = {"2": "2.7", "3": "3.7"} + @lru_cache def get_build_script_files(self): """ Dict of files to be copied to the container image for use in building. @@ -359,6 +365,7 @@ class CondaBuildPack(BaseImage): self.kernel_env_cutoff_version ) + @lru_cache def get_preassemble_script_files(self): """preassembly only requires environment.yml @@ -372,6 +379,7 @@ class CondaBuildPack(BaseImage): assemble_files[environment_yml] = environment_yml return assemble_files + @lru_cache def get_env_scripts(self): """Return series of build-steps specific to this source repository.""" scripts = [] @@ -438,12 +446,14 @@ class CondaBuildPack(BaseImage): ] return scripts + @lru_cache def get_preassemble_scripts(self): scripts = super().get_preassemble_scripts() if self._should_preassemble_env: scripts.extend(self.get_env_scripts()) return scripts + @lru_cache def get_assemble_scripts(self): scripts = super().get_assemble_scripts() if not self._should_preassemble_env: diff --git a/repo2docker/buildpacks/julia/julia_project.py b/repo2docker/buildpacks/julia/julia_project.py index b7eab5c3..c9f72a1e 100644 --- a/repo2docker/buildpacks/julia/julia_project.py +++ b/repo2docker/buildpacks/julia/julia_project.py @@ -1,6 +1,7 @@ """Generates a Dockerfile based on an input matrix for Julia""" import functools import os +from functools import lru_cache import requests import semver @@ -19,7 +20,7 @@ class JuliaProjectTomlBuildPack(PythonBuildPack): # Note that these must remain ordered, in order for the find_semver_match() # function to behave correctly. @property - @functools.lru_cache(maxsize=1) + @lru_cache(maxsize=1) def all_julias(self): try: json = requests.get( @@ -67,6 +68,7 @@ class JuliaProjectTomlBuildPack(PythonBuildPack): raise RuntimeError("Failed to find a matching Julia version: {compat}") return match + @lru_cache def get_build_env(self): """Get additional environment settings for Julia and Jupyter @@ -109,9 +111,11 @@ class JuliaProjectTomlBuildPack(PythonBuildPack): else: return "${REPO_DIR}" + @lru_cache def get_env(self): return super().get_env() + [("JULIA_PROJECT", self.project_dir)] + @lru_cache def get_path(self): """Adds path to Julia binaries to user's PATH. @@ -122,6 +126,7 @@ class JuliaProjectTomlBuildPack(PythonBuildPack): """ return super().get_path() + ["${JULIA_PATH}/bin"] + @lru_cache def get_build_scripts(self): """ Return series of build-steps common to "ALL" Julia repositories @@ -150,6 +155,7 @@ class JuliaProjectTomlBuildPack(PythonBuildPack): ), ] + @lru_cache def get_assemble_scripts(self): """ Return series of build-steps specific to "this" Julia repository diff --git a/repo2docker/buildpacks/julia/julia_require.py b/repo2docker/buildpacks/julia/julia_require.py index ad3f5448..10046800 100644 --- a/repo2docker/buildpacks/julia/julia_require.py +++ b/repo2docker/buildpacks/julia/julia_require.py @@ -1,6 +1,7 @@ """Generates a Dockerfile based on an input matrix with REQUIRE for legacy Julia""" import os +from functools import lru_cache from ...semver import parse_version as V from ..python import PythonBuildPack @@ -54,6 +55,7 @@ class JuliaRequireBuildPack(PythonBuildPack): self._julia_version = julia_version return self._julia_version + @lru_cache def get_build_env(self): """Get additional environment settings for Julia and Jupyter @@ -102,6 +104,7 @@ class JuliaRequireBuildPack(PythonBuildPack): ("JUPYTER", jupyter), ] + @lru_cache def get_path(self): """Adds path to Julia binaries to user's PATH. @@ -112,6 +115,7 @@ class JuliaRequireBuildPack(PythonBuildPack): """ return super().get_path() + ["${JULIA_HOME}"] + @lru_cache def get_build_scripts(self): """ Return series of build-steps common to "ALL" Julia repositories @@ -149,6 +153,7 @@ class JuliaRequireBuildPack(PythonBuildPack): ), ] + @lru_cache def get_assemble_scripts(self): """ Return series of build-steps specific to "this" Julia repository @@ -176,6 +181,7 @@ class JuliaRequireBuildPack(PythonBuildPack): ) ] + @lru_cache def get_build_script_files(self): files = { "julia/install-repo-dependencies.jl": "/tmp/install-repo-dependencies.jl" diff --git a/repo2docker/buildpacks/nix/__init__.py b/repo2docker/buildpacks/nix/__init__.py index 9dbb412a..93b6c6fc 100644 --- a/repo2docker/buildpacks/nix/__init__.py +++ b/repo2docker/buildpacks/nix/__init__.py @@ -1,16 +1,19 @@ """BuildPack for nixpkgs environments""" import os +from functools import lru_cache -from ..base import BaseImage, BuildPack +from ..base import BaseImage class NixBuildPack(BaseImage): """A nix Package Manager BuildPack""" + @lru_cache def get_path(self): """Return paths to be added to PATH environemnt variable""" return super().get_path() + ["/home/${NB_USER}/.nix-profile/bin"] + @lru_cache def get_env(self): """Ordered list of environment variables to be set for this image""" @@ -20,6 +23,7 @@ class NixBuildPack(BaseImage): ("GIT_SSL_CAINFO", "/etc/ssl/certs/ca-certificates.crt"), ] + @lru_cache def get_build_scripts(self): """ Return series of build-steps common to all nix repositories. @@ -55,6 +59,7 @@ class NixBuildPack(BaseImage): ), ] + @lru_cache def get_build_script_files(self): """Dict of files to be copied to the container image for use in building""" return { @@ -62,6 +67,7 @@ class NixBuildPack(BaseImage): "nix/nix-shell-wrapper": "/usr/local/bin/nix-shell-wrapper", } + @lru_cache def get_assemble_scripts(self): """Return series of build-steps specific to this source repository.""" return super().get_assemble_scripts() + [ @@ -75,6 +81,7 @@ class NixBuildPack(BaseImage): ) ] + @lru_cache def get_start_script(self): """The path to a script to be executed as ENTRYPOINT""" # the shell wrapper script duplicates the behaviour of other buildpacks diff --git a/repo2docker/buildpacks/pipfile/__init__.py b/repo2docker/buildpacks/pipfile/__init__.py index 7f8ec53b..29ba6ef3 100644 --- a/repo2docker/buildpacks/pipfile/__init__.py +++ b/repo2docker/buildpacks/pipfile/__init__.py @@ -8,6 +8,7 @@ same as the Python or Conda build packs. import json import os import re +from functools import lru_cache import toml @@ -75,6 +76,7 @@ class PipfileBuildPack(CondaBuildPack): self._python_version = self.major_pythons["3"] return self._python_version + @lru_cache def get_preassemble_script_files(self): """Return files needed for preassembly""" files = super().get_preassemble_script_files() @@ -84,6 +86,7 @@ class PipfileBuildPack(CondaBuildPack): files[path] = path return files + @lru_cache def get_preassemble_scripts(self): """scripts to run prior to staging the repo contents""" scripts = super().get_preassemble_scripts() @@ -101,6 +104,7 @@ class PipfileBuildPack(CondaBuildPack): ) return scripts + @lru_cache def get_assemble_scripts(self): """Return series of build-steps specific to this repository.""" # If we have either Pipfile.lock, Pipfile, or runtime.txt declare the diff --git a/repo2docker/buildpacks/python/__init__.py b/repo2docker/buildpacks/python/__init__.py index a96a6cab..89a78375 100644 --- a/repo2docker/buildpacks/python/__init__.py +++ b/repo2docker/buildpacks/python/__init__.py @@ -1,5 +1,6 @@ """Generates Dockerfiles based on an input matrix based on Python.""" import os +from functools import lru_cache from ...utils import is_local_pip_requirement, open_guess_encoding from ..conda import CondaBuildPack @@ -93,6 +94,7 @@ class PythonBuildPack(CondaBuildPack): # allow assembly from subset return True + @lru_cache def get_preassemble_script_files(self): assemble_files = super().get_preassemble_script_files() for name in ("requirements.txt", "requirements3.txt"): @@ -101,6 +103,7 @@ class PythonBuildPack(CondaBuildPack): assemble_files[requirements_txt] = requirements_txt return assemble_files + @lru_cache def get_preassemble_scripts(self): """Return scripts to run before adding the full repository""" scripts = super().get_preassemble_scripts() @@ -108,6 +111,7 @@ class PythonBuildPack(CondaBuildPack): scripts.extend(self._get_pip_scripts()) return scripts + @lru_cache def get_assemble_scripts(self): """Return series of build steps that require the full repository""" # If we have a runtime.txt & that's set to python-2.7, diff --git a/repo2docker/buildpacks/r.py b/repo2docker/buildpacks/r.py index 8d8ae6b1..a3c8bade 100644 --- a/repo2docker/buildpacks/r.py +++ b/repo2docker/buildpacks/r.py @@ -1,6 +1,7 @@ import datetime import os import re +from functools import lru_cache import requests @@ -142,6 +143,7 @@ class RBuildPack(PythonBuildPack): self._runtime = f"r-{str(self._checkpoint_date)}" return True + @lru_cache def get_env(self): """ Set custom env vars needed for RStudio to load @@ -156,6 +158,7 @@ class RBuildPack(PythonBuildPack): ("LD_LIBRARY_PATH", "${R_HOME}/lib:${LD_LIBRARY_PATH}"), ] + @lru_cache def get_path(self): """ Return paths to be added to the PATH environment variable. @@ -165,6 +168,7 @@ class RBuildPack(PythonBuildPack): """ return super().get_path() + ["/usr/lib/rstudio-server/bin/"] + @lru_cache def get_build_env(self): """ Return environment variables to be set. @@ -178,6 +182,7 @@ class RBuildPack(PythonBuildPack): ("R_LIBS_USER", "${APP_BASE}/rlibs") ] + @lru_cache def get_packages(self): """ Return list of packages to be installed. @@ -195,6 +200,7 @@ class RBuildPack(PythonBuildPack): return super().get_packages().union(packages) + @lru_cache def get_rspm_snapshot_url(self, snapshot_date, max_days_prior=7): for i in range(max_days_prior): snapshots = requests.post( @@ -219,6 +225,7 @@ class RBuildPack(PythonBuildPack): ) ) + @lru_cache def get_mran_snapshot_url(self, snapshot_date, max_days_prior=7): for i in range(max_days_prior): try_date = snapshot_date - datetime.timedelta(days=i) @@ -233,6 +240,7 @@ class RBuildPack(PythonBuildPack): ) ) + @lru_cache def get_cran_mirror_url(self, snapshot_date): # Date after which we will use rspm + binary packages instead of MRAN + source packages rspm_cutoff_date = datetime.date(2022, 1, 1) @@ -242,6 +250,7 @@ class RBuildPack(PythonBuildPack): else: return self.get_mran_snapshot_url(snapshot_date) + @lru_cache def get_devtools_snapshot_url(self): """ Return url of snapshot to use for getting devtools install @@ -255,6 +264,7 @@ class RBuildPack(PythonBuildPack): # necessary apt packages. return "https://packagemanager.rstudio.com/all/__linux__/bionic/2022-01-04+Y3JhbiwyOjQ1MjYyMTU7NzlBRkJEMzg" + @lru_cache def get_build_scripts(self): """ Return series of build-steps common to all R repositories @@ -349,6 +359,7 @@ class RBuildPack(PythonBuildPack): return super().get_build_scripts() + scripts + @lru_cache def get_preassemble_script_files(self): files = super().get_preassemble_script_files() installR_path = self.binder_path("install.R") @@ -357,6 +368,7 @@ class RBuildPack(PythonBuildPack): return files + @lru_cache def get_preassemble_scripts(self): """Install contents of install.R @@ -382,6 +394,7 @@ class RBuildPack(PythonBuildPack): return super().get_preassemble_scripts() + scripts + @lru_cache def get_assemble_scripts(self): """Install the dependencies of or the repository itself""" assemble_scripts = super().get_assemble_scripts()