diff --git a/repo2docker/buildpacks/conda/__init__.py b/repo2docker/buildpacks/conda/__init__.py index 2aa45c37..87865250 100644 --- a/repo2docker/buildpacks/conda/__init__.py +++ b/repo2docker/buildpacks/conda/__init__.py @@ -6,6 +6,7 @@ from collections import Mapping from ruamel.yaml import YAML from ..base import BaseImage +from ...utils import is_local_pip_requirement # pattern for parsing conda dependency line PYTHON_REGEX = re.compile(r"python\s*=+\s*([\d\.]*)") @@ -127,6 +128,50 @@ class CondaBuildPack(BaseImage): files.update(super().get_build_script_files()) return files + _environment_yaml = None + + @property + def environment_yaml(self): + if self._environment_yaml is not None: + return self._environment_yaml + + environment_yml = self.binder_path("environment.yml") + if not os.path.exists(environment_yml): + self._environment_yaml = {} + return self._environment_yaml + + with open(environment_yml) as f: + env = YAML().load(f) + # check if the env file is empty, if so instantiate an empty dictionary. + if env is None: + env = {} + # check if the env file provided a dick-like thing not a list or other data structure. + if not isinstance(env, Mapping): + raise TypeError( + "environment.yml should contain a dictionary. Got %r" % type(env) + ) + self._environment_yaml = env + + return self._environment_yaml + + @property + def _should_preassemble_env(self): + """Check for local pip requirements in environment.yaml + + If there are any local references, e.g. `-e .`, + stage the whole repo prior to installation. + """ + dependencies = self.environment_yaml.get("dependencies", []) + pip_requirements = None + for dep in dependencies: + if isinstance(dep, dict) and dep.get("pip"): + pip_requirements = dep["pip"] + if isinstance(pip_requirements, list): + for line in pip_requirements: + if is_local_pip_requirement(line): + return False + return True + @property def python_version(self): """Detect the Python version for a given `environment.yml` @@ -135,31 +180,17 @@ class CondaBuildPack(BaseImage): or a Falsy empty string '' if not found. """ - environment_yml = self.binder_path("environment.yml") - if not os.path.exists(environment_yml): - return "" - if not hasattr(self, "_python_version"): py_version = None - with open(environment_yml) as f: - env = YAML().load(f) - # check if the env file is empty, if so instantiate an empty dictionary. - if env is None: - env = {} - # check if the env file provided a dick-like thing not a list or other data structure. - if not isinstance(env, Mapping): - raise TypeError( - "environment.yml should contain a dictionary. Got %r" - % type(env) - ) - for dep in env.get("dependencies", []): - if not isinstance(dep, str): - continue - match = PYTHON_REGEX.match(dep) - if not match: - continue - py_version = match.group(1) - break + env = self.environment_yaml + for dep in env.get("dependencies", []): + if not isinstance(dep, str): + continue + match = PYTHON_REGEX.match(dep) + if not match: + continue + py_version = match.group(1) + break # extract major.minor if py_version: @@ -185,19 +216,20 @@ class CondaBuildPack(BaseImage): repo contents change """ assemble_files = super().get_preassemble_script_files() - environment_yml = self.binder_path("environment.yml") - if os.path.exists(environment_yml): - assemble_files[environment_yml] = environment_yml + if self._should_preassemble_env: + environment_yml = self.binder_path("environment.yml") + if os.path.exists(environment_yml): + assemble_files[environment_yml] = environment_yml return assemble_files - def get_preassemble_scripts(self): + def get_env_scripts(self): """Return series of build-steps specific to this source repository. """ - assembly_scripts = [] + scripts = [] environment_yml = self.binder_path("environment.yml") env_prefix = "${KERNEL_PYTHON_PREFIX}" if self.py2 else "${NB_PYTHON_PREFIX}" if os.path.exists(environment_yml): - assembly_scripts.append( + scripts.append( ( "${NB_USER}", r""" @@ -209,7 +241,19 @@ class CondaBuildPack(BaseImage): ), ) ) - return super().get_preassemble_scripts() + assembly_scripts + return scripts + + def get_preassemble_scripts(self): + scripts = super().get_preassemble_scripts() + if self._should_preassemble_env: + scripts.extend(self.get_env_scripts()) + return scripts + + def get_assemble_scripts(self): + scripts = super().get_assemble_scripts() + if not self._should_preassemble_env: + scripts.extend(self.get_env_scripts()) + return scripts def detect(self): """Check if current repo should be built with the Conda BuildPack. diff --git a/repo2docker/buildpacks/python/__init__.py b/repo2docker/buildpacks/python/__init__.py index 7b6fc5af..72029d78 100644 --- a/repo2docker/buildpacks/python/__init__.py +++ b/repo2docker/buildpacks/python/__init__.py @@ -2,6 +2,7 @@ import os from ..conda import CondaBuildPack +from ...utils import is_local_pip_requirement class PythonBuildPack(CondaBuildPack): @@ -34,31 +35,6 @@ class PythonBuildPack(CondaBuildPack): self._python_version = py_version return self._python_version - def _is_local_requirement(self, line): - """Return whether a line in a requirements.txt file references a local file""" - # trim comments and skip empty lines - line = line.split("#", 1)[0].strip() - if not line: - return False - if line.startswith(("-r", "-c")): - # local -r or -c references break isolation - return True - # strip off `-e, etc.` - if line.startswith("-"): - line = line.split(None, 1)[1] - if "file://" in line: - # file references break isolation - return True - if "://" in line: - # handle git://../local/file - path = line.split("://", 1)[1] - else: - path = line - if path.startswith("."): - # references a local file - return True - return False - def _get_pip_scripts(self): """Get pip install scripts @@ -112,7 +88,7 @@ class PythonBuildPack(CondaBuildPack): continue with open(requirements_txt) as f: for line in f: - if self._is_local_requirement(line): + if is_local_pip_requirement(line): return False # didn't find any local references, diff --git a/repo2docker/utils.py b/repo2docker/utils.py index a53ba5ef..1c0a4334 100644 --- a/repo2docker/utils.py +++ b/repo2docker/utils.py @@ -431,3 +431,29 @@ def normalize_doi(val): (e.g. https://doi.org/10.1234/jshd123)""" m = doi_regexp.match(val) return m.group(2) + + +def is_local_pip_requirement(line): + """Return whether a pip requirement (e.g. in requirements.txt file) references a local file""" + # trim comments and skip empty lines + line = line.split("#", 1)[0].strip() + if not line: + return False + if line.startswith(("-r", "-c")): + # local -r or -c references break isolation + return True + # strip off `-e, etc.` + if line.startswith("-"): + line = line.split(None, 1)[1] + if "file://" in line: + # file references break isolation + return True + if "://" in line: + # handle git://../local/file + path = line.split("://", 1)[1] + else: + path = line + if path.startswith("."): + # references a local file + return True + return False