kopia lustrzana https://github.com/jupyterhub/repo2docker
check for local pip requirements in conda environments
Min RK
rodzic
4f3f192d2f
commit
3da5543bfc
|
|
@ -6,6 +6,7 @@ from collections import Mapping
|
|||
from ruamel.yaml import YAML
|
||||
|
||||
from ..base import BaseImage
|
||||
from ...utils import is_local_pip_requirement
|
||||
|
||||
# pattern for parsing conda dependency line
|
||||
PYTHON_REGEX = re.compile(r"python\s*=+\s*([\d\.]*)")
|
||||
|
|
@ -127,6 +128,50 @@ class CondaBuildPack(BaseImage):
|
|||
files.update(super().get_build_script_files())
|
||||
return files
|
||||
|
||||
_environment_yaml = None
|
||||
|
||||
@property
|
||||
def environment_yaml(self):
|
||||
if self._environment_yaml is not None:
|
||||
return self._environment_yaml
|
||||
|
||||
environment_yml = self.binder_path("environment.yml")
|
||||
if not os.path.exists(environment_yml):
|
||||
self._environment_yaml = {}
|
||||
return self._environment_yaml
|
||||
|
||||
with open(environment_yml) as f:
|
||||
env = YAML().load(f)
|
||||
# check if the env file is empty, if so instantiate an empty dictionary.
|
||||
if env is None:
|
||||
env = {}
|
||||
# check if the env file provided a dick-like thing not a list or other data structure.
|
||||
if not isinstance(env, Mapping):
|
||||
raise TypeError(
|
||||
"environment.yml should contain a dictionary. Got %r" % type(env)
|
||||
)
|
||||
self._environment_yaml = env
|
||||
|
||||
return self._environment_yaml
|
||||
|
||||
@property
|
||||
def _should_preassemble_env(self):
|
||||
"""Check for local pip requirements in environment.yaml
|
||||
|
||||
If there are any local references, e.g. `-e .`,
|
||||
stage the whole repo prior to installation.
|
||||
"""
|
||||
dependencies = self.environment_yaml.get("dependencies", [])
|
||||
pip_requirements = None
|
||||
for dep in dependencies:
|
||||
if isinstance(dep, dict) and dep.get("pip"):
|
||||
pip_requirements = dep["pip"]
|
||||
if isinstance(pip_requirements, list):
|
||||
for line in pip_requirements:
|
||||
if is_local_pip_requirement(line):
|
||||
return False
|
||||
return True
|
||||
|
||||
@property
|
||||
def python_version(self):
|
||||
"""Detect the Python version for a given `environment.yml`
|
||||
|
|
@ -135,31 +180,17 @@ class CondaBuildPack(BaseImage):
|
|||
or a Falsy empty string '' if not found.
|
||||
|
||||
"""
|
||||
environment_yml = self.binder_path("environment.yml")
|
||||
if not os.path.exists(environment_yml):
|
||||
return ""
|
||||
|
||||
if not hasattr(self, "_python_version"):
|
||||
py_version = None
|
||||
with open(environment_yml) as f:
|
||||
env = YAML().load(f)
|
||||
# check if the env file is empty, if so instantiate an empty dictionary.
|
||||
if env is None:
|
||||
env = {}
|
||||
# check if the env file provided a dick-like thing not a list or other data structure.
|
||||
if not isinstance(env, Mapping):
|
||||
raise TypeError(
|
||||
"environment.yml should contain a dictionary. Got %r"
|
||||
% type(env)
|
||||
)
|
||||
for dep in env.get("dependencies", []):
|
||||
if not isinstance(dep, str):
|
||||
continue
|
||||
match = PYTHON_REGEX.match(dep)
|
||||
if not match:
|
||||
continue
|
||||
py_version = match.group(1)
|
||||
break
|
||||
env = self.environment_yaml
|
||||
for dep in env.get("dependencies", []):
|
||||
if not isinstance(dep, str):
|
||||
continue
|
||||
match = PYTHON_REGEX.match(dep)
|
||||
if not match:
|
||||
continue
|
||||
py_version = match.group(1)
|
||||
break
|
||||
|
||||
# extract major.minor
|
||||
if py_version:
|
||||
|
|
@ -185,19 +216,20 @@ class CondaBuildPack(BaseImage):
|
|||
repo contents change
|
||||
"""
|
||||
assemble_files = super().get_preassemble_script_files()
|
||||
environment_yml = self.binder_path("environment.yml")
|
||||
if os.path.exists(environment_yml):
|
||||
assemble_files[environment_yml] = environment_yml
|
||||
if self._should_preassemble_env:
|
||||
environment_yml = self.binder_path("environment.yml")
|
||||
if os.path.exists(environment_yml):
|
||||
assemble_files[environment_yml] = environment_yml
|
||||
return assemble_files
|
||||
|
||||
def get_preassemble_scripts(self):
|
||||
def get_env_scripts(self):
|
||||
"""Return series of build-steps specific to this source repository.
|
||||
"""
|
||||
assembly_scripts = []
|
||||
scripts = []
|
||||
environment_yml = self.binder_path("environment.yml")
|
||||
env_prefix = "${KERNEL_PYTHON_PREFIX}" if self.py2 else "${NB_PYTHON_PREFIX}"
|
||||
if os.path.exists(environment_yml):
|
||||
assembly_scripts.append(
|
||||
scripts.append(
|
||||
(
|
||||
"${NB_USER}",
|
||||
r"""
|
||||
|
|
@ -209,7 +241,19 @@ class CondaBuildPack(BaseImage):
|
|||
),
|
||||
)
|
||||
)
|
||||
return super().get_preassemble_scripts() + assembly_scripts
|
||||
return scripts
|
||||
|
||||
def get_preassemble_scripts(self):
|
||||
scripts = super().get_preassemble_scripts()
|
||||
if self._should_preassemble_env:
|
||||
scripts.extend(self.get_env_scripts())
|
||||
return scripts
|
||||
|
||||
def get_assemble_scripts(self):
|
||||
scripts = super().get_assemble_scripts()
|
||||
if not self._should_preassemble_env:
|
||||
scripts.extend(self.get_env_scripts())
|
||||
return scripts
|
||||
|
||||
def detect(self):
|
||||
"""Check if current repo should be built with the Conda BuildPack.
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
import os
|
||||
|
||||
from ..conda import CondaBuildPack
|
||||
from ...utils import is_local_pip_requirement
|
||||
|
||||
|
||||
class PythonBuildPack(CondaBuildPack):
|
||||
|
|
@ -34,31 +35,6 @@ class PythonBuildPack(CondaBuildPack):
|
|||
self._python_version = py_version
|
||||
return self._python_version
|
||||
|
||||
def _is_local_requirement(self, line):
|
||||
"""Return whether a line in a requirements.txt file references a local file"""
|
||||
# trim comments and skip empty lines
|
||||
line = line.split("#", 1)[0].strip()
|
||||
if not line:
|
||||
return False
|
||||
if line.startswith(("-r", "-c")):
|
||||
# local -r or -c references break isolation
|
||||
return True
|
||||
# strip off `-e, etc.`
|
||||
if line.startswith("-"):
|
||||
line = line.split(None, 1)[1]
|
||||
if "file://" in line:
|
||||
# file references break isolation
|
||||
return True
|
||||
if "://" in line:
|
||||
# handle git://../local/file
|
||||
path = line.split("://", 1)[1]
|
||||
else:
|
||||
path = line
|
||||
if path.startswith("."):
|
||||
# references a local file
|
||||
return True
|
||||
return False
|
||||
|
||||
def _get_pip_scripts(self):
|
||||
"""Get pip install scripts
|
||||
|
||||
|
|
@ -112,7 +88,7 @@ class PythonBuildPack(CondaBuildPack):
|
|||
continue
|
||||
with open(requirements_txt) as f:
|
||||
for line in f:
|
||||
if self._is_local_requirement(line):
|
||||
if is_local_pip_requirement(line):
|
||||
return False
|
||||
|
||||
# didn't find any local references,
|
||||
|
|
|
|||
|
|
@ -431,3 +431,29 @@ def normalize_doi(val):
|
|||
(e.g. https://doi.org/10.1234/jshd123)"""
|
||||
m = doi_regexp.match(val)
|
||||
return m.group(2)
|
||||
|
||||
|
||||
def is_local_pip_requirement(line):
|
||||
"""Return whether a pip requirement (e.g. in requirements.txt file) references a local file"""
|
||||
# trim comments and skip empty lines
|
||||
line = line.split("#", 1)[0].strip()
|
||||
if not line:
|
||||
return False
|
||||
if line.startswith(("-r", "-c")):
|
||||
# local -r or -c references break isolation
|
||||
return True
|
||||
# strip off `-e, etc.`
|
||||
if line.startswith("-"):
|
||||
line = line.split(None, 1)[1]
|
||||
if "file://" in line:
|
||||
# file references break isolation
|
||||
return True
|
||||
if "://" in line:
|
||||
# handle git://../local/file
|
||||
path = line.split("://", 1)[1]
|
||||
else:
|
||||
path = line
|
||||
if path.startswith("."):
|
||||
# references a local file
|
||||
return True
|
||||
return False
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue