From d6e66c886d7c88985d00f2e8fcb4e4525082605b Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 16 Jul 2019 09:02:59 +0200 Subject: [PATCH] preassemble python --- repo2docker/buildpacks/python/__init__.py | 114 +++++++++++++--------- 1 file changed, 67 insertions(+), 47 deletions(-) diff --git a/repo2docker/buildpacks/python/__init__.py b/repo2docker/buildpacks/python/__init__.py index 0d9a5b39..7b6fc5af 100644 --- a/repo2docker/buildpacks/python/__init__.py +++ b/repo2docker/buildpacks/python/__init__.py @@ -34,80 +34,47 @@ class PythonBuildPack(CondaBuildPack): self._python_version = py_version return self._python_version - def get_assemble_files(self): - assemble_files = super().get_assemble_files() - for name in ('requirements.txt', 'requirements3.txt'): - requirements_txt = self.binder_path(name) - if os.path.exists(requirements_txt): - assemble_files.append(requirements_txt) - return assemble_files - def _is_local_requirement(self, line): """Return whether a line in a requirements.txt file references a local file""" # trim comments and skip empty lines - line = line.split('#', 1)[0].strip() + line = line.split("#", 1)[0].strip() if not line: return False - if line.startswith(('-r', '-c')): + if line.startswith(("-r", "-c")): # local -r or -c references break isolation return True # strip off `-e, etc.` - if line.startswith('-'): + if line.startswith("-"): line = line.split(None, 1)[1] - if 'file://' in line: + if "file://" in line: # file references break isolation return True - if '://' in line: + if "://" in line: # handle git://../local/file - path = line.split('://', 1)[1] + path = line.split("://", 1)[1] else: path = line - if path.startswith('.'): + if path.startswith("."): # references a local file return True return False - @property - def assemble_from_subset(self): - """Peek in requirements.txt to determine if we can assemble from only env files + def _get_pip_scripts(self): + """Get pip install scripts - If there are any local references, e.g. `-e .`, - stage the whole repo prior to installation. + added to preassemble unless local references are found, + in which case this happens in assemble. """ - if not os.path.exists('binder') and os.path.exists('setup.py'): - # can't install from subset if we're using setup.py - return False - for name in ('requirements.txt', 'requirements3.txt'): - requirements_txt = self.binder_path(name) - if not os.path.exists(requirements_txt): - continue - with open(requirements_txt) as f: - for line in f: - if self._is_local_requirement(line): - return False - - # didn't find any local references, - # allow assembly from subset - return True - - def get_assemble_scripts(self): - """Return series of build-steps specific to this repository. - """ - # If we have a runtime.txt & that's set to python-2.7, - # requirements.txt will be installed in the *kernel* env - # and requirements3.txt (if it exists) - # will be installed in the python 3 notebook server env. - assemble_scripts = super().get_assemble_scripts() - setup_py = "setup.py" # KERNEL_PYTHON_PREFIX is the env with the kernel, # whether it's distinct from the notebook or the same. pip = "${KERNEL_PYTHON_PREFIX}/bin/pip" + scripts = [] if self.py2: # using python 2 kernel, # requirements3.txt allows installation in the notebook server env nb_requirements_file = self.binder_path("requirements3.txt") if os.path.exists(nb_requirements_file): - assemble_scripts.append( + scripts.append( ( "${NB_USER}", # want the $NB_PYHTON_PREFIX environment variable, not for @@ -121,12 +88,65 @@ class PythonBuildPack(CondaBuildPack): # install requirements.txt in the kernel env requirements_file = self.binder_path("requirements.txt") if os.path.exists(requirements_file): - assemble_scripts.append( + scripts.append( ( "${NB_USER}", '{} install --no-cache-dir -r "{}"'.format(pip, requirements_file), ) ) + return scripts + + @property + def _should_preassemble_pip(self): + """Peek in requirements.txt to determine if we can assemble from only env files + + If there are any local references, e.g. `-e .`, + stage the whole repo prior to installation. + """ + if not os.path.exists("binder") and os.path.exists("setup.py"): + # can't install from subset if we're using setup.py + return False + for name in ("requirements.txt", "requirements3.txt"): + requirements_txt = self.binder_path(name) + if not os.path.exists(requirements_txt): + continue + with open(requirements_txt) as f: + for line in f: + if self._is_local_requirement(line): + return False + + # didn't find any local references, + # allow assembly from subset + return True + + def get_preassemble_script_files(self): + assemble_files = super().get_preassemble_script_files() + for name in ("requirements.txt", "requirements3.txt"): + requirements_txt = self.binder_path(name) + if os.path.exists(requirements_txt): + assemble_files[requirements_txt] = requirements_txt + return assemble_files + + def get_preassemble_scripts(self): + """Return scripts to run before adding the full repository""" + scripts = super().get_preassemble_scripts() + if self._should_preassemble_pip: + scripts.extend(self._get_pip_scripts()) + return scripts + + def get_assemble_scripts(self): + """Return series of build steps that require the full repository""" + # If we have a runtime.txt & that's set to python-2.7, + # requirements.txt will be installed in the *kernel* env + # and requirements3.txt (if it exists) + # will be installed in the python 3 notebook server env. + assemble_scripts = super().get_assemble_scripts() + setup_py = "setup.py" + # KERNEL_PYTHON_PREFIX is the env with the kernel, + # whether it's distinct from the notebook or the same. + pip = "${KERNEL_PYTHON_PREFIX}/bin/pip" + if not self._should_preassemble_pip: + assemble_scripts.extend(self._get_pip_scripts()) # setup.py exists *and* binder dir is not used if not self.binder_dir and os.path.exists(setup_py):