preassemble python

repo2docker/buildpacks/python/__init__.py

@@ -34,80 +34,47 @@ class PythonBuildPack(CondaBuildPack):
         self._python_version = py_version
         return self._python_version
 
-    def get_assemble_files(self):
-        assemble_files = super().get_assemble_files()
-        for name in ('requirements.txt', 'requirements3.txt'):
-            requirements_txt = self.binder_path(name)
-            if os.path.exists(requirements_txt):
-                assemble_files.append(requirements_txt)
-        return assemble_files
-
     def _is_local_requirement(self, line):
         """Return whether a line in a requirements.txt file references a local file"""
         # trim comments and skip empty lines
-        line = line.split('#', 1)[0].strip()
+        line = line.split("#", 1)[0].strip()
         if not line:
             return False
-        if line.startswith(('-r', '-c')):
+        if line.startswith(("-r", "-c")):
             # local -r or -c references break isolation
             return True
         # strip off `-e, etc.`
-        if line.startswith('-'):
+        if line.startswith("-"):
             line = line.split(None, 1)[1]
-        if 'file://' in line:
+        if "file://" in line:
             # file references break isolation
             return True
-        if '://' in line:
+        if "://" in line:
             # handle git://../local/file
-            path = line.split('://', 1)[1]
+            path = line.split("://", 1)[1]
         else:
             path = line
-        if path.startswith('.'):
+        if path.startswith("."):
             # references a local file
             return True
         return False
 
-    @property
+    def _get_pip_scripts(self):
-    def assemble_from_subset(self):
+        """Get pip install scripts
-        """Peek in requirements.txt to determine if we can assemble from only env files
 
-        If there are any local references, e.g. `-e .`,
+        added to preassemble unless local references are found,
-        stage the whole repo prior to installation.
+        in which case this happens in assemble.
         """
-        if not os.path.exists('binder') and os.path.exists('setup.py'):
-            # can't install from subset if we're using setup.py
-            return False
-        for name in ('requirements.txt', 'requirements3.txt'):
-            requirements_txt = self.binder_path(name)
-            if not os.path.exists(requirements_txt):
-                continue
-            with open(requirements_txt) as f:
-                for line in f:
-                    if self._is_local_requirement(line):
-                        return False
-
-        # didn't find any local references,
-        # allow assembly from subset
-        return True
-
-    def get_assemble_scripts(self):
-        """Return series of build-steps specific to this repository.
-        """
-        # If we have a runtime.txt & that's set to python-2.7,
-        # requirements.txt will be installed in the *kernel* env
-        # and requirements3.txt (if it exists)
-        # will be installed in the python 3 notebook server env.
-        assemble_scripts = super().get_assemble_scripts()
-        setup_py = "setup.py"
         # KERNEL_PYTHON_PREFIX is the env with the kernel,
         # whether it's distinct from the notebook or the same.
         pip = "${KERNEL_PYTHON_PREFIX}/bin/pip"
+        scripts = []
         if self.py2:
             # using python 2 kernel,
             # requirements3.txt allows installation in the notebook server env
             nb_requirements_file = self.binder_path("requirements3.txt")
             if os.path.exists(nb_requirements_file):
-                assemble_scripts.append(
+                scripts.append(
                     (
                         "${NB_USER}",
                         # want the $NB_PYHTON_PREFIX environment variable, not for
@@ -121,12 +88,65 @@ class PythonBuildPack(CondaBuildPack):
         # install requirements.txt in the kernel env
         requirements_file = self.binder_path("requirements.txt")
         if os.path.exists(requirements_file):
-            assemble_scripts.append(
+            scripts.append(
                 (
                     "${NB_USER}",
                     '{} install --no-cache-dir -r "{}"'.format(pip, requirements_file),
                 )
             )
+        return scripts
+
+    @property
+    def _should_preassemble_pip(self):
+        """Peek in requirements.txt to determine if we can assemble from only env files
+
+        If there are any local references, e.g. `-e .`,
+        stage the whole repo prior to installation.
+        """
+        if not os.path.exists("binder") and os.path.exists("setup.py"):
+            # can't install from subset if we're using setup.py
+            return False
+        for name in ("requirements.txt", "requirements3.txt"):
+            requirements_txt = self.binder_path(name)
+            if not os.path.exists(requirements_txt):
+                continue
+            with open(requirements_txt) as f:
+                for line in f:
+                    if self._is_local_requirement(line):
+                        return False
+
+        # didn't find any local references,
+        # allow assembly from subset
+        return True
+
+    def get_preassemble_script_files(self):
+        assemble_files = super().get_preassemble_script_files()
+        for name in ("requirements.txt", "requirements3.txt"):
+            requirements_txt = self.binder_path(name)
+            if os.path.exists(requirements_txt):
+                assemble_files[requirements_txt] = requirements_txt
+        return assemble_files
+
+    def get_preassemble_scripts(self):
+        """Return scripts to run before adding the full repository"""
+        scripts = super().get_preassemble_scripts()
+        if self._should_preassemble_pip:
+            scripts.extend(self._get_pip_scripts())
+        return scripts
+
+    def get_assemble_scripts(self):
+        """Return series of build steps that require the full repository"""
+        # If we have a runtime.txt & that's set to python-2.7,
+        # requirements.txt will be installed in the *kernel* env
+        # and requirements3.txt (if it exists)
+        # will be installed in the python 3 notebook server env.
+        assemble_scripts = super().get_assemble_scripts()
+        setup_py = "setup.py"
+        # KERNEL_PYTHON_PREFIX is the env with the kernel,
+        # whether it's distinct from the notebook or the same.
+        pip = "${KERNEL_PYTHON_PREFIX}/bin/pip"
+        if not self._should_preassemble_pip:
+            assemble_scripts.extend(self._get_pip_scripts())
 
         # setup.py exists *and* binder dir is not used
         if not self.binder_dir and os.path.exists(setup_py):