Split detectors.py into individual files

These were buildpacks now anyway, not detectors.

repo2docker/app.py

@@ -25,7 +25,7 @@ from docker.utils import kwargs_from_env
 
 import subprocess
 
-from .detectors import (
+from .buildpacks import (
     PythonBuildPack, DockerBuildPack, LegacyBinderDockerBuildPack,
     CondaBuildPack, JuliaBuildPack, Python2BuildPack, BaseImage
 )

repo2docker/buildpacks/__init__.py

@@ -0,0 +1,6 @@
+from .base import BuildPack, BaseImage
+from .python import Python2BuildPack, PythonBuildPack
+from .conda import CondaBuildPack
+from .julia import JuliaBuildPack
+from .docker import DockerBuildPack
+from .legacy import LegacyBinderDockerBuildPack

repo2docker/buildpacks/base.py

@@ -1,6 +1,3 @@
-"""
-Generates a variety of Dockerfiles based on an input matrix
-"""
 import textwrap
 from traitlets.config import LoggingConfigurable
 from traitlets import Unicode, Set, List, Dict, Tuple, default
@@ -385,7 +382,7 @@ class BuildPack(LoggingConfigurable):
 
         for src in sorted(self.build_script_files):
             src_parts = src.split('/')
-            src_path = os.path.join(os.path.dirname(__file__), 'files', *src_parts)
+            src_path = os.path.join(os.path.dirname(__file__), '..', 'files', *src_parts)
             tar.add(src_path, src, filter=_filter_tar)
 
         tar.add('.', 'src/', filter=_filter_tar)
@@ -453,315 +450,3 @@ class BaseImage(BuildPack):
                                  post_build, DOC_URL+'#system-post-build-scripts'))
             return [post_build]
         return []
-
-class PythonBuildPack(BuildPack):
-    name = "python3.5"
-    version = "0.1"
-
-    packages = {
-        'python3',
-        'python3-venv',
-        'python3-dev',
-    }
-
-    env = [
-        ("VENV_PATH", "${APP_BASE}/venv"),
-        # Prefix to use for installing kernels and finding jupyter binary
-        ("NB_PYTHON_PREFIX", "${VENV_PATH}"),
-    ]
-
-    path = [
-        "${VENV_PATH}/bin"
-    ]
-
-    build_scripts = [
-        (
-            "root",
-            r"""
-            mkdir -p ${VENV_PATH} && \
-            chown -R ${NB_USER}:${NB_USER} ${VENV_PATH}
-            """
-        ),
-        (
-            "${NB_USER}",
-            r"""
-            python3 -m venv ${VENV_PATH}
-            """
-        ),
-        (
-            "${NB_USER}",
-            r"""
-            pip install --no-cache-dir \
-                notebook==5.2.2 \
-                ipywidgets==6.0.0 \
-                jupyterlab==0.28 && \
-            jupyter nbextension enable --py widgetsnbextension --sys-prefix && \
-            jupyter serverextension enable --py jupyterlab --sys-prefix
-            """
-        )
-    ]
-
-    @default('assemble_scripts')
-    def setup_assembly(self):
-        # If we have a runtime.txt & that's set to python-2.7,
-        # we will *not* install requirements.txt but will find &
-        # install a requirements3.txt file if it exists.
-        # This way, when using python2 venv, requirements.txt will
-        # be installed in the python2 venv, and requirements3.txt
-        # will be installed in python3 venv. This is less of a
-        # surprise than requiring python2 to be requirements2.txt tho.
-        try:
-            with open(self.binder_path('runtime.txt')) as f:
-                runtime = f.read().strip()
-        except FileNotFoundError:
-            runtime = 'python-3.5'
-        if runtime == 'python-2.7':
-            requirements_file = self.binder_path('requirements3.txt')
-        else:
-            requirements_file = self.binder_path('requirements.txt')
-        if os.path.exists(requirements_file):
-            return [(
-                '${NB_USER}',
-                'pip3 install --no-cache-dir -r "{}"'.format(requirements_file)
-            )]
-        return []
-
-    def detect(self):
-        return os.path.exists('requirements.txt') and super().detect()
-
-class CondaBuildPack(BuildPack):
-    name = "conda"
-    version = "0.1"
-    env = [
-        ('CONDA_DIR', '${APP_BASE}/conda'),
-        ('NB_PYTHON_PREFIX', '${CONDA_DIR}')
-    ]
-
-    path = ['${CONDA_DIR}/bin']
-
-    build_script_files = {
-        'conda/install-miniconda.bash': '/tmp/install-miniconda.bash',
-        'conda/environment.yml': '/tmp/environment.yml'
-    }
-
-    build_scripts = [
-        (
-            "root",
-            r"""
-            bash /tmp/install-miniconda.bash && \
-            rm /tmp/install-miniconda.bash /tmp/environment.yml
-            """
-        )
-    ]
-
-    @default('assemble_scripts')
-    def setup_assembly(self):
-        assembly_scripts = []
-        environment_yml = self.binder_path('environment.yml')
-        if os.path.exists(environment_yml):
-            assembly_scripts.append((
-                '${NB_USER}',
-                r"""
-                conda env update -v -n root -f "{}" && \
-                conda clean -tipsy
-                """.format(environment_yml)
-            ))
-        return assembly_scripts
-
-    def detect(self):
-        return os.path.exists(self.binder_path('environment.yml')) and super().detect()
-
-
-class Python2BuildPack(BuildPack):
-    name = "python2.7"
-    version = "0.1"
-
-    packages = {
-        'python',
-        'python-dev',
-        'virtualenv'
-    }
-
-    env = [
-        ('VENV2_PATH', '${APP_BASE}/venv2')
-    ]
-
-    path = [
-        "${VENV2_PATH}/bin"
-    ]
-
-    build_scripts = [
-        (
-            "root",
-            r"""
-            mkdir -p ${VENV2_PATH} && \
-            chown -R ${NB_USER}:${NB_USER} ${VENV2_PATH}
-            """
-        ),
-        (
-            "${NB_USER}",
-            r"""
-            virtualenv -p python2 ${VENV2_PATH}
-            """
-        ),
-        (
-            "${NB_USER}",
-            r"""
-            pip2 install --no-cache-dir \
-                 ipykernel==4.6.1 && \
-            python2 -m ipykernel install --prefix=${NB_PYTHON_PREFIX}
-            """
-        )
-    ]
-
-    @default('assemble_scripts')
-    def setup_assembly(self):
-        return [
-            (
-                '${NB_USER}',
-                'pip2 install --no-cache-dir -r requirements.txt'
-            )
-        ]
-
-    def detect(self):
-        requirements_txt = self.binder_path('requirements.txt')
-        runtime_txt = self.binder_path('runtime.txt')
-        if os.path.exists(requirements_txt) and os.path.exists(runtime_txt):
-            with open(runtime_txt) as f:
-                runtime = f.read().strip()
-            if runtime == 'python-2.7':
-                return True
-        return False
-
-class JuliaBuildPack(BuildPack):
-    name = "julia"
-    version = "0.1"
-    env = [
-        ('JULIA_PATH', '${APP_BASE}/julia'),
-        ('JULIA_HOME', '${JULIA_PATH}/bin'),
-        ('JULIA_PKGDIR', '${JULIA_PATH}/pkg'),
-        ('JULIA_VERSION', '0.6.0'),
-        ('JUPYTER', '${NB_PYTHON_PREFIX}/bin/jupyter')
-    ]
-
-    path = [
-        '${JULIA_PATH}/bin'
-    ]
-
-    build_scripts = [
-        (
-            "root",
-            r"""
-            mkdir -p ${JULIA_PATH} && \
-            curl -sSL "https://julialang-s3.julialang.org/bin/linux/x64/${JULIA_VERSION%[.-]*}/julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | tar -xz -C ${JULIA_PATH} --strip-components 1
-            """
-        ),
-        (
-            "root",
-            r"""
-            mkdir -p ${JULIA_PKGDIR} && \
-            chown ${NB_USER}:${NB_USER} ${JULIA_PKGDIR}
-            """
-        ),
-        (
-            "${NB_USER}",
-            # HACK: Can't seem to tell IJulia to install in sys-prefix
-            # FIXME: Find way to get it to install under /srv and not $HOME?
-            r"""
-            julia -e 'Pkg.init(); Pkg.add("IJulia"); using IJulia;' && \
-            mv ${HOME}/.local/share/jupyter/kernels/julia-0.6  ${NB_PYTHON_PREFIX}/share/jupyter/kernels/julia-0.6
-            """
-        )
-    ]
-
-    @default('assemble_scripts')
-    def setup_assembly(self):
-        require = self.binder_path('REQUIRE')
-        return [(
-            "${NB_USER}",
-            # Pre-compile all libraries if they've opted into it. `using {libraryname}` does the
-            # right thing
-            r"""
-            cat "%(require)s" >> ${JULIA_PKGDIR}/v0.6/REQUIRE && \
-            julia -e ' \
-               Pkg.resolve(); \
-               for pkg in keys(Pkg.Reqs.parse("%(require)s")) \
-                pkg != "julia" && eval(:(using $(Symbol(pkg)))) \
-               end \
-            '
-            """ % { "require" : require }
-        )]
-
-    def detect(self):
-        return os.path.exists(self.binder_path('REQUIRE')) and super()
-
-
-class DockerBuildPack(BuildPack):
-    name = "Dockerfile"
-    dockerfile = "Dockerfile"
-
-    def detect(self):
-        return os.path.exists(self.binder_path('Dockerfile'))
-
-    def render(self):
-        Dockerfile = self.binder_path('Dockerfile')
-        with open(Dockerfile) as f:
-            return f.read()
-
-    def build(self, image_spec):
-        client = docker.APIClient(version='auto', **docker.utils.kwargs_from_env())
-        for line in client.build(
-                path=os.getcwd(),
-                dockerfile=self.binder_path(self.dockerfile),
-                tag=image_spec,
-                buildargs={},
-                decode=True,
-                forcerm=True,
-                rm=True
-        ):
-            yield line
-
-class LegacyBinderDockerBuildPack(DockerBuildPack):
-
-    name = 'Legacy Binder Dockerfile'
-    dockerfile = '._binder.Dockerfile'
-
-    dockerfile_appendix = Unicode(dedent(r"""
-    USER root
-    COPY . /home/main/notebooks
-    RUN chown -R main:main /home/main/notebooks
-    USER main
-    WORKDIR /home/main/notebooks
-    ENV PATH /home/main/anaconda2/envs/python3/bin:$PATH
-    RUN conda install -yq -n python3 notebook==5.0.0 ipykernel==4.6.0 && \
-        conda remove -yq -n python3 nb_conda_kernels && \
-        conda install -yq -n root ipykernel==4.6.0 && \
-        /home/main/anaconda2/envs/python3/bin/ipython kernel install --sys-prefix && \
-        /home/main/anaconda2/bin/ipython kernel install --prefix=/home/main/anaconda2/envs/python3 && \
-        /home/main/anaconda2/bin/ipython kernel install --sys-prefix
-    ENV JUPYTER_PATH /home/main/anaconda2/share/jupyter:$JUPYTER_PATH
-    CMD jupyter notebook --ip 0.0.0.0
-    """), config=True)
-
-    def render(self):
-        with open('Dockerfile') as f:
-            return f.read() + self.dockerfile_appendix
-
-    def build(self, image_spec):
-        with open(self.dockerfile, 'w') as f:
-            f.write(self.render())
-        return super().build(image_spec)
-
-    def detect(self):
-        try:
-            with open('Dockerfile', 'r') as f:
-                for line in f:
-                    if line.startswith('FROM'):
-                        if 'andrewosh/binder-base' in line.split('#')[0].lower():
-                            return True
-                        else:
-                            return False
-        except FileNotFoundError:
-            pass
-
-        return False

repo2docker/buildpacks/conda.py

@@ -0,0 +1,58 @@
+"""
+Generates a variety of Dockerfiles based on an input matrix
+"""
+import textwrap
+from traitlets.config import LoggingConfigurable
+from traitlets import Unicode, Set, List, Dict, Tuple, default
+from textwrap import dedent
+import jinja2
+import tarfile
+import io
+import os
+import stat
+import re
+import docker
+from .base import BuildPack
+
+
+class CondaBuildPack(BuildPack):
+    name = "conda"
+    version = "0.1"
+    env = [
+        ('CONDA_DIR', '${APP_BASE}/conda'),
+        ('NB_PYTHON_PREFIX', '${CONDA_DIR}')
+    ]
+
+    path = ['${CONDA_DIR}/bin']
+
+    build_script_files = {
+        'conda/install-miniconda.bash': '/tmp/install-miniconda.bash',
+        'conda/environment.yml': '/tmp/environment.yml'
+    }
+
+    build_scripts = [
+        (
+            "root",
+            r"""
+            bash /tmp/install-miniconda.bash && \
+            rm /tmp/install-miniconda.bash /tmp/environment.yml
+            """
+        )
+    ]
+
+    @default('assemble_scripts')
+    def setup_assembly(self):
+        assembly_scripts = []
+        environment_yml = self.binder_path('environment.yml')
+        if os.path.exists(environment_yml):
+            assembly_scripts.append((
+                '${NB_USER}',
+                r"""
+                conda env update -v -n root -f "{}" && \
+                conda clean -tipsy
+                """.format(environment_yml)
+            ))
+        return assembly_scripts
+
+    def detect(self):
+        return os.path.exists(self.binder_path('environment.yml')) and super().detect()

repo2docker/buildpacks/docker.py

@@ -0,0 +1,41 @@
+"""
+Generates a variety of Dockerfiles based on an input matrix
+"""
+import textwrap
+from traitlets.config import LoggingConfigurable
+from traitlets import Unicode, Set, List, Dict, Tuple, default
+from textwrap import dedent
+import jinja2
+import tarfile
+import io
+import os
+import stat
+import re
+import docker
+from .base import BuildPack
+
+
+class DockerBuildPack(BuildPack):
+    name = "Dockerfile"
+    dockerfile = "Dockerfile"
+
+    def detect(self):
+        return os.path.exists(self.binder_path('Dockerfile'))
+
+    def render(self):
+        Dockerfile = self.binder_path('Dockerfile')
+        with open(Dockerfile) as f:
+            return f.read()
+
+    def build(self, image_spec):
+        client = docker.APIClient(version='auto', **docker.utils.kwargs_from_env())
+        for line in client.build(
+                path=os.getcwd(),
+                dockerfile=self.binder_path(self.dockerfile),
+                tag=image_spec,
+                buildargs={},
+                decode=True,
+                forcerm=True,
+                rm=True
+        ):
+            yield line

repo2docker/buildpacks/julia.py

@@ -0,0 +1,78 @@
+"""
+Generates a variety of Dockerfiles based on an input matrix
+"""
+import textwrap
+from traitlets.config import LoggingConfigurable
+from traitlets import Unicode, Set, List, Dict, Tuple, default
+from textwrap import dedent
+import jinja2
+import tarfile
+import io
+import os
+import stat
+import re
+import docker
+from .base import BuildPack
+
+
+class JuliaBuildPack(BuildPack):
+    name = "julia"
+    version = "0.1"
+    env = [
+        ('JULIA_PATH', '${APP_BASE}/julia'),
+        ('JULIA_HOME', '${JULIA_PATH}/bin'),
+        ('JULIA_PKGDIR', '${JULIA_PATH}/pkg'),
+        ('JULIA_VERSION', '0.6.0'),
+        ('JUPYTER', '${NB_PYTHON_PREFIX}/bin/jupyter')
+    ]
+
+    path = [
+        '${JULIA_PATH}/bin'
+    ]
+
+    build_scripts = [
+        (
+            "root",
+            r"""
+            mkdir -p ${JULIA_PATH} && \
+            curl -sSL "https://julialang-s3.julialang.org/bin/linux/x64/${JULIA_VERSION%[.-]*}/julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | tar -xz -C ${JULIA_PATH} --strip-components 1
+            """
+        ),
+        (
+            "root",
+            r"""
+            mkdir -p ${JULIA_PKGDIR} && \
+            chown ${NB_USER}:${NB_USER} ${JULIA_PKGDIR}
+            """
+        ),
+        (
+            "${NB_USER}",
+            # HACK: Can't seem to tell IJulia to install in sys-prefix
+            # FIXME: Find way to get it to install under /srv and not $HOME?
+            r"""
+            julia -e 'Pkg.init(); Pkg.add("IJulia"); using IJulia;' && \
+            mv ${HOME}/.local/share/jupyter/kernels/julia-0.6  ${NB_PYTHON_PREFIX}/share/jupyter/kernels/julia-0.6
+            """
+        )
+    ]
+
+    @default('assemble_scripts')
+    def setup_assembly(self):
+        require = self.binder_path('REQUIRE')
+        return [(
+            "${NB_USER}",
+            # Pre-compile all libraries if they've opted into it. `using {libraryname}` does the
+            # right thing
+            r"""
+            cat "%(require)s" >> ${JULIA_PKGDIR}/v0.6/REQUIRE && \
+            julia -e ' \
+               Pkg.resolve(); \
+               for pkg in keys(Pkg.Reqs.parse("%(require)s")) \
+                pkg != "julia" && eval(:(using $(Symbol(pkg)))) \
+               end \
+            '
+            """ % { "require" : require }
+        )]
+
+    def detect(self):
+        return os.path.exists(self.binder_path('REQUIRE')) and super()

repo2docker/buildpacks/legacy.py

@@ -0,0 +1,60 @@
+"""
+Generates a variety of Dockerfiles based on an input matrix
+"""
+import textwrap
+from traitlets.config import LoggingConfigurable
+from traitlets import Unicode, Set, List, Dict, Tuple, default
+from textwrap import dedent
+import jinja2
+import tarfile
+import io
+import os
+import stat
+import re
+import docker
+from .docker import DockerBuildPack
+
+class LegacyBinderDockerBuildPack(DockerBuildPack):
+
+    name = 'Legacy Binder Dockerfile'
+    dockerfile = '._binder.Dockerfile'
+
+    dockerfile_appendix = Unicode(dedent(r"""
+    USER root
+    COPY . /home/main/notebooks
+    RUN chown -R main:main /home/main/notebooks
+    USER main
+    WORKDIR /home/main/notebooks
+    ENV PATH /home/main/anaconda2/envs/python3/bin:$PATH
+    RUN conda install -yq -n python3 notebook==5.0.0 ipykernel==4.6.0 && \
+        conda remove -yq -n python3 nb_conda_kernels && \
+        conda install -yq -n root ipykernel==4.6.0 && \
+        /home/main/anaconda2/envs/python3/bin/ipython kernel install --sys-prefix && \
+        /home/main/anaconda2/bin/ipython kernel install --prefix=/home/main/anaconda2/envs/python3 && \
+        /home/main/anaconda2/bin/ipython kernel install --sys-prefix
+    ENV JUPYTER_PATH /home/main/anaconda2/share/jupyter:$JUPYTER_PATH
+    CMD jupyter notebook --ip 0.0.0.0
+    """), config=True)
+
+    def render(self):
+        with open('Dockerfile') as f:
+            return f.read() + self.dockerfile_appendix
+
+    def build(self, image_spec):
+        with open(self.dockerfile, 'w') as f:
+            f.write(self.render())
+        return super().build(image_spec)
+
+    def detect(self):
+        try:
+            with open('Dockerfile', 'r') as f:
+                for line in f:
+                    if line.startswith('FROM'):
+                        if 'andrewosh/binder-base' in line.split('#')[0].lower():
+                            return True
+                        else:
+                            return False
+        except FileNotFoundError:
+            pass
+
+        return False

repo2docker/buildpacks/python.py

@@ -0,0 +1,153 @@
+"""
+Generates a variety of Dockerfiles based on an input matrix
+"""
+import textwrap
+from traitlets.config import LoggingConfigurable
+from traitlets import Unicode, Set, List, Dict, Tuple, default
+from textwrap import dedent
+import jinja2
+import tarfile
+import io
+import os
+import stat
+import re
+import docker
+from .base import BuildPack
+
+
+class PythonBuildPack(BuildPack):
+    name = "python3.5"
+    version = "0.1"
+
+    packages = {
+        'python3',
+        'python3-venv',
+        'python3-dev',
+    }
+
+    env = [
+        ("VENV_PATH", "${APP_BASE}/venv"),
+        # Prefix to use for installing kernels and finding jupyter binary
+        ("NB_PYTHON_PREFIX", "${VENV_PATH}"),
+    ]
+
+    path = [
+        "${VENV_PATH}/bin"
+    ]
+
+    build_scripts = [
+        (
+            "root",
+            r"""
+            mkdir -p ${VENV_PATH} && \
+            chown -R ${NB_USER}:${NB_USER} ${VENV_PATH}
+            """
+        ),
+        (
+            "${NB_USER}",
+            r"""
+            python3 -m venv ${VENV_PATH}
+            """
+        ),
+        (
+            "${NB_USER}",
+            r"""
+            pip install --no-cache-dir \
+                notebook==5.2.2 \
+                ipywidgets==6.0.0 \
+                jupyterlab==0.28 && \
+            jupyter nbextension enable --py widgetsnbextension --sys-prefix && \
+            jupyter serverextension enable --py jupyterlab --sys-prefix
+            """
+        )
+    ]
+
+    @default('assemble_scripts')
+    def setup_assembly(self):
+        # If we have a runtime.txt & that's set to python-2.7,
+        # we will *not* install requirements.txt but will find &
+        # install a requirements3.txt file if it exists.
+        # This way, when using python2 venv, requirements.txt will
+        # be installed in the python2 venv, and requirements3.txt
+        # will be installed in python3 venv. This is less of a
+        # surprise than requiring python2 to be requirements2.txt tho.
+        try:
+            with open(self.binder_path('runtime.txt')) as f:
+                runtime = f.read().strip()
+        except FileNotFoundError:
+            runtime = 'python-3.5'
+        if runtime == 'python-2.7':
+            requirements_file = self.binder_path('requirements3.txt')
+        else:
+            requirements_file = self.binder_path('requirements.txt')
+        if os.path.exists(requirements_file):
+            return [(
+                '${NB_USER}',
+                'pip3 install --no-cache-dir -r "{}"'.format(requirements_file)
+            )]
+        return []
+
+    def detect(self):
+        return os.path.exists('requirements.txt') and super().detect()
+
+
+class Python2BuildPack(BuildPack):
+    name = "python2.7"
+    version = "0.1"
+
+    packages = {
+        'python',
+        'python-dev',
+        'virtualenv'
+    }
+
+    env = [
+        ('VENV2_PATH', '${APP_BASE}/venv2')
+    ]
+
+    path = [
+        "${VENV2_PATH}/bin"
+    ]
+
+    build_scripts = [
+        (
+            "root",
+            r"""
+            mkdir -p ${VENV2_PATH} && \
+            chown -R ${NB_USER}:${NB_USER} ${VENV2_PATH}
+            """
+        ),
+        (
+            "${NB_USER}",
+            r"""
+            virtualenv -p python2 ${VENV2_PATH}
+            """
+        ),
+        (
+            "${NB_USER}",
+            r"""
+            pip2 install --no-cache-dir \
+                 ipykernel==4.6.1 && \
+            python2 -m ipykernel install --prefix=${NB_PYTHON_PREFIX}
+            """
+        )
+    ]
+
+    @default('assemble_scripts')
+    def setup_assembly(self):
+        return [
+            (
+                '${NB_USER}',
+                'pip2 install --no-cache-dir -r requirements.txt'
+            )
+        ]
+
+    def detect(self):
+        requirements_txt = self.binder_path('requirements.txt')
+        runtime_txt = self.binder_path('runtime.txt')
+        if os.path.exists(requirements_txt) and os.path.exists(runtime_txt):
+            with open(runtime_txt) as f:
+                runtime = f.read().strip()
+            if runtime == 'python-2.7':
+                return True
+        return False