Split detectors.py into individual files

These were buildpacks now anyway, not detectors.
pull/155/head
yuvipanda 2017-11-29 23:20:24 -08:00
rodzic c7ca099ad0
commit 1ddaf0bcb1
8 zmienionych plików z 398 dodań i 317 usunięć

Wyświetl plik

@ -25,7 +25,7 @@ from docker.utils import kwargs_from_env
import subprocess
from .detectors import (
from .buildpacks import (
PythonBuildPack, DockerBuildPack, LegacyBinderDockerBuildPack,
CondaBuildPack, JuliaBuildPack, Python2BuildPack, BaseImage
)

Wyświetl plik

@ -0,0 +1,6 @@
from .base import BuildPack, BaseImage
from .python import Python2BuildPack, PythonBuildPack
from .conda import CondaBuildPack
from .julia import JuliaBuildPack
from .docker import DockerBuildPack
from .legacy import LegacyBinderDockerBuildPack

Wyświetl plik

@ -1,6 +1,3 @@
"""
Generates a variety of Dockerfiles based on an input matrix
"""
import textwrap
from traitlets.config import LoggingConfigurable
from traitlets import Unicode, Set, List, Dict, Tuple, default
@ -385,7 +382,7 @@ class BuildPack(LoggingConfigurable):
for src in sorted(self.build_script_files):
src_parts = src.split('/')
src_path = os.path.join(os.path.dirname(__file__), 'files', *src_parts)
src_path = os.path.join(os.path.dirname(__file__), '..', 'files', *src_parts)
tar.add(src_path, src, filter=_filter_tar)
tar.add('.', 'src/', filter=_filter_tar)
@ -453,315 +450,3 @@ class BaseImage(BuildPack):
post_build, DOC_URL+'#system-post-build-scripts'))
return [post_build]
return []
class PythonBuildPack(BuildPack):
name = "python3.5"
version = "0.1"
packages = {
'python3',
'python3-venv',
'python3-dev',
}
env = [
("VENV_PATH", "${APP_BASE}/venv"),
# Prefix to use for installing kernels and finding jupyter binary
("NB_PYTHON_PREFIX", "${VENV_PATH}"),
]
path = [
"${VENV_PATH}/bin"
]
build_scripts = [
(
"root",
r"""
mkdir -p ${VENV_PATH} && \
chown -R ${NB_USER}:${NB_USER} ${VENV_PATH}
"""
),
(
"${NB_USER}",
r"""
python3 -m venv ${VENV_PATH}
"""
),
(
"${NB_USER}",
r"""
pip install --no-cache-dir \
notebook==5.2.2 \
ipywidgets==6.0.0 \
jupyterlab==0.28 && \
jupyter nbextension enable --py widgetsnbextension --sys-prefix && \
jupyter serverextension enable --py jupyterlab --sys-prefix
"""
)
]
@default('assemble_scripts')
def setup_assembly(self):
# If we have a runtime.txt & that's set to python-2.7,
# we will *not* install requirements.txt but will find &
# install a requirements3.txt file if it exists.
# This way, when using python2 venv, requirements.txt will
# be installed in the python2 venv, and requirements3.txt
# will be installed in python3 venv. This is less of a
# surprise than requiring python2 to be requirements2.txt tho.
try:
with open(self.binder_path('runtime.txt')) as f:
runtime = f.read().strip()
except FileNotFoundError:
runtime = 'python-3.5'
if runtime == 'python-2.7':
requirements_file = self.binder_path('requirements3.txt')
else:
requirements_file = self.binder_path('requirements.txt')
if os.path.exists(requirements_file):
return [(
'${NB_USER}',
'pip3 install --no-cache-dir -r "{}"'.format(requirements_file)
)]
return []
def detect(self):
return os.path.exists('requirements.txt') and super().detect()
class CondaBuildPack(BuildPack):
name = "conda"
version = "0.1"
env = [
('CONDA_DIR', '${APP_BASE}/conda'),
('NB_PYTHON_PREFIX', '${CONDA_DIR}')
]
path = ['${CONDA_DIR}/bin']
build_script_files = {
'conda/install-miniconda.bash': '/tmp/install-miniconda.bash',
'conda/environment.yml': '/tmp/environment.yml'
}
build_scripts = [
(
"root",
r"""
bash /tmp/install-miniconda.bash && \
rm /tmp/install-miniconda.bash /tmp/environment.yml
"""
)
]
@default('assemble_scripts')
def setup_assembly(self):
assembly_scripts = []
environment_yml = self.binder_path('environment.yml')
if os.path.exists(environment_yml):
assembly_scripts.append((
'${NB_USER}',
r"""
conda env update -v -n root -f "{}" && \
conda clean -tipsy
""".format(environment_yml)
))
return assembly_scripts
def detect(self):
return os.path.exists(self.binder_path('environment.yml')) and super().detect()
class Python2BuildPack(BuildPack):
name = "python2.7"
version = "0.1"
packages = {
'python',
'python-dev',
'virtualenv'
}
env = [
('VENV2_PATH', '${APP_BASE}/venv2')
]
path = [
"${VENV2_PATH}/bin"
]
build_scripts = [
(
"root",
r"""
mkdir -p ${VENV2_PATH} && \
chown -R ${NB_USER}:${NB_USER} ${VENV2_PATH}
"""
),
(
"${NB_USER}",
r"""
virtualenv -p python2 ${VENV2_PATH}
"""
),
(
"${NB_USER}",
r"""
pip2 install --no-cache-dir \
ipykernel==4.6.1 && \
python2 -m ipykernel install --prefix=${NB_PYTHON_PREFIX}
"""
)
]
@default('assemble_scripts')
def setup_assembly(self):
return [
(
'${NB_USER}',
'pip2 install --no-cache-dir -r requirements.txt'
)
]
def detect(self):
requirements_txt = self.binder_path('requirements.txt')
runtime_txt = self.binder_path('runtime.txt')
if os.path.exists(requirements_txt) and os.path.exists(runtime_txt):
with open(runtime_txt) as f:
runtime = f.read().strip()
if runtime == 'python-2.7':
return True
return False
class JuliaBuildPack(BuildPack):
name = "julia"
version = "0.1"
env = [
('JULIA_PATH', '${APP_BASE}/julia'),
('JULIA_HOME', '${JULIA_PATH}/bin'),
('JULIA_PKGDIR', '${JULIA_PATH}/pkg'),
('JULIA_VERSION', '0.6.0'),
('JUPYTER', '${NB_PYTHON_PREFIX}/bin/jupyter')
]
path = [
'${JULIA_PATH}/bin'
]
build_scripts = [
(
"root",
r"""
mkdir -p ${JULIA_PATH} && \
curl -sSL "https://julialang-s3.julialang.org/bin/linux/x64/${JULIA_VERSION%[.-]*}/julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | tar -xz -C ${JULIA_PATH} --strip-components 1
"""
),
(
"root",
r"""
mkdir -p ${JULIA_PKGDIR} && \
chown ${NB_USER}:${NB_USER} ${JULIA_PKGDIR}
"""
),
(
"${NB_USER}",
# HACK: Can't seem to tell IJulia to install in sys-prefix
# FIXME: Find way to get it to install under /srv and not $HOME?
r"""
julia -e 'Pkg.init(); Pkg.add("IJulia"); using IJulia;' && \
mv ${HOME}/.local/share/jupyter/kernels/julia-0.6 ${NB_PYTHON_PREFIX}/share/jupyter/kernels/julia-0.6
"""
)
]
@default('assemble_scripts')
def setup_assembly(self):
require = self.binder_path('REQUIRE')
return [(
"${NB_USER}",
# Pre-compile all libraries if they've opted into it. `using {libraryname}` does the
# right thing
r"""
cat "%(require)s" >> ${JULIA_PKGDIR}/v0.6/REQUIRE && \
julia -e ' \
Pkg.resolve(); \
for pkg in keys(Pkg.Reqs.parse("%(require)s")) \
pkg != "julia" && eval(:(using $(Symbol(pkg)))) \
end \
'
""" % { "require" : require }
)]
def detect(self):
return os.path.exists(self.binder_path('REQUIRE')) and super()
class DockerBuildPack(BuildPack):
name = "Dockerfile"
dockerfile = "Dockerfile"
def detect(self):
return os.path.exists(self.binder_path('Dockerfile'))
def render(self):
Dockerfile = self.binder_path('Dockerfile')
with open(Dockerfile) as f:
return f.read()
def build(self, image_spec):
client = docker.APIClient(version='auto', **docker.utils.kwargs_from_env())
for line in client.build(
path=os.getcwd(),
dockerfile=self.binder_path(self.dockerfile),
tag=image_spec,
buildargs={},
decode=True,
forcerm=True,
rm=True
):
yield line
class LegacyBinderDockerBuildPack(DockerBuildPack):
name = 'Legacy Binder Dockerfile'
dockerfile = '._binder.Dockerfile'
dockerfile_appendix = Unicode(dedent(r"""
USER root
COPY . /home/main/notebooks
RUN chown -R main:main /home/main/notebooks
USER main
WORKDIR /home/main/notebooks
ENV PATH /home/main/anaconda2/envs/python3/bin:$PATH
RUN conda install -yq -n python3 notebook==5.0.0 ipykernel==4.6.0 && \
conda remove -yq -n python3 nb_conda_kernels && \
conda install -yq -n root ipykernel==4.6.0 && \
/home/main/anaconda2/envs/python3/bin/ipython kernel install --sys-prefix && \
/home/main/anaconda2/bin/ipython kernel install --prefix=/home/main/anaconda2/envs/python3 && \
/home/main/anaconda2/bin/ipython kernel install --sys-prefix
ENV JUPYTER_PATH /home/main/anaconda2/share/jupyter:$JUPYTER_PATH
CMD jupyter notebook --ip 0.0.0.0
"""), config=True)
def render(self):
with open('Dockerfile') as f:
return f.read() + self.dockerfile_appendix
def build(self, image_spec):
with open(self.dockerfile, 'w') as f:
f.write(self.render())
return super().build(image_spec)
def detect(self):
try:
with open('Dockerfile', 'r') as f:
for line in f:
if line.startswith('FROM'):
if 'andrewosh/binder-base' in line.split('#')[0].lower():
return True
else:
return False
except FileNotFoundError:
pass
return False

Wyświetl plik

@ -0,0 +1,58 @@
"""
Generates a variety of Dockerfiles based on an input matrix
"""
import textwrap
from traitlets.config import LoggingConfigurable
from traitlets import Unicode, Set, List, Dict, Tuple, default
from textwrap import dedent
import jinja2
import tarfile
import io
import os
import stat
import re
import docker
from .base import BuildPack
class CondaBuildPack(BuildPack):
name = "conda"
version = "0.1"
env = [
('CONDA_DIR', '${APP_BASE}/conda'),
('NB_PYTHON_PREFIX', '${CONDA_DIR}')
]
path = ['${CONDA_DIR}/bin']
build_script_files = {
'conda/install-miniconda.bash': '/tmp/install-miniconda.bash',
'conda/environment.yml': '/tmp/environment.yml'
}
build_scripts = [
(
"root",
r"""
bash /tmp/install-miniconda.bash && \
rm /tmp/install-miniconda.bash /tmp/environment.yml
"""
)
]
@default('assemble_scripts')
def setup_assembly(self):
assembly_scripts = []
environment_yml = self.binder_path('environment.yml')
if os.path.exists(environment_yml):
assembly_scripts.append((
'${NB_USER}',
r"""
conda env update -v -n root -f "{}" && \
conda clean -tipsy
""".format(environment_yml)
))
return assembly_scripts
def detect(self):
return os.path.exists(self.binder_path('environment.yml')) and super().detect()

Wyświetl plik

@ -0,0 +1,41 @@
"""
Generates a variety of Dockerfiles based on an input matrix
"""
import textwrap
from traitlets.config import LoggingConfigurable
from traitlets import Unicode, Set, List, Dict, Tuple, default
from textwrap import dedent
import jinja2
import tarfile
import io
import os
import stat
import re
import docker
from .base import BuildPack
class DockerBuildPack(BuildPack):
name = "Dockerfile"
dockerfile = "Dockerfile"
def detect(self):
return os.path.exists(self.binder_path('Dockerfile'))
def render(self):
Dockerfile = self.binder_path('Dockerfile')
with open(Dockerfile) as f:
return f.read()
def build(self, image_spec):
client = docker.APIClient(version='auto', **docker.utils.kwargs_from_env())
for line in client.build(
path=os.getcwd(),
dockerfile=self.binder_path(self.dockerfile),
tag=image_spec,
buildargs={},
decode=True,
forcerm=True,
rm=True
):
yield line

Wyświetl plik

@ -0,0 +1,78 @@
"""
Generates a variety of Dockerfiles based on an input matrix
"""
import textwrap
from traitlets.config import LoggingConfigurable
from traitlets import Unicode, Set, List, Dict, Tuple, default
from textwrap import dedent
import jinja2
import tarfile
import io
import os
import stat
import re
import docker
from .base import BuildPack
class JuliaBuildPack(BuildPack):
name = "julia"
version = "0.1"
env = [
('JULIA_PATH', '${APP_BASE}/julia'),
('JULIA_HOME', '${JULIA_PATH}/bin'),
('JULIA_PKGDIR', '${JULIA_PATH}/pkg'),
('JULIA_VERSION', '0.6.0'),
('JUPYTER', '${NB_PYTHON_PREFIX}/bin/jupyter')
]
path = [
'${JULIA_PATH}/bin'
]
build_scripts = [
(
"root",
r"""
mkdir -p ${JULIA_PATH} && \
curl -sSL "https://julialang-s3.julialang.org/bin/linux/x64/${JULIA_VERSION%[.-]*}/julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | tar -xz -C ${JULIA_PATH} --strip-components 1
"""
),
(
"root",
r"""
mkdir -p ${JULIA_PKGDIR} && \
chown ${NB_USER}:${NB_USER} ${JULIA_PKGDIR}
"""
),
(
"${NB_USER}",
# HACK: Can't seem to tell IJulia to install in sys-prefix
# FIXME: Find way to get it to install under /srv and not $HOME?
r"""
julia -e 'Pkg.init(); Pkg.add("IJulia"); using IJulia;' && \
mv ${HOME}/.local/share/jupyter/kernels/julia-0.6 ${NB_PYTHON_PREFIX}/share/jupyter/kernels/julia-0.6
"""
)
]
@default('assemble_scripts')
def setup_assembly(self):
require = self.binder_path('REQUIRE')
return [(
"${NB_USER}",
# Pre-compile all libraries if they've opted into it. `using {libraryname}` does the
# right thing
r"""
cat "%(require)s" >> ${JULIA_PKGDIR}/v0.6/REQUIRE && \
julia -e ' \
Pkg.resolve(); \
for pkg in keys(Pkg.Reqs.parse("%(require)s")) \
pkg != "julia" && eval(:(using $(Symbol(pkg)))) \
end \
'
""" % { "require" : require }
)]
def detect(self):
return os.path.exists(self.binder_path('REQUIRE')) and super()

Wyświetl plik

@ -0,0 +1,60 @@
"""
Generates a variety of Dockerfiles based on an input matrix
"""
import textwrap
from traitlets.config import LoggingConfigurable
from traitlets import Unicode, Set, List, Dict, Tuple, default
from textwrap import dedent
import jinja2
import tarfile
import io
import os
import stat
import re
import docker
from .docker import DockerBuildPack
class LegacyBinderDockerBuildPack(DockerBuildPack):
name = 'Legacy Binder Dockerfile'
dockerfile = '._binder.Dockerfile'
dockerfile_appendix = Unicode(dedent(r"""
USER root
COPY . /home/main/notebooks
RUN chown -R main:main /home/main/notebooks
USER main
WORKDIR /home/main/notebooks
ENV PATH /home/main/anaconda2/envs/python3/bin:$PATH
RUN conda install -yq -n python3 notebook==5.0.0 ipykernel==4.6.0 && \
conda remove -yq -n python3 nb_conda_kernels && \
conda install -yq -n root ipykernel==4.6.0 && \
/home/main/anaconda2/envs/python3/bin/ipython kernel install --sys-prefix && \
/home/main/anaconda2/bin/ipython kernel install --prefix=/home/main/anaconda2/envs/python3 && \
/home/main/anaconda2/bin/ipython kernel install --sys-prefix
ENV JUPYTER_PATH /home/main/anaconda2/share/jupyter:$JUPYTER_PATH
CMD jupyter notebook --ip 0.0.0.0
"""), config=True)
def render(self):
with open('Dockerfile') as f:
return f.read() + self.dockerfile_appendix
def build(self, image_spec):
with open(self.dockerfile, 'w') as f:
f.write(self.render())
return super().build(image_spec)
def detect(self):
try:
with open('Dockerfile', 'r') as f:
for line in f:
if line.startswith('FROM'):
if 'andrewosh/binder-base' in line.split('#')[0].lower():
return True
else:
return False
except FileNotFoundError:
pass
return False

Wyświetl plik

@ -0,0 +1,153 @@
"""
Generates a variety of Dockerfiles based on an input matrix
"""
import textwrap
from traitlets.config import LoggingConfigurable
from traitlets import Unicode, Set, List, Dict, Tuple, default
from textwrap import dedent
import jinja2
import tarfile
import io
import os
import stat
import re
import docker
from .base import BuildPack
class PythonBuildPack(BuildPack):
name = "python3.5"
version = "0.1"
packages = {
'python3',
'python3-venv',
'python3-dev',
}
env = [
("VENV_PATH", "${APP_BASE}/venv"),
# Prefix to use for installing kernels and finding jupyter binary
("NB_PYTHON_PREFIX", "${VENV_PATH}"),
]
path = [
"${VENV_PATH}/bin"
]
build_scripts = [
(
"root",
r"""
mkdir -p ${VENV_PATH} && \
chown -R ${NB_USER}:${NB_USER} ${VENV_PATH}
"""
),
(
"${NB_USER}",
r"""
python3 -m venv ${VENV_PATH}
"""
),
(
"${NB_USER}",
r"""
pip install --no-cache-dir \
notebook==5.2.2 \
ipywidgets==6.0.0 \
jupyterlab==0.28 && \
jupyter nbextension enable --py widgetsnbextension --sys-prefix && \
jupyter serverextension enable --py jupyterlab --sys-prefix
"""
)
]
@default('assemble_scripts')
def setup_assembly(self):
# If we have a runtime.txt & that's set to python-2.7,
# we will *not* install requirements.txt but will find &
# install a requirements3.txt file if it exists.
# This way, when using python2 venv, requirements.txt will
# be installed in the python2 venv, and requirements3.txt
# will be installed in python3 venv. This is less of a
# surprise than requiring python2 to be requirements2.txt tho.
try:
with open(self.binder_path('runtime.txt')) as f:
runtime = f.read().strip()
except FileNotFoundError:
runtime = 'python-3.5'
if runtime == 'python-2.7':
requirements_file = self.binder_path('requirements3.txt')
else:
requirements_file = self.binder_path('requirements.txt')
if os.path.exists(requirements_file):
return [(
'${NB_USER}',
'pip3 install --no-cache-dir -r "{}"'.format(requirements_file)
)]
return []
def detect(self):
return os.path.exists('requirements.txt') and super().detect()
class Python2BuildPack(BuildPack):
name = "python2.7"
version = "0.1"
packages = {
'python',
'python-dev',
'virtualenv'
}
env = [
('VENV2_PATH', '${APP_BASE}/venv2')
]
path = [
"${VENV2_PATH}/bin"
]
build_scripts = [
(
"root",
r"""
mkdir -p ${VENV2_PATH} && \
chown -R ${NB_USER}:${NB_USER} ${VENV2_PATH}
"""
),
(
"${NB_USER}",
r"""
virtualenv -p python2 ${VENV2_PATH}
"""
),
(
"${NB_USER}",
r"""
pip2 install --no-cache-dir \
ipykernel==4.6.1 && \
python2 -m ipykernel install --prefix=${NB_PYTHON_PREFIX}
"""
)
]
@default('assemble_scripts')
def setup_assembly(self):
return [
(
'${NB_USER}',
'pip2 install --no-cache-dir -r requirements.txt'
)
]
def detect(self):
requirements_txt = self.binder_path('requirements.txt')
runtime_txt = self.binder_path('runtime.txt')
if os.path.exists(requirements_txt) and os.path.exists(runtime_txt):
with open(runtime_txt) as f:
runtime = f.read().strip()
if runtime == 'python-2.7':
return True
return False