Merge s2i-builders repo to the builder repo

These are fairly tightly coupled so let's make them in the same repo

.gitignore

@@ -0,0 +1 @@
+.#*

singleuser-builder-conda/Dockerfile

@@ -0,0 +1,33 @@
+FROM ubuntu:16.10
+
+MAINTAINER Yuvi Panda <yuvipanda@gmail.com>
+
+LABEL io.openshift.s2i.scripts-url=image:///usr/libexec/s2i
+
+RUN apt-get update && \
+    apt-get install --yes --no-install-recommends \
+            wget \
+            tar \
+            ca-certificates \
+            bzip2 \
+            git && \
+    apt-get purge && apt-get clean
+
+ENV CONDA_DIR=/opt/conda \
+    PATH=/opt/conda/bin:$PATH \
+    NB_USER=jovyan \
+    HOME=/home/jovyan
+
+RUN adduser --disabled-password --gecos "Default Jupyter user" ${NB_USER}
+
+WORKDIR $HOME
+ADD install-miniconda.bash /tmp/
+ADD environment.yml /tmp/
+
+RUN bash /tmp/install-miniconda.bash && \
+    rm /tmp/install-miniconda.bash /tmp/environment.yml
+
+USER $NB_USER
+COPY ./s2i/bin/ /usr/libexec/s2i
+
+EXPOSE 8888

singleuser-builder-conda/Makefile

@@ -0,0 +1,12 @@
+
+IMAGE_NAME = jupyterhub/singleuser-builder-conda
+VERSION = $(shell cat version)
+
+.PHONY: build
+build:
+	docker build -t $(IMAGE_NAME):$(VERSION) .
+
+.PHONY: test
+test:
+	docker build -t $(IMAGE_NAME)-candidate:$(VERSION) .
+	IMAGE_NAME=$(IMAGE_NAME)-candidate:$(VERSION) test/run

singleuser-builder-conda/README.md

@@ -0,0 +1,12 @@
+
+# JupyterHub singleuser builder
+
+This is a builder image for use with [s2i](https://github.com/openshift/source-to-image). It
+builds a source repository (such as a github repository) into a docker image that is suitable
+for use with [JupyterHub](http://github.com/jupyterhub/jupyterhub).
+
+It is based off Ubuntu 16.10, and uses [virtualenv](https://pypi.python.org/pypi/virtualenv) to
+provide a custom python3.5 environment.
+
+It looks for a `requirements.txt` file in the source repository, and installs it into the virtualenv.
+It also installs a number of default notebook related modules in there

singleuser-builder-conda/environment.yml

@@ -0,0 +1,8 @@
+dependencies:
+  - python==3.6.1
+  - notebook==5.0.0
+  - ipython==6.0.0
+  - ipykernel==4.6.0
+  - ipywidgets==6.0.0
+  - pip:
+    - jupyterhub==0.7.2

singleuser-builder-conda/install-miniconda.bash

@@ -0,0 +1,39 @@
+#!/bin/bash
+# This downloads and installs a pinned version of miniconda
+set -ex
+
+cd $(dirname $0)
+CONDA_VERSION=4.3.14
+URL="https://repo.continuum.io/miniconda/Miniconda3-${CONDA_VERSION}-Linux-x86_64.sh"
+INSTALLER_PATH=/tmp/miniconda-installer.sh
+
+wget --quiet $URL -O ${INSTALLER_PATH}
+chmod +x ${INSTALLER_PATH}
+
+# Only MD5 checksums are available for miniconda
+# Can be obtained from https://repo.continuum.io/miniconda/
+MD5SUM="fc6fc37479e3e3fcf3f9ba52cae98991"
+
+if ! echo "${MD5SUM}  ${INSTALLER_PATH}" | md5sum  --quiet -c -; then
+    echo "md5sum mismatch for ${INSTALLER_PATH}, exiting!"
+    exit 1
+fi
+
+bash ${INSTALLER_PATH} -b -p ${CONDA_DIR}
+
+# Allow easy direct installs from conda forge
+${CONDA_DIR}/bin/conda config --system --add channels conda-forge
+
+# Do not attempt to auto update conda or dependencies
+${CONDA_DIR}/bin/conda config --system --set auto_update_conda false
+${CONDA_DIR}/bin/conda config --system --set update_dependencies false
+${CONDA_DIR}/bin/conda config --system --set show_channel_urls true
+
+${CONDA_DIR}/bin/conda env update -n root -f /tmp/environment.yml
+# Clean things out!
+${CONDA_DIR}/bin/conda clean -tipsy
+
+# Remove the big installer so we don't increase docker image size too much
+rm ${INSTALLER_PATH}
+
+chown -R $NB_USER:$NB_USER ${CONDA_DIR}

singleuser-builder-conda/s2i/bin/assemble

@@ -0,0 +1,26 @@
+#!/bin/bash -e
+#
+# S2I assemble script for the jupyterhub/singleuser-builder image.
+# The 'assemble' script builds your application source so that it is ready to run.
+#
+# For more information refer to the documentation:
+#	https://github.com/openshift/source-to-image/blob/master/docs/builder_image.md
+#
+
+if [[ "$1" == "-h" ]]; then
+	exec /usr/libexec/s2i/usage
+fi
+
+# Restore artifacts from the previous build (if they exist).
+echo "---> Installing application source..."
+# HACK: We copy it to current directory too, since people wanna see the
+# notebooks and stuff in there. Figure out a way to tweak this?
+cp -Rf /tmp/src/. .
+
+echo "---> Building application from source..."
+if [ -f environment.yml ]; then
+    conda env update -n root -f environment.yml
+    conda clean -tipsy
+elif [ -f requirements.txt ]; then
+    pip install -r requirements.txt
+fi

singleuser-builder-conda/s2i/bin/run

@@ -0,0 +1,10 @@
+#!/bin/bash -e
+#
+# S2I run script for the 'ubuntu1610-python35-venv' image.
+# The run script executes the server that runs your application.
+#
+# For more information see the documentation:
+#	https://github.com/openshift/source-to-image/blob/master/docs/builder_image.md
+#
+
+exec jupyter notebook --ip=0.0.0.0

singleuser-builder-conda/s2i/bin/save-artifacts

@@ -0,0 +1,10 @@
+#!/bin/sh -e
+#
+# S2I save-artifacts script for the 'ubuntu1610-python35-venv' image.
+# The save-artifacts script streams a tar archive to standard output.
+# The archive contains the files and folders you want to re-use in the next build.
+#
+# For more information see the documentation:
+#	https://github.com/openshift/source-to-image/blob/master/docs/builder_image.md
+#
+# tar cf - <list of files and folders>

singleuser-builder-conda/s2i/bin/usage

@@ -0,0 +1,12 @@
+#!/bin/bash -e
+cat <<EOF
+This is the ubuntu1610-python35-venv S2I image:
+To use it, install S2I: https://github.com/openshift/source-to-image
+
+Sample invocation:
+
+s2i build <source code path/URL> ubuntu1610-python35-venv <application image>
+
+You can then run the resulting image via:
+docker run <application image>
+EOF

singleuser-builder-conda/test/run

@@ -0,0 +1,160 @@
+#!/bin/bash
+#
+# The 'run' performs a simple test that verifies the S2I image.
+# The main focus here is to exercise the S2I scripts.
+#
+# For more information see the documentation:
+# https://github.com/openshift/source-to-image/blob/master/docs/builder_image.md
+#
+# IMAGE_NAME specifies a name of the candidate image used for testing.
+# The image has to be available before this script is executed.
+#
+IMAGE_NAME=${IMAGE_NAME-ubuntu1610-python35-venv-candidate}
+
+# Determining system utility executables (darwin compatibility check)
+READLINK_EXEC="readlink"
+MKTEMP_EXEC="mktemp"
+if [[ "$OSTYPE" =~ 'darwin' ]]; then
+  ! type -a "greadlink" &>"/dev/null" || READLINK_EXEC="greadlink"
+  ! type -a "gmktemp" &>"/dev/null" || MKTEMP_EXEC="gmktemp"
+fi
+
+test_dir="$($READLINK_EXEC -zf $(dirname "${BASH_SOURCE[0]}"))"
+image_dir=$($READLINK_EXEC -zf ${test_dir}/..)
+scripts_url="file://${image_dir}/.s2i/bin"
+cid_file=$($MKTEMP_EXEC -u --suffix=.cid)
+
+# Since we built the candidate image locally, we don't want S2I to attempt to pull
+# it from Docker hub
+s2i_args="--pull-policy=never --loglevel=2"
+
+# Port the image exposes service to be tested
+test_port=8080
+
+image_exists() {
+  docker inspect $1 &>/dev/null
+}
+
+container_exists() {
+  image_exists $(cat $cid_file)
+}
+
+container_ip() {
+  if [ ! -z "$DOCKER_HOST" ] && [[ "$OSTYPE" =~ 'darwin' ]]; then
+    docker-machine ip
+  else
+    docker inspect --format="{{ .NetworkSettings.IPAddress }}" $(cat $cid_file)
+  fi
+}
+
+container_port() {
+  if [ ! -z "$DOCKER_HOST" ] && [[ "$OSTYPE" =~ 'darwin' ]]; then
+    docker inspect --format="{{(index .NetworkSettings.Ports \"$test_port/tcp\" 0).HostPort}}" "$(cat "${cid_file}")"
+  else
+    echo $test_port
+  fi
+}
+
+run_s2i_build() {
+  s2i build --incremental=true ${s2i_args} file://${test_dir}/test-app ${IMAGE_NAME} ${IMAGE_NAME}-testapp
+}
+
+prepare() {
+  if ! image_exists ${IMAGE_NAME}; then
+    echo "ERROR: The image ${IMAGE_NAME} must exist before this script is executed."
+    exit 1
+  fi
+  # s2i build requires the application is a valid 'Git' repository
+  pushd ${test_dir}/test-app >/dev/null
+  git init
+  git config user.email "build@localhost" && git config user.name "builder"
+  git add -A && git commit -m "Sample commit"
+  popd >/dev/null
+  run_s2i_build
+}
+
+run_test_application() {
+  docker run --rm --cidfile=${cid_file} -p ${test_port} ${IMAGE_NAME}-testapp
+}
+
+cleanup() {
+  if [ -f $cid_file ]; then
+    if container_exists; then
+      docker stop $(cat $cid_file)
+    fi
+  fi
+  if image_exists ${IMAGE_NAME}-testapp; then
+    docker rmi ${IMAGE_NAME}-testapp
+  fi
+}
+
+check_result() {
+  local result="$1"
+  if [[ "$result" != "0" ]]; then
+    echo "S2I image '${IMAGE_NAME}' test FAILED (exit code: ${result})"
+    cleanup
+    exit $result
+  fi
+}
+
+wait_for_cid() {
+  local max_attempts=10
+  local sleep_time=1
+  local attempt=1
+  local result=1
+  while [ $attempt -le $max_attempts ]; do
+    [ -f $cid_file ] && break
+    echo "Waiting for container to start..."
+    attempt=$(( $attempt + 1 ))
+    sleep $sleep_time
+  done
+}
+
+test_usage() {
+  echo "Testing 's2i usage'..."
+  s2i usage ${s2i_args} ${IMAGE_NAME} &>/dev/null
+}
+
+test_connection() {
+  echo "Testing HTTP connection (http://$(container_ip):$(container_port))"
+  local max_attempts=10
+  local sleep_time=1
+  local attempt=1
+  local result=1
+  while [ $attempt -le $max_attempts ]; do
+    echo "Sending GET request to http://$(container_ip):$(container_port)/"
+    response_code=$(curl -s -w %{http_code} -o /dev/null http://$(container_ip):$(container_port)/)
+    status=$?
+    if [ $status -eq 0 ]; then
+      if [ $response_code -eq 200 ]; then
+        result=0
+      fi
+      break
+    fi
+    attempt=$(( $attempt + 1 ))
+    sleep $sleep_time
+  done
+  return $result
+}
+
+# Build the application image twice to ensure the 'save-artifacts' and
+# 'restore-artifacts' scripts are working properly
+prepare
+run_s2i_build
+check_result $?
+
+# Verify the 'usage' script is working properly
+test_usage
+check_result $?
+
+# Verify that the HTTP connection can be established to test application container
+run_test_application &
+
+# Wait for the container to write its CID file
+wait_for_cid
+
+test_connection
+check_result $?
+
+cleanup
+

singleuser-builder-conda/test/test-app/index.html

@@ -0,0 +1,11 @@
+
+<!doctype html>
+<html>
+	<head>
+		<title>Hello World!</title>
+	</head>
+	<body>
+		<h1>Hello World!</h1>
+	</body>
+</html>
+ 

singleuser-builder-conda/version

@@ -0,0 +1 @@
+v0.1.5

singleuser-builder/Dockerfile

@@ -0,0 +1,32 @@
+FROM ubuntu:16.10
+
+MAINTAINER Yuvi Panda <yuvipanda@gmail.com>
+
+LABEL io.openshift.s2i.scripts-url=image:///usr/libexec/s2i
+
+ENV APP_DIR /srv/app
+ENV PATH ${APP_DIR}/venv/bin:$PATH
+
+RUN apt-get update && \
+    apt-get install --yes --no-install-recommends \
+            python3 \
+            python3-venv \
+            tar \
+            git && \
+    apt-get purge && apt-get clean
+
+RUN adduser --disabled-password --gecos "Default Jupyter user" jovyan
+
+RUN mkdir -p ${APP_DIR} && chown -R jovyan:jovyan ${APP_DIR}
+
+WORKDIR /home/jovyan
+
+USER jovyan
+RUN python3 -m venv ${APP_DIR}/venv
+
+RUN ${APP_DIR}/venv/bin/pip install --no-cache-dir notebook==5.0.0 jupyterhub==0.7.2 ipywidgets==5.2.3 && \
+    ${APP_DIR}/venv/bin/jupyter nbextension enable --py widgetsnbextension
+
+COPY ./s2i/bin/ /usr/libexec/s2i
+
+EXPOSE 8888

singleuser-builder/Dockerfile.py2

@@ -0,0 +1,34 @@
+FROM ubuntu:16.10
+
+MAINTAINER Yuvi Panda <yuvipanda@gmail.com>
+
+LABEL io.openshift.s2i.scripts-url=image:///usr/libexec/s2i
+
+ENV APP_DIR /srv/app
+ENV PATH ${APP_DIR}/venv/bin:$PATH
+
+RUN apt-get update && \
+    apt-get install --yes --no-install-recommends \
+            python \
+            python-dev \
+            build-essential \
+            virtualenv \
+            tar \
+            git && \
+    apt-get purge && apt-get clean
+
+RUN adduser --disabled-password --gecos "Default Jupyter user" jovyan
+
+RUN mkdir -p ${APP_DIR} && chown -R jovyan:jovyan ${APP_DIR}
+
+WORKDIR /home/jovyan
+
+USER jovyan
+RUN virtualenv ${APP_DIR}/venv
+
+RUN ${APP_DIR}/venv/bin/pip install --no-cache-dir notebook==5.0.0 git+https://github.com/yuvipanda/jupyterhub-legacy-singleuser@master ipywidgets==5.2.3 && \
+    ${APP_DIR}/venv/bin/jupyter nbextension enable --py widgetsnbextension
+
+COPY ./s2i/bin/ /usr/libexec/s2i
+
+EXPOSE 8888

singleuser-builder/Makefile

@@ -0,0 +1,7 @@
+IMAGE_PREFIX = jupyterhub/singleuser-builder-venv-
+VERSION = $(shell cat version)
+
+.PHONY: build
+build:
+	docker build -t $(IMAGE_PREFIX)3.5:$(VERSION) . -f Dockerfile
+	docker build -t $(IMAGE_PREFIX)2.7:$(VERSION) . -f Dockerfile.py2

singleuser-builder/README.md

@@ -0,0 +1,12 @@
+
+# JupyterHub singleuser builder
+
+This is a builder image for use with [s2i](https://github.com/openshift/source-to-image). It
+builds a source repository (such as a github repository) into a docker image that is suitable
+for use with [JupyterHub](http://github.com/jupyterhub/jupyterhub).
+
+It is based off Ubuntu 16.10, and uses [virtualenv](https://pypi.python.org/pypi/virtualenv) to
+provide a custom python3.5 environment.
+
+It looks for a `requirements.txt` file in the source repository, and installs it into the virtualenv.
+It also installs a number of default notebook related modules in there

singleuser-builder/s2i/bin/assemble

@@ -0,0 +1,33 @@
+#!/bin/bash -e
+#
+# S2I assemble script for the jupyterhub/singleuser-builder image.
+# The 'assemble' script builds your application source so that it is ready to run.
+#
+# For more information refer to the documentation:
+#	https://github.com/openshift/source-to-image/blob/master/docs/builder_image.md
+#
+
+if [[ "$1" == "-h" ]]; then
+	exec /usr/libexec/s2i/usage
+fi
+
+# Restore artifacts from the previous build (if they exist).
+#
+echo "---> Installing application source..."
+mkdir -p ${APP_DIR}/src
+cp -Rf /tmp/src/. ${APP_DIR}/src
+
+# HACK: We copy it to current directory too, since people wanna see the
+# notebooks and stuff in there. Figure out a way to tweak this?
+cp -Rf /tmp/src/. .
+
+echo "---> Building application from source..."
+if [ -f ${APP_DIR}/src/requirements.txt ]; then
+    # HACK: If numpy is present in requirements.txt, install it first!
+    # This is because some packages (tiffiles *cough*) need numpy to be fully installed
+    # for their setup.py to work, and this... causes all sorts of problems.
+    if grep --quiet numpy ${APP_DIR}/src/requirements.txt; then
+        ${APP_DIR}/venv/bin/pip install $(grep numpy ${APP_DIR}/src/requirements.txt)
+    fi
+    ${APP_DIR}/venv/bin/pip install --no-cache-dir -r ${APP_DIR}/src/requirements.txt
+fi

singleuser-builder/s2i/bin/run

@@ -0,0 +1,10 @@
+#!/bin/bash -e
+#
+# S2I run script for the 'ubuntu1610-python35-venv' image.
+# The run script executes the server that runs your application.
+#
+# For more information see the documentation:
+#	https://github.com/openshift/source-to-image/blob/master/docs/builder_image.md
+#
+
+exec ${APP_DIR}/venv/bin/jupyter notebook --ip=0.0.0.0

singleuser-builder/s2i/bin/save-artifacts

@@ -0,0 +1,10 @@
+#!/bin/sh -e
+#
+# S2I save-artifacts script for the 'ubuntu1610-python35-venv' image.
+# The save-artifacts script streams a tar archive to standard output.
+# The archive contains the files and folders you want to re-use in the next build.
+#
+# For more information see the documentation:
+#	https://github.com/openshift/source-to-image/blob/master/docs/builder_image.md
+#
+# tar cf - <list of files and folders>

singleuser-builder/s2i/bin/usage

@@ -0,0 +1,12 @@
+#!/bin/bash -e
+cat <<EOF
+This is the ubuntu1610-python35-venv S2I image:
+To use it, install S2I: https://github.com/openshift/source-to-image
+
+Sample invocation:
+
+s2i build <source code path/URL> ubuntu1610-python35-venv <application image>
+
+You can then run the resulting image via:
+docker run <application image>
+EOF

singleuser-builder/test/run

@@ -0,0 +1,160 @@
+#!/bin/bash
+#
+# The 'run' performs a simple test that verifies the S2I image.
+# The main focus here is to exercise the S2I scripts.
+#
+# For more information see the documentation:
+# https://github.com/openshift/source-to-image/blob/master/docs/builder_image.md
+#
+# IMAGE_NAME specifies a name of the candidate image used for testing.
+# The image has to be available before this script is executed.
+#
+IMAGE_NAME=${IMAGE_NAME-ubuntu1610-python35-venv-candidate}
+
+# Determining system utility executables (darwin compatibility check)
+READLINK_EXEC="readlink"
+MKTEMP_EXEC="mktemp"
+if [[ "$OSTYPE" =~ 'darwin' ]]; then
+  ! type -a "greadlink" &>"/dev/null" || READLINK_EXEC="greadlink"
+  ! type -a "gmktemp" &>"/dev/null" || MKTEMP_EXEC="gmktemp"
+fi
+
+test_dir="$($READLINK_EXEC -zf $(dirname "${BASH_SOURCE[0]}"))"
+image_dir=$($READLINK_EXEC -zf ${test_dir}/..)
+scripts_url="file://${image_dir}/.s2i/bin"
+cid_file=$($MKTEMP_EXEC -u --suffix=.cid)
+
+# Since we built the candidate image locally, we don't want S2I to attempt to pull
+# it from Docker hub
+s2i_args="--pull-policy=never --loglevel=2"
+
+# Port the image exposes service to be tested
+test_port=8080
+
+image_exists() {
+  docker inspect $1 &>/dev/null
+}
+
+container_exists() {
+  image_exists $(cat $cid_file)
+}
+
+container_ip() {
+  if [ ! -z "$DOCKER_HOST" ] && [[ "$OSTYPE" =~ 'darwin' ]]; then
+    docker-machine ip
+  else
+    docker inspect --format="{{ .NetworkSettings.IPAddress }}" $(cat $cid_file)
+  fi
+}
+
+container_port() {
+  if [ ! -z "$DOCKER_HOST" ] && [[ "$OSTYPE" =~ 'darwin' ]]; then
+    docker inspect --format="{{(index .NetworkSettings.Ports \"$test_port/tcp\" 0).HostPort}}" "$(cat "${cid_file}")"
+  else
+    echo $test_port
+  fi
+}
+
+run_s2i_build() {
+  s2i build --incremental=true ${s2i_args} file://${test_dir}/test-app ${IMAGE_NAME} ${IMAGE_NAME}-testapp
+}
+
+prepare() {
+  if ! image_exists ${IMAGE_NAME}; then
+    echo "ERROR: The image ${IMAGE_NAME} must exist before this script is executed."
+    exit 1
+  fi
+  # s2i build requires the application is a valid 'Git' repository
+  pushd ${test_dir}/test-app >/dev/null
+  git init
+  git config user.email "build@localhost" && git config user.name "builder"
+  git add -A && git commit -m "Sample commit"
+  popd >/dev/null
+  run_s2i_build
+}
+
+run_test_application() {
+  docker run --rm --cidfile=${cid_file} -p ${test_port} ${IMAGE_NAME}-testapp
+}
+
+cleanup() {
+  if [ -f $cid_file ]; then
+    if container_exists; then
+      docker stop $(cat $cid_file)
+    fi
+  fi
+  if image_exists ${IMAGE_NAME}-testapp; then
+    docker rmi ${IMAGE_NAME}-testapp
+  fi
+}
+
+check_result() {
+  local result="$1"
+  if [[ "$result" != "0" ]]; then
+    echo "S2I image '${IMAGE_NAME}' test FAILED (exit code: ${result})"
+    cleanup
+    exit $result
+  fi
+}
+
+wait_for_cid() {
+  local max_attempts=10
+  local sleep_time=1
+  local attempt=1
+  local result=1
+  while [ $attempt -le $max_attempts ]; do
+    [ -f $cid_file ] && break
+    echo "Waiting for container to start..."
+    attempt=$(( $attempt + 1 ))
+    sleep $sleep_time
+  done
+}
+
+test_usage() {
+  echo "Testing 's2i usage'..."
+  s2i usage ${s2i_args} ${IMAGE_NAME} &>/dev/null
+}
+
+test_connection() {
+  echo "Testing HTTP connection (http://$(container_ip):$(container_port))"
+  local max_attempts=10
+  local sleep_time=1
+  local attempt=1
+  local result=1
+  while [ $attempt -le $max_attempts ]; do
+    echo "Sending GET request to http://$(container_ip):$(container_port)/"
+    response_code=$(curl -s -w %{http_code} -o /dev/null http://$(container_ip):$(container_port)/)
+    status=$?
+    if [ $status -eq 0 ]; then
+      if [ $response_code -eq 200 ]; then
+        result=0
+      fi
+      break
+    fi
+    attempt=$(( $attempt + 1 ))
+    sleep $sleep_time
+  done
+  return $result
+}
+
+# Build the application image twice to ensure the 'save-artifacts' and
+# 'restore-artifacts' scripts are working properly
+prepare
+run_s2i_build
+check_result $?
+
+# Verify the 'usage' script is working properly
+test_usage
+check_result $?
+
+# Verify that the HTTP connection can be established to test application container
+run_test_application &
+
+# Wait for the container to write its CID file
+wait_for_cid
+
+test_connection
+check_result $?
+
+cleanup
+

singleuser-builder/test/test-app/index.html

@@ -0,0 +1,11 @@
+
+<!doctype html>
+<html>
+	<head>
+		<title>Hello World!</title>
+	</head>
+	<body>
+		<h1>Hello World!</h1>
+	</body>
+</html>
+ 

singleuser-builder/version

@@ -0,0 +1 @@
+v0.1.2