From 938179d1548e377fd41b67f681c036d6b8e3b687 Mon Sep 17 00:00:00 2001 From: vsoch Date: Mon, 16 Jul 2018 01:35:02 -0700 Subject: [PATCH 01/15] adding config --- .circleci/config.yml | 168 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..71b7d64d --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,168 @@ +version: 2 +jobs: + + # If you want to deploy your container to Docker Hub, you must + # 1. define the CONTAINER_NAME environment variable for the project + # 2. Also define DOCKER_USER and DOCKER_PASS + setup: + environment: + - TZ: "/usr/share/zoneinfo/America/Los_Angeles" + docker: + - image: docker:18.01.0-ce-git + - image: circleci/python:3.6.1 + steps: + - run: + name: Test if the user has defined the repository name + command: apk add --no-cache pigz python3 + + build: + environment: + - TZ: "/usr/share/zoneinfo/America/Los_Angeles" + docker: + - image: docker:18.01.0-ce-git + working_directory: /tmp/src + steps: + - run: + name: Install parallel gzip and python3 + command: apk add --no-cache pigz python3 + - restore_cache: + keys: + - docker-v1-{{ .Branch }} + paths: + - /tmp/cache/container.tar.gz + - restore_cache: + key: dependency-cache-repo2docker + - checkout + - setup_remote_docker + - run: + name: Load Docker image layer cache + no_output_timeout: 30m + command: | + docker info + set +o pipefail + if [ -f /tmp/cache/container.tar.gz ]; then + pigz -d --stdout /tmp/cache/container.tar.gz | docker load + fi + docker images + - run: # install and activate virtual environment with pip + name: Install Repo2Docker + command: | + python3 -m venv venv + . venv/bin/activate + pip install jupyter-repo2docker + export VERSION=$(python3 -c "import repo2docker; print(repo2docker.__version__)") + echo "repo2docker version: ${VERSION}" + - run: + name: Generate Docker Image + no_output_timeout: 60m + command: | + echo "1. Preparing building environment" + echo "Working directory is ${PWD}" + # If not set, define CONTAINER_NAME + if [ ! -n "${CONTAINER_NAME:-}" ] + then + CONTAINER_NAME="${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}" + fi + # If not set, use testing repo name + if [ ! -n "{REPO_NAME}" ] + then + ${REPO_NAME}="https://www.github.com/${CONTAINER_NAME}" + fi + echo "Container name is ${CONTAINER_NAME}" + echo "Repository name is ${REPO_NAME}" + echo "2. Running jupyter-repo2docker..." + echo "jupyter-repo2docker --debug --image-name ${CONTAINER_NAME} ${REPO_NAME}" + jupyter-repo2docker --debug --image-name "${CONTAINER_NAME}" "${REPO_NAME}" + - run: + name: Docker Save + no_output_timeout: 40m + command: | + # If not set, define CONTAINER_NAME + if [ ! -n "${CONTAINER_NAME:-}" ] + then + CONTAINER_NAME="${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}" + fi + echo "Saving ${CONTAINER_NAME} to container.tar.gz" + mkdir -p /tmp/cache + docker save ${CONTAINER_NAME}:latest \ + | pigz -2 -p 3 > /tmp/cache/container.tar.gz + - persist_to_workspace: + root: /tmp + paths: + - cache/container.tar.gz + - src + + update_cache: + machine: + # Ubuntu 14.04 with Docker 17.10.0-ce + image: circleci/classic:201711-01 + working_directory: /tmp/src + steps: + - attach_workspace: + at: /tmp + - save_cache: + key: docker-v1-{{ .Branch }} + paths: + - /tmp/cache/container.tar.gz + + deploy: + machine: + image: circleci/classic:201711-01 + working_directory: /tmp/src + steps: + - attach_workspace: + at: /tmp + - run: + name: Load Docker image layer cache + no_output_timeout: 30m + command: | + echo "Working directory is ${PWD}" + docker info + set +o pipefail + if [ -f /tmp/cache/container.tar.gz ]; then + sudo apt update && sudo apt -y install pigz + pigz -d --stdout /tmp/cache/container.tar.gz | docker load + docker images + fi + - run: + name: Deploy to Docker Hub + no_output_timeout: 40m + command: | + echo "Looking for Docker deployment options" + # If not set, define CONTAINER_NAME + if [[ ! -n "${CONTAINER_NAME:-}" ]] + then + CONTAINER_NAME="${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}" + fi + echo "Container name set to ${CONTAINER_NAME}" + if [[ -n "$DOCKER_PASS" ]]; then + docker login -u $DOCKER_USER -p $DOCKER_PASS + docker push ${CONTAINER_NAME}:latest + fi +workflows: + version: 2 + build_deploy: + jobs: + - build: + filters: + tags: + only: /.*/ + + - update_cache: + requires: + - build + filters: + branches: + ignore: /docs?\/.*/ + tags: + only: /.*/ + + # This is where we upload the container to some final resting spot :) + - deploy: + requires: + - build + filters: + branches: + only: master + tags: + only: /.*/ From e1ca357909c4e69e2aa4ad1a28996d385b586c09 Mon Sep 17 00:00:00 2001 From: vsoch Date: Mon, 16 Jul 2018 01:37:04 -0700 Subject: [PATCH 02/15] install deps in same step --- .circleci/config.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 71b7d64d..5a0d77b9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,18 +44,16 @@ jobs: pigz -d --stdout /tmp/cache/container.tar.gz | docker load fi docker images - - run: # install and activate virtual environment with pip - name: Install Repo2Docker + - run: + name: Generate Docker Image + no_output_timeout: 60m command: | + echo "0. Installing dependencies..." python3 -m venv venv . venv/bin/activate pip install jupyter-repo2docker export VERSION=$(python3 -c "import repo2docker; print(repo2docker.__version__)") echo "repo2docker version: ${VERSION}" - - run: - name: Generate Docker Image - no_output_timeout: 60m - command: | echo "1. Preparing building environment" echo "Working directory is ${PWD}" # If not set, define CONTAINER_NAME From 4733d3a3ea457957475e5f852a70e7aad63a83c8 Mon Sep 17 00:00:00 2001 From: vsoch Date: Mon, 16 Jul 2018 01:39:29 -0700 Subject: [PATCH 03/15] try setting username --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5a0d77b9..feee9397 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,7 +45,7 @@ jobs: fi docker images - run: - name: Generate Docker Image + name: Run jupyter-repo2docker to generate Docker Container no_output_timeout: 60m command: | echo "0. Installing dependencies..." @@ -70,7 +70,7 @@ jobs: echo "Repository name is ${REPO_NAME}" echo "2. Running jupyter-repo2docker..." echo "jupyter-repo2docker --debug --image-name ${CONTAINER_NAME} ${REPO_NAME}" - jupyter-repo2docker --debug --image-name "${CONTAINER_NAME}" "${REPO_NAME}" + USER=username jupyter-repo2docker --debug --image-name "${CONTAINER_NAME}" "${REPO_NAME}" - run: name: Docker Save no_output_timeout: 40m From 75ac3cc18b95635ff374121802e73a3ec7952bf9 Mon Sep 17 00:00:00 2001 From: vsoch Date: Mon, 16 Jul 2018 01:40:32 -0700 Subject: [PATCH 04/15] try setting username and userid --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index feee9397..cfbfe7a5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,7 +70,7 @@ jobs: echo "Repository name is ${REPO_NAME}" echo "2. Running jupyter-repo2docker..." echo "jupyter-repo2docker --debug --image-name ${CONTAINER_NAME} ${REPO_NAME}" - USER=username jupyter-repo2docker --debug --image-name "${CONTAINER_NAME}" "${REPO_NAME}" + jupyter-repo2docker --debug --user-name username --user-id 1000 --image-name "${CONTAINER_NAME}" "${REPO_NAME}" - run: name: Docker Save no_output_timeout: 40m From 4e15dec3f594a2f6d8e6bc28d10130c81b814e2b Mon Sep 17 00:00:00 2001 From: vsoch Date: Mon, 16 Jul 2018 02:00:16 -0700 Subject: [PATCH 05/15] adding example with save --- .circleci/config.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cfbfe7a5..5a7baf1f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,8 +2,11 @@ version: 2 jobs: # If you want to deploy your container to Docker Hub, you must - # 1. define the CONTAINER_NAME environment variable for the project - # 2. Also define DOCKER_USER and DOCKER_PASS + # 1. define the CONTAINER_NAME environment variable for the project + # If not defined, will use the repo that is being built from. + # 2. define the REPO_NAME with the juypter notebook you want to build. If not + # defined, will use https://www.github.com/{CONTAINER_NAME} + # 2. Also define DOCKER_USER and DOCKER_PASS (for deploy, if wanted) setup: environment: - TZ: "/usr/share/zoneinfo/America/Los_Angeles" @@ -70,7 +73,11 @@ jobs: echo "Repository name is ${REPO_NAME}" echo "2. Running jupyter-repo2docker..." echo "jupyter-repo2docker --debug --image-name ${CONTAINER_NAME} ${REPO_NAME}" - jupyter-repo2docker --debug --user-name username --user-id 1000 --image-name "${CONTAINER_NAME}" "${REPO_NAME}" + jupyter-repo2docker --debug --user-name username --user-id 1000 --no-run --image-name "${CONTAINER_NAME}" "${REPO_NAME}" + echo "3. Running without server (example)" + docker run -td --name repo2docker --entrypoint /bin/bash "${CONTAINER_NAME}" + docker ps + docker images - run: name: Docker Save no_output_timeout: 40m From b5c471560bcd1a590f1f63916e9b24431ab8222a Mon Sep 17 00:00:00 2001 From: vsoch Date: Mon, 16 Jul 2018 02:06:38 -0700 Subject: [PATCH 06/15] also deploy on branch non-master --- .circleci/config.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5a7baf1f..2484710c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -72,7 +72,7 @@ jobs: echo "Container name is ${CONTAINER_NAME}" echo "Repository name is ${REPO_NAME}" echo "2. Running jupyter-repo2docker..." - echo "jupyter-repo2docker --debug --image-name ${CONTAINER_NAME} ${REPO_NAME}" + echo "jupyter-repo2docker --debug --user-name username --user-id 1000 --no-run --image-name ${CONTAINER_NAME} ${REPO_NAME}" jupyter-repo2docker --debug --user-name username --user-id 1000 --no-run --image-name "${CONTAINER_NAME}" "${REPO_NAME}" echo "3. Running without server (example)" docker run -td --name repo2docker --entrypoint /bin/bash "${CONTAINER_NAME}" @@ -166,8 +166,5 @@ workflows: - deploy: requires: - build - filters: - branches: - only: master tags: only: /.*/ From f93a4073f1d2095fabcf5ed18dc919e345198f8d Mon Sep 17 00:00:00 2001 From: vsoch Date: Mon, 16 Jul 2018 02:07:58 -0700 Subject: [PATCH 07/15] oups --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2484710c..571c8e6f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -166,5 +166,6 @@ workflows: - deploy: requires: - build + filters: tags: only: /.*/ From 67621204c2b9ebe6db5a5f13dc0794d9a376fc61 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Mon, 16 Jul 2018 05:35:12 -0700 Subject: [PATCH 08/15] adding deploy docs --- docs/source/deploy.rst | 92 ++++++++++++++++++++++++++++++++++++++++++ docs/source/index.rst | 1 + 2 files changed, 93 insertions(+) create mode 100644 docs/source/deploy.rst diff --git a/docs/source/deploy.rst b/docs/source/deploy.rst new file mode 100644 index 00000000..d1d9a5b6 --- /dev/null +++ b/docs/source/deploy.rst @@ -0,0 +1,92 @@ +.. _usage: + +Deploy Using ``repo2docker`` +========================== + +You can deploy a `Docker `_ container +to `Docker Hub `_ directly from a GIthub repository +that has a Jupyter notebook. Here are instructions to do this. + +Getting Started +--------------- +Today you will be doing the following: + + 1. Fork and clone the repo2docker Github repository to obtain the hidden ``.circleci`` folder. + 2. creating an image repository on Docker Hub + 3. connecting your repository to CircleCI + 4. push, commit, or create a pull request to trigger a build. + +You don't need to install any dependencies on your host to build the container, it will be done +on a continuous integration server, and the container built and available to you +to pull from Docker Hub. + + +Step 1. Clone the Repository +............................ +First, fork the `repo2docker `_ Github +repository to your account, and clone the branch. + + + git clone https://www.github.com//repo2docker + # or + git clone git@github.com:/repo2docker.git + + +Step 2. Docker Hub +.................. +Go to `Docker Hub `_, log in, and click the big blue +button that says "create repository" (not an automated build). Choose an organization +and name that you like, and remember it! We will be adding it, along with your +Docker credentials, to be encrypted CircleCI environment variables. + + +Step 3. Connect to CircleCI +........................... +If you navigate to the main `app page `_ you +should be able to click "Add Projects" and then select your repository. If you don't +see it on the list, then select a different organization in the top left. Once +you find the repository, you can click the button to "Start Building" adn accept +the defaults. + +Before you push or trigger a build, let's set up the following environment variables. +Also in the project interface on CirleCi, click the gears icon next to the project +name to get to your project settings. Under settings, click on the "Environment +Variables" tab. In this section, you want to define the following: + +1. ``CONTAINER_NAME`` should be the name of the Docker Hub repository you just created. +2. ``DOCKER_USER`` and ``DOCKER_PASS`` should be your credentials (to allowing pushing) +3. ``REPO_NAME`` should be the full Github url (or other) of the repository with the notebook. + +If you don't define the ``CONTAINER_NAME`` it will default to be the repository where it is +building from, which you should only do if the Docker Hub repository is named equivalently. +If you don't define either of the variables from 2. for the Docker credentials, your +image will build but not be deployed to Docker Hub. Finally, if you don't define the ``REPO_NAME`` +it will again use the name of the repository defined for the ``CONTAINER_NAME``. + +Step 4. Push Away, Merrill! +........................... + +Once the environment variables are set up, you can push or issue a pull request +to see circle build the workflow. Remember that you only need the ``.circleci/config.yml`` +and not any other files in the repository. If your notebook is hosted in the same repo, +you might want to add these, along with your requirements.txt, etc. + +.. tip:: + By default, new builds on CircleCI will not build for + pull requests and you can change this default in the settings. You can easily add + filters (or other criteria and actions) to be performed during or after the build + by editing the ``.circleci/config.yml`` file in your repository. + + +Step 5. Use Your Container! +........................... + +You should then be able to pull your new container, and run it! Here is an example: + + docker pull vanessa/repo2docker + docker run -it --name repo2docker -p 8888:8888 vanessa/repo2docker jupyter notebook --ip 0.0.0.0 + +You can then enter the url and token provided in the browser to access your notebook. When you are done and need to stop and remove the container: + + docker stop repo2docker + docker rm repo2docker diff --git a/docs/source/index.rst b/docs/source/index.rst index 63aeafbc..f37cd487 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -22,6 +22,7 @@ Site Contents usage config_files faq + deploy design architecture dev_newbuildpack From c3e6fb68cb6df8e0425b282ac839ee2772e64a7c Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Mon, 16 Jul 2018 05:38:49 -0700 Subject: [PATCH 09/15] replacing notebook example with pytudes --- docs/source/usage.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/source/usage.rst b/docs/source/usage.rst index a88dc7b5..ad73a179 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -21,14 +21,14 @@ repo2docker is called with this command:: where ```` is a URL or path to the source repository. For example, use the following to build an image of the -`Python Data Science Handbook `_:: +`Python Pytudes notebook `_:: - jupyter-repo2docker https://github.com/jakevdp/PythonDataScienceHandbook + jupyter-repo2docker https://github.com/norvig/pytudes To build a particular branch and commit, use the argument ``--ref`` to specify the ``branch-name`` or ``commit-hash``:: - jupyter-repo2docker https://github.com/jakevdp/PythonDataScienceHandbook --ref 599aa0fe3f882c0001670e676e5a8d43b92c35fc + jupyter-repo2docker https://github.com/norvig/pytudes --ref 9ced85dd9a84859d0767369e58f33912a214a3cf .. tip:: For reproducibile research, we recommend specifying a commit-hash to @@ -47,8 +47,8 @@ Python 3.6 unless you include the version of Python in your Python 2.7, 3.5, and 3.6. In the case of this repo, a Python version is not specified in their configuation files and Python 3.6 is installed. -`Python Data Science Handbook `_ -uses a `requirements.txt file `_ +`Python Pytudes Repository `_ +uses a `requirements.txt file `_ to specify its Python environment. ``repo2docker`` uses ``pip`` to install dependencies listed in the ``requirement.txt`` in the image. To learn more about configuration files in ``repo2docker`` visit :ref:`config-files`. @@ -86,7 +86,7 @@ To debug the docker image being built, pass the ``--debug`` parameter: .. code-block:: bash - jupyter-repo2docker --debug https://github.com/jakevdp/PythonDataScienceHandbook + jupyter-repo2docker --debug https://github.com/norvig/pytudes This will print the generated ``Dockerfile``, build it, and run it. @@ -97,4 +97,4 @@ be used by docker directly. .. code-block:: bash - jupyter-repo2docker --no-build --debug https://github.com/jakevdp/PythonDataScienceHandbook + jupyter-repo2docker --no-build --debug https://github.com/norvig/pytudes From 9d3725d9d08aa477e4b724e24526c9427e7bd71b Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Mon, 16 Jul 2018 06:35:49 -0700 Subject: [PATCH 10/15] testing addition of DOCKER_TAG instead of latest --- .circleci/config.yml | 28 ++++++++++++++++++++++------ docs/source/deploy.rst | 5 +++-- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 571c8e6f..54e24a54 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,9 +4,11 @@ jobs: # If you want to deploy your container to Docker Hub, you must # 1. define the CONTAINER_NAME environment variable for the project # If not defined, will use the repo that is being built from. - # 2. define the REPO_NAME with the juypter notebook you want to build. If not + # 2. Define the DOCKER_TAG if you have preference. If not defined, will use + # the commit associated with the build. + # 3. define the REPO_NAME with the juypter notebook you want to build. If not # defined, will use https://www.github.com/{CONTAINER_NAME} - # 2. Also define DOCKER_USER and DOCKER_PASS (for deploy, if wanted) + # 4. Also define DOCKER_USER and DOCKER_PASS (for deploy, if wanted) setup: environment: - TZ: "/usr/share/zoneinfo/America/Los_Angeles" @@ -59,6 +61,11 @@ jobs: echo "repo2docker version: ${VERSION}" echo "1. Preparing building environment" echo "Working directory is ${PWD}" + # If not set, define DOCKER_TAG + if [ ! -n "${DOCKER_TAG:-}" ] + then + DOCKER_TAG=$(echo "${CIRCLE_SHA1}" | cut -c1-10) + fi # If not set, define CONTAINER_NAME if [ ! -n "${CONTAINER_NAME:-}" ] then @@ -82,14 +89,19 @@ jobs: name: Docker Save no_output_timeout: 40m command: | + # If not set, use commit id as tag + if [ ! -n "${DOCKER_TAG:-}" ] + then + DOCKER_TAG=$(echo "${CIRCLE_SHA1}" | cut -c1-10) + fi # If not set, define CONTAINER_NAME if [ ! -n "${CONTAINER_NAME:-}" ] then CONTAINER_NAME="${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}" fi - echo "Saving ${CONTAINER_NAME} to container.tar.gz" + echo "Saving ${CONTAINER_NAME}:${DOCKER_TAG} to container.tar.gz" mkdir -p /tmp/cache - docker save ${CONTAINER_NAME}:latest \ + docker save ${CONTAINER_NAME}:${DOCKER_TAG} \ | pigz -2 -p 3 > /tmp/cache/container.tar.gz - persist_to_workspace: root: /tmp @@ -134,15 +146,19 @@ jobs: no_output_timeout: 40m command: | echo "Looking for Docker deployment options" + if [ ! -n "${DOCKER_TAG:-}" ] + then + DOCKER_TAG=$(echo "${CIRCLE_SHA1}" | cut -c1-10) + fi # If not set, define CONTAINER_NAME if [[ ! -n "${CONTAINER_NAME:-}" ]] then CONTAINER_NAME="${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}" fi - echo "Container name set to ${CONTAINER_NAME}" + echo "Container name set to ${CONTAINER_NAME}:${DOCKER_TAG}" if [[ -n "$DOCKER_PASS" ]]; then docker login -u $DOCKER_USER -p $DOCKER_PASS - docker push ${CONTAINER_NAME}:latest + docker push ${CONTAINER_NAME}:${DOCKER_TAG} fi workflows: version: 2 diff --git a/docs/source/deploy.rst b/docs/source/deploy.rst index d1d9a5b6..143c0d80 100644 --- a/docs/source/deploy.rst +++ b/docs/source/deploy.rst @@ -54,8 +54,9 @@ name to get to your project settings. Under settings, click on the "Environment Variables" tab. In this section, you want to define the following: 1. ``CONTAINER_NAME`` should be the name of the Docker Hub repository you just created. -2. ``DOCKER_USER`` and ``DOCKER_PASS`` should be your credentials (to allowing pushing) -3. ``REPO_NAME`` should be the full Github url (or other) of the repository with the notebook. +2. ``DOCKER_TAG`` is the tag you want to use. If not defined, will use first 10 characters of commit. +3. ``DOCKER_USER`` and ``DOCKER_PASS`` should be your credentials (to allowing pushing) +4. ``REPO_NAME`` should be the full Github url (or other) of the repository with the notebook. If you don't define the ``CONTAINER_NAME`` it will default to be the repository where it is building from, which you should only do if the Docker Hub repository is named equivalently. From 7b0352bef459f314ba3d39117fb95f8dba637bee Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Mon, 16 Jul 2018 06:40:55 -0700 Subject: [PATCH 11/15] forgot to add tag to build step! --- .circleci/config.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 54e24a54..240a03c8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,10 +79,8 @@ jobs: echo "Container name is ${CONTAINER_NAME}" echo "Repository name is ${REPO_NAME}" echo "2. Running jupyter-repo2docker..." - echo "jupyter-repo2docker --debug --user-name username --user-id 1000 --no-run --image-name ${CONTAINER_NAME} ${REPO_NAME}" - jupyter-repo2docker --debug --user-name username --user-id 1000 --no-run --image-name "${CONTAINER_NAME}" "${REPO_NAME}" - echo "3. Running without server (example)" - docker run -td --name repo2docker --entrypoint /bin/bash "${CONTAINER_NAME}" + echo "jupyter-repo2docker --debug --user-name username --user-id 1000 --no-run --image-name ${CONTAINER_NAME}:${DOCKER_TAG} ${REPO_NAME}" + jupyter-repo2docker --debug --user-name username --user-id 1000 --no-run --image-name "${CONTAINER_NAME}:${DOCKER_TAG}" "${REPO_NAME}" docker ps docker images - run: From 30d80dfa919962455d86f488a8d79dacdf456743 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Tue, 17 Jul 2018 07:55:37 -0700 Subject: [PATCH 12/15] destroying circle, pewpewpew --- .circleci/config.yml | 185 ------------------------------------------- 1 file changed, 185 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 240a03c8..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,185 +0,0 @@ -version: 2 -jobs: - - # If you want to deploy your container to Docker Hub, you must - # 1. define the CONTAINER_NAME environment variable for the project - # If not defined, will use the repo that is being built from. - # 2. Define the DOCKER_TAG if you have preference. If not defined, will use - # the commit associated with the build. - # 3. define the REPO_NAME with the juypter notebook you want to build. If not - # defined, will use https://www.github.com/{CONTAINER_NAME} - # 4. Also define DOCKER_USER and DOCKER_PASS (for deploy, if wanted) - setup: - environment: - - TZ: "/usr/share/zoneinfo/America/Los_Angeles" - docker: - - image: docker:18.01.0-ce-git - - image: circleci/python:3.6.1 - steps: - - run: - name: Test if the user has defined the repository name - command: apk add --no-cache pigz python3 - - build: - environment: - - TZ: "/usr/share/zoneinfo/America/Los_Angeles" - docker: - - image: docker:18.01.0-ce-git - working_directory: /tmp/src - steps: - - run: - name: Install parallel gzip and python3 - command: apk add --no-cache pigz python3 - - restore_cache: - keys: - - docker-v1-{{ .Branch }} - paths: - - /tmp/cache/container.tar.gz - - restore_cache: - key: dependency-cache-repo2docker - - checkout - - setup_remote_docker - - run: - name: Load Docker image layer cache - no_output_timeout: 30m - command: | - docker info - set +o pipefail - if [ -f /tmp/cache/container.tar.gz ]; then - pigz -d --stdout /tmp/cache/container.tar.gz | docker load - fi - docker images - - run: - name: Run jupyter-repo2docker to generate Docker Container - no_output_timeout: 60m - command: | - echo "0. Installing dependencies..." - python3 -m venv venv - . venv/bin/activate - pip install jupyter-repo2docker - export VERSION=$(python3 -c "import repo2docker; print(repo2docker.__version__)") - echo "repo2docker version: ${VERSION}" - echo "1. Preparing building environment" - echo "Working directory is ${PWD}" - # If not set, define DOCKER_TAG - if [ ! -n "${DOCKER_TAG:-}" ] - then - DOCKER_TAG=$(echo "${CIRCLE_SHA1}" | cut -c1-10) - fi - # If not set, define CONTAINER_NAME - if [ ! -n "${CONTAINER_NAME:-}" ] - then - CONTAINER_NAME="${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}" - fi - # If not set, use testing repo name - if [ ! -n "{REPO_NAME}" ] - then - ${REPO_NAME}="https://www.github.com/${CONTAINER_NAME}" - fi - echo "Container name is ${CONTAINER_NAME}" - echo "Repository name is ${REPO_NAME}" - echo "2. Running jupyter-repo2docker..." - echo "jupyter-repo2docker --debug --user-name username --user-id 1000 --no-run --image-name ${CONTAINER_NAME}:${DOCKER_TAG} ${REPO_NAME}" - jupyter-repo2docker --debug --user-name username --user-id 1000 --no-run --image-name "${CONTAINER_NAME}:${DOCKER_TAG}" "${REPO_NAME}" - docker ps - docker images - - run: - name: Docker Save - no_output_timeout: 40m - command: | - # If not set, use commit id as tag - if [ ! -n "${DOCKER_TAG:-}" ] - then - DOCKER_TAG=$(echo "${CIRCLE_SHA1}" | cut -c1-10) - fi - # If not set, define CONTAINER_NAME - if [ ! -n "${CONTAINER_NAME:-}" ] - then - CONTAINER_NAME="${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}" - fi - echo "Saving ${CONTAINER_NAME}:${DOCKER_TAG} to container.tar.gz" - mkdir -p /tmp/cache - docker save ${CONTAINER_NAME}:${DOCKER_TAG} \ - | pigz -2 -p 3 > /tmp/cache/container.tar.gz - - persist_to_workspace: - root: /tmp - paths: - - cache/container.tar.gz - - src - - update_cache: - machine: - # Ubuntu 14.04 with Docker 17.10.0-ce - image: circleci/classic:201711-01 - working_directory: /tmp/src - steps: - - attach_workspace: - at: /tmp - - save_cache: - key: docker-v1-{{ .Branch }} - paths: - - /tmp/cache/container.tar.gz - - deploy: - machine: - image: circleci/classic:201711-01 - working_directory: /tmp/src - steps: - - attach_workspace: - at: /tmp - - run: - name: Load Docker image layer cache - no_output_timeout: 30m - command: | - echo "Working directory is ${PWD}" - docker info - set +o pipefail - if [ -f /tmp/cache/container.tar.gz ]; then - sudo apt update && sudo apt -y install pigz - pigz -d --stdout /tmp/cache/container.tar.gz | docker load - docker images - fi - - run: - name: Deploy to Docker Hub - no_output_timeout: 40m - command: | - echo "Looking for Docker deployment options" - if [ ! -n "${DOCKER_TAG:-}" ] - then - DOCKER_TAG=$(echo "${CIRCLE_SHA1}" | cut -c1-10) - fi - # If not set, define CONTAINER_NAME - if [[ ! -n "${CONTAINER_NAME:-}" ]] - then - CONTAINER_NAME="${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}" - fi - echo "Container name set to ${CONTAINER_NAME}:${DOCKER_TAG}" - if [[ -n "$DOCKER_PASS" ]]; then - docker login -u $DOCKER_USER -p $DOCKER_PASS - docker push ${CONTAINER_NAME}:${DOCKER_TAG} - fi -workflows: - version: 2 - build_deploy: - jobs: - - build: - filters: - tags: - only: /.*/ - - - update_cache: - requires: - - build - filters: - branches: - ignore: /docs?\/.*/ - tags: - only: /.*/ - - # This is where we upload the container to some final resting spot :) - - deploy: - requires: - - build - filters: - tags: - only: /.*/ From c7a22f9fef4d8ef9e5f2baf0f6de066fe3577c4d Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Tue, 17 Jul 2018 08:07:02 -0700 Subject: [PATCH 13/15] fixing up deploy.rst to point to updated repository continuous-build --- docs/source/deploy.rst | 62 ++++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/docs/source/deploy.rst b/docs/source/deploy.rst index 143c0d80..af01310f 100644 --- a/docs/source/deploy.rst +++ b/docs/source/deploy.rst @@ -1,17 +1,18 @@ .. _usage: -Deploy Using ``repo2docker`` -========================== +Deploy ``repo2docker`` +====================== -You can deploy a `Docker `_ container -to `Docker Hub `_ directly from a GIthub repository +We've created for you the `continuous-build `_ +repository so that you can deploy a `Docker `_ container +to `Docker Hub `_ directly from a Github repository that has a Jupyter notebook. Here are instructions to do this. Getting Started --------------- Today you will be doing the following: - 1. Fork and clone the repo2docker Github repository to obtain the hidden ``.circleci`` folder. + 1. Fork and clone the continuous-build Github repository to obtain the hidden ``.circleci`` folder. 2. creating an image repository on Docker Hub 3. connecting your repository to CircleCI 4. push, commit, or create a pull request to trigger a build. @@ -23,16 +24,49 @@ to pull from Docker Hub. Step 1. Clone the Repository ............................ -First, fork the `repo2docker `_ Github +First, fork the `continuous-build `_ Github repository to your account, and clone the branch. - - git clone https://www.github.com//repo2docker + git clone https://www.github.com//continuous-build # or - git clone git@github.com:/repo2docker.git + git clone git@github.com:/continuous-build.git -Step 2. Docker Hub +Step 2. Choose your Configuration +................................. + +The hidden folder `.circleci/config.yml` has instructions for `CircleCI `_ +to automatically discover and build your repo2docker jupyter notebook container. +The default template provided in the repository in this folder will do the most basic steps, +including: + +1. clone of the repository with the notebook that you specify +2. build +3. push to Docker Hub + +Also provided (or will be provided as they are requested) are other configuration templates in +the subfolders of that repository. The README.md in each template example folder describes its +functions. If you have a request for a template that is not provided, please +`let us know `_. +We will add templates as they are requested to do additional tasks like test containers, run +nbconvert, etc. + +Thus, if I have a repository named ``myrepo`` and I want to use the default configuration on circleCI, +I would copy it there from the ``continuous-build`` folder. In the example below, I'm +creating a new folder called "myrepo" and then copying the entire folder there. + + mkdir -p myrepo + cp -R continuous-build/.circleci myrepo/ + +You would then logically create a Github repository in the "myrepo" folder, +add the circleci configuration folder, and continue on to the next steps. + + cd myrepo + git init + git add .circleci + + +Step 3. Docker Hub .................. Go to `Docker Hub `_, log in, and click the big blue button that says "create repository" (not an automated build). Choose an organization @@ -40,7 +74,7 @@ and name that you like, and remember it! We will be adding it, along with your Docker credentials, to be encrypted CircleCI environment variables. -Step 3. Connect to CircleCI +Step 4. Connect to CircleCI ........................... If you navigate to the main `app page `_ you should be able to click "Add Projects" and then select your repository. If you don't @@ -56,15 +90,15 @@ Variables" tab. In this section, you want to define the following: 1. ``CONTAINER_NAME`` should be the name of the Docker Hub repository you just created. 2. ``DOCKER_TAG`` is the tag you want to use. If not defined, will use first 10 characters of commit. 3. ``DOCKER_USER`` and ``DOCKER_PASS`` should be your credentials (to allowing pushing) -4. ``REPO_NAME`` should be the full Github url (or other) of the repository with the notebook. +4. ``REPO_NAME`` should be the full Github url (or other) of the repository with the notebook. This doesn't have to coincide with the repository you are using to do the build (e.g., "myrepo" in our example). If you don't define the ``CONTAINER_NAME`` it will default to be the repository where it is building from, which you should only do if the Docker Hub repository is named equivalently. -If you don't define either of the variables from 2. for the Docker credentials, your +If you don't define either of the variables from step 3. for the Docker credentials, your image will build but not be deployed to Docker Hub. Finally, if you don't define the ``REPO_NAME`` it will again use the name of the repository defined for the ``CONTAINER_NAME``. -Step 4. Push Away, Merrill! +Step 5. Push Away, Merrill! ........................... Once the environment variables are set up, you can push or issue a pull request From 6a5fdc4f6e93dec43eafd8bc918991110390cd30 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Tue, 17 Jul 2018 08:13:54 -0700 Subject: [PATCH 14/15] good job @vsoch, wrong repo name! --- docs/source/deploy.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/deploy.rst b/docs/source/deploy.rst index af01310f..96d464ca 100644 --- a/docs/source/deploy.rst +++ b/docs/source/deploy.rst @@ -3,7 +3,7 @@ Deploy ``repo2docker`` ====================== -We've created for you the `continuous-build `_ +We've created for you the `continuous-build `_ repository so that you can deploy a `Docker `_ container to `Docker Hub `_ directly from a Github repository that has a Jupyter notebook. Here are instructions to do this. @@ -24,7 +24,7 @@ to pull from Docker Hub. Step 1. Clone the Repository ............................ -First, fork the `continuous-build `_ Github +First, fork the `continuous-build `_ Github repository to your account, and clone the branch. git clone https://www.github.com//continuous-build @@ -47,7 +47,7 @@ including: Also provided (or will be provided as they are requested) are other configuration templates in the subfolders of that repository. The README.md in each template example folder describes its functions. If you have a request for a template that is not provided, please -`let us know `_. +`let us know `_. We will add templates as they are requested to do additional tasks like test containers, run nbconvert, etc. From 9f991f3e38c0ccaa867abbede82eecb2f80a6457 Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Wed, 18 Jul 2018 12:11:20 -0700 Subject: [PATCH 15/15] adjusting deploy.rst --- docs/source/deploy.rst | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/source/deploy.rst b/docs/source/deploy.rst index 96d464ca..32ec86d8 100644 --- a/docs/source/deploy.rst +++ b/docs/source/deploy.rst @@ -1,10 +1,10 @@ .. _usage: -Deploy ``repo2docker`` -====================== +Using ``repo2docker`` as part of your Continuous Integration +============================================================ We've created for you the `continuous-build `_ -repository so that you can deploy a `Docker `_ container +repository so that you can push a `Docker `_ container to `Docker Hub `_ directly from a Github repository that has a Jupyter notebook. Here are instructions to do this. @@ -44,9 +44,8 @@ including: 2. build 3. push to Docker Hub -Also provided (or will be provided as they are requested) are other configuration templates in -the subfolders of that repository. The README.md in each template example folder describes its -functions. If you have a request for a template that is not provided, please +This repository aims to provide templates for your use. +If you have a request for a new template, please `let us know `_. We will add templates as they are requested to do additional tasks like test containers, run nbconvert, etc. @@ -70,7 +69,8 @@ Step 3. Docker Hub .................. Go to `Docker Hub `_, log in, and click the big blue button that says "create repository" (not an automated build). Choose an organization -and name that you like, and remember it! We will be adding it, along with your +and name that you like (in the traditional format ``/``), and +remember it! We will be adding it, along with your Docker credentials, to be encrypted CircleCI environment variables. @@ -95,7 +95,7 @@ Variables" tab. In this section, you want to define the following: If you don't define the ``CONTAINER_NAME`` it will default to be the repository where it is building from, which you should only do if the Docker Hub repository is named equivalently. If you don't define either of the variables from step 3. for the Docker credentials, your -image will build but not be deployed to Docker Hub. Finally, if you don't define the ``REPO_NAME`` +image will build but not be pushed to Docker Hub. Finally, if you don't define the ``REPO_NAME`` it will again use the name of the repository defined for the ``CONTAINER_NAME``. Step 5. Push Away, Merrill! @@ -118,9 +118,15 @@ Step 5. Use Your Container! You should then be able to pull your new container, and run it! Here is an example: + docker pull / + docker run -it --name repo2docker -p 8888:8888 / jupyter notebook --ip 0.0.0.0 + + +For a pre-built working example, try the following: + docker pull vanessa/repo2docker docker run -it --name repo2docker -p 8888:8888 vanessa/repo2docker jupyter notebook --ip 0.0.0.0 - + You can then enter the url and token provided in the browser to access your notebook. When you are done and need to stop and remove the container: docker stop repo2docker