From e9a31df758885185d9a6d6fc88c2810de93cbd89 Mon Sep 17 00:00:00 2001 From: Evert Rol Date: Mon, 1 Oct 2018 17:41:09 +0200 Subject: [PATCH 1/6] Add an edit-mode option This adds an option to run from a local repository in edit mode, where changes in a running Docker container (for example, through a notebook) are reflected in the local repository. Implements the feature suggested in #357 --- repo2docker/app.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/repo2docker/app.py b/repo2docker/app.py index 575c9db4..c594b4ed 100644 --- a/repo2docker/app.py +++ b/repo2docker/app.py @@ -342,6 +342,13 @@ class Repo2Docker(Application): default=[] ) + argparser.add_argument( + '--editable', '-E', + dest='editable', + action='store_true', + help='Use the local repository in edit mode', + ) + argparser.add_argument( '--appendix', type=str, @@ -392,6 +399,8 @@ class Repo2Docker(Application): self.repo = args.repo self.ref = None self.cleanup_checkout = False + if args.editable: + self.volumes[os.path.abspath(args.repo)] = '.' else: self.repo_type = 'remote' self.repo = args.repo From fd21926039468dca502acaf1ba6d9f10aa67bc71 Mon Sep 17 00:00:00 2001 From: Evert Rol Date: Wed, 3 Oct 2018 11:35:22 +0200 Subject: [PATCH 2/6] Add FAQ entry for editable local repositories --- docs/source/faq.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/source/faq.md b/docs/source/faq.md index e1ed8070..7c5c7e41 100644 --- a/docs/source/faq.md +++ b/docs/source/faq.md @@ -70,3 +70,31 @@ or similar traditional docker command. Check out the [binder-examples](http://github.com/binder-examples/) GitHub organization for example repositories you can copy & modify for your own use! + +## Can I use repo2docker to edit a local repository within a Docker environment? + +Yes: use the `--editable` or `-E` flag (don't confuse it with the `-e` +flag for environment variables), and run repo2docker on a local +repository: `repo2docker -E my-repository/.`. + +This builds a Docker container from the files in that repository +(using, for example, a `requirements.txt` file or `Dockerfile`), then +runs that container, while connecting the home directory inside the +container to the local repository outside the container. For example, +in case there is a notebook file (`.ipynb`), this will open in a local +webbrowser, and one can edit it and save it. The resulting notebook is +updated in both the Docker container and the local repository. Once +the container is exited, the changed file will still be in the local +repository. + +This allows for easy testing of the container while debugging some +items, as well as using a fully customizable container to edit, for +example, notebooks. + +**note** + +Editable mode is a convenience option that will mount the repository +to container working directory (usually `/home/$USER`). If you need to +mount to a different location in the container, use the `--volumes` +option instead. Similarly, for a fully customized user Dockerfile, +this option is not guaranteed to work. From 6847e716704e9316fb259b2e9ce52325ba802898 Mon Sep 17 00:00:00 2001 From: Evert Rol Date: Thu, 4 Oct 2018 11:24:44 +0200 Subject: [PATCH 3/6] Add test for editable option This test only verifies the option one way: a change inside the container (a new file) is reflected in the local host repository outside the container. A further test where a modification at the host level is reflected in the container, is still neede. --- tests/dockerfile/editable/Dockerfile | 14 ++++++++++++ tests/dockerfile/editable/README.rst | 19 ++++++++++++++++ tests/dockerfile/editable/change.sh | 6 +++++ tests/dockerfile/editable/verify | 3 +++ tests/test_editable.py | 34 ++++++++++++++++++++++++++++ 5 files changed, 76 insertions(+) create mode 100644 tests/dockerfile/editable/Dockerfile create mode 100644 tests/dockerfile/editable/README.rst create mode 100755 tests/dockerfile/editable/change.sh create mode 100755 tests/dockerfile/editable/verify create mode 100644 tests/test_editable.py diff --git a/tests/dockerfile/editable/Dockerfile b/tests/dockerfile/editable/Dockerfile new file mode 100644 index 00000000..122f0c03 --- /dev/null +++ b/tests/dockerfile/editable/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.5 + +RUN pip install --no-cache notebook + +CMD "/bin/sh" + +ADD change.sh /usr/local/bin/change.sh +ADD verify verify + +ARG NB_UID +ENV HOME /tmp +WORKDIR ${HOME} + +USER $NB_UID diff --git a/tests/dockerfile/editable/README.rst b/tests/dockerfile/editable/README.rst new file mode 100644 index 00000000..14169b1a --- /dev/null +++ b/tests/dockerfile/editable/README.rst @@ -0,0 +1,19 @@ +Docker - Edit mode +------------------ + +Using the --editable option with a local repository, one can modify a +file or create a new file in the container, and this change is +reflected in the respective host directory. It is essentially a +shortcut for `--mount +type=bind,source=,target=.` (where the target +resolves into the container working directory). + +This is tested by running the change.sh script inside the container +(using the 'cmd' argument to the Repo2Docker app), which creates a new +file, and then verifying on the host side the new file is created with +the proper contents. + +In practice, this can be used to run a notebook from inside a +container (which provides the proper environment), making changes as +necessary, which are then immediately reflected in the host +repository. diff --git a/tests/dockerfile/editable/change.sh b/tests/dockerfile/editable/change.sh new file mode 100755 index 00000000..8f15b4af --- /dev/null +++ b/tests/dockerfile/editable/change.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +cat < newfile +new contents +EOF +exit 0 diff --git a/tests/dockerfile/editable/verify b/tests/dockerfile/editable/verify new file mode 100755 index 00000000..0a44b7de --- /dev/null +++ b/tests/dockerfile/editable/verify @@ -0,0 +1,3 @@ +#!/bin/bash + +/usr/local/bin/change.sh diff --git a/tests/test_editable.py b/tests/test_editable.py new file mode 100644 index 00000000..4ed4a7ab --- /dev/null +++ b/tests/test_editable.py @@ -0,0 +1,34 @@ +import pytest +import os +import time +from conftest import make_test_func + + +DIR = os.path.join(os.path.dirname(__file__), 'dockerfile', 'editable') + + +@pytest.fixture(scope="module") +def run_repo2docker(): + def run_test(args): + return make_test_func(args)() + return run_test + + +def test_editable(run_repo2docker): + """Run a local repository in edit mode. Verify a new file has been + created afterwards""" + newfile = os.path.join(DIR, 'newfile') + try: + # If the file didn't get properly cleaned up last time, we + # need to do that now + os.remove(newfile) + except FileNotFoundError: + pass + argv = ['--editable', DIR, '/usr/local/bin/change.sh'] + run_repo2docker(argv) + try: + with open(newfile) as fp: + contents = fp.read() + assert contents == "new contents\n" + finally: + os.remove(newfile) From c834e2768637a26c52f684d2b0661349ac3480b0 Mon Sep 17 00:00:00 2001 From: Evert Rol Date: Thu, 4 Oct 2018 13:22:39 +0200 Subject: [PATCH 4/6] Add test for host file creation in edit mode This will test whether an externally (host) created file is seen inside the container. This means starting the container, creating a temporary file on the host, then running a simple verification command (`ls`) to detect the file. Once the temporary file is removed on the host side, it should be removed in the container as well. --- tests/test_editable.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_editable.py b/tests/test_editable.py index 4ed4a7ab..a6dd7f20 100644 --- a/tests/test_editable.py +++ b/tests/test_editable.py @@ -1,7 +1,10 @@ import pytest import os import time +import re +import tempfile from conftest import make_test_func +from repo2docker.app import Repo2Docker DIR = os.path.join(os.path.dirname(__file__), 'dockerfile', 'editable') @@ -32,3 +35,30 @@ def test_editable(run_repo2docker): assert contents == "new contents\n" finally: os.remove(newfile) + + +def test_editable_by_host(): + """Test whether a new file created by the host environment, is + detected in the container""" + + app = Repo2Docker() + app.initialize(['--editable', DIR]) + app.run = False + app.start() # This just build the image and does not run it. + container = app.start_container() + # give the container a chance to start + time.sleep(1) + try: + with tempfile.NamedTemporaryFile(dir=DIR, prefix='testfile', suffix='.txt'): + status, output = container.exec_run(['sh', '-c', 'ls testfile????????.txt']) + assert status == 0 + assert re.match(br'^testfile\w{8}\.txt\n$', output) is not None + # File should be removed in the container as well + status, output = container.exec_run(['sh', '-c', 'ls testfile????????.txt']) + assert status != 1 + assert re.match(br'^testfile\w{8}\.txt\n$', output) is None + + finally: + # stop the container + container.stop() + app.wait_for_container(container) From 871b0629a4462e6e0608ff79fa5144b1548db012 Mon Sep 17 00:00:00 2001 From: Evert Rol Date: Thu, 4 Oct 2018 15:13:42 +0200 Subject: [PATCH 5/6] Remove dockerfile/editable test from main tests The dockerfile test is used by test_editable, but should not be used by the generated tests. Removing the verify script will prevent it being run as part of the generated tests. --- tests/dockerfile/editable/Dockerfile | 1 - tests/dockerfile/editable/verify | 3 --- 2 files changed, 4 deletions(-) delete mode 100755 tests/dockerfile/editable/verify diff --git a/tests/dockerfile/editable/Dockerfile b/tests/dockerfile/editable/Dockerfile index 122f0c03..8baa17d8 100644 --- a/tests/dockerfile/editable/Dockerfile +++ b/tests/dockerfile/editable/Dockerfile @@ -5,7 +5,6 @@ RUN pip install --no-cache notebook CMD "/bin/sh" ADD change.sh /usr/local/bin/change.sh -ADD verify verify ARG NB_UID ENV HOME /tmp diff --git a/tests/dockerfile/editable/verify b/tests/dockerfile/editable/verify deleted file mode 100755 index 0a44b7de..00000000 --- a/tests/dockerfile/editable/verify +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -/usr/local/bin/change.sh From 754edd01c4b63e4dbc52ce704af76081c118bf25 Mon Sep 17 00:00:00 2001 From: Evert Rol Date: Mon, 8 Oct 2018 14:18:41 +0200 Subject: [PATCH 6/6] Move test fixture to conftest.py --- tests/conftest.py | 8 ++++++++ tests/test_editable.py | 8 -------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4e05a690..1af3ec9a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -68,6 +68,14 @@ def make_test_func(args): return test +@pytest.fixture() +def run_repo2docker(): + def run_test(args): + return make_test_func(args)() + return run_test + + + class Repo2DockerTest(pytest.Function): """A pytest.Item for running repo2docker""" def __init__(self, name, parent, args): diff --git a/tests/test_editable.py b/tests/test_editable.py index a6dd7f20..1bbd05d4 100644 --- a/tests/test_editable.py +++ b/tests/test_editable.py @@ -1,4 +1,3 @@ -import pytest import os import time import re @@ -10,13 +9,6 @@ from repo2docker.app import Repo2Docker DIR = os.path.join(os.path.dirname(__file__), 'dockerfile', 'editable') -@pytest.fixture(scope="module") -def run_repo2docker(): - def run_test(args): - return make_test_func(args)() - return run_test - - def test_editable(run_repo2docker): """Run a local repository in edit mode. Verify a new file has been created afterwards"""