Merge pull request #172 from yuvipanda/volumes

Allow mounting arbitrary volumes into the repo2docker container

.travis.yml

@@ -31,6 +31,7 @@ env:
   - REPO_TYPE=julia
   - REPO_TYPE=dockerfile
   - REPO_TYPE=external/*
+  - REPO_TYPE=*.py
   global:
   - secure: gX7IOkbjlvcDwIH24sOLhutINx6TZRwujEusMWh1dqgYG2D69qQai/mTrRXO9PGRrsvQwIBk4RcILKAiZnk5O2Z1hLoIHk/oU2mNUmE44dDm4Xf/VTTdeYhjeOTR9B+KJ9NVwPxuSEDSND3lD7yFfvCqNXykipEhBtTliLupjWVxxXnaz0aZTYHUPJwanxdUc06AphSPwZjtm1m3qMUU8v7UdTGGAdW3NlgkKw0Xx2x5W31fW676vskC/GNQAbcRociYipuhSFWV4lu+6d8XF2xVO97xtzf54tBQzt6RgVfAKtiqkEIYSzJQBBpkQ6SM6yg+fQoQpOo8jPU9ZBjvaoopUG9vn8HRS/OtQrDcG3kEFnFAnaes8Iqtidp1deTn27LIlfCTl7kTFOp8yaaNlIMHJTJKTEMRhfdDlBYx7qiH8e9d/z37lupzY2loLHeNHdMRS1uYsfacZsmrnu9vAdpQmP1LuHivBPZEvgerinADaJiekelWOIEn956pDrno/YgnzP0i9LEBYnbbunqT8oEzLintNt5CXGdhkiG60j38McKCIn4sD6jbMMwgsqVFdClCBersyorKhOs7P8at5vX4xf8fMiKPC8LZPzYVIQYzCjmwSOFQ+Rzmz5gSj+DRTANKfHpzZCKZEF6amBYMGE1O5osF8m6M10vtW9ToK+s=
   - secure: Cfhb0BUT54JjEZD8n44Jj+o1lt5p32Lfg7W/euTyZ61YylDx0+XEYTzfWcwxOzH9fLpWr6dDrBMGHA/FPqsWA5BkoGdiBJ1OOVy2tmDRButctobWM3SVwa+Rhh8bZWlK8yKT2S3n6CtK4mesmjzdbUShL7YnKOSl8LBaTT5Y5oT8Oxsq51pfg8fJUImim8H20t8H7emaEzZorF4OSGRtajcAgukt5YoAqTEVDq+bFRBHZalxkcRqLhsGe3CCWa28kjGTL4MPZpCI6/AXIXHzihfG3rGq40ZT8jZ9GPP3MBgkiJWtFiTC9h16G34b/JI/TD40zCmoW9/9oVjRK4UlLGCAv6bgzFhCRof2abhB9NTZDniNzkO0T15uHs3VLbLCPYB0xYyClAFxm2P6e8WPChyENKfTNh+803IKFFo4JaTjOnKzi89N72v5+bT6ghP932nmjJr1AO65xjw63CeDmaLoHDY73n11DibybWQgEeiNzJuSzbIHyqMPhW5XqeroEjKKstdPHtVfOViI9ywjEMy0HCPsspaVI7Aow0Iv8E4Ajvd32W7z0h0fSCx/i25hEOAo2vhBsmQKJA7IquB3N88M11L874h/8J+oc/osW1EB5z7Ukke5YCq94Qh3qImSIhJULXMMc1QjEqYsqhLXtiMG2HUge0Y5hwwnnbEIRMQ=

repo2docker/app.py

@@ -16,10 +16,11 @@ import argparse
 import tempfile
 from pythonjsonlogger import jsonlogger
 import escapism
+import pwd
 
 
 from traitlets.config import Application
-from traitlets import Unicode, List, default, Tuple
+from traitlets import Unicode, List, default, Tuple, Dict, Int
 import docker
 from docker.utils import kwargs_from_env
 
@@ -101,6 +102,63 @@ class Repo2Docker(Application):
         config=True
     )
 
+    volumes = Dict(
+        {},
+        help="""
+        Volumes to mount when running the container.
+
+        Only used when running, not during build!
+
+        Should be a key value pair, with the key being the volume source &
+        value being the destination. Both can be relative - sources are
+        resolved relative to the current working directory on the host,
+        destination is resolved relative to the working directory of the image -
+        ($HOME by default)
+        """,
+        config=True
+    )
+
+    user_id = Int(
+        help="""
+        UID of the user to create inside the built image.
+
+        Should be a uid that is not currently used by anything in the image.
+        Defaults to uid of currently running user, since that is the most
+        common case when running r2d manually.
+
+        Might not affect Dockerfile builds.
+        """,
+        config=True
+    )
+
+    @default('user_id')
+    def _user_id_default(self):
+        """
+        Default user_id to current running user.
+        """
+        return os.geteuid()
+
+    user_name = Unicode(
+        'jovyan',
+        help="""
+        Username of the user to create inside the built image.
+
+        Should be a username that is not currently used by anything in the image,
+        and should conform to the restrictions on user names for Linux.
+
+        Defaults to username of currently running user, since that is the most
+        common case when running r2d manually.
+        """,
+        config=True
+    )
+
+    @default('user_name')
+    def _user_name_default(self):
+        """
+        Default user_name to current running user.
+        """
+        return pwd.getpwuid(os.getuid()).pw_name
+
     def fetch(self, url, ref, checkout_path):
         try:
             for line in execute_cmd(['git', 'clone', url, checkout_path],
@@ -200,6 +258,25 @@ class Repo2Docker(Application):
             help='Push docker image to repository'
         )
 
+        argparser.add_argument(
+            '--volume', '-v',
+            dest='volumes',
+            action='append',
+            help='Volumes to mount inside the container, in form src:dest',
+            default=[]
+        )
+
+        argparser.add_argument(
+            '--user-id',
+            help='User ID of the primary user in the image',
+            type=int
+        )
+
+        argparser.add_argument(
+            '--user-name',
+            help='Username of the primary user in the image',
+        )
+
         return argparser
 
     def json_excepthook(self, etype, evalue, traceback):
@@ -272,8 +349,22 @@ class Repo2Docker(Application):
             self.run = False
             self.push = False
 
+        if args.volumes and not args.run:
+            # Can't mount if we aren't running
+            print("To Mount volumes with -v, you also need to run the container")
+            sys.exit(1)
+
+        for v in args.volumes:
+            src, dest = v.split(':')
+            self.volumes[src] = dest
+
         self.run_cmd = args.cmd
 
+        if args.user_id:
+            self.user_id = args.user_id
+        if args.user_name:
+            self.user_name = args.user_name
+
         if args.build_memory_limit:
             self.build_memory_limit = args.build_memory_limit
 
@@ -310,11 +401,27 @@ class Repo2Docker(Application):
         else:
             run_cmd = self.run_cmd
             ports = {}
+        container_volumes = {}
+        if self.volumes:
+            api_client = docker.APIClient(
+                version='auto',
+                **docker.utils.kwargs_from_env()
+            )
+            image = api_client.inspect_image(self.output_image_spec)
+            image_workdir = image['ContainerConfig']['WorkingDir']
+
+            for k, v in self.volumes.items():
+                container_volumes[os.path.abspath(k)] = {
+                    'bind': v if v.startswith('/') else os.path.join(image_workdir, v),
+                    'mode': 'rw'
+                }
+
         container = client.containers.run(
             self.output_image_spec,
             ports=ports,
             detach=True,
-            command=run_cmd
+            command=run_cmd,
+            volumes=container_volumes
         )
         while container.status == 'created':
             time.sleep(0.5)
@@ -377,9 +484,13 @@ class Repo2Docker(Application):
                            extra=dict(phase='building'))
 
             if self.build:
+                build_args = {
+                    'NB_USER': self.user_name,
+                    'NB_UID': str(self.user_id)
+                }
                 self.log.info('Using %s builder\n', bp.name,
                               extra=dict(phase='building'))
-                for l in picked_buildpack.build(self.output_image_spec, self.build_memory_limit):
+                for l in picked_buildpack.build(self.output_image_spec, self.build_memory_limit, build_args):
                     if 'stream' in l:
                         self.log.info(l['stream'],
                                       extra=dict(phase='building'))

repo2docker/buildpacks/base.py

@@ -30,8 +30,9 @@ ENV LANGUAGE en_US.UTF-8
 ENV SHELL /bin/bash
 
 # Set up user
-ENV NB_USER jovyan
-ENV NB_UID 1000
+ARG NB_USER
+ARG NB_UID
+ENV USER ${NB_USER}
 ENV HOME /home/${NB_USER}
 
 RUN adduser --disabled-password \
@@ -360,7 +361,7 @@ class BuildPack(LoggingConfigurable):
             post_build_scripts=self.post_build_scripts,
         )
 
-    def build(self, image_spec, memory_limit):
+    def build(self, image_spec, memory_limit, build_args):
         tarf = io.BytesIO()
         tar = tarfile.open(fileobj=tarf, mode='w')
         dockerfile_tarinfo = tarfile.TarInfo("Dockerfile")
@@ -406,7 +407,7 @@ class BuildPack(LoggingConfigurable):
                 fileobj=tarf,
                 tag=image_spec,
                 custom_context=True,
-                buildargs={},
+                buildargs=build_args,
                 decode=True,
                 forcerm=True,
                 rm=True,

repo2docker/buildpacks/docker.py

@@ -18,7 +18,7 @@ class DockerBuildPack(BuildPack):
         with open(Dockerfile) as f:
             return f.read()
 
-    def build(self, image_spec, memory_limit):
+    def build(self, image_spec, memory_limit, build_args):
         limits = {
             # Always disable memory swap for building, since mostly
             # nothing good can come of that.
@@ -31,7 +31,7 @@ class DockerBuildPack(BuildPack):
                 path=os.getcwd(),
                 dockerfile=self.binder_path(self.dockerfile),
                 tag=image_spec,
-                buildargs={},
+                buildargs=build_args,
                 decode=True,
                 forcerm=True,
                 rm=True,

repo2docker/buildpacks/legacy.py

@@ -31,10 +31,10 @@ class LegacyBinderDockerBuildPack(DockerBuildPack):
         with open('Dockerfile') as f:
             return f.read() + self.dockerfile_appendix
 
-    def build(self, image_spec, memory_limit):
+    def build(self, image_spec, memory_limit, build_args):
         with open(self.dockerfile, 'w') as f:
             f.write(self.render())
-        return super().build(image_spec, memory_limit)
+        return super().build(image_spec, memory_limit, build_args)
 
     def detect(self):
         try:

tests/users.py

@@ -0,0 +1,36 @@
+"""
+Test that User name and ID mapping works
+"""
+import os
+import subprocess
+import tempfile
+import time
+
+def test_user():
+    """
+    Validate user id and name setting
+    """
+    ts = str(time.time())
+    # FIXME: Use arbitrary login here, We need it now since we wanna put things to volume.
+    username = os.getlogin()
+    userid = str(os.geteuid())
+    with tempfile.TemporaryDirectory() as tmpdir:
+        subprocess.check_call([
+            'repo2docker',
+            '-v', '{}:/home/{}'.format(tmpdir, username),
+            '--user-id', userid,
+            '--user-name', username,
+            tmpdir,
+            '--',
+            '/bin/bash',
+            '-c', 'id -u > id && pwd > pwd && whoami > name && echo -n $USER > env_user'.format(ts)
+        ])
+
+        with open(os.path.join(tmpdir, 'id')) as f:
+            assert f.read().strip() == userid
+        with open(os.path.join(tmpdir, 'pwd')) as f:
+            assert f.read().strip() == '/home/{}'.format(username)
+        with open(os.path.join(tmpdir, 'name')) as f:
+            assert f.read().strip() == username
+        with open(os.path.join(tmpdir, 'name')) as f:
+            assert f.read().strip() == username

tests/volumes.py

@@ -0,0 +1,54 @@
+"""
+Test that volume mounts work when running
+"""
+import os
+import subprocess
+import tempfile
+import time
+
+def test_volume_abspath():
+    """
+    Validate that you can bind mount a volume onto an absolute dir & write to it
+    """
+    ts = str(time.time())
+    with tempfile.TemporaryDirectory() as tmpdir:
+        username = os.getlogin()
+        subprocess.check_call([
+            'repo2docker',
+            '-v', '{}:/home/{}'.format(tmpdir, username),
+            '--user-id', str(os.geteuid()),
+            '--user-name', username,
+            tmpdir,
+            '--',
+            '/bin/bash',
+            '-c', 'echo -n {} > ts'.format(ts)
+        ])
+
+        with open(os.path.join(tmpdir, 'ts')) as f:
+            assert f.read() == ts
+
+
+def test_volume_relpath():
+    """
+    Validate that you can bind mount a volume onto an relative path & write to it
+    """
+    curdir = os.getcwd()
+    try:
+        ts = str(time.time())
+        with tempfile.TemporaryDirectory() as tmpdir:
+            os.chdir(tmpdir)
+            subprocess.check_call([
+                'repo2docker',
+                '-v', '.:.',
+                '--user-id', str(os.geteuid()),
+                '--user-name', os.getlogin(),
+                tmpdir,
+                '--',
+                '/bin/bash',
+                '-c', 'echo -n {} > ts'.format(ts)
+            ])
+
+            with open(os.path.join(tmpdir, 'ts')) as f:
+                assert f.read() == ts
+    finally:
+        os.chdir(curdir)