Merge pull request #180 from mukundans91/addPortMapping

add ability for port mapping to work with non jupyter-workflow

Dockerfile

@@ -1,5 +1,7 @@
 FROM python:3.6.3
 
+# Used for testing purpose in ports.py
+EXPOSE 52000
 RUN mkdir /tmp/src
 ADD . /tmp/src
 RUN pip3 install --no-cache-dir /tmp/src

repo2docker/app.py

@@ -30,7 +30,7 @@ from .buildpacks import (
     PythonBuildPack, DockerBuildPack, LegacyBinderDockerBuildPack,
     CondaBuildPack, JuliaBuildPack, Python2BuildPack, BaseImage
 )
-from .utils import execute_cmd, ByteSpecification, maybe_cleanup, is_valid_docker_image_name
+from .utils import execute_cmd, ByteSpecification, maybe_cleanup, is_valid_docker_image_name, validate_and_generate_port_mapping
 from . import __version__
 
 
@@ -264,6 +264,20 @@ class Repo2Docker(Application):
             help='Do not run container after it has been built'
         )
 
+        argparser.add_argument(
+            '--publish', '-p',
+            dest='ports',
+            action='append',
+            help='Specify port mappings for the image. Needs a command to run in the container.'
+        )
+
+        argparser.add_argument(
+            '--publish-all', '-P',
+            dest='all_ports',
+            action='store_true',
+            help='Publish all exposed ports to random host ports.'
+        )
+
         argparser.add_argument(
             '--no-clean',
             dest='clean',
@@ -389,6 +403,21 @@ class Repo2Docker(Application):
 
         self.run_cmd = args.cmd
 
+        if args.all_ports and not self.run:
+            print('To publish user defined port mappings, the container must also be run')
+            sys.exit(1)
+
+        if args.ports and not self.run:
+            print('To publish user defined port mappings, the container must also be run')
+            sys.exit(1)
+
+        if args.ports and not self.run_cmd:
+            print('To publish user defined port mapping, user must specify the command to run in the container')
+            sys.exit(1)
+
+        self.ports = validate_and_generate_port_mapping(args.ports)
+        self.all_ports = args.all_ports
+
         if args.user_id:
             self.user_id = args.user_id
         if args.user_name:
@@ -427,15 +456,19 @@ class Repo2Docker(Application):
 
     def run_image(self):
         client = docker.from_env(version='auto')
-        port = self._get_free_port()
         if not self.run_cmd:
             port = str(self._get_free_port())
+
             run_cmd = ['jupyter', 'notebook', '--ip', '0.0.0.0',
                        '--port', port]
             ports = {'%s/tcp' % port: port}
         else:
+            # run_cmd given by user, if port is also given then pass it on
             run_cmd = self.run_cmd
-            ports = {}
+            if self.ports:
+                ports = self.ports
+            else:
+                ports = {}
         container_volumes = {}
         if self.volumes:
             api_client = docker.APIClient(
@@ -453,6 +486,7 @@ class Repo2Docker(Application):
 
         container = client.containers.run(
             self.output_image_spec,
+            publish_all_ports=self.all_ports,
             ports=ports,
             detach=True,
             command=run_cmd,
@@ -488,6 +522,7 @@ class Repo2Docker(Application):
         s.close()
         return port
 
+
     def start(self):
         if self.repo_type == 'local':
             checkout_path = self.repo

repo2docker/utils.py

@@ -3,6 +3,7 @@ from functools import partial
 import shutil
 import subprocess
 import re
+import sys
 
 from traitlets import Integer
 
@@ -57,6 +58,75 @@ def maybe_cleanup(path, cleanup=False):
         shutil.rmtree(path, ignore_errors=True)
 
 
+def validate_and_generate_port_mapping(port_mapping):
+    """
+    Validate the port mapping list provided as argument and split into as dictionary of key being continer port and the
+    values being None, or 'host_port' or ['interface_ip','host_port']
+
+
+    Args:
+        port_mapping (list): List of strings of format 'host_port:container_port'
+                             with optional tcp udp values and host network interface
+
+    Returns:
+        List of validated tuples of form ('host_port:container_port') with optional tcp udp values and host network interface
+
+    Raises:
+        Exception on invalid port mapping
+
+    Note:
+        One limitation cannot bind single container_port to multiple host_ports (docker-py supports this but repo2docker
+        does not)
+
+    Examples:
+        Valid port mappings are
+        127.0.0.1:90:900
+        :999 - To match to any host port
+        999:999/tcp - bind 999 host port to 999 tcp container port
+
+        Invalid port mapping
+        127.0.0.1::999 --- even though docker accepts it
+        other invalid ip address combinations
+    """
+    reg_regex = re.compile('''^(
+                                    ( # or capturing group
+                                    (?: # start capturing ip address of network interface
+                                    (?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3} # first three parts
+                                       (?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?) # last part of the ip address
+                                    :(?:6553[0-5]|655[0-2][0-9]|65[0-4](\d){2}|6[0-4](\d){3}|[1-5](\d){4}|(\d){1,4})
+                                    )?
+                                    |   # host ip with port or only port
+                                    (?:6553[0-5]|655[0-2][0-9]|65[0-4](\d){2}|6[0-4](\d){3}|[1-5](\d){4}|(\d){0,4})
+                                    )
+                                    :
+                                    (?:6553[0-5]|655[0-2][0-9]|65[0-4](\d){2}|6[0-4](\d){3}|[1-5](\d){4}|(\d){0,4})
+                                    (?:/udp|/tcp)?
+                               )$''', re.VERBOSE)
+    ports = {}
+    if not port_mapping:
+        return None
+    for p in port_mapping:
+        if reg_regex.match(p) is None:
+            raise Exception('Invalid port mapping ' + str(p))
+        # Do a reverse split twice on the separator :
+        port_host = str(p).rsplit(':', 2)
+        host = None
+        if len(port_host) == 3:
+            # host, optional host_port and container port information given
+            host = port_host[0]
+            host_port = port_host[1]
+            container_port = port_host[2]
+        else:
+            host_port = port_host[0] if len(port_host[0]) > 0 else None
+            container_port = port_host[1]
+
+
+        if host is None:
+            ports[str(container_port)] = host_port
+        else:
+            ports[str(container_port)] = (host, host_port)
+    return ports
+
 def is_valid_docker_image_name(image_name):
     """
     Function that constructs a regex representing the docker image name and tests it against the given image_name

tests/argumentvalidation.py

@@ -5,6 +5,7 @@ Tests that runs validity checks on arguments passed in from shell
 import os
 import subprocess
 
+
 def validate_arguments(builddir, args_list, expected):
     try:
         cmd = ['repo2docker']
@@ -101,6 +102,7 @@ def test_image_name_valid_name_success():
 
     assert validate_arguments(builddir, args_list, None)
 
+
 def test_volume_no_build_fail():
     """
     Test to check if repo2docker fails when both --no-build and -v arguments are given
@@ -120,6 +122,7 @@ def test_volume_no_run_fail():
 
     assert not validate_arguments(builddir, args_list, 'To Mount volumes with -v, you also need to run the container')
 
+
 def test_env_no_run_fail():
     """
     Test to check if repo2docker fails when both --no-run and -e arguments are given 
@@ -129,3 +132,58 @@ def test_env_no_run_fail():
 
     assert not validate_arguments(builddir, args_list, 'To specify environment variables, you also need to run the container')
 
+
+def test_port_mapping_no_run_fail():
+    """
+    Test to check if repo2docker fails when both --no-run and --publish arguments are specified.
+    """
+    builddir = os.path.dirname(__file__)
+    args_list = ['--no-run', '--publish', '8000:8000']
+
+    assert not validate_arguments(builddir, args_list, 'To publish user defined port mappings, the container must also be run')
+
+
+def test_all_ports_mapping_no_run_fail():
+    """
+    Test to check if repo2docker fails when both --no-run and -P arguments are specified.
+    """
+    builddir = os.path.dirname(__file__)
+    args_list = ['--no-run', '-P']
+
+    assert not validate_arguments(builddir, args_list, 'To publish user defined port mappings, the container must also be run')
+
+
+def test_invalid_port_mapping_fail():
+    """
+    Test to check if r2d fails when an invalid port is specified in the port mapping
+    """
+    builddir = os.path.dirname(__file__)
+    # Specifying builddir here itself to simulate passing in a run command
+    # builddir passed in the function will be an argument for the run command
+    args_list = ['-p', '75000:80', builddir, 'ls']
+
+    assert not validate_arguments(builddir, args_list, 'Invalid port mapping')
+
+
+def test_invalid_protocol_port_mapping_fail():
+    """
+    Test to check if r2d fails when an invalid protocol is specified in the port mapping
+    """
+    builddir = os.path.dirname(__file__)
+    # Specifying builddir here itself to simulate passing in a run command
+    # builddir passed in the function will be an argument for the run command
+    args_list = ['-p', '80/tpc:8000', builddir, 'ls']
+
+    assert not validate_arguments(builddir, args_list, 'Invalid port mapping')
+
+
+def test_invalid_container_port_protocol_mapping_fail():
+    """
+    Test to check if r2d fails when an invalid protocol is specified in the container port in port mapping
+    """
+    builddir = os.path.dirname(__file__)
+    # Specifying builddir here itself to simulate passing in a run command
+    # builddir passed in the function will be an argument for the run command
+    args_list = ['-p', '80:8000/upd', builddir, 'ls']
+
+    assert not validate_arguments(builddir, args_list, 'Invalid port mapping')

tests/ports.py

@@ -0,0 +1,137 @@
+"""
+Test Port mappings work on running non-jupyter workflows
+"""
+import subprocess
+import requests
+import time
+import os
+import tempfile
+import signal
+import random
+
+
+def read_port_mapping_response(host, port, protocol = None):
+    """
+    Deploy container and test if port mappings work as expected
+
+    Args:
+        host: the host interface to bind to.
+        port: the random host port to bind to
+        protocol: the protocol to use valid values /tcp or /udp
+    """
+    builddir = os.path.dirname(__file__)
+    port_protocol = '8000'
+    if protocol:
+        port_protocol += protocol
+    host_port = port
+    if host:
+        host_port = host + ':' + port
+    else:
+        host = 'localhost'
+    with tempfile.TemporaryDirectory() as tmpdir:
+        username = os.getlogin()
+
+        # Deploy a test container using r2d in a subprocess
+        # Added the -v volumes to be able to poll for changes within the container from the
+        # host (In this case container starting up)
+        proc = subprocess.Popen(['repo2docker',
+                                 '-p',
+                                 host_port + ':' + port_protocol,
+                                 '-v', '{}:/home'.format(tmpdir),
+                                 '--user-id', str(os.geteuid()),
+                                 '--user-name', username,
+                                 '.',
+                                 '/bin/bash', '-c', 'echo \'hi\' > /home/ts && python -m http.server 8000'],
+                                 cwd=builddir + "/../",
+                                 stderr=subprocess.STDOUT)
+        try:
+            # Wait till docker builds image and starts up
+            while not os.path.exists(os.path.join(tmpdir, 'ts')):
+                if proc.poll() is not None:
+                    # Break loop on errors from the subprocess
+                    raise Exception("Process running r2d exited")
+
+            # Sleep to wait for python http server to start
+            time.sleep(20)
+            resp = requests.request("GET", 'http://' + host + ':' + port)
+
+            # Check if the response is correct
+            assert b'Directory listing' in resp.content
+        finally:
+            if proc.poll() is None:
+                # If the subprocess running the container is still running, interrupt it to close it
+                os.kill(proc.pid, signal.SIGINT)
+                time.sleep(10)
+
+
+def test_all_port_mapping_response():
+    """
+    Deploy container and test if all port expose works as expected
+    """
+    builddir = os.path.dirname(__file__)
+    with tempfile.TemporaryDirectory() as tmpdir:
+        username = os.getlogin()
+
+        # Deploy a test container using r2d in a subprocess
+        # Added the -v volumes to be able to poll for changes within the container from the
+        # host (In this case container starting up)
+        proc = subprocess.Popen(['repo2docker',
+                                 "--image-name",
+                                 "testallport:0.1",
+                                 '-P',
+                                 '-v', '{}:/home'.format(tmpdir),
+                                 '--user-id', str(os.geteuid()),
+                                 '--user-name', username,
+                                 '.',
+                                 '/bin/bash', '-c', 'echo \'hi\' > /home/ts && python -m http.server 52000'],
+                                 cwd=builddir + "/../",
+                                 stderr=subprocess.STDOUT)
+
+        try:
+            # Wait till docker builds image and starts up
+            while not os.path.exists(os.path.join(tmpdir, 'ts')):
+                if proc.poll() is not None:
+                    # Break loop on errors from the subprocess
+                    raise Exception("Process running r2d exited")
+
+            # Sleep to wait for python http server to start
+            time.sleep(20)
+            port = subprocess.check_output("docker ps -f ancestor=testallport:0.1 --format '{{.Ports}}' | cut -f 1 -d - | cut -d: -f 2",
+                                            shell=True).decode("utf-8")
+            port = port.strip("\n\t")
+            resp = requests.request("GET", 'http://localhost' + ':' + port)
+
+            # Check if the response is correct
+            assert b'Directory listing' in resp.content
+        finally:
+            if proc.poll() is None:
+                # If the subprocess running the container is still running, interrupt it to close it
+                os.kill(proc.pid, signal.SIGINT)
+                time.sleep(10)
+
+
+def test_port_mapping_random_port():
+    """
+    Test a simple random port bind
+    """
+    port = str(random.randint(50000, 51000))
+    host = None
+    read_port_mapping_response(host, port)
+
+
+def test_port_mapping_particular_interface():
+    """
+    Test if binding to a single interface is possible
+    """
+    port = str(random.randint(50000, 51000))
+    host = '127.0.0.1'
+    read_port_mapping_response(host, port)
+
+
+def test_port_mapping_protocol():
+    """
+    Test if a particular protocol can be used
+    """
+    port = str(random.randint(50000, 51000))
+    host = None
+    read_port_mapping_response(host, port, '/tcp')