add ability for port mapping to work with non jupyter-workflow

Added tests for port mapping
Added tests for publish all exposed ports
Added argument validation for ports argument
Added short form for argument publish. Changed variable name port to ports.

Dockerfile

@@ -1,5 +1,7 @@
 FROM python:3.6.3
 
+# Used for testing purpose in ports.py
+EXPOSE 52000
 RUN mkdir /tmp/src
 ADD . /tmp/src
 RUN pip3 install --no-cache-dir /tmp/src

repo2docker/app.py

@@ -30,7 +30,7 @@ from .buildpacks import (
     PythonBuildPack, DockerBuildPack, LegacyBinderDockerBuildPack,
     CondaBuildPack, JuliaBuildPack, Python2BuildPack, BaseImage
 )
-from .utils import execute_cmd, ByteSpecification, maybe_cleanup, is_valid_docker_image_name
+from .utils import execute_cmd, ByteSpecification, maybe_cleanup, is_valid_docker_image_name, validate_and_generate_port_mapping
 from . import __version__
 
 
@@ -264,6 +264,20 @@ class Repo2Docker(Application):
             help='Do not run container after it has been built'
         )
 
+        argparser.add_argument(
+            '--publish', '-p',
+            dest='ports',
+            action='append',
+            help='Specify port mappings for the image. Needs a command to run in the container.'
+        )
+
+        argparser.add_argument(
+            '--publish-all', '-P',
+            dest='all_ports',
+            action='store_true',
+            help='Publish all exposed ports to random host ports.'
+        )
+
         argparser.add_argument(
             '--no-clean',
             dest='clean',
@@ -389,6 +403,21 @@ class Repo2Docker(Application):
 
         self.run_cmd = args.cmd
 
+        if args.all_ports and not self.run:
+            print('To publish user defined port mappings, the container must also be run')
+            sys.exit(1)
+
+        if args.ports and not self.run:
+            print('To publish user defined port mappings, the container must also be run')
+            sys.exit(1)
+
+        if args.ports and not self.run_cmd:
+            print('To publish user defined port mapping, user must specify the command to run in the container')
+            sys.exit(1)
+
+        self.ports = validate_and_generate_port_mapping(args.ports)
+        self.all_ports = args.all_ports
+
         if args.user_id:
             self.user_id = args.user_id
         if args.user_name:
@@ -427,15 +456,19 @@ class Repo2Docker(Application):
 
     def run_image(self):
         client = docker.from_env(version='auto')
-        port = self._get_free_port()
         if not self.run_cmd:
             port = str(self._get_free_port())
+
             run_cmd = ['jupyter', 'notebook', '--ip', '0.0.0.0',
                        '--port', port]
             ports = {'%s/tcp' % port: port}
         else:
+            # run_cmd given by user, if port is also given then pass it on
             run_cmd = self.run_cmd
-            ports = {}
+            if self.ports:
+                ports = self.ports
+            else:
+                ports = {}
         container_volumes = {}
         if self.volumes:
             api_client = docker.APIClient(
@@ -453,6 +486,7 @@ class Repo2Docker(Application):
 
         container = client.containers.run(
             self.output_image_spec,
+            publish_all_ports=self.all_ports,
             ports=ports,
             detach=True,
             command=run_cmd,
@@ -488,6 +522,7 @@ class Repo2Docker(Application):
         s.close()
         return port
 
+
     def start(self):
         if self.repo_type == 'local':
             checkout_path = self.repo

repo2docker/utils.py

@@ -3,6 +3,7 @@ from functools import partial
 import shutil
 import subprocess
 import re
+import sys
 
 from traitlets import Integer
 
@@ -57,6 +58,75 @@ def maybe_cleanup(path, cleanup=False):
         shutil.rmtree(path, ignore_errors=True)
 
 
+def validate_and_generate_port_mapping(port_mapping):
+    """
+    Validate the port mapping list provided as argument and split into as dictionary of key being continer port and the
+    values being None, or 'host_port' or ['interface_ip','host_port']
+
+
+    Args:
+        port_mapping (list): List of strings of format 'host_port:container_port'
+                             with optional tcp udp values and host network interface
+
+    Returns:
+        List of validated tuples of form ('host_port:container_port') with optional tcp udp values and host network interface
+
+    Raises:
+        Exception on invalid port mapping
+
+    Note:
+        One limitation cannot bind single container_port to multiple host_ports (docker-py supports this but repo2docker
+        does not)
+
+    Examples:
+        Valid port mappings are
+        127.0.0.1:90:900
+        :999 - To match to any host port
+        999:999/tcp - bind 999 host port to 999 tcp container port
+
+        Invalid port mapping
+        127.0.0.1::999 --- even though docker accepts it
+        other invalid ip address combinations
+    """
+    reg_regex = re.compile('''^(
+                                    ( # or capturing group
+                                    (?: # start capturing ip address of network interface
+                                    (?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3} # first three parts
+                                       (?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?) # last part of the ip address
+                                    :(?:6553[0-5]|655[0-2][0-9]|65[0-4](\d){2}|6[0-4](\d){3}|[1-5](\d){4}|(\d){1,4})
+                                    )?
+                                    |   # host ip with port or only port
+                                    (?:6553[0-5]|655[0-2][0-9]|65[0-4](\d){2}|6[0-4](\d){3}|[1-5](\d){4}|(\d){0,4})
+                                    )
+                                    :
+                                    (?:6553[0-5]|655[0-2][0-9]|65[0-4](\d){2}|6[0-4](\d){3}|[1-5](\d){4}|(\d){0,4})
+                                    (?:/udp|/tcp)?
+                               )$''', re.VERBOSE)
+    ports = {}
+    if not port_mapping:
+        return None
+    for p in port_mapping:
+        if reg_regex.match(p) is None:
+            raise Exception('Invalid port mapping ' + str(p))
+        # Do a reverse split twice on the separator :
+        port_host = str(p).rsplit(':', 2)
+        host = None
+        if len(port_host) == 3:
+            # host, optional host_port and container port information given
+            host = port_host[0]
+            host_port = port_host[1]
+            container_port = port_host[2]
+        else:
+            host_port = port_host[0] if len(port_host[0]) > 0 else None
+            container_port = port_host[1]
+
+
+        if host is None:
+            ports[str(container_port)] = host_port
+        else:
+            ports[str(container_port)] = (host, host_port)
+    return ports
+
 def is_valid_docker_image_name(image_name):
     """
     Function that constructs a regex representing the docker image name and tests it against the given image_name

tests/argumentvalidation.py

@@ -5,6 +5,7 @@ Tests that runs validity checks on arguments passed in from shell
 import os
 import subprocess
 
+
 def validate_arguments(builddir, args_list, expected):
     try:
         cmd = ['repo2docker']
@@ -101,6 +102,7 @@ def test_image_name_valid_name_success():
 
     assert validate_arguments(builddir, args_list, None)
 
+
 def test_volume_no_build_fail():
     """
     Test to check if repo2docker fails when both --no-build and -v arguments are given
@@ -120,6 +122,7 @@ def test_volume_no_run_fail():
 
     assert not validate_arguments(builddir, args_list, 'To Mount volumes with -v, you also need to run the container')
 
+
 def test_env_no_run_fail():
     """
     Test to check if repo2docker fails when both --no-run and -e arguments are given 
@@ -129,3 +132,58 @@ def test_env_no_run_fail():
 
     assert not validate_arguments(builddir, args_list, 'To specify environment variables, you also need to run the container')
 
+
+def test_port_mapping_no_run_fail():
+    """
+    Test to check if repo2docker fails when both --no-run and --publish arguments are specified.
+    """
+    builddir = os.path.dirname(__file__)
+    args_list = ['--no-run', '--publish', '8000:8000']
+
+    assert not validate_arguments(builddir, args_list, 'To publish user defined port mappings, the container must also be run')
+
+
+def test_all_ports_mapping_no_run_fail():
+    """
+    Test to check if repo2docker fails when both --no-run and -P arguments are specified.
+    """
+    builddir = os.path.dirname(__file__)
+    args_list = ['--no-run', '-P']
+
+    assert not validate_arguments(builddir, args_list, 'To publish user defined port mappings, the container must also be run')
+
+
+def test_invalid_port_mapping_fail():
+    """
+    Test to check if r2d fails when an invalid port is specified in the port mapping
+    """
+    builddir = os.path.dirname(__file__)
+    # Specifying builddir here itself to simulate passing in a run command
+    # builddir passed in the function will be an argument for the run command
+    args_list = ['-p', '75000:80', builddir, 'ls']
+
+    assert not validate_arguments(builddir, args_list, 'Invalid port mapping')
+
+
+def test_invalid_protocol_port_mapping_fail():
+    """
+    Test to check if r2d fails when an invalid protocol is specified in the port mapping
+    """
+    builddir = os.path.dirname(__file__)
+    # Specifying builddir here itself to simulate passing in a run command
+    # builddir passed in the function will be an argument for the run command
+    args_list = ['-p', '80/tpc:8000', builddir, 'ls']
+
+    assert not validate_arguments(builddir, args_list, 'Invalid port mapping')
+
+
+def test_invalid_container_port_protocol_mapping_fail():
+    """
+    Test to check if r2d fails when an invalid protocol is specified in the container port in port mapping
+    """
+    builddir = os.path.dirname(__file__)
+    # Specifying builddir here itself to simulate passing in a run command
+    # builddir passed in the function will be an argument for the run command
+    args_list = ['-p', '80:8000/upd', builddir, 'ls']
+
+    assert not validate_arguments(builddir, args_list, 'Invalid port mapping')

tests/ports.py

@@ -0,0 +1,137 @@
+"""
+Test Port mappings work on running non-jupyter workflows
+"""
+import subprocess
+import requests
+import time
+import os
+import tempfile
+import signal
+import random
+
+
+def read_port_mapping_response(host, port, protocol = None):
+    """
+    Deploy container and test if port mappings work as expected
+
+    Args:
+        host: the host interface to bind to.
+        port: the random host port to bind to
+        protocol: the protocol to use valid values /tcp or /udp
+    """
+    builddir = os.path.dirname(__file__)
+    port_protocol = '8000'
+    if protocol:
+        port_protocol += protocol
+    host_port = port
+    if host:
+        host_port = host + ':' + port
+    else:
+        host = 'localhost'
+    with tempfile.TemporaryDirectory() as tmpdir:
+        username = os.getlogin()
+
+        # Deploy a test container using r2d in a subprocess
+        # Added the -v volumes to be able to poll for changes within the container from the
+        # host (In this case container starting up)
+        proc = subprocess.Popen(['repo2docker',
+                                 '-p',
+                                 host_port + ':' + port_protocol,
+                                 '-v', '{}:/home'.format(tmpdir),
+                                 '--user-id', str(os.geteuid()),
+                                 '--user-name', username,
+                                 '.',
+                                 '/bin/bash', '-c', 'echo \'hi\' > /home/ts && python -m http.server 8000'],
+                                 cwd=builddir + "/../",
+                                 stderr=subprocess.STDOUT)
+        try:
+            # Wait till docker builds image and starts up
+            while not os.path.exists(os.path.join(tmpdir, 'ts')):
+                if proc.poll() is not None:
+                    # Break loop on errors from the subprocess
+                    raise Exception("Process running r2d exited")
+
+            # Sleep to wait for python http server to start
+            time.sleep(20)
+            resp = requests.request("GET", 'http://' + host + ':' + port)
+
+            # Check if the response is correct
+            assert b'Directory listing' in resp.content
+        finally:
+            if proc.poll() is None:
+                # If the subprocess running the container is still running, interrupt it to close it
+                os.kill(proc.pid, signal.SIGINT)
+                time.sleep(10)
+
+
+def test_all_port_mapping_response():
+    """
+    Deploy container and test if all port expose works as expected
+    """
+    builddir = os.path.dirname(__file__)
+    with tempfile.TemporaryDirectory() as tmpdir:
+        username = os.getlogin()
+
+        # Deploy a test container using r2d in a subprocess
+        # Added the -v volumes to be able to poll for changes within the container from the
+        # host (In this case container starting up)
+        proc = subprocess.Popen(['repo2docker',
+                                 "--image-name",
+                                 "testallport:0.1",
+                                 '-P',
+                                 '-v', '{}:/home'.format(tmpdir),
+                                 '--user-id', str(os.geteuid()),
+                                 '--user-name', username,
+                                 '.',
+                                 '/bin/bash', '-c', 'echo \'hi\' > /home/ts && python -m http.server 52000'],
+                                 cwd=builddir + "/../",
+                                 stderr=subprocess.STDOUT)
+
+        try:
+            # Wait till docker builds image and starts up
+            while not os.path.exists(os.path.join(tmpdir, 'ts')):
+                if proc.poll() is not None:
+                    # Break loop on errors from the subprocess
+                    raise Exception("Process running r2d exited")
+
+            # Sleep to wait for python http server to start
+            time.sleep(20)
+            port = subprocess.check_output("docker ps -f ancestor=testallport:0.1 --format '{{.Ports}}' | cut -f 1 -d - | cut -d: -f 2",
+                                            shell=True).decode("utf-8")
+            port = port.strip("\n\t")
+            resp = requests.request("GET", 'http://localhost' + ':' + port)
+
+            # Check if the response is correct
+            assert b'Directory listing' in resp.content
+        finally:
+            if proc.poll() is None:
+                # If the subprocess running the container is still running, interrupt it to close it
+                os.kill(proc.pid, signal.SIGINT)
+                time.sleep(10)
+
+
+def test_port_mapping_random_port():
+    """
+    Test a simple random port bind
+    """
+    port = str(random.randint(50000, 51000))
+    host = None
+    read_port_mapping_response(host, port)
+
+
+def test_port_mapping_particular_interface():
+    """
+    Test if binding to a single interface is possible
+    """
+    port = str(random.randint(50000, 51000))
+    host = '127.0.0.1'
+    read_port_mapping_response(host, port)
+
+
+def test_port_mapping_protocol():
+    """
+    Test if a particular protocol can be used
+    """
+    port = str(random.randint(50000, 51000))
+    host = None
+    read_port_mapping_response(host, port, '/tcp')