diff --git a/Dockerfile b/Dockerfile index 37e9a76e..91e21487 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ FROM python:3.6.3 +# Used for testing purpose in ports.py +EXPOSE 52000 RUN mkdir /tmp/src ADD . /tmp/src RUN pip3 install --no-cache-dir /tmp/src diff --git a/repo2docker/app.py b/repo2docker/app.py index b0f32602..aaea2946 100644 --- a/repo2docker/app.py +++ b/repo2docker/app.py @@ -30,7 +30,7 @@ from .buildpacks import ( PythonBuildPack, DockerBuildPack, LegacyBinderDockerBuildPack, CondaBuildPack, JuliaBuildPack, Python2BuildPack, BaseImage ) -from .utils import execute_cmd, ByteSpecification, maybe_cleanup, is_valid_docker_image_name +from .utils import execute_cmd, ByteSpecification, maybe_cleanup, is_valid_docker_image_name, validate_and_generate_port_mapping from . import __version__ @@ -264,6 +264,20 @@ class Repo2Docker(Application): help='Do not run container after it has been built' ) + argparser.add_argument( + '--publish', '-p', + dest='ports', + action='append', + help='Specify port mappings for the image. Needs a command to run in the container.' + ) + + argparser.add_argument( + '--publish-all', '-P', + dest='all_ports', + action='store_true', + help='Publish all exposed ports to random host ports.' + ) + argparser.add_argument( '--no-clean', dest='clean', @@ -389,6 +403,21 @@ class Repo2Docker(Application): self.run_cmd = args.cmd + if args.all_ports and not self.run: + print('To publish user defined port mappings, the container must also be run') + sys.exit(1) + + if args.ports and not self.run: + print('To publish user defined port mappings, the container must also be run') + sys.exit(1) + + if args.ports and not self.run_cmd: + print('To publish user defined port mapping, user must specify the command to run in the container') + sys.exit(1) + + self.ports = validate_and_generate_port_mapping(args.ports) + self.all_ports = args.all_ports + if args.user_id: self.user_id = args.user_id if args.user_name: @@ -427,15 +456,19 @@ class Repo2Docker(Application): def run_image(self): client = docker.from_env(version='auto') - port = self._get_free_port() if not self.run_cmd: port = str(self._get_free_port()) + run_cmd = ['jupyter', 'notebook', '--ip', '0.0.0.0', '--port', port] ports = {'%s/tcp' % port: port} else: + # run_cmd given by user, if port is also given then pass it on run_cmd = self.run_cmd - ports = {} + if self.ports: + ports = self.ports + else: + ports = {} container_volumes = {} if self.volumes: api_client = docker.APIClient( @@ -453,6 +486,7 @@ class Repo2Docker(Application): container = client.containers.run( self.output_image_spec, + publish_all_ports=self.all_ports, ports=ports, detach=True, command=run_cmd, @@ -488,6 +522,7 @@ class Repo2Docker(Application): s.close() return port + def start(self): if self.repo_type == 'local': checkout_path = self.repo diff --git a/repo2docker/utils.py b/repo2docker/utils.py index 504f3949..c9cf85f5 100644 --- a/repo2docker/utils.py +++ b/repo2docker/utils.py @@ -3,6 +3,7 @@ from functools import partial import shutil import subprocess import re +import sys from traitlets import Integer @@ -57,6 +58,75 @@ def maybe_cleanup(path, cleanup=False): shutil.rmtree(path, ignore_errors=True) +def validate_and_generate_port_mapping(port_mapping): + """ + Validate the port mapping list provided as argument and split into as dictionary of key being continer port and the + values being None, or 'host_port' or ['interface_ip','host_port'] + + + Args: + port_mapping (list): List of strings of format 'host_port:container_port' + with optional tcp udp values and host network interface + + Returns: + List of validated tuples of form ('host_port:container_port') with optional tcp udp values and host network interface + + Raises: + Exception on invalid port mapping + + Note: + One limitation cannot bind single container_port to multiple host_ports (docker-py supports this but repo2docker + does not) + + Examples: + Valid port mappings are + 127.0.0.1:90:900 + :999 - To match to any host port + 999:999/tcp - bind 999 host port to 999 tcp container port + + Invalid port mapping + 127.0.0.1::999 --- even though docker accepts it + other invalid ip address combinations + """ + reg_regex = re.compile('''^( + ( # or capturing group + (?: # start capturing ip address of network interface + (?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3} # first three parts + (?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?) # last part of the ip address + :(?:6553[0-5]|655[0-2][0-9]|65[0-4](\d){2}|6[0-4](\d){3}|[1-5](\d){4}|(\d){1,4}) + )? + | # host ip with port or only port + (?:6553[0-5]|655[0-2][0-9]|65[0-4](\d){2}|6[0-4](\d){3}|[1-5](\d){4}|(\d){0,4}) + ) + : + (?:6553[0-5]|655[0-2][0-9]|65[0-4](\d){2}|6[0-4](\d){3}|[1-5](\d){4}|(\d){0,4}) + (?:/udp|/tcp)? + )$''', re.VERBOSE) + ports = {} + if not port_mapping: + return None + for p in port_mapping: + if reg_regex.match(p) is None: + raise Exception('Invalid port mapping ' + str(p)) + # Do a reverse split twice on the separator : + port_host = str(p).rsplit(':', 2) + host = None + if len(port_host) == 3: + # host, optional host_port and container port information given + host = port_host[0] + host_port = port_host[1] + container_port = port_host[2] + else: + host_port = port_host[0] if len(port_host[0]) > 0 else None + container_port = port_host[1] + + + if host is None: + ports[str(container_port)] = host_port + else: + ports[str(container_port)] = (host, host_port) + return ports + def is_valid_docker_image_name(image_name): """ Function that constructs a regex representing the docker image name and tests it against the given image_name diff --git a/tests/argumentvalidation.py b/tests/argumentvalidation.py index ced73636..014104eb 100644 --- a/tests/argumentvalidation.py +++ b/tests/argumentvalidation.py @@ -5,6 +5,7 @@ Tests that runs validity checks on arguments passed in from shell import os import subprocess + def validate_arguments(builddir, args_list, expected): try: cmd = ['repo2docker'] @@ -101,6 +102,7 @@ def test_image_name_valid_name_success(): assert validate_arguments(builddir, args_list, None) + def test_volume_no_build_fail(): """ Test to check if repo2docker fails when both --no-build and -v arguments are given @@ -120,6 +122,7 @@ def test_volume_no_run_fail(): assert not validate_arguments(builddir, args_list, 'To Mount volumes with -v, you also need to run the container') + def test_env_no_run_fail(): """ Test to check if repo2docker fails when both --no-run and -e arguments are given @@ -129,3 +132,58 @@ def test_env_no_run_fail(): assert not validate_arguments(builddir, args_list, 'To specify environment variables, you also need to run the container') + +def test_port_mapping_no_run_fail(): + """ + Test to check if repo2docker fails when both --no-run and --publish arguments are specified. + """ + builddir = os.path.dirname(__file__) + args_list = ['--no-run', '--publish', '8000:8000'] + + assert not validate_arguments(builddir, args_list, 'To publish user defined port mappings, the container must also be run') + + +def test_all_ports_mapping_no_run_fail(): + """ + Test to check if repo2docker fails when both --no-run and -P arguments are specified. + """ + builddir = os.path.dirname(__file__) + args_list = ['--no-run', '-P'] + + assert not validate_arguments(builddir, args_list, 'To publish user defined port mappings, the container must also be run') + + +def test_invalid_port_mapping_fail(): + """ + Test to check if r2d fails when an invalid port is specified in the port mapping + """ + builddir = os.path.dirname(__file__) + # Specifying builddir here itself to simulate passing in a run command + # builddir passed in the function will be an argument for the run command + args_list = ['-p', '75000:80', builddir, 'ls'] + + assert not validate_arguments(builddir, args_list, 'Invalid port mapping') + + +def test_invalid_protocol_port_mapping_fail(): + """ + Test to check if r2d fails when an invalid protocol is specified in the port mapping + """ + builddir = os.path.dirname(__file__) + # Specifying builddir here itself to simulate passing in a run command + # builddir passed in the function will be an argument for the run command + args_list = ['-p', '80/tpc:8000', builddir, 'ls'] + + assert not validate_arguments(builddir, args_list, 'Invalid port mapping') + + +def test_invalid_container_port_protocol_mapping_fail(): + """ + Test to check if r2d fails when an invalid protocol is specified in the container port in port mapping + """ + builddir = os.path.dirname(__file__) + # Specifying builddir here itself to simulate passing in a run command + # builddir passed in the function will be an argument for the run command + args_list = ['-p', '80:8000/upd', builddir, 'ls'] + + assert not validate_arguments(builddir, args_list, 'Invalid port mapping') diff --git a/tests/ports.py b/tests/ports.py new file mode 100644 index 00000000..425d6317 --- /dev/null +++ b/tests/ports.py @@ -0,0 +1,137 @@ +""" +Test Port mappings work on running non-jupyter workflows +""" +import subprocess +import requests +import time +import os +import tempfile +import signal +import random + + +def read_port_mapping_response(host, port, protocol = None): + """ + Deploy container and test if port mappings work as expected + + Args: + host: the host interface to bind to. + port: the random host port to bind to + protocol: the protocol to use valid values /tcp or /udp + """ + builddir = os.path.dirname(__file__) + port_protocol = '8000' + if protocol: + port_protocol += protocol + host_port = port + if host: + host_port = host + ':' + port + else: + host = 'localhost' + with tempfile.TemporaryDirectory() as tmpdir: + username = os.getlogin() + + # Deploy a test container using r2d in a subprocess + # Added the -v volumes to be able to poll for changes within the container from the + # host (In this case container starting up) + proc = subprocess.Popen(['repo2docker', + '-p', + host_port + ':' + port_protocol, + '-v', '{}:/home'.format(tmpdir), + '--user-id', str(os.geteuid()), + '--user-name', username, + '.', + '/bin/bash', '-c', 'echo \'hi\' > /home/ts && python -m http.server 8000'], + cwd=builddir + "/../", + stderr=subprocess.STDOUT) + try: + # Wait till docker builds image and starts up + while not os.path.exists(os.path.join(tmpdir, 'ts')): + if proc.poll() is not None: + # Break loop on errors from the subprocess + raise Exception("Process running r2d exited") + + # Sleep to wait for python http server to start + time.sleep(20) + resp = requests.request("GET", 'http://' + host + ':' + port) + + # Check if the response is correct + assert b'Directory listing' in resp.content + finally: + if proc.poll() is None: + # If the subprocess running the container is still running, interrupt it to close it + os.kill(proc.pid, signal.SIGINT) + time.sleep(10) + + +def test_all_port_mapping_response(): + """ + Deploy container and test if all port expose works as expected + """ + builddir = os.path.dirname(__file__) + with tempfile.TemporaryDirectory() as tmpdir: + username = os.getlogin() + + # Deploy a test container using r2d in a subprocess + # Added the -v volumes to be able to poll for changes within the container from the + # host (In this case container starting up) + proc = subprocess.Popen(['repo2docker', + "--image-name", + "testallport:0.1", + '-P', + '-v', '{}:/home'.format(tmpdir), + '--user-id', str(os.geteuid()), + '--user-name', username, + '.', + '/bin/bash', '-c', 'echo \'hi\' > /home/ts && python -m http.server 52000'], + cwd=builddir + "/../", + stderr=subprocess.STDOUT) + + try: + # Wait till docker builds image and starts up + while not os.path.exists(os.path.join(tmpdir, 'ts')): + if proc.poll() is not None: + # Break loop on errors from the subprocess + raise Exception("Process running r2d exited") + + # Sleep to wait for python http server to start + time.sleep(20) + port = subprocess.check_output("docker ps -f ancestor=testallport:0.1 --format '{{.Ports}}' | cut -f 1 -d - | cut -d: -f 2", + shell=True).decode("utf-8") + port = port.strip("\n\t") + resp = requests.request("GET", 'http://localhost' + ':' + port) + + # Check if the response is correct + assert b'Directory listing' in resp.content + finally: + if proc.poll() is None: + # If the subprocess running the container is still running, interrupt it to close it + os.kill(proc.pid, signal.SIGINT) + time.sleep(10) + + +def test_port_mapping_random_port(): + """ + Test a simple random port bind + """ + port = str(random.randint(50000, 51000)) + host = None + read_port_mapping_response(host, port) + + +def test_port_mapping_particular_interface(): + """ + Test if binding to a single interface is possible + """ + port = str(random.randint(50000, 51000)) + host = '127.0.0.1' + read_port_mapping_response(host, port) + + +def test_port_mapping_protocol(): + """ + Test if a particular protocol can be used + """ + port = str(random.randint(50000, 51000)) + host = None + read_port_mapping_response(host, port, '/tcp')