diff --git a/.travis.yml b/.travis.yml
index 2358d1a0..4db0e631 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,9 +17,9 @@ script:
 # possible issues with MANIFEST.in
 - pushd tests;
   if [ ${REPO_TYPE} == "r" ] || [ ${REPO_TYPE} == "stencila-r" ] || [ ${REPO_TYPE} == "stencila-py" ]; then
-    travis_wait 30 pytest --cov repo2docker -v ${REPO_TYPE} || exit 1;
+    travis_wait 30 pytest --durations 10 --cov repo2docker -v ${REPO_TYPE} || exit 1;
   else
-    travis_retry pytest --cov repo2docker -v ${REPO_TYPE} || exit 1;
+    travis_retry pytest --durations 10 --cov repo2docker -v ${REPO_TYPE} || exit 1;
   fi;
   popd;
 - pip install -r docs/doc-requirements.txt
@@ -44,6 +44,7 @@ jobs:
         branch: master
 env:
   matrix:
+  - REPO_TYPE=unit
   - REPO_TYPE=base
   - REPO_TYPE=conda
   - REPO_TYPE=venv
@@ -53,9 +54,7 @@ env:
   - REPO_TYPE=r
   - REPO_TYPE=nix
   - REPO_TYPE=dockerfile
-  - REPO_TYPE=external/*
-  - REPO_TYPE=contentproviders/*.py
-  - REPO_TYPE=test_args.py
+  - REPO_TYPE=external
   global:
   - secure: gX7IOkbjlvcDwIH24sOLhutINx6TZRwujEusMWh1dqgYG2D69qQai/mTrRXO9PGRrsvQwIBk4RcILKAiZnk5O2Z1hLoIHk/oU2mNUmE44dDm4Xf/VTTdeYhjeOTR9B+KJ9NVwPxuSEDSND3lD7yFfvCqNXykipEhBtTliLupjWVxxXnaz0aZTYHUPJwanxdUc06AphSPwZjtm1m3qMUU8v7UdTGGAdW3NlgkKw0Xx2x5W31fW676vskC/GNQAbcRociYipuhSFWV4lu+6d8XF2xVO97xtzf54tBQzt6RgVfAKtiqkEIYSzJQBBpkQ6SM6yg+fQoQpOo8jPU9ZBjvaoopUG9vn8HRS/OtQrDcG3kEFnFAnaes8Iqtidp1deTn27LIlfCTl7kTFOp8yaaNlIMHJTJKTEMRhfdDlBYx7qiH8e9d/z37lupzY2loLHeNHdMRS1uYsfacZsmrnu9vAdpQmP1LuHivBPZEvgerinADaJiekelWOIEn956pDrno/YgnzP0i9LEBYnbbunqT8oEzLintNt5CXGdhkiG60j38McKCIn4sD6jbMMwgsqVFdClCBersyorKhOs7P8at5vX4xf8fMiKPC8LZPzYVIQYzCjmwSOFQ+Rzmz5gSj+DRTANKfHpzZCKZEF6amBYMGE1O5osF8m6M10vtW9ToK+s=
   - secure: Cfhb0BUT54JjEZD8n44Jj+o1lt5p32Lfg7W/euTyZ61YylDx0+XEYTzfWcwxOzH9fLpWr6dDrBMGHA/FPqsWA5BkoGdiBJ1OOVy2tmDRButctobWM3SVwa+Rhh8bZWlK8yKT2S3n6CtK4mesmjzdbUShL7YnKOSl8LBaTT5Y5oT8Oxsq51pfg8fJUImim8H20t8H7emaEzZorF4OSGRtajcAgukt5YoAqTEVDq+bFRBHZalxkcRqLhsGe3CCWa28kjGTL4MPZpCI6/AXIXHzihfG3rGq40ZT8jZ9GPP3MBgkiJWtFiTC9h16G34b/JI/TD40zCmoW9/9oVjRK4UlLGCAv6bgzFhCRof2abhB9NTZDniNzkO0T15uHs3VLbLCPYB0xYyClAFxm2P6e8WPChyENKfTNh+803IKFFo4JaTjOnKzi89N72v5+bT6ghP932nmjJr1AO65xjw63CeDmaLoHDY73n11DibybWQgEeiNzJuSzbIHyqMPhW5XqeroEjKKstdPHtVfOViI9ywjEMy0HCPsspaVI7Aow0Iv8E4Ajvd32W7z0h0fSCx/i25hEOAo2vhBsmQKJA7IquB3N88M11L874h/8J+oc/osW1EB5z7Ukke5YCq94Qh3qImSIhJULXMMc1QjEqYsqhLXtiMG2HUge0Y5hwwnnbEIRMQ=
diff --git a/repo2docker/__main__.py b/repo2docker/__main__.py
index 8a3e82ec..6c5bd4c6 100644
--- a/repo2docker/__main__.py
+++ b/repo2docker/__main__.py
@@ -230,7 +230,7 @@ def make_r2d(argv=None):
         if os.path.isdir(args.repo):
             r2d.volumes[os.path.abspath(args.repo)] = '.'
         else:
-            r2d.log.error('Can not mount "{}" in editable mode '
+            r2d.log.error('Cannot mount "{}" in editable mode '
                             'as it is not a directory'.format(args.repo),
                             extra=dict(phase='failed'))
             sys.exit(1)
diff --git a/repo2docker/app.py b/repo2docker/app.py
index 949bd22b..ef637a79 100644
--- a/repo2docker/app.py
+++ b/repo2docker/app.py
@@ -399,10 +399,10 @@ class Repo2Docker(Application):
             )
 
         if self.dry_run and (self.run or self.push):
-            raise ValueError("Can not push or run image if we are not building it")
+            raise ValueError("Cannot push or run image if we are not building it")
 
         if self.volumes and not self.run:
-            raise ValueError("Can not mount volumes if container is not run")
+            raise ValueError("Cannot mount volumes if container is not run")
 
     def push_image(self):
         """Push docker image to registry"""
diff --git a/tests/memlimit.py b/tests/memlimit.py
deleted file mode 100644
index 50c909dd..00000000
--- a/tests/memlimit.py
+++ /dev/null
@@ -1,100 +0,0 @@
-"""
-Test that build time memory limits are actually enforced.
-
-We give the container image at least 128M of RAM (so base things like
-apt and pip can run), and then try to allocate & use 256MB in postBuild.
-This should fail!
-"""
-import os
-import subprocess
-import time
-
-def does_build(builddir, mem_limit, mem_allocate_mb):
-    mem_allocate_mb_file = os.path.join(builddir, 'mem_allocate_mb')
-
-    # Cache bust so we actually do a rebuild each time this is run!
-    with open(os.path.join(builddir, 'cachebust'), 'w') as cachebust:
-        cachebust.write(str(time.time()))
-
-    try:
-        # we don't have an easy way to pass env vars or whatever to
-        # postBuild from here, so we write a file into the repo that is
-        # read by the postBuild script!
-        with open(mem_allocate_mb_file, 'w') as f:
-            f.write(str(mem_allocate_mb))
-        try:
-            output = subprocess.check_output(
-                [
-                    'repo2docker',
-                    '--no-run',
-                    '--build-memory-limit', '{}M'.format(mem_limit),
-                    builddir
-                ],
-                stderr=subprocess.STDOUT,
-            ).decode()
-            print(output)
-            return True
-        except subprocess.CalledProcessError as e:
-            output = e.output.decode()
-            print(output)
-            if "/postBuild' returned a non-zero code: 137" in output:
-                return False
-            else:
-                raise
-    finally:
-        os.remove(mem_allocate_mb_file)
-
-
-
-def test_memlimit_nondockerfile_fail():
-    """
-    Test if memory limited builds are working for non dockerfile builds
-    """
-    basedir = os.path.dirname(__file__)
-    assert not does_build(
-        os.path.join(basedir, 'memlimit/non-dockerfile'),
-        128,
-        256
-    )
-    assert does_build(
-        os.path.join(basedir, 'memlimit/non-dockerfile'),
-        512,
-        256
-    )
-
-
-def test_memlimit_dockerfile_fail():
-    """
-    Test if memory limited builds are working for dockerfile builds
-    """
-    basedir = os.path.dirname(__file__)
-    assert not does_build(
-        os.path.join(basedir, 'memlimit/dockerfile'),
-        128,
-        256
-    )
-
-    assert does_build(
-        os.path.join(basedir, 'memlimit/dockerfile'),
-        512,
-        256
-    )
-
-
-def test_memlimit_same_postbuild():
-    """
-    Validate that the postBuild files for dockerfile & nondockerfile are same
-
-    Until https://github.com/jupyter/repo2docker/issues/160 gets fixed.
-    """
-    basedir = os.path.dirname(__file__)
-    filepaths = [
-        os.path.join(basedir, 'memlimit', t, "postBuild")
-        for t in ("dockerfile", "non-dockerfile")
-    ]
-    file_contents = []
-    for fp in filepaths:
-        with open(fp) as f:
-            file_contents.append(f.read())
-    # Make sure they're all the same
-    assert len(set(file_contents)) == 1
diff --git a/tests/ports.py b/tests/ports.py
deleted file mode 100644
index 425d6317..00000000
--- a/tests/ports.py
+++ /dev/null
@@ -1,137 +0,0 @@
-"""
-Test Port mappings work on running non-jupyter workflows
-"""
-import subprocess
-import requests
-import time
-import os
-import tempfile
-import signal
-import random
-
-
-def read_port_mapping_response(host, port, protocol = None):
-    """
-    Deploy container and test if port mappings work as expected
-
-    Args:
-        host: the host interface to bind to.
-        port: the random host port to bind to
-        protocol: the protocol to use valid values /tcp or /udp
-    """
-    builddir = os.path.dirname(__file__)
-    port_protocol = '8000'
-    if protocol:
-        port_protocol += protocol
-    host_port = port
-    if host:
-        host_port = host + ':' + port
-    else:
-        host = 'localhost'
-    with tempfile.TemporaryDirectory() as tmpdir:
-        username = os.getlogin()
-
-        # Deploy a test container using r2d in a subprocess
-        # Added the -v volumes to be able to poll for changes within the container from the
-        # host (In this case container starting up)
-        proc = subprocess.Popen(['repo2docker',
-                                 '-p',
-                                 host_port + ':' + port_protocol,
-                                 '-v', '{}:/home'.format(tmpdir),
-                                 '--user-id', str(os.geteuid()),
-                                 '--user-name', username,
-                                 '.',
-                                 '/bin/bash', '-c', 'echo \'hi\' > /home/ts && python -m http.server 8000'],
-                                 cwd=builddir + "/../",
-                                 stderr=subprocess.STDOUT)
-        try:
-            # Wait till docker builds image and starts up
-            while not os.path.exists(os.path.join(tmpdir, 'ts')):
-                if proc.poll() is not None:
-                    # Break loop on errors from the subprocess
-                    raise Exception("Process running r2d exited")
-
-            # Sleep to wait for python http server to start
-            time.sleep(20)
-            resp = requests.request("GET", 'http://' + host + ':' + port)
-
-            # Check if the response is correct
-            assert b'Directory listing' in resp.content
-        finally:
-            if proc.poll() is None:
-                # If the subprocess running the container is still running, interrupt it to close it
-                os.kill(proc.pid, signal.SIGINT)
-                time.sleep(10)
-
-
-def test_all_port_mapping_response():
-    """
-    Deploy container and test if all port expose works as expected
-    """
-    builddir = os.path.dirname(__file__)
-    with tempfile.TemporaryDirectory() as tmpdir:
-        username = os.getlogin()
-
-        # Deploy a test container using r2d in a subprocess
-        # Added the -v volumes to be able to poll for changes within the container from the
-        # host (In this case container starting up)
-        proc = subprocess.Popen(['repo2docker',
-                                 "--image-name",
-                                 "testallport:0.1",
-                                 '-P',
-                                 '-v', '{}:/home'.format(tmpdir),
-                                 '--user-id', str(os.geteuid()),
-                                 '--user-name', username,
-                                 '.',
-                                 '/bin/bash', '-c', 'echo \'hi\' > /home/ts && python -m http.server 52000'],
-                                 cwd=builddir + "/../",
-                                 stderr=subprocess.STDOUT)
-
-        try:
-            # Wait till docker builds image and starts up
-            while not os.path.exists(os.path.join(tmpdir, 'ts')):
-                if proc.poll() is not None:
-                    # Break loop on errors from the subprocess
-                    raise Exception("Process running r2d exited")
-
-            # Sleep to wait for python http server to start
-            time.sleep(20)
-            port = subprocess.check_output("docker ps -f ancestor=testallport:0.1 --format '{{.Ports}}' | cut -f 1 -d - | cut -d: -f 2",
-                                            shell=True).decode("utf-8")
-            port = port.strip("\n\t")
-            resp = requests.request("GET", 'http://localhost' + ':' + port)
-
-            # Check if the response is correct
-            assert b'Directory listing' in resp.content
-        finally:
-            if proc.poll() is None:
-                # If the subprocess running the container is still running, interrupt it to close it
-                os.kill(proc.pid, signal.SIGINT)
-                time.sleep(10)
-
-
-def test_port_mapping_random_port():
-    """
-    Test a simple random port bind
-    """
-    port = str(random.randint(50000, 51000))
-    host = None
-    read_port_mapping_response(host, port)
-
-
-def test_port_mapping_particular_interface():
-    """
-    Test if binding to a single interface is possible
-    """
-    port = str(random.randint(50000, 51000))
-    host = '127.0.0.1'
-    read_port_mapping_response(host, port)
-
-
-def test_port_mapping_protocol():
-    """
-    Test if a particular protocol can be used
-    """
-    port = str(random.randint(50000, 51000))
-    host = None
-    read_port_mapping_response(host, port, '/tcp')
diff --git a/tests/test_cache_from.py b/tests/test_cache_from.py
deleted file mode 100644
index 6b7c9640..00000000
--- a/tests/test_cache_from.py
+++ /dev/null
@@ -1,82 +0,0 @@
-"""
-Test that --cache-from is passed in to docker API properly.
-"""
-import os
-import docker
-from unittest.mock import MagicMock, patch
-from repo2docker.buildpacks import BaseImage, DockerBuildPack, LegacyBinderDockerBuildPack
-from tempfile import TemporaryDirectory
-
-def test_cache_from_base(monkeypatch):
-    FakeDockerClient = MagicMock()
-    cache_from = [
-        'image-1:latest'
-    ]
-    fake_log_value = {'stream': 'fake'}
-    fake_client = MagicMock(spec=docker.APIClient)
-    fake_client.build.return_value = iter([fake_log_value])
-
-    with TemporaryDirectory() as d:
-        # Test base image build pack
-        monkeypatch.chdir(d)
-        for line in BaseImage().build(fake_client, 'image-2', '1Gi', {}, cache_from):
-            assert line == fake_log_value
-        called_args, called_kwargs = fake_client.build.call_args
-        assert 'cache_from' in called_kwargs
-        assert called_kwargs['cache_from'] == cache_from
-
-
-
-def test_cache_from_docker(monkeypatch):
-    FakeDockerClient = MagicMock()
-    cache_from = [
-        'image-1:latest'
-    ]
-    fake_log_value = {'stream': 'fake'}
-    fake_client = MagicMock(spec=docker.APIClient)
-    fake_client.build.return_value = iter([fake_log_value])
-
-    with TemporaryDirectory() as d:
-        # Test docker image
-        with open(os.path.join(d, 'Dockerfile'), 'w') as f:
-            f.write('FROM scratch\n')
-
-        for line in DockerBuildPack().build(fake_client, 'image-2', '1Gi', {}, cache_from):
-            assert line == fake_log_value
-        called_args, called_kwargs = fake_client.build.call_args
-        assert 'cache_from' in called_kwargs
-        assert called_kwargs['cache_from'] == cache_from
-
-        # Test legacy docker image
-        with open(os.path.join(d, 'Dockerfile'), 'w') as f:
-            f.write('FROM andrewosh/binder-base\n')
-
-        for line in LegacyBinderDockerBuildPack().build(fake_client, 'image-2', '1Gi', {}, cache_from):
-            print(line)
-            assert line == fake_log_value
-        called_args, called_kwargs = fake_client.build.call_args
-        assert 'cache_from' in called_kwargs
-        assert called_kwargs['cache_from'] == cache_from
-
-
-def test_cache_from_legacy(monkeypatch):
-    FakeDockerClient = MagicMock()
-    cache_from = [
-        'image-1:latest'
-    ]
-    fake_log_value = {'stream': 'fake'}
-    fake_client = MagicMock(spec=docker.APIClient)
-    fake_client.build.return_value = iter([fake_log_value])
-
-    with TemporaryDirectory() as d:
-        # Test legacy docker image
-        with open(os.path.join(d, 'Dockerfile'), 'w') as f:
-            f.write('FROM andrewosh/binder-base\n')
-
-        for line in LegacyBinderDockerBuildPack().build(fake_client, 'image-2', '1Gi', {}, cache_from):
-            assert line == fake_log_value
-        called_args, called_kwargs = fake_client.build.call_args
-        assert 'cache_from' in called_kwargs
-        assert called_kwargs['cache_from'] == cache_from
-
-
diff --git a/tests/contentproviders/test_git.py b/tests/unit/contentproviders/test_git.py
similarity index 100%
rename from tests/contentproviders/test_git.py
rename to tests/unit/contentproviders/test_git.py
diff --git a/tests/contentproviders/test_local.py b/tests/unit/contentproviders/test_local.py
similarity index 100%
rename from tests/contentproviders/test_local.py
rename to tests/unit/contentproviders/test_local.py
diff --git a/tests/test_args.py b/tests/unit/test_args.py
similarity index 97%
rename from tests/test_args.py
rename to tests/unit/test_args.py
index 2fba2027..d4b1a565 100644
--- a/tests/test_args.py
+++ b/tests/unit/test_args.py
@@ -81,4 +81,4 @@ def test_invalid_image_name():
     Test validating image names
     """
     with pytest.raises(SystemExit):
-        make_r2d(['--image-name', '_invalid', '.'])
\ No newline at end of file
+        make_r2d(['--image-name', '_invalid', '.'])
diff --git a/tests/argumentvalidation.py b/tests/unit/test_argumentvalidation.py
similarity index 74%
rename from tests/argumentvalidation.py
rename to tests/unit/test_argumentvalidation.py
index f38c056c..b8ae7e20 100644
--- a/tests/argumentvalidation.py
+++ b/tests/unit/test_argumentvalidation.py
@@ -5,8 +5,23 @@ Tests that runs validity checks on arguments passed in from shell
 import os
 import subprocess
 
+import pytest
 
-def validate_arguments(builddir, args_list, expected, disable_dockerd=False):
+
+here = os.path.dirname(os.path.abspath(__file__))
+test_dir = os.path.dirname(here)
+docker_simple = os.path.join(test_dir, 'dockerfile', 'simple')
+
+# default to building in the cwd (a temporary directory)
+builddir = '.'
+
+
+@pytest.fixture
+def temp_cwd(tmpdir):
+    tmpdir.chdir()
+
+
+def validate_arguments(builddir, args_list='.', expected=None, disable_dockerd=False):
     try:
         cmd = ['repo2docker']
         for k in args_list:
@@ -19,19 +34,20 @@ def validate_arguments(builddir, args_list, expected, disable_dockerd=False):
         return True
     except subprocess.CalledProcessError as e:
         output = e.output.decode()
-        if expected in output:
+        if expected is not None:
+            assert expected in output
             return False
         else:
+            print(output)
             raise
 
 
-def test_image_name_fail():
+def test_image_name_fail(temp_cwd):
     """
     Test to check if repo2docker throws image_name validation error on --image-name argument containing
     uppercase characters and _ characters in incorrect positions.
     """
 
-    builddir = os.path.dirname(__file__)
     image_name = 'Test/Invalid_name:1.0.0'
     args_list = ['--no-run', '--no-build', '--image-name', image_name]
     expected = (
@@ -42,12 +58,11 @@ def test_image_name_fail():
     assert not validate_arguments(builddir, args_list, expected)
 
 
-def test_image_name_underscore_fail():
+def test_image_name_underscore_fail(temp_cwd):
     """
     Test to check if repo2docker throws image_name validation error on --image-name argument starts with _.
     """
 
-    builddir = os.path.dirname(__file__)
     image_name = '_test/invalid_name:1.0.0'
     args_list = ['--no-run', '--no-build', '--image-name', image_name]
     expected = (
@@ -58,12 +73,11 @@ def test_image_name_underscore_fail():
     assert not validate_arguments(builddir, args_list, expected)
 
 
-def test_image_name_double_dot_fail():
+def test_image_name_double_dot_fail(temp_cwd):
     """
     Test to check if repo2docker throws image_name validation error on --image-name argument contains consecutive dots.
     """
 
-    builddir = os.path.dirname(__file__)
     image_name = 'test..com/invalid_name:1.0.0'
     args_list = ['--no-run', '--no-build', '--image-name', image_name]
     expected = (
@@ -74,13 +88,12 @@ def test_image_name_double_dot_fail():
     assert not validate_arguments(builddir, args_list, expected)
 
 
-def test_image_name_valid_restircted_registry_domain_name_fail():
+def test_image_name_valid_restircted_registry_domain_name_fail(temp_cwd):
     """
     Test to check if repo2docker throws image_name validation error on -image-name argument being invalid. Based on the
     regex definitions first part of registry domain cannot contain uppercase characters
     """
 
-    builddir = os.path.dirname(__file__)
     image_name = 'Test.com/valid_name:1.0.0'
     args_list = ['--no-run', '--no-build', '--image-name', image_name]
     expected = (
@@ -92,85 +105,87 @@ def test_image_name_valid_restircted_registry_domain_name_fail():
     assert not validate_arguments(builddir, args_list, expected)
 
 
-def test_image_name_valid_registry_domain_name_success():
+def test_image_name_valid_registry_domain_name_success(temp_cwd):
     """
     Test to check if repo2docker runs with a valid --image-name argument.
     """
 
-    builddir = os.path.dirname(__file__) + '/dockerfile/simple/'
+    builddir = docker_simple
     image_name = 'test.COM/valid_name:1.0.0'
     args_list = ['--no-run', '--no-build', '--image-name', image_name]
 
     assert validate_arguments(builddir, args_list, None)
 
 
-def test_image_name_valid_name_success():
+def test_image_name_valid_name_success(temp_cwd):
     """
     Test to check if repo2docker runs with a valid --image-name argument.
     """
 
-    builddir = os.path.dirname(__file__) + '/dockerfile/simple/'
+    builddir = docker_simple
     image_name = 'test.com/valid_name:1.0.0'
     args_list = ['--no-run', '--no-build', '--image-name', image_name]
 
     assert validate_arguments(builddir, args_list, None)
 
 
-def test_volume_no_build_fail():
+def test_volume_no_build_fail(temp_cwd):
     """
     Test to check if repo2docker fails when both --no-build and -v arguments are given
     """
-    builddir = os.path.dirname(__file__)
     args_list = ['--no-build', '-v', '/data:/data']
 
-    assert not validate_arguments(builddir, args_list, 'To Mount volumes with -v, you also need to run the container')
+    assert not validate_arguments(
+        builddir,
+        args_list,
+        'Cannot mount volumes if container is not run',
+    )
 
 
-def test_volume_no_run_fail():
+def test_volume_no_run_fail(temp_cwd):
     """
     Test to check if repo2docker fails when both --no-run and -v arguments are given
     """
-    builddir = os.path.dirname(__file__)
     args_list = ['--no-run', '-v', '/data:/data']
 
-    assert not validate_arguments(builddir, args_list, 'To Mount volumes with -v, you also need to run the container')
+    assert not validate_arguments(
+        builddir,
+        args_list,
+        'Cannot mount volumes if container is not run',
+    )
 
 
-def test_env_no_run_fail():
+def test_env_no_run_fail(temp_cwd):
     """
     Test to check if repo2docker fails when both --no-run and -e arguments are given
     """
-    builddir = os.path.dirname(__file__)
     args_list = ['--no-run', '-e', 'FOO=bar', '--']
 
     assert not validate_arguments(builddir, args_list, 'To specify environment variables, you also need to run the container')
 
 
-def test_port_mapping_no_run_fail():
+def test_port_mapping_no_run_fail(temp_cwd):
     """
     Test to check if repo2docker fails when both --no-run and --publish arguments are specified.
     """
-    builddir = os.path.dirname(__file__)
     args_list = ['--no-run', '--publish', '8000:8000']
 
     assert not validate_arguments(builddir, args_list, 'To publish user defined port mappings, the container must also be run')
 
 
-def test_all_ports_mapping_no_run_fail():
+def test_all_ports_mapping_no_run_fail(temp_cwd):
     """
     Test to check if repo2docker fails when both --no-run and -P arguments are specified.
     """
-    builddir = os.path.dirname(__file__)
     args_list = ['--no-run', '-P']
 
     assert not validate_arguments(builddir, args_list, 'To publish user defined port mappings, the container must also be run')
 
 
-def test_invalid_port_mapping_fail():
+def test_invalid_port_mapping_fail(temp_cwd):
     """
     Test to check if r2d fails when an invalid port is specified in the port mapping
     """
-    builddir = os.path.dirname(__file__)
     # Specifying builddir here itself to simulate passing in a run command
     # builddir passed in the function will be an argument for the run command
     args_list = ['-p', '75000:80', builddir, 'ls']
@@ -178,11 +193,10 @@ def test_invalid_port_mapping_fail():
     assert not validate_arguments(builddir, args_list, 'Invalid port mapping')
 
 
-def test_invalid_protocol_port_mapping_fail():
+def test_invalid_protocol_port_mapping_fail(temp_cwd):
     """
     Test to check if r2d fails when an invalid protocol is specified in the port mapping
     """
-    builddir = os.path.dirname(__file__)
     # Specifying builddir here itself to simulate passing in a run command
     # builddir passed in the function will be an argument for the run command
     args_list = ['-p', '80/tpc:8000', builddir, 'ls']
@@ -190,11 +204,10 @@ def test_invalid_protocol_port_mapping_fail():
     assert not validate_arguments(builddir, args_list, 'Invalid port mapping')
 
 
-def test_invalid_container_port_protocol_mapping_fail():
+def test_invalid_container_port_protocol_mapping_fail(temp_cwd):
     """
     Test to check if r2d fails when an invalid protocol is specified in the container port in port mapping
     """
-    builddir = os.path.dirname(__file__)
     # Specifying builddir here itself to simulate passing in a run command
     # builddir passed in the function will be an argument for the run command
     args_list = ['-p', '80:8000/upd', builddir, 'ls']
@@ -202,31 +215,39 @@ def test_invalid_container_port_protocol_mapping_fail():
     assert not validate_arguments(builddir, args_list, 'Invalid port mapping')
 
 
-def test_docker_handle_fail():
+@pytest.mark.xfail(reason="Regression in new arg parsing")
+def test_docker_handle_fail(temp_cwd):
     """
     Test to check if r2d fails with minimal error message on not being able to connect to docker daemon
     """
     args_list = []
-    builddir = os.path.dirname(__file__) + '/../'
 
-    assert not validate_arguments(builddir, args_list, "Docker client initialization error. Check if docker is running on the host.", True)
+    assert not validate_arguments(
+        builddir,
+        args_list,
+        "Docker client initialization error. Check if docker is running on the host.",
+        disable_dockerd=True,
+    )
 
 
-def test_docker_handle_debug_fail():
+def test_docker_handle_debug_fail(temp_cwd):
     """
     Test to check if r2d fails with stack trace on not being able to connect to docker daemon and debug enabled
     """
     args_list = ['--debug']
-    builddir = os.path.dirname(__file__) + '/../'
 
-    assert not validate_arguments(builddir, args_list, "docker.errors.DockerException", True)
+    assert not validate_arguments(
+        builddir,
+        args_list,
+        "docker.errors.DockerException",
+        disable_dockerd=True,
+    )
 
 
-def test_docker_no_build_success():
+def test_docker_no_build_success(temp_cwd):
     """
     Test to check if r2d succeeds with --no-build argument with not being able to connect to docker daemon
     """
     args_list = ['--no-build', '--no-run']
-    builddir = os.path.dirname(__file__) + '/../'
 
-    assert validate_arguments(builddir, args_list, "", True)
+    assert validate_arguments(builddir, args_list, disable_dockerd=True)
diff --git a/tests/unit/test_cache_from.py b/tests/unit/test_cache_from.py
new file mode 100644
index 00000000..6718e42b
--- /dev/null
+++ b/tests/unit/test_cache_from.py
@@ -0,0 +1,70 @@
+"""
+Test that --cache-from is passed in to docker API properly.
+"""
+
+from unittest.mock import MagicMock
+
+import docker
+
+from repo2docker.buildpacks import BaseImage, DockerBuildPack, LegacyBinderDockerBuildPack
+
+
+def test_cache_from_base(tmpdir):
+    FakeDockerClient = MagicMock()
+    cache_from = [
+        'image-1:latest'
+    ]
+    fake_log_value = {'stream': 'fake'}
+    fake_client = MagicMock(spec=docker.APIClient)
+    fake_client.build.return_value = iter([fake_log_value])
+
+    # Test base image build pack
+    tmpdir.chdir()
+    for line in BaseImage().build(fake_client, 'image-2', '1Gi', {}, cache_from):
+        assert line == fake_log_value
+    called_args, called_kwargs = fake_client.build.call_args
+    assert 'cache_from' in called_kwargs
+    assert called_kwargs['cache_from'] == cache_from
+
+
+
+def test_cache_from_docker(tmpdir):
+    FakeDockerClient = MagicMock()
+    cache_from = [
+        'image-1:latest'
+    ]
+    fake_log_value = {'stream': 'fake'}
+    fake_client = MagicMock(spec=docker.APIClient)
+    fake_client.build.return_value = iter([fake_log_value])
+
+    tmpdir.chdir()
+    # test dockerfile
+    with tmpdir.join("Dockerfile").open('w') as f:
+        f.write('FROM scratch\n')
+
+    for line in DockerBuildPack().build(fake_client, 'image-2', '1Gi', {}, cache_from):
+        assert line == fake_log_value
+    called_args, called_kwargs = fake_client.build.call_args
+    assert 'cache_from' in called_kwargs
+    assert called_kwargs['cache_from'] == cache_from
+
+
+def test_cache_from_legacy(tmpdir):
+    FakeDockerClient = MagicMock()
+    cache_from = [
+        'image-1:latest'
+    ]
+    fake_log_value = {'stream': 'fake'}
+    fake_client = MagicMock(spec=docker.APIClient)
+    fake_client.build.return_value = iter([fake_log_value])
+
+    # Test legacy docker image
+    with tmpdir.join("Dockerfile").open('w') as f:
+        f.write('FROM andrewosh/binder-base\n')
+
+    for line in LegacyBinderDockerBuildPack().build(fake_client, 'image-2', '1Gi', {}, cache_from):
+        assert line == fake_log_value
+    called_args, called_kwargs = fake_client.build.call_args
+    assert 'cache_from' in called_kwargs
+    assert called_kwargs['cache_from'] == cache_from
+
diff --git a/tests/test_clone_depth.py b/tests/unit/test_clone_depth.py
similarity index 67%
rename from tests/test_clone_depth.py
rename to tests/unit/test_clone_depth.py
index 4ef8c90e..cef95e87 100644
--- a/tests/test_clone_depth.py
+++ b/tests/unit/test_clone_depth.py
@@ -21,15 +21,16 @@ def test_clone_depth():
     """Test a remote repository, without a refspec"""
 
     with TemporaryDirectory() as d:
-        app = Repo2Docker()
-        argv = [URL]
-        app.initialize(argv)
-        app.build = False
-        app.run = False
-        # turn of automatic clean up of the checkout so we can inspect it
-        # we also set the work directory explicitly so we know where to look
-        app.cleanup_checkout = False
-        app.git_workdir = d
+        app = Repo2Docker(
+            repo=URL,
+            dry_run=True,
+            run=False,
+            # turn of automatic clean up of the checkout so we can inspect it
+            # we also set the work directory explicitly so we know where to look
+            cleanup_checkout=False,
+            git_workdir=d,
+        )
+        app.initialize()
         app.start()
 
         cmd = ['git', 'rev-parse', 'HEAD']
@@ -46,15 +47,17 @@ def test_clone_depth_full():
     """Test a remote repository, with a refspec of 'master'"""
 
     with TemporaryDirectory() as d:
-        app = Repo2Docker()
-        argv = ['--ref', 'master', URL]
-        app.initialize(argv)
-        app.run = False
-        app.build = False
-        # turn of automatic clean up of the checkout so we can inspect it
-        # we also set the work directory explicitly so we know where to look
-        app.cleanup_checkout = False
-        app.git_workdir = d
+        app = Repo2Docker(
+            repo=URL,
+            ref='master',
+            dry_run=True,
+            run=False,
+            # turn of automatic clean up of the checkout so we can inspect it
+            # we also set the work directory explicitly so we know where to look
+            cleanup_checkout=False,
+            git_workdir=d,
+        )
+        app.initialize()
         app.start()
 
         # Building the image has already put us in the cloned repository directory
@@ -72,16 +75,17 @@ def test_clone_depth_full2():
     """Test a remote repository, with a refspec of the master commit hash"""
 
     with TemporaryDirectory() as d:
-        app = Repo2Docker()
-        argv = ['--ref', '703322e', URL]
-
-        app.initialize(argv)
-        app.run = False
-        app.build = False
-        # turn of automatic clean up of the checkout so we can inspect it
-        # we also set the work directory explicitly so we know where to look
-        app.cleanup_checkout = False
-        app.git_workdir = d
+        app = Repo2Docker(
+            repo=URL,
+            ref='703322e',
+            dry_run=True,
+            run=False,
+            # turn of automatic clean up of the checkout so we can inspect it
+            # we also set the work directory explicitly so we know where to look
+            cleanup_checkout=False,
+            git_workdir=d,
+        )
+        app.initialize()
         app.start()
 
         # Building the image has already put us in the cloned repository directory
@@ -99,16 +103,17 @@ def test_clone_depth_mid():
     """Test a remote repository, with a refspec of a commit hash halfway"""
 
     with TemporaryDirectory() as d:
-        app = Repo2Docker()
-        argv = ['--ref', '8bc4f21', URL]
-
-        app.initialize(argv)
-        app.run = False
-        app.build = False
-        # turn of automatic clean up of the checkout so we can inspect it
-        # we also set the work directory explicitly so we know where to look
-        app.cleanup_checkout = False
-        app.git_workdir = d
+        app = Repo2Docker(
+            repo=URL,
+            ref='8bc4f21',
+            dry_run=True,
+            run=False,
+            # turn of automatic clean up of the checkout so we can inspect it
+            # we also set the work directory explicitly so we know where to look
+            cleanup_checkout=False,
+            git_workdir=d,
+        )
+        app.initialize()
         app.start()
 
         # Building the image has already put us in the cloned repository directory
diff --git a/tests/test_connect_url.py b/tests/unit/test_connect_url.py
similarity index 84%
rename from tests/test_connect_url.py
rename to tests/unit/test_connect_url.py
index 5c5e1acf..4769f1e3 100644
--- a/tests/test_connect_url.py
+++ b/tests/unit/test_connect_url.py
@@ -7,17 +7,11 @@ from repo2docker.app import Repo2Docker
 
 def test_connect_url(tmpdir):
     tmpdir.chdir()
-    #q = tmpdir.join("environment.yml")
-    #q.write("dependencies:\n"
-    #        "  -  notebook==5.6.0")
     p = tmpdir.join("requirements.txt")
-    p.write("notebook==5.6.0")
+    p.write("notebook>=5.6.0")
 
-    app = Repo2Docker()
-    argv = [str(tmpdir), ]
-    app.initialize(argv)
-    app.debug = True
-    app.run = False
+    app = Repo2Docker(repo=str(tmpdir), run=False)
+    app.initialize()
     app.start()  # This just build the image and does not run it.
     container = app.start_container()
     container_url = 'http://{}:{}/api'.format(app.hostname, app.port)
diff --git a/tests/test_editable.py b/tests/unit/test_editable.py
similarity index 87%
rename from tests/test_editable.py
rename to tests/unit/test_editable.py
index 1bbd05d4..87d66ee6 100644
--- a/tests/test_editable.py
+++ b/tests/unit/test_editable.py
@@ -1,12 +1,14 @@
 import os
-import time
 import re
 import tempfile
-from conftest import make_test_func
+import time
+
 from repo2docker.app import Repo2Docker
+from repo2docker.__main__ import make_r2d
 
+from conftest import make_test_func
 
-DIR = os.path.join(os.path.dirname(__file__), 'dockerfile', 'editable')
+DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'dockerfile', 'editable')
 
 
 def test_editable(run_repo2docker):
@@ -33,10 +35,9 @@ def test_editable_by_host():
     """Test whether a new file created by the host environment, is
     detected in the container"""
 
-    app = Repo2Docker()
-    app.initialize(['--editable', DIR])
-    app.run = False
-    app.start()  # This just build the image and does not run it.
+    app = make_r2d(['--editable', DIR])
+    app.initialize()
+    app.build()
     container = app.start_container()
     # give the container a chance to start
     time.sleep(1)
diff --git a/tests/env.py b/tests/unit/test_env.py
similarity index 100%
rename from tests/env.py
rename to tests/unit/test_env.py
diff --git a/tests/test_env_yml.py b/tests/unit/test_env_yml.py
similarity index 100%
rename from tests/test_env_yml.py
rename to tests/unit/test_env_yml.py
diff --git a/tests/test_labels.py b/tests/unit/test_labels.py
similarity index 88%
rename from tests/test_labels.py
rename to tests/unit/test_labels.py
index 123eb9e3..2b667749 100644
--- a/tests/test_labels.py
+++ b/tests/unit/test_labels.py
@@ -26,22 +26,19 @@ def test_buildpack_labels_rendered():
     (None, None, 'local'),
 ])
 def test_Repo2Docker_labels(ref, repo, expected_repo_label, tmpdir):
-    if repo is None:
-        repo = str(tmpdir)
-    if ref is not None:
-        argv = ['--ref', ref, repo]
-    else:
-        argv = [repo]
-
-    app = Repo2Docker()
+    app = Repo2Docker(dry_run=True)
     # Add mock BuildPack to app
     mock_buildpack = Mock()
     mock_buildpack.return_value.labels = {}
     app.buildpacks = [mock_buildpack]
 
-    app.initialize(argv)
-    app.build = False
-    app.run = False
+    if repo is None:
+        repo = str(tmpdir)
+    app.repo = repo
+    if ref is not None:
+        app.ref = ref
+
+    app.initialize()
     app.start()
     expected_labels = {
         'repo2docker.ref': ref,
diff --git a/tests/unit/test_memlimit.py b/tests/unit/test_memlimit.py
new file mode 100644
index 00000000..b8c53c6c
--- /dev/null
+++ b/tests/unit/test_memlimit.py
@@ -0,0 +1,85 @@
+"""
+Test that build time memory limits are actually enforced.
+
+We give the container image at least 128M of RAM (so base things like
+apt and pip can run), and then try to allocate & use 256MB in postBuild.
+This should fail!
+"""
+
+import os
+import shutil
+import time
+
+import pytest
+
+from repo2docker.app import Repo2Docker
+
+
+basedir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+def does_build(tmpdir, build_src_dir, mem_limit, mem_allocate_mb):
+    builddir = tmpdir.join('build')
+    shutil.copytree(build_src_dir, builddir)
+    builddir.chdir()
+    print(os.getcwd(), os.listdir('.'))
+    mem_allocate_mb_file = os.path.join(builddir, 'mem_allocate_mb')
+
+    # Cache bust so we actually do a rebuild each time this is run!
+    with builddir.join('cachebust').open('w') as cachebust:
+        cachebust.write(str(time.time()))
+
+    # we don't have an easy way to pass env vars or whatever to
+    # postBuild from here, so we write a file into the repo that is
+    # read by the postBuild script!
+    with open(mem_allocate_mb_file, 'w') as f:
+        f.write(str(mem_allocate_mb))
+    r2d = Repo2Docker(build_memory_limit=str(mem_limit) + 'M')
+    r2d.initialize()
+    try:
+        r2d.build()
+    except Exception:
+        return False
+    else:
+        return True
+
+
+@pytest.mark.parametrize(
+    'test, mem_limit, mem_allocate_mb, expected',
+    [
+        ('dockerfile', 128, 256, False),
+        ('dockerfile', 512, 256, True),
+        ('non-dockerfile', 128, 256, False),
+        ('non-dockerfile', 512, 256, True),
+    ]
+)
+def test_memlimit_nondockerfile(tmpdir, test, mem_limit, mem_allocate_mb, expected):
+    """
+    Test if memory limited builds are working for non dockerfile builds
+    """
+    success = does_build(
+        tmpdir,
+        os.path.join(basedir, 'memlimit', test),
+        mem_limit,
+        mem_allocate_mb,
+    )
+    assert success == expected
+
+
+
+def test_memlimit_same_postbuild():
+    """
+    Validate that the postBuild files for dockerfile & nondockerfile are same
+
+    Until https://github.com/jupyter/repo2docker/issues/160 gets fixed.
+    """
+    filepaths = [
+        os.path.join(basedir, 'memlimit', t, "postBuild")
+        for t in ("dockerfile", "non-dockerfile")
+    ]
+    file_contents = []
+    for fp in filepaths:
+        with open(fp) as f:
+            file_contents.append(f.read())
+    # Make sure they're all the same
+    assert len(set(file_contents)) == 1
diff --git a/tests/unit/test_ports.py b/tests/unit/test_ports.py
new file mode 100644
index 00000000..f2c3ae97
--- /dev/null
+++ b/tests/unit/test_ports.py
@@ -0,0 +1,124 @@
+"""
+Test Port mappings work on running non-jupyter workflows
+"""
+
+import requests
+import time
+import os
+import tempfile
+import random
+
+import docker
+import pytest
+
+from repo2docker.app import Repo2Docker
+
+
+def read_port_mapping_response(request, tmpdir, host=None, port='',
+                               all_ports=False, protocol=None):
+    """
+    Deploy container and test if port mappings work as expected
+
+    Args:
+        request: pytest request fixture
+        host: the host interface to bind to
+        port: the random host port to bind to
+        protocol: the protocol to use valid values /tcp or /udp
+    """
+    port_protocol = '8888'
+    if protocol:
+        port_protocol += protocol
+    host_port = port
+    if host:
+        host_port = (host, port)
+    else:
+        host = 'localhost'
+
+    if port:
+        ports = {port_protocol: host_port}
+    else:
+        ports = {}
+
+    # run in an empty temporary directory
+    td = tempfile.TemporaryDirectory()
+    # cleanup at the end of the test
+    request.addfinalizer(td.cleanup)
+    tmpdir.chdir()
+
+    username = os.getlogin()
+    tmpdir.mkdir('username')
+    r2d = Repo2Docker(
+        repo=str(tmpdir.mkdir('repo')),
+        user_id=os.geteuid(),
+        user_name=username,
+        all_ports=all_ports,
+        ports=ports,
+        run=True,
+        run_cmd=['python', '-m', 'http.server', '8888'],
+    )
+    r2d.initialize()
+    r2d.build()
+    # create container
+    container = r2d.start_container()
+
+    # register cleanup first thing so we don't leave it lying around
+    def _cleanup():
+        container.reload()
+        if container.status == 'running':
+            container.kill()
+        try:
+            container.remove()
+        except docker.errors.NotFound:
+            pass
+    request.addfinalizer(_cleanup)
+
+    container.reload()
+    assert container.status == 'running'
+    port_mapping = container.attrs['NetworkSettings']['Ports']
+    if all_ports:
+        port = port_mapping['8888/tcp'][0]['HostPort']
+
+    url = 'http://{}:{}'.format(host, port)
+    for i in range(5):
+        try:
+            r = requests.get(url)
+            r.raise_for_status()
+        except Exception as e:
+            print("No response from {}: {}".format(url, e))
+            container.reload()
+            assert container.status == 'running'
+            time.sleep(3)
+            continue
+        else:
+            break
+    else:
+        pytest.fail("Never succeded in talking to %s" % url)
+    assert 'Directory listing' in r.text
+
+
+def test_all_port_mapping_response(request, tmpdir):
+    """
+    Deploy container and test if all port exposed works as expected
+    """
+    read_port_mapping_response(request, tmpdir, all_ports=True)
+
+
+@pytest.mark.parametrize(
+    'host, protocol',
+    [
+        (None, None),
+        ('127.0.0.1', None),
+        (None, '/tcp'),
+    ]
+)
+def test_port_mapping(request, tmpdir, host, protocol):
+    """Test a port mapping"""
+    port = str(random.randint(50000, 51000))
+    read_port_mapping_response(
+        request,
+        tmpdir,
+        host=host,
+        port=port,
+        protocol=protocol,
+    )
+
diff --git a/tests/test_subdir.py b/tests/unit/test_subdir.py
similarity index 61%
rename from tests/test_subdir.py
rename to tests/unit/test_subdir.py
index 49f8e81f..89f0884b 100644
--- a/tests/test_subdir.py
+++ b/tests/unit/test_subdir.py
@@ -27,18 +27,10 @@ def test_subdir_invalid(caplog):
     # test an error is raised when requesting a non existent subdir
     #caplog.set_level(logging.INFO, logger='Repo2Docker')
 
-    app = Repo2Docker()
-    argv = ['--subdir', 'invalid-sub-dir', TEST_REPO]
-    app.initialize(argv)
-    app.debug = True
-    # no build does not imply no run
-    app.build = False
-    app.run = False
-    with pytest.raises(SystemExit) as excinfo:
-        app.start()  # Just build the image and do not run it.
-
-    # The build should fail
-    assert excinfo.value.code == 1
-
-    # Can't get this to record the logs?
-    #assert caplog.text == "Subdirectory tests/conda/invalid does not exist"
+    app = Repo2Docker(
+        repo=TEST_REPO,
+        subdir='invalid-sub-dir',
+    )
+    app.initialize()
+    with pytest.raises(FileNotFoundError):
+        app.build()  # Just build the image and do not run it.
diff --git a/tests/users.py b/tests/unit/test_users.py
similarity index 96%
rename from tests/users.py
rename to tests/unit/test_users.py
index a3714e2c..b7fd07d7 100644
--- a/tests/users.py
+++ b/tests/unit/test_users.py
@@ -15,6 +15,7 @@ def test_user():
     username = os.getlogin()
     userid = str(os.geteuid())
     with tempfile.TemporaryDirectory() as tmpdir:
+        tmpdir = os.path.realpath(tmpdir)
         subprocess.check_call([
             'repo2docker',
             '-v', '{}:/home/{}'.format(tmpdir, username),
diff --git a/tests/test_utils.py b/tests/unit/test_utils.py
similarity index 87%
rename from tests/test_utils.py
rename to tests/unit/test_utils.py
index 8c9dc1a0..54c58640 100644
--- a/tests/test_utils.py
+++ b/tests/unit/test_utils.py
@@ -40,12 +40,12 @@ def test_capture_cmd_capture_fail():
             assert line == 'test\n'
 
 
-def test_chdir():
-    with TemporaryDirectory() as d:
-        cur_cwd = os.getcwd()
-        with utils.chdir(d):
-            assert os.getcwd() == d
-        assert os.getcwd() == cur_cwd
+def test_chdir(tmpdir):
+    d = str(tmpdir.mkdir('cwd'))
+    cur_cwd = os.getcwd()
+    with utils.chdir(d):
+        assert os.getcwd() == d
+    assert os.getcwd() == cur_cwd
 
 
 def test_byte_spec_validation():
@@ -63,4 +63,4 @@ def test_byte_spec_validation():
         bs.validate(None, 'NK')
 
     with pytest.raises(traitlets.TraitError):
-        bs.validate(None, '1m')
\ No newline at end of file
+        bs.validate(None, '1m')
diff --git a/tests/volumes.py b/tests/unit/test_volumes.py
similarity index 97%
rename from tests/volumes.py
rename to tests/unit/test_volumes.py
index 5d283562..edd65267 100644
--- a/tests/volumes.py
+++ b/tests/unit/test_volumes.py
@@ -12,6 +12,8 @@ def test_volume_abspath():
     """
     ts = str(time.time())
     with tempfile.TemporaryDirectory() as tmpdir:
+        tmpdir = os.path.realpath(tmpdir)
+
         username = os.getlogin()
         subprocess.check_call([
             'repo2docker',