diff --git a/.travis.yml b/.travis.yml
index 625c2f2e..dd8f22a2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,7 +7,7 @@ python:
 
 install:
   - pip install -e .
-  - pip install pytest-xdist
+  - pip install pytest-xdist pyyaml
 
 script:
   # Run at least 4 tests in parallel. These tests are all fully
@@ -19,3 +19,4 @@ env:
   - REPO_TYPE=venv
   - REPO_TYPE=julia
   - REPO_TYPE=dockerfile
+  - REPO_TYPE=external/*
diff --git a/tests/conftest.py b/tests/conftest.py
index dc8a0a82..2d1e7acc 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -9,17 +9,20 @@ success.
 """
 import pytest
 import subprocess
+import yaml
+import shlex
 
 def pytest_collect_file(parent, path):
     if path.basename == 'verify':
-        return Repo(path, parent)
+        return LocalRepo(path, parent)
+    elif path.basename.endswith('.repos.yaml'):
+        return RemoteRepoList(path, parent)
 
-class Repo(pytest.File):
+class LocalRepo(pytest.File):
     def collect(self):
-        yield RepoTest(self.fspath.basename, self, self.fspath)
+        yield LocalRepoTest(self.fspath.basename, self, self.fspath)
 
-
-class RepoTest(pytest.Item):
+class LocalRepoTest(pytest.Item):
     def __init__(self, name, parent, path):
         super().__init__(name, parent)
         self.path = path
@@ -30,3 +33,27 @@ class RepoTest(pytest.Item):
             str(self.path.dirname),
             './verify'
         ])
+
+
+class RemoteRepoList(pytest.File):
+    def collect(self):
+        with self.fspath.open() as f:
+            repos = yaml.safe_load(f)
+        for repo in repos:
+            yield RemoteRepoTest(repo['name'], self, repo['url'], repo['ref'], repo['verify'])
+
+
+class RemoteRepoTest(pytest.Item):
+    def __init__(self, name, parent, url, ref, verify):
+        super().__init__(name, parent)
+        self.url = url
+        self.ref = ref
+        self.verify = verify
+
+    def runtest(self):
+        subprocess.check_call([
+            'jupyter-repo2docker',
+            '--ref', self.ref,
+            self.url,
+            '--',
+        ] + shlex.split(self.verify))
diff --git a/tests/external/datasci.repos.yaml b/tests/external/datasci.repos.yaml
new file mode 100644
index 00000000..aeeed4e0
--- /dev/null
+++ b/tests/external/datasci.repos.yaml
@@ -0,0 +1,5 @@
+# A bunch of external datascience related repos we care about!
+- name: Jake's Data Science Book
+  url: https://github.com/jakevdp/PythonDataScienceHandbook
+  ref: de0cc6bd31
+  verify: python -c 'import matplotlib'
diff --git a/tests/external/reproductions.repos.yaml b/tests/external/reproductions.repos.yaml
new file mode 100644
index 00000000..668424b2
--- /dev/null
+++ b/tests/external/reproductions.repos.yaml
@@ -0,0 +1,5 @@
+# A bunch of external repos that reproduce something cool we care about
+- name: LIGO Gravitational Waves
+  url: https://github.com/minrk/ligo-binder/
+  ref: b8259dac9eb
+  verify: python -c 'import matplotlib'