Add Python integration test

.travis.yml

@@ -1,37 +1,16 @@
 sudo: required
-language: bash
+language: python
+python:
+  - "3.4"
 services:
   - docker
 before_install:
-  - wget -nc -P "$TRAVIS_BUILD_DIR/import" "http://download.geofabrik.de/europe/liechtenstein-160601.osm.pbf"
+  - wget -nc -P "$TRAVIS_BUILD_DIR/import" "http://download.geofabrik.de/europe/albania-160601.osm.pbf"
+  - wget -nc -O "planet.mbtiles" -P "$TRAVIS_BUILD_DIR/export" "https://osm2vectortiles-downloads.os.zhdk.cloud.switch.ch/v2.0/planet_z0-z5.mbtiles"
   - docker-compose pull import-external export import-osm postgis
   # Avoid building really expensive images
   - make fast
+install: "pip install -r ./tools/integration-test/requirements.txt"
 script:
   # Test import
-  - docker-compose up -d postgis rabbitmq mock-s3
-  - sleep 10
-  - docker-compose run import-external
-  - docker-compose run import-osm
-  - docker-compose run import-sql
-  # Test local export
-  - docker-compose run export && mv export/tiles.mbtiles export/planet.mbtiles
-  # Test distributed export
-  - docker-compose run -e TILE_X=539 -e TILE_Y=359 -e TILE_Z=10 -e JOB_ZOOM=11 generate-jobs
-  - docker-compose scale export-worker=2
-  - sleep 60
-  - docker-compose scale export-worker=0
-  - docker-compose run -d merge-jobs
-  - sleep 10
-  - docker-compose stop merge-jobs
-  # Test changed tiles
-  - docker-compose run update-osm-diff
-  - docker-compose run import-osm-diff
-  - docker-compose run changed-tiles
-  - docker-compose run generate-diff-jobs
-  - docker-compose scale export-worker=1
-  - sleep 30
-  - docker-compose scale export-worker=0
-  - docker-compose run merge-jobs
-  - sleep 10
-  - docker-compose stop merge-jobs
+  - py.test -x ./tools/integration-test/integration_test.py

Makefile

@@ -2,7 +2,7 @@
 
 all: postgis export-mbtiles import-osm import-external import-sql create-extracts changed-tiles generate-jobs merge-jobs compare-visual mapping-qa-report
 
-fast: postgis export-mbtiles import-osm import-sql create-extracts changed-tiles generate-jobs merge-jobs
+fast: postgis export-mbtiles import-osm import-sql create-extracts changed-tiles generate-jobs merge-jobs integration-test
 
 postgis:
 	docker build -t osm2vectortiles/postgis src/postgis
@@ -36,3 +36,6 @@ compare-visual:
 
 mapping-qa-report:
 	docker build -t osm2vectortiles/mapping-qa-report tools/mapping-qa-report
+
+integration-test:
+	docker build -t osm2vectortiles/integration-test tools/integration-test

docker-compose.yml

@@ -47,6 +47,11 @@ merge-osm-diff:
   command: ./import-mergediffs.sh
   volumes:
    - ./import:/data/import
+integration-test:
+  image: "osm2vectortiles/integration-test"
+  volumes:
+   - /var/run/docker.sock:/var/run/docker.sock:ro
+   - ./:/usr/src/osm2vectortiles
 import-external:
   image: "osm2vectortiles/import-external"
   links:

tools/integration-test/integration_test.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python
+""""
+Integration test for entire OSM2VectorTiles workflow.
+Primary purpose is to verify that everything works together
+not to verify correctness.
+"""
+
+import os
+import time
+import subprocess
+import pytest
+from mbtoolbox.verify import list_required_tiles, missing_tiles
+from mbtoolbox.mbtiles import MBTiles
+
+PARENT_PROJECT_DIR = os.path.join(os.path.realpath(__file__), '../../../')
+PROJECT_DIR = os.path.abspath(os.getenv('PROJECT_DIR', PARENT_PROJECT_DIR))
+ALBANIA_BBOX = '19.6875,40.97989806962015,20.390625,41.50857729743933'
+ALBANIA_TIRANA_TILE = (284, 191, 9)
+
+
+class DockerCompose(object):
+    def __init__(self, project_dir=PROJECT_DIR):
+        self.project_dir = project_dir
+
+    def compose(self, args):
+        subprocess.check_call(['docker-compose'] + args, cwd=self.project_dir)
+
+    def run(self, args):
+        self.compose(['run'] + args)
+
+    def scale(self, container, count):
+        self.compose(['scale', '{}={}'.format(container, count)])
+
+    def up(self, container):
+        self.compose(['up', '-d', container])
+
+    def stop(self, container):
+        self.compose(['stop', container])
+
+    def remove_all(self):
+        self.compose(['stop'])
+        self.compose(['rm', '-v', '--all', '--force'])
+
+
+@pytest.mark.run(order=1)
+def test_postgis_startup():
+    print(PROJECT_DIR)
+    dc = DockerCompose()
+    dc.remove_all()
+    dc.up('postgis')
+    time.sleep(5)
+
+
+@pytest.mark.run(order=2)
+def test_import_external():
+    dc = DockerCompose()
+    dc.run(['import-external'])
+
+
+@pytest.mark.run(order=3)
+def test_import_osm():
+    dc = DockerCompose()
+    dc.run(['import-osm'])
+
+
+@pytest.mark.run(order=4)
+def test_import_sql():
+    dc = DockerCompose()
+    dc.run(['import-sql'])
+
+
+@pytest.mark.run(order=5)
+def test_local_export():
+    "Test export of local Liechtenstein bbox and verify all tiles are present"
+    dc = DockerCompose()
+
+    def export_bbox(bbox, min_zoom, max_zoom):
+        dc.run([
+            '-e', 'BBOX={}'.format(bbox),
+            '-e', 'MIN_ZOOM={}'.format(min_zoom),
+            '-e', 'MAX_ZOOM={}'.format(max_zoom),
+            'export'
+        ])
+
+    def verify_no_tiles_missing(x, y, min_z, max_z):
+        exported_mbtiles = os.path.join(PROJECT_DIR, 'export/tiles.mbtiles')
+        mbtiles = MBTiles(exported_mbtiles, 'tms')
+        required_tiles = list_required_tiles(x, y, min_z, max_z)
+        assert list(missing_tiles(mbtiles, required_tiles)) == []
+
+    tile_x, tile_y, tile_z = ALBANIA_TIRANA_TILE
+    export_bbox(ALBANIA_BBOX, tile_z, 14)
+
+    # There are missing tiles on z14 because
+    # Albania does not have data at some places
+    verify_no_tiles_missing(tile_x, tile_y, tile_z, 13)
+    exported_mbtiles = os.path.join(PROJECT_DIR, 'export/tiles.mbtiles')
+    tiles = find_missing_tiles(exported_mbtiles, tile_x, tile_y, tile_z, 13)
+    assert tiles == []
+
+
+def find_missing_tiles(mbtiles_file, x, y, min_z, max_z):
+    mbtiles = MBTiles(mbtiles_file, 'tms')
+    required_tiles = list_required_tiles(x, y, min_z, max_z)
+    return list(missing_tiles(mbtiles, required_tiles))
+
+
+@pytest.mark.run(order=6)
+def test_distributed_worker():
+    dc = DockerCompose()
+
+    def schedule_tile_jobs(x, y, z, job_zoom):
+        dc.run([
+            '-e', 'TILE_X={}'.format(x),
+            '-e', 'TILE_Y={}'.format(y),
+            '-e', 'TILE_Z={}'.format(z),
+            '-e', 'JOB_ZOOM={}'.format(job_zoom),
+            'generate-jobs'
+        ])
+
+    dc.up('rabbitmq')
+    dc.up('mock-s3')
+    time.sleep(5)
+
+    tile_x, tile_y, tile_z = ALBANIA_TIRANA_TILE
+    schedule_tile_jobs(tile_x, tile_y, tile_z, tile_z + 2)
+
+    dc.scale('export-worker', 2)
+    time.sleep(60)
+    dc.scale('export-worker', 0)
+
+    dc.up('merge-jobs')
+    time.sleep(10)
+    dc.stop('merge-jobs')
+
+    # Merge jobs will merge all results into the existing planet.mbtiles
+    # if MBTiles contains all the Albania tiles the export was successful
+    exported_mbtiles = os.path.join(PROJECT_DIR, 'export/planet.mbtiles')
+    tiles = find_missing_tiles(exported_mbtiles, tile_x, tile_y, tile_z, 13)
+    assert tiles == []
+
+
+@pytest.mark.run(order=7)
+def test_diff_update():
+    dc = DockerCompose()
+
+    # Pull the latest diffs
+    baseurl = 'http://download.geofabrik.de/europe/albania-updates/'
+    dc.run(['-e', 'OSM_UPDATE_BASEURL={}'.format(baseurl), 'update-osm-diff'])
+
+    # Import diffs and calculate the changed tiles
+    dc.run(['import-osm-diff'])
+    dc.run(['changed-tiles'])
+
+    # Read and verify that at least one tile is marked dirty
+    tile_file = os.path.join(PROJECT_DIR, 'export/tiles.txt')
+    num_lines = sum(1 for line in open(tile_file))
+    assert num_lines > 0
+
+
+@pytest.mark.run(order=8)
+def test_diff_jobs():
+    dc = DockerCompose()
+
+    # Schedule changed tiles as jobs
+    dc.run(['generate-diff-jobs'])
+
+    dc.scale('export-worker', 2)
+    time.sleep(60)
+    dc.scale('export-worker', 0)
+
+    dc.up('merge-jobs')
+    time.sleep(10)
+    dc.stop('merge-jobs')
+
+    # Test if the MBTiles is still complete
+    # This does not verify whether new data has been added successfully
+    exported_mbtiles = os.path.join(PROJECT_DIR, 'export/planet.mbtiles')
+    tile_x, tile_y, tile_z = ALBANIA_TIRANA_TILE
+    tiles = find_missing_tiles(exported_mbtiles, tile_x, tile_y, tile_z, 13)
+    assert tiles == []

tools/integration-test/requirements.txt

@@ -0,0 +1,4 @@
+pytest==2.9.1
+docker-compose==1.7.1
+pytest-ordering==0.4
+-e git+https://github.com/lukasmartinelli/mbtoolbox.git@#egg=mbtoolbox