diff --git a/.travis.yml b/.travis.yml index 5a7aecb..609f737 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,37 +1,16 @@ sudo: required -language: bash +language: python +python: + - "3.4" services: - docker before_install: - - wget -nc -P "$TRAVIS_BUILD_DIR/import" "http://download.geofabrik.de/europe/liechtenstein-160601.osm.pbf" + - wget -nc -P "$TRAVIS_BUILD_DIR/import" "http://download.geofabrik.de/europe/albania-160601.osm.pbf" + - wget -nc -O "planet.mbtiles" -P "$TRAVIS_BUILD_DIR/export" "https://osm2vectortiles-downloads.os.zhdk.cloud.switch.ch/v2.0/planet_z0-z5.mbtiles" - docker-compose pull import-external export import-osm postgis # Avoid building really expensive images - make fast +install: "pip install -r ./tools/integration-test/requirements.txt" script: # Test import - - docker-compose up -d postgis rabbitmq mock-s3 - - sleep 10 - - docker-compose run import-external - - docker-compose run import-osm - - docker-compose run import-sql - # Test local export - - docker-compose run export && mv export/tiles.mbtiles export/planet.mbtiles - # Test distributed export - - docker-compose run -e TILE_X=539 -e TILE_Y=359 -e TILE_Z=10 -e JOB_ZOOM=11 generate-jobs - - docker-compose scale export-worker=2 - - sleep 60 - - docker-compose scale export-worker=0 - - docker-compose run -d merge-jobs - - sleep 10 - - docker-compose stop merge-jobs - # Test changed tiles - - docker-compose run update-osm-diff - - docker-compose run import-osm-diff - - docker-compose run changed-tiles - - docker-compose run generate-diff-jobs - - docker-compose scale export-worker=1 - - sleep 30 - - docker-compose scale export-worker=0 - - docker-compose run merge-jobs - - sleep 10 - - docker-compose stop merge-jobs + - py.test -x ./tools/integration-test/integration_test.py diff --git a/Makefile b/Makefile index da83d28..50932c6 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: postgis export-mbtiles import-osm import-external import-sql create-extracts changed-tiles generate-jobs merge-jobs compare-visual mapping-qa-report -fast: postgis export-mbtiles import-osm import-sql create-extracts changed-tiles generate-jobs merge-jobs +fast: postgis export-mbtiles import-osm import-sql create-extracts changed-tiles generate-jobs merge-jobs integration-test postgis: docker build -t osm2vectortiles/postgis src/postgis @@ -36,3 +36,6 @@ compare-visual: mapping-qa-report: docker build -t osm2vectortiles/mapping-qa-report tools/mapping-qa-report + +integration-test: + docker build -t osm2vectortiles/integration-test tools/integration-test diff --git a/docker-compose.yml b/docker-compose.yml index d9b3fd6..0444f57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,6 +47,11 @@ merge-osm-diff: command: ./import-mergediffs.sh volumes: - ./import:/data/import +integration-test: + image: "osm2vectortiles/integration-test" + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./:/usr/src/osm2vectortiles import-external: image: "osm2vectortiles/import-external" links: diff --git a/tools/integration-test/integration_test.py b/tools/integration-test/integration_test.py new file mode 100644 index 0000000..86a04f0 --- /dev/null +++ b/tools/integration-test/integration_test.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python +"""" +Integration test for entire OSM2VectorTiles workflow. +Primary purpose is to verify that everything works together +not to verify correctness. +""" + +import os +import time +import subprocess +import pytest +from mbtoolbox.verify import list_required_tiles, missing_tiles +from mbtoolbox.mbtiles import MBTiles + +PARENT_PROJECT_DIR = os.path.join(os.path.realpath(__file__), '../../../') +PROJECT_DIR = os.path.abspath(os.getenv('PROJECT_DIR', PARENT_PROJECT_DIR)) +ALBANIA_BBOX = '19.6875,40.97989806962015,20.390625,41.50857729743933' +ALBANIA_TIRANA_TILE = (284, 191, 9) + + +class DockerCompose(object): + def __init__(self, project_dir=PROJECT_DIR): + self.project_dir = project_dir + + def compose(self, args): + subprocess.check_call(['docker-compose'] + args, cwd=self.project_dir) + + def run(self, args): + self.compose(['run'] + args) + + def scale(self, container, count): + self.compose(['scale', '{}={}'.format(container, count)]) + + def up(self, container): + self.compose(['up', '-d', container]) + + def stop(self, container): + self.compose(['stop', container]) + + def remove_all(self): + self.compose(['stop']) + self.compose(['rm', '-v', '--all', '--force']) + + +@pytest.mark.run(order=1) +def test_postgis_startup(): + print(PROJECT_DIR) + dc = DockerCompose() + dc.remove_all() + dc.up('postgis') + time.sleep(5) + + +@pytest.mark.run(order=2) +def test_import_external(): + dc = DockerCompose() + dc.run(['import-external']) + + +@pytest.mark.run(order=3) +def test_import_osm(): + dc = DockerCompose() + dc.run(['import-osm']) + + +@pytest.mark.run(order=4) +def test_import_sql(): + dc = DockerCompose() + dc.run(['import-sql']) + + +@pytest.mark.run(order=5) +def test_local_export(): + "Test export of local Liechtenstein bbox and verify all tiles are present" + dc = DockerCompose() + + def export_bbox(bbox, min_zoom, max_zoom): + dc.run([ + '-e', 'BBOX={}'.format(bbox), + '-e', 'MIN_ZOOM={}'.format(min_zoom), + '-e', 'MAX_ZOOM={}'.format(max_zoom), + 'export' + ]) + + def verify_no_tiles_missing(x, y, min_z, max_z): + exported_mbtiles = os.path.join(PROJECT_DIR, 'export/tiles.mbtiles') + mbtiles = MBTiles(exported_mbtiles, 'tms') + required_tiles = list_required_tiles(x, y, min_z, max_z) + assert list(missing_tiles(mbtiles, required_tiles)) == [] + + tile_x, tile_y, tile_z = ALBANIA_TIRANA_TILE + export_bbox(ALBANIA_BBOX, tile_z, 14) + + # There are missing tiles on z14 because + # Albania does not have data at some places + verify_no_tiles_missing(tile_x, tile_y, tile_z, 13) + exported_mbtiles = os.path.join(PROJECT_DIR, 'export/tiles.mbtiles') + tiles = find_missing_tiles(exported_mbtiles, tile_x, tile_y, tile_z, 13) + assert tiles == [] + + +def find_missing_tiles(mbtiles_file, x, y, min_z, max_z): + mbtiles = MBTiles(mbtiles_file, 'tms') + required_tiles = list_required_tiles(x, y, min_z, max_z) + return list(missing_tiles(mbtiles, required_tiles)) + + +@pytest.mark.run(order=6) +def test_distributed_worker(): + dc = DockerCompose() + + def schedule_tile_jobs(x, y, z, job_zoom): + dc.run([ + '-e', 'TILE_X={}'.format(x), + '-e', 'TILE_Y={}'.format(y), + '-e', 'TILE_Z={}'.format(z), + '-e', 'JOB_ZOOM={}'.format(job_zoom), + 'generate-jobs' + ]) + + dc.up('rabbitmq') + dc.up('mock-s3') + time.sleep(5) + + tile_x, tile_y, tile_z = ALBANIA_TIRANA_TILE + schedule_tile_jobs(tile_x, tile_y, tile_z, tile_z + 2) + + dc.scale('export-worker', 2) + time.sleep(60) + dc.scale('export-worker', 0) + + dc.up('merge-jobs') + time.sleep(10) + dc.stop('merge-jobs') + + # Merge jobs will merge all results into the existing planet.mbtiles + # if MBTiles contains all the Albania tiles the export was successful + exported_mbtiles = os.path.join(PROJECT_DIR, 'export/planet.mbtiles') + tiles = find_missing_tiles(exported_mbtiles, tile_x, tile_y, tile_z, 13) + assert tiles == [] + + +@pytest.mark.run(order=7) +def test_diff_update(): + dc = DockerCompose() + + # Pull the latest diffs + baseurl = 'http://download.geofabrik.de/europe/albania-updates/' + dc.run(['-e', 'OSM_UPDATE_BASEURL={}'.format(baseurl), 'update-osm-diff']) + + # Import diffs and calculate the changed tiles + dc.run(['import-osm-diff']) + dc.run(['changed-tiles']) + + # Read and verify that at least one tile is marked dirty + tile_file = os.path.join(PROJECT_DIR, 'export/tiles.txt') + num_lines = sum(1 for line in open(tile_file)) + assert num_lines > 0 + + +@pytest.mark.run(order=8) +def test_diff_jobs(): + dc = DockerCompose() + + # Schedule changed tiles as jobs + dc.run(['generate-diff-jobs']) + + dc.scale('export-worker', 2) + time.sleep(60) + dc.scale('export-worker', 0) + + dc.up('merge-jobs') + time.sleep(10) + dc.stop('merge-jobs') + + # Test if the MBTiles is still complete + # This does not verify whether new data has been added successfully + exported_mbtiles = os.path.join(PROJECT_DIR, 'export/planet.mbtiles') + tile_x, tile_y, tile_z = ALBANIA_TIRANA_TILE + tiles = find_missing_tiles(exported_mbtiles, tile_x, tile_y, tile_z, 13) + assert tiles == [] diff --git a/tools/integration-test/requirements.txt b/tools/integration-test/requirements.txt new file mode 100644 index 0000000..ae2316b --- /dev/null +++ b/tools/integration-test/requirements.txt @@ -0,0 +1,4 @@ +pytest==2.9.1 +docker-compose==1.7.1 +pytest-ordering==0.4 +-e git+https://github.com/lukasmartinelli/mbtoolbox.git@#egg=mbtoolbox