Add highway tests (#1294)

This PR adds unit tests for the transportation layer, and fixes several bugs in the unit testing code. It verifies the correct zoom association to the various highway classes, and verifies that updates to highway attributes are populated. Additionally, the documentation for unit tests in the CONTRIB guide is updated to note the specific make commands for developers to run.
2021-11-11 02:27:01 -05:00 · 2021-11-11 02:27:01 -05:00 · d7ea45349e
commit d7ea45349e
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -46,6 +46,8 @@ When you are making PR that adds new spatial features to OpenMapTiles schema, pl

 It is recommended that you create a [unit test](TESTING.md) when modifying the behavior of the SQL layer.  This will ensure that your changes are working as expected when importing or updating OSM data into an OpenMapTiles database.

+To run the unit tests, run `make clean destroy-db && make test-sql`
+
 # Verifying that updates still work

 When testing a PR, you shoud also verify that the update process completes without error.  The following procedure will run the update process.
--- a/3
+++ b/3
@ -612,13 +612,14 @@ data/changes.osc.gz: init-dirs
 	@echo " UPDATE unit test data..."
 	$(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools sh -c 'osmconvert tests/update/*.osc --merge-versions -o=data/changes.osc && gzip -f data/changes.osc'

-test-sql: clean refresh-docker-images destroy-db start-db-nowait build/import-tests.osm.pbf data/changes.state.txt data/last.state.txt data/changes.repl.json build/mapping.yaml data/changes.osc.gz
+test-sql: clean refresh-docker-images destroy-db start-db-nowait build/import-tests.osm.pbf data/changes.state.txt data/last.state.txt data/changes.repl.json build/mapping.yaml data/changes.osc.gz build/openmaptiles.tm2source/data.yml build/mapping.yaml build-sql
 	$(eval area := changes)

 	@echo "Load IMPORT test data"
 	sed -ir "s/^[#]*\s*MAX_ZOOM=.*/MAX_ZOOM=14/" .env
 	sed -ir "s/^[#]*\s*DIFF_MODE=.*/DIFF_MODE=false/" .env
 	$(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools sh -c 'pgwait && import-osm build/import-tests.osm.pbf'
+	$(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) import-data

 	@echo "Apply OpenMapTiles SQL schema to test data @ Zoom 14..."
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c 'pgwait && import-sql' | \
--- a/tests/import/500_import-highway.osm
+++ b/tests/import/500_import-highway.osm
@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<osm version="0.6" generator="Manual">
+  <node id="500001" visible="true" timestamp="2019-01-01T00:00:00Z" version="1" lat="33.0" lon="-90.00" />
+  <node id="500002" visible="true" timestamp="2019-01-01T00:00:00Z" version="1" lat="33.5" lon="-90.00" />
+  <node id="500011" visible="true" timestamp="2019-01-01T00:00:00Z" version="1" lat="33.0" lon="-90.01" />
+  <node id="500012" visible="true" timestamp="2019-01-01T00:00:00Z" version="1" lat="33.5" lon="-90.01" />
+  <node id="500021" visible="true" timestamp="2019-01-01T00:00:00Z" version="1" lat="33.0" lon="-90.02" />
+  <node id="500022" visible="true" timestamp="2019-01-01T00:00:00Z" version="1" lat="33.5" lon="-90.02" />
+  <node id="500031" visible="true" timestamp="2019-01-01T00:00:00Z" version="1" lat="33.0" lon="-90.03" />
+  <node id="500032" visible="true" timestamp="2019-01-01T00:00:00Z" version="1" lat="33.5" lon="-90.03" />
+  <node id="500041" visible="true" timestamp="2019-01-01T00:00:00Z" version="1" lat="33.0" lon="-90.04" />
+  <node id="500042" visible="true" timestamp="2019-01-01T00:00:00Z" version="1" lat="33.5" lon="-90.04" />
+  <node id="500051" visible="true" timestamp="2019-01-01T00:00:00Z" version="1" lat="33.0" lon="-90.05" />
+  <node id="500052" visible="true" timestamp="2019-01-01T00:00:00Z" version="1" lat="33.5" lon="-90.05" />
+  <node id="500061" visible="true" timestamp="2019-01-01T00:00:00Z" version="1" lat="33.0" lon="-90.06" />
+  <node id="500062" visible="true" timestamp="2019-01-01T00:00:00Z" version="1" lat="33.5" lon="-90.06" />
+
+  <way id="5000" version="1" timestamp="2019-01-01T00:00:00Z" visible="true">
+    <nd ref="500001" />
+    <nd ref="500002" />
+    <tag k="highway" v="motorway"/>
+    <tag k="name" v="OpenMapTiles Motorway"/>
+  </way>
+  <way id="5001" version="1" timestamp="2019-01-01T00:00:00Z" visible="true">
+    <nd ref="500011" />
+    <nd ref="500012" />
+    <tag k="highway" v="trunk"/>
+    <tag k="name" v="OpenMapTiles Trunk"/>
+  </way>
+  <way id="5002" version="1" timestamp="2019-01-01T00:00:00Z" visible="true">
+    <nd ref="500021" />
+    <nd ref="500022" />
+    <tag k="highway" v="primary"/>
+    <tag k="name" v="OpenMapTiles Primary"/>
+  </way>
+  <way id="5003" version="1" timestamp="2019-01-01T00:00:00Z" visible="true">
+    <nd ref="500031" />
+    <nd ref="500032" />
+    <tag k="highway" v="secondary"/>
+    <tag k="name" v="OpenMapTiles Secondary"/>
+    <tag k="bridge" v="yes"/>
+    <tag k="oneway" v="yes"/>
+    <tag k="toll" v="yes"/>
+    <tag k="layer" v="1"/>
+    <tag k="bicycle" v="no"/>
+    <tag k="horse" v="no"/>
+    <tag k="foot" v="no"/>
+  </way>
+  <way id="5004" version="1" timestamp="2019-01-01T00:00:00Z" visible="true">
+    <nd ref="500041" />
+    <nd ref="500042" />
+    <tag k="highway" v="tertiary"/>
+    <tag k="name" v="OpenMapTiles Tertiary"/>
+  </way>
+  <way id="5005" version="1" timestamp="2019-01-01T00:00:00Z" visible="true">
+    <nd ref="500051" />
+    <nd ref="500052" />
+    <tag k="highway" v="service"/>
+    <tag k="service" v="driveway"/>
+    <tag k="name" v="OpenMapTiles Service"/>
+  </way>
+  <way id="5006" version="1" timestamp="2019-01-01T00:00:00Z" visible="true">
+    <nd ref="500061" />
+    <nd ref="500062" />
+    <tag k="highway" v="track"/>
+    <tag k="name" v="OpenMapTiles Track"/>
+  </way>
+</osm>
--- a/tests/test-post-import.sql
+++ b/tests/test-post-import.sql
@ -47,7 +47,7 @@ BEGIN
    INSERT INTO omt_test_failures VALUES(200, 'import', 'osm_aerodrome_label expected 3, got ' || cnt);
  END IF;

-  SELECT COUNT(*) INTO cnt FROM osm_aerodrome_label_point WHERE ele=123;
+  SELECT COUNT(*) INTO cnt FROM osm_aerodrome_label_point WHERE ele='123';
  IF cnt <> 1 THEN
    INSERT INTO omt_test_failures VALUES(200, 'import', 'osm_aerodrome_label ele=123 expected 1, got ' || cnt);
  END IF;
@ -61,12 +61,76 @@ BEGIN
  -- Test 400
  SELECT COUNT(DISTINCT relation_id) INTO cnt FROM osm_border_linestring WHERE admin_level=8;
  IF cnt <> 1 THEN
-    INSERT INTO omt_test_failures VALUES(400, 'update', 'osm_border_linestring city count expected 1, got ' || cnt);
+    INSERT INTO omt_test_failures VALUES(400, 'import', 'osm_border_linestring city count expected 1, got ' || cnt);
  END IF;

  SELECT COUNT(DISTINCT relation_id) INTO cnt FROM osm_border_linestring WHERE admin_level=2;
  IF cnt <> 1 THEN
-    INSERT INTO omt_test_failures VALUES(400, 'update', 'osm_border_linestring country count expected 1, got ' || cnt);
+    INSERT INTO omt_test_failures VALUES(400, 'import', 'osm_border_linestring country count expected 1, got ' || cnt);
+  END IF;
+
+  -- Test 500
+
+  -- Verify that road classifications show up at the right zoom level
+  SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z4 WHERE highway='motorway';
+  IF cnt <> 1 THEN
+    INSERT INTO omt_test_failures VALUES(500, 'import', 'osm_transportation_linestring z4 motorway count expected 1, got ' || cnt);
+  END IF;
+
+  SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z4 WHERE highway='trunk';
+  IF cnt <> 0 THEN
+    INSERT INTO omt_test_failures VALUES(500, 'import', 'osm_transportation_linestring z4 trunk count expected 0, got ' || cnt);
+  END IF;
+
+  SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z5 WHERE highway='trunk';
+  IF cnt <> 1 THEN
+    INSERT INTO omt_test_failures VALUES(500, 'import', 'osm_transportation_linestring z5 trunk count expected 1, got ' || cnt);
+  END IF;
+
+  SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z6 WHERE highway='primary';
+  IF cnt <> 0 THEN
+    INSERT INTO omt_test_failures VALUES(500, 'import', 'osm_transportation_linestring z6 primary count expected 0, got ' || cnt);
+  END IF;
+
+  SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z7 WHERE highway='primary';
+  IF cnt <> 1 THEN
+    INSERT INTO omt_test_failures VALUES(500, 'import', 'osm_transportation_linestring z7 primary count expected 1, got ' || cnt);
+  END IF;
+
+  SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z8 WHERE highway='secondary';
+  IF cnt <> 0 THEN
+    INSERT INTO omt_test_failures VALUES(500, 'import', 'osm_transportation_linestring z8 secondary count expected 0, got ' || cnt);
+  END IF;
+
+  SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z9 WHERE highway='secondary';
+  IF cnt <> 1 THEN
+    INSERT INTO omt_test_failures VALUES(500, 'import', 'osm_transportation_linestring z9 secondary count expected 1, got ' || cnt);
+  END IF;
+
+  SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z10 WHERE highway='tertiary';
+  IF cnt <> 0 THEN
+    INSERT INTO omt_test_failures VALUES(500, 'import', 'osm_transportation_linestring z10 tertiary count expected 0, got ' || cnt);
+  END IF;
+
+  SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z11 WHERE highway='tertiary';
+  IF cnt <> 1 THEN
+    INSERT INTO omt_test_failures VALUES(500, 'import', 'osm_transportation_linestring z11 tertiary count expected 1, got ' || cnt);
+  END IF;
+
+  SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z11 WHERE highway IN ('service', 'track');
+  IF cnt <> 0 THEN
+    INSERT INTO omt_test_failures VALUES(500, 'import', 'osm_transportation_linestring z11 minor road count expected 0, got ' || cnt);
+  END IF;
+
+  SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z9
+    WHERE is_bridge = TRUE
+      AND toll = TRUE
+      AND layer = 1
+      AND bicycle = 'no'
+      AND foot = 'no'
+      AND horse = 'no';
+  IF cnt <> 1 THEN
+    INSERT INTO omt_test_failures VALUES(500, 'import', 'osm_transportation_linestring z9 import tags expected 1, got ' || cnt);
  END IF;

 END;
--- a/tests/test-post-update.sql
+++ b/tests/test-post-update.sql
@ -34,12 +34,12 @@ BEGIN
 -- Test 300: Verify landuse modified
  SELECT COUNT(*) INTO cnt FROM osm_landcover_polygon WHERE mapping_key='natural' AND subclass='scrub';
  IF cnt <> 1 THEN
-    INSERT INTO omt_test_failures VALUES(300, 'import', 'osm_landcover_polygon natural=scrub expected 1, got ' || cnt);
+    INSERT INTO omt_test_failures VALUES(300, 'update', 'osm_landcover_polygon natural=scrub expected 1, got ' || cnt);
  END IF;

  SELECT COUNT(*) INTO cnt FROM osm_landcover_polygon WHERE mapping_key='natural' AND subclass='wood';
  IF cnt <> 0 THEN
-    INSERT INTO omt_test_failures VALUES(300, 'import', 'osm_landcover_polygon natural=wood expected 0, got ' || cnt);
+    INSERT INTO omt_test_failures VALUES(300, 'update', 'osm_landcover_polygon natural=wood expected 0, got ' || cnt);
  END IF;

  -- Test 400: Verify new city added
@ -48,6 +48,19 @@ BEGIN
    INSERT INTO omt_test_failures VALUES(400, 'update', 'osm_border_linestring city count expected 2, got ' || cnt);
  END IF;

+  -- Test 500: Verify tags changed
+  SELECT COUNT(*) INTO cnt FROM osm_transportation_merge_linestring_gen_z9
+    WHERE is_tunnel = TRUE
+      AND is_bridge = FALSE
+      AND toll = FALSE
+      AND layer = -1
+      AND bicycle = 'yes'
+      AND foot = 'yes'
+      AND horse = 'yes';
+  IF cnt <> 1 THEN
+    INSERT INTO omt_test_failures VALUES(500, 'update', 'osm_transportation_linestring z9 update tags expected 1, got ' || cnt);
+  END IF;
+
 END;

 $$;
--- a/tests/update/500_update-highway.osc
+++ b/tests/update/500_update-highway.osc
@ -0,0 +1,23 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<osmChange version="0.6" generator="Manual" timestamp="2020-01-02T00:00:00Z">
+  <!--
+      Test 500: Highways
+        Change tags
+  -->
+  <modify>
+    <!-- Change tags -->
+    <way id="5003" version="2" timestamp="2019-01-01T00:00:00Z" visible="true">
+      <nd ref="500031" />
+      <nd ref="500032" />
+      <tag k="highway" v="secondary"/>
+      <tag k="name" v="OpenMapTiles Secondary Updated"/>
+      <tag k="tunnel" v="yes"/>
+      <tag k="oneway" v="no"/>
+      <tag k="toll" v="no"/>
+      <tag k="layer" v="-1"/>
+      <tag k="bicycle" v="yes"/>
+      <tag k="horse" v="yes"/>
+      <tag k="foot" v="yes"/>
+    </way>
+  </modify>
+</osmChange>