Use one pass docs image generation (#751)

quicker and cleaner diagram image generation.
Remove etl-graph and mapping-graph targets - redundant

Also, the obsolete "fields" is still in Imposm's code and both names are accepted,
but "fields" is not documented anywhere (PR submitted), and could be removed at any moment.

Our docs were not supporting it until this PR, so renaming it at the same time.

Several images have been updated due to a more inclusive mapping scan
Requires https://github.com/openmaptiles/openmaptiles-tools/pull/147 (merged)

CONTRIBUTING.md

@@ -32,15 +32,11 @@ Your pull request must:
  * Use clear commit messages.
  * Be possible to merge automatically.
 
-When you modify rules of importing data in `mapping.yaml` or `*.sql`, please update also:
+When you modify import data rules in `mapping.yaml` or `*.sql`, please update:
 
 1. field description in `[layer].yaml`
 2. comments starting with `#etldoc`
-3. if needed, generate new `mapping_diagram.png` or `etl_diagram.png` using commands below:
-```
-make mapping-graph-[layer]
-make etl-graph-[layer]
-```
+3. regenerate documentation graphs with `make generate-devdoc`
 4. update layer description on https://openmaptiles.org/schema/ (https://github.com/openmaptiles/www.openmaptiles.org/tree/master/layers)
 5. check if OMT styles are affected by the PR and if there is a need for style updates
 

Makefile

@@ -7,9 +7,11 @@ DC_USER_OPTS?=$(DC_OPTS) -u $$(id -u $${USER}):$$(id -g $${USER})
 # If running in the test mode, compare files rather than copy them
 TEST_MODE?=no
 ifeq ($(TEST_MODE),yes)
-  COPY_TO_GIT=diff
+  # create images in ./build/devdoc and compare them to ./layers
+  GRAPH_PARAMS=./build/devdoc ./layers
 else
-  COPY_TO_GIT=cp
+  # update graphs in the ./layers dir
+  GRAPH_PARAMS=./layers
 endif
 
 .PHONY: all
@@ -36,9 +38,7 @@ help:
 	@echo "  make psql-vacuum-analyze             # PostgreSQL: VACUUM ANALYZE"
 	@echo "  make psql-analyze                    # PostgreSQL: ANALYZE"
 	@echo "  make generate-qareports              # generate reports [./build/qareports]"
-	@echo "  make generate-devdoc                 # generate devdoc including graphs for all layers  [./build/devdoc]"
-	@echo "  make etl-graph                       # hint for generating a single etl graph"
-	@echo "  make mapping-graph                   # hint for generating a single mapping graph"
+	@echo "  make generate-devdoc                 # generate devdoc including graphs for all layers  [./layers/...]"
 	@echo "  make import-sql-dev                  # start import-sql /bin/bash terminal"
 	@echo "  make import-osm-dev                  # start import-osm /bin/bash terminal (imposm3)"
 	@echo "  make clean-docker                    # remove docker containers, PG data volume"
@@ -172,41 +172,13 @@ start-postserve: db-start
 generate-qareports:
 	./qa/run.sh
 
-build/devdoc:
-	mkdir -p ./build/devdoc
-
-
-layers = $(notdir $(wildcard layers/*)) # all layers
-
-.PHONY: etl-graph
-etl-graph:
-	@echo 'Use'
-	@echo '   make etl-graph-[layer]	to generate etl graph for [layer]'
-	@echo '   example: make etl-graph-poi'
-	@echo 'Valid layers: $(layers)'
-
-# generate etl graph for a certain layer, e.g. etl-graph-building, etl-graph-place
-etl-graph-%: layers/% build/devdoc
-	docker-compose run $(DC_USER_OPTS) openmaptiles-tools generate-etlgraph layers/$*/$*.yaml ./build/devdoc
-	@$(COPY_TO_GIT) ./build/devdoc/etl_$*.png layers/$*/etl_diagram.png
-
-
-mappingLayers = $(notdir $(patsubst %/mapping.yaml,%, $(wildcard layers/*/mapping.yaml))) # layers with mapping.yaml
-
-# generate mapping graph for a certain layer, e.g. mapping-graph-building, mapping-graph-place
-.PHONY: mapping-graph
-mapping-graph:
-	@echo 'Use'
-	@echo '   make mapping-graph-[layer]	to generate mapping graph for [layer]'
-	@echo '   example: make mapping-graph-poi'
-	@echo 'Valid layers: $(mappingLayers)'
-
-mapping-graph-%: ./layers/%/mapping.yaml build/devdoc
-	docker-compose run $(DC_USER_OPTS) openmaptiles-tools generate-mapping-graph layers/$*/$*.yaml ./build/devdoc/mapping-diagram-$*
-	@$(COPY_TO_GIT) ./build/devdoc/mapping-diagram-$*.png layers/$*/mapping_diagram.png
-
 # generate all etl and mapping graphs
-generate-devdoc: $(addprefix etl-graph-,$(layers)) $(addprefix mapping-graph-,$(mappingLayers))
+.PHONY: generate-devdoc
+generate-devdoc:
+	mkdir -p ./build/devdoc && \
+	docker-compose run $(DC_USER_OPTS) openmaptiles-tools-latest sh -c \
+			'generate-etlgraph openmaptiles.yaml $(GRAPH_PARAMS) && \
+			 generate-mapping-graph openmaptiles.yaml $(GRAPH_PARAMS)'
 
 .PHONY: import-sql-dev
 import-sql-dev:

layers/aerodrome_label/mapping.yaml

@@ -3,7 +3,7 @@ tables:
   # etldoc: imposm3 -> osm_aerodrome_label_point
   aerodrome_label_point:
     type: geometry
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry

layers/aeroway/mapping.yaml

@@ -50,7 +50,7 @@ tables:
   # etldoc: imposm3 -> osm_aeroway_polygon
   aeroway_polygon:
     type: polygon
-    fields:
+    columns:
     - *ref
     - name: osm_id
       type: id
@@ -67,7 +67,7 @@ tables:
   # etldoc: imposm3 -> osm_aeroway_linestring
   aeroway_linestring:
     type: linestring
-    fields:
+    columns:
     - *ref
     - name: osm_id
       type: id

layers/boundary/mapping.yaml

@@ -76,7 +76,7 @@ tables:
   # etldoc: imposm3 -> osm_border_disp_relation
   border_disp_relation:
     type: relation_member
-    fields:
+    columns:
     - name: relation_id
       type: id
     - name: osm_id

layers/building/mapping.yaml

@@ -8,7 +8,7 @@ generalized_tables:
 tables:
   # etldoc: imposm3 -> osm_building_polygon
   building_polygon:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -69,7 +69,7 @@ tables:
 
   # etldoc: imposm3 -> osm_building_street
   building_street:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -161,7 +161,7 @@ tables:
 
   # etldoc: imposm3 -> osm_building_associatedstreet
   building_associatedstreet:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -253,7 +253,7 @@ tables:
 
   # etldoc: imposm3 -> osm_building_relation
   building_relation:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -345,7 +345,7 @@ tables:
 
   # etldoc: imposm3 -> osm_building_multipolygon
   building_multipolygon:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry

layers/housenumber/mapping.yaml

@@ -4,7 +4,7 @@ tables:
   # etldoc: imposm3 -> osm_housenumber_point
   housenumber_point:
     type: geometry
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry

layers/landcover/mapping.yaml

@@ -46,7 +46,7 @@ generalized_tables:
 tables:
   # etldoc: imposm3 -> osm_landcover_polygon
   landcover_polygon:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry

layers/landuse/mapping.yaml

@@ -38,7 +38,7 @@ tables:
   # etldoc: imposm3 -> osm_landuse_polygon
   landuse_polygon:
     type: polygon
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry

layers/mountain_peak/mapping.yaml

@@ -3,7 +3,7 @@ tables:
   # etldoc: imposm3 -> osm_peak_point
   peak_point:
     type: point
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry

layers/park/mapping.yaml

@@ -53,7 +53,7 @@ tables:
   park_polygon:
     type: polygon
     _resolve_wikidata: false
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry

layers/place/mapping.yaml

@@ -20,7 +20,7 @@ tables:
   # etldoc: imposm3 -> osm_continent_point
   continent_point:
     type: point
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -40,7 +40,7 @@ tables:
   # etldoc: imposm3 -> osm_country_point
   country_point:
     type: point
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -70,7 +70,7 @@ tables:
   # etldoc: imposm3 -> osm_island_polygon
   island_polygon:
     type: polygon
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -93,7 +93,7 @@ tables:
   # etldoc: imposm3 -> osm_island_point
   island_point:
     type: point
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -114,7 +114,7 @@ tables:
   # etldoc: imposm3 -> osm_state_point
   state_point:
     type: point
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -144,7 +144,7 @@ tables:
   # etldoc: imposm3 -> osm_city_point
   city_point:
     type: point
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry

layers/poi/mapping.yaml

@@ -381,11 +381,11 @@ tables:
   # etldoc: imposm3 -> osm_poi_point
   poi_point:
     type: point
-    fields: *poi_fields
+    columns: *poi_fields
     mapping: *poi_mapping
 
   # etldoc: imposm3 -> osm_poi_polygon
   poi_polygon:
     type: polygon
-    fields: *poi_fields
+    columns: *poi_fields
     mapping: *poi_mapping

layers/transportation/mapping.yaml

@@ -159,7 +159,7 @@ tables:
   highway_linestring:
     type: linestring
     _resolve_wikidata: false
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -233,7 +233,7 @@ tables:
   railway_linestring:
     type: linestring
     _resolve_wikidata: false
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -276,7 +276,7 @@ tables:
   aerialway_linestring:
     type: linestring
     _resolve_wikidata: false
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -309,7 +309,7 @@ tables:
   shipway_linestring:
     type: linestring
     _resolve_wikidata: false
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry
@@ -340,7 +340,7 @@ tables:
 # etldoc: imposm3 -> osm_highway_polygon
   highway_polygon:
     type: polygon
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry

layers/water/mapping.yaml

@@ -49,7 +49,7 @@ tables:
 
   # etldoc: imposm3 -> osm_water_polygon
   water_polygon:
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry

layers/water_name/mapping.yaml

@@ -2,7 +2,7 @@ tables:
   # etldoc: imposm3 -> osm_marine_point
   marine_point:
     type: point
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry

layers/waterway/mapping.yaml

@@ -28,7 +28,7 @@ tables:
   # etldoc: imposm3 -> osm_waterway_linestring
   waterway_linestring:
     type: linestring
-    fields:
+    columns:
     - name: osm_id
       type: id
     - name: geometry