diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9e614928..535f3c16 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,15 +32,11 @@ Your pull request must: * Use clear commit messages. * Be possible to merge automatically. -When you modify rules of importing data in `mapping.yaml` or `*.sql`, please update also: +When you modify import data rules in `mapping.yaml` or `*.sql`, please update: 1. field description in `[layer].yaml` 2. comments starting with `#etldoc` -3. if needed, generate new `mapping_diagram.png` or `etl_diagram.png` using commands below: -``` -make mapping-graph-[layer] -make etl-graph-[layer] -``` +3. regenerate documentation graphs with `make generate-devdoc` 4. update layer description on https://openmaptiles.org/schema/ (https://github.com/openmaptiles/www.openmaptiles.org/tree/master/layers) 5. check if OMT styles are affected by the PR and if there is a need for style updates diff --git a/Makefile b/Makefile index e6c3b88e..86587ef8 100644 --- a/Makefile +++ b/Makefile @@ -7,9 +7,11 @@ DC_USER_OPTS?=$(DC_OPTS) -u $$(id -u $${USER}):$$(id -g $${USER}) # If running in the test mode, compare files rather than copy them TEST_MODE?=no ifeq ($(TEST_MODE),yes) - COPY_TO_GIT=diff + # create images in ./build/devdoc and compare them to ./layers + GRAPH_PARAMS=./build/devdoc ./layers else - COPY_TO_GIT=cp + # update graphs in the ./layers dir + GRAPH_PARAMS=./layers endif .PHONY: all @@ -36,9 +38,7 @@ help: @echo " make psql-vacuum-analyze # PostgreSQL: VACUUM ANALYZE" @echo " make psql-analyze # PostgreSQL: ANALYZE" @echo " make generate-qareports # generate reports [./build/qareports]" - @echo " make generate-devdoc # generate devdoc including graphs for all layers [./build/devdoc]" - @echo " make etl-graph # hint for generating a single etl graph" - @echo " make mapping-graph # hint for generating a single mapping graph" + @echo " make generate-devdoc # generate devdoc including graphs for all layers [./layers/...]" @echo " make import-sql-dev # start import-sql /bin/bash terminal" @echo " make import-osm-dev # start import-osm /bin/bash terminal (imposm3)" @echo " make clean-docker # remove docker containers, PG data volume" @@ -172,41 +172,13 @@ start-postserve: db-start generate-qareports: ./qa/run.sh -build/devdoc: - mkdir -p ./build/devdoc - - -layers = $(notdir $(wildcard layers/*)) # all layers - -.PHONY: etl-graph -etl-graph: - @echo 'Use' - @echo ' make etl-graph-[layer] to generate etl graph for [layer]' - @echo ' example: make etl-graph-poi' - @echo 'Valid layers: $(layers)' - -# generate etl graph for a certain layer, e.g. etl-graph-building, etl-graph-place -etl-graph-%: layers/% build/devdoc - docker-compose run $(DC_USER_OPTS) openmaptiles-tools generate-etlgraph layers/$*/$*.yaml ./build/devdoc - @$(COPY_TO_GIT) ./build/devdoc/etl_$*.png layers/$*/etl_diagram.png - - -mappingLayers = $(notdir $(patsubst %/mapping.yaml,%, $(wildcard layers/*/mapping.yaml))) # layers with mapping.yaml - -# generate mapping graph for a certain layer, e.g. mapping-graph-building, mapping-graph-place -.PHONY: mapping-graph -mapping-graph: - @echo 'Use' - @echo ' make mapping-graph-[layer] to generate mapping graph for [layer]' - @echo ' example: make mapping-graph-poi' - @echo 'Valid layers: $(mappingLayers)' - -mapping-graph-%: ./layers/%/mapping.yaml build/devdoc - docker-compose run $(DC_USER_OPTS) openmaptiles-tools generate-mapping-graph layers/$*/$*.yaml ./build/devdoc/mapping-diagram-$* - @$(COPY_TO_GIT) ./build/devdoc/mapping-diagram-$*.png layers/$*/mapping_diagram.png - # generate all etl and mapping graphs -generate-devdoc: $(addprefix etl-graph-,$(layers)) $(addprefix mapping-graph-,$(mappingLayers)) +.PHONY: generate-devdoc +generate-devdoc: + mkdir -p ./build/devdoc && \ + docker-compose run $(DC_USER_OPTS) openmaptiles-tools-latest sh -c \ + 'generate-etlgraph openmaptiles.yaml $(GRAPH_PARAMS) && \ + generate-mapping-graph openmaptiles.yaml $(GRAPH_PARAMS)' .PHONY: import-sql-dev import-sql-dev: diff --git a/layers/aerodrome_label/mapping.yaml b/layers/aerodrome_label/mapping.yaml index 46cdb71d..ff0f22c0 100644 --- a/layers/aerodrome_label/mapping.yaml +++ b/layers/aerodrome_label/mapping.yaml @@ -3,7 +3,7 @@ tables: # etldoc: imposm3 -> osm_aerodrome_label_point aerodrome_label_point: type: geometry - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/aeroway/mapping.yaml b/layers/aeroway/mapping.yaml index 7c90897d..2e6bf121 100644 --- a/layers/aeroway/mapping.yaml +++ b/layers/aeroway/mapping.yaml @@ -50,7 +50,7 @@ tables: # etldoc: imposm3 -> osm_aeroway_polygon aeroway_polygon: type: polygon - fields: + columns: - *ref - name: osm_id type: id @@ -67,7 +67,7 @@ tables: # etldoc: imposm3 -> osm_aeroway_linestring aeroway_linestring: type: linestring - fields: + columns: - *ref - name: osm_id type: id diff --git a/layers/boundary/mapping.yaml b/layers/boundary/mapping.yaml index 9b3e67b3..aacb1b5f 100644 --- a/layers/boundary/mapping.yaml +++ b/layers/boundary/mapping.yaml @@ -76,7 +76,7 @@ tables: # etldoc: imposm3 -> osm_border_disp_relation border_disp_relation: type: relation_member - fields: + columns: - name: relation_id type: id - name: osm_id diff --git a/layers/building/mapping.yaml b/layers/building/mapping.yaml index 118eceb3..d2f96939 100644 --- a/layers/building/mapping.yaml +++ b/layers/building/mapping.yaml @@ -8,7 +8,7 @@ generalized_tables: tables: # etldoc: imposm3 -> osm_building_polygon building_polygon: - fields: + columns: - name: osm_id type: id - name: geometry @@ -69,7 +69,7 @@ tables: # etldoc: imposm3 -> osm_building_street building_street: - fields: + columns: - name: osm_id type: id - name: geometry @@ -161,7 +161,7 @@ tables: # etldoc: imposm3 -> osm_building_associatedstreet building_associatedstreet: - fields: + columns: - name: osm_id type: id - name: geometry @@ -253,7 +253,7 @@ tables: # etldoc: imposm3 -> osm_building_relation building_relation: - fields: + columns: - name: osm_id type: id - name: geometry @@ -345,7 +345,7 @@ tables: # etldoc: imposm3 -> osm_building_multipolygon building_multipolygon: - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/housenumber/mapping.yaml b/layers/housenumber/mapping.yaml index cf5f2557..174335bb 100644 --- a/layers/housenumber/mapping.yaml +++ b/layers/housenumber/mapping.yaml @@ -4,7 +4,7 @@ tables: # etldoc: imposm3 -> osm_housenumber_point housenumber_point: type: geometry - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/landcover/mapping.yaml b/layers/landcover/mapping.yaml index 952111cf..142f3dd5 100644 --- a/layers/landcover/mapping.yaml +++ b/layers/landcover/mapping.yaml @@ -46,7 +46,7 @@ generalized_tables: tables: # etldoc: imposm3 -> osm_landcover_polygon landcover_polygon: - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/landuse/mapping.yaml b/layers/landuse/mapping.yaml index 038313e7..b3970f36 100644 --- a/layers/landuse/mapping.yaml +++ b/layers/landuse/mapping.yaml @@ -38,7 +38,7 @@ tables: # etldoc: imposm3 -> osm_landuse_polygon landuse_polygon: type: polygon - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/mountain_peak/mapping.yaml b/layers/mountain_peak/mapping.yaml index 7b9cc688..84a0146e 100644 --- a/layers/mountain_peak/mapping.yaml +++ b/layers/mountain_peak/mapping.yaml @@ -3,7 +3,7 @@ tables: # etldoc: imposm3 -> osm_peak_point peak_point: type: point - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/park/mapping.yaml b/layers/park/mapping.yaml index dcfe5fbd..8cde10d1 100644 --- a/layers/park/mapping.yaml +++ b/layers/park/mapping.yaml @@ -53,7 +53,7 @@ tables: park_polygon: type: polygon _resolve_wikidata: false - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/place/mapping.yaml b/layers/place/mapping.yaml index 15d7ef5a..29d1f3ba 100644 --- a/layers/place/mapping.yaml +++ b/layers/place/mapping.yaml @@ -20,7 +20,7 @@ tables: # etldoc: imposm3 -> osm_continent_point continent_point: type: point - fields: + columns: - name: osm_id type: id - name: geometry @@ -40,7 +40,7 @@ tables: # etldoc: imposm3 -> osm_country_point country_point: type: point - fields: + columns: - name: osm_id type: id - name: geometry @@ -70,7 +70,7 @@ tables: # etldoc: imposm3 -> osm_island_polygon island_polygon: type: polygon - fields: + columns: - name: osm_id type: id - name: geometry @@ -93,7 +93,7 @@ tables: # etldoc: imposm3 -> osm_island_point island_point: type: point - fields: + columns: - name: osm_id type: id - name: geometry @@ -114,7 +114,7 @@ tables: # etldoc: imposm3 -> osm_state_point state_point: type: point - fields: + columns: - name: osm_id type: id - name: geometry @@ -144,7 +144,7 @@ tables: # etldoc: imposm3 -> osm_city_point city_point: type: point - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/poi/mapping.yaml b/layers/poi/mapping.yaml index 3fb327b3..a50e11d3 100644 --- a/layers/poi/mapping.yaml +++ b/layers/poi/mapping.yaml @@ -381,11 +381,11 @@ tables: # etldoc: imposm3 -> osm_poi_point poi_point: type: point - fields: *poi_fields + columns: *poi_fields mapping: *poi_mapping # etldoc: imposm3 -> osm_poi_polygon poi_polygon: type: polygon - fields: *poi_fields + columns: *poi_fields mapping: *poi_mapping diff --git a/layers/transportation/mapping.yaml b/layers/transportation/mapping.yaml index e634c76c..7ad30f9e 100644 --- a/layers/transportation/mapping.yaml +++ b/layers/transportation/mapping.yaml @@ -159,7 +159,7 @@ tables: highway_linestring: type: linestring _resolve_wikidata: false - fields: + columns: - name: osm_id type: id - name: geometry @@ -233,7 +233,7 @@ tables: railway_linestring: type: linestring _resolve_wikidata: false - fields: + columns: - name: osm_id type: id - name: geometry @@ -276,7 +276,7 @@ tables: aerialway_linestring: type: linestring _resolve_wikidata: false - fields: + columns: - name: osm_id type: id - name: geometry @@ -309,7 +309,7 @@ tables: shipway_linestring: type: linestring _resolve_wikidata: false - fields: + columns: - name: osm_id type: id - name: geometry @@ -340,7 +340,7 @@ tables: # etldoc: imposm3 -> osm_highway_polygon highway_polygon: type: polygon - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/transportation/mapping_diagram.png b/layers/transportation/mapping_diagram.png index eacdce72..959e43b4 100644 Binary files a/layers/transportation/mapping_diagram.png and b/layers/transportation/mapping_diagram.png differ diff --git a/layers/transportation_name/mapping_diagram.png b/layers/transportation_name/mapping_diagram.png index eacdce72..959e43b4 100644 Binary files a/layers/transportation_name/mapping_diagram.png and b/layers/transportation_name/mapping_diagram.png differ diff --git a/layers/water/mapping.yaml b/layers/water/mapping.yaml index 68c5bfc1..2bca61c1 100644 --- a/layers/water/mapping.yaml +++ b/layers/water/mapping.yaml @@ -49,7 +49,7 @@ tables: # etldoc: imposm3 -> osm_water_polygon water_polygon: - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/water_name/mapping.yaml b/layers/water_name/mapping.yaml index 7d27ac4c..558ad61a 100644 --- a/layers/water_name/mapping.yaml +++ b/layers/water_name/mapping.yaml @@ -2,7 +2,7 @@ tables: # etldoc: imposm3 -> osm_marine_point marine_point: type: point - fields: + columns: - name: osm_id type: id - name: geometry diff --git a/layers/water_name/mapping_diagram.png b/layers/water_name/mapping_diagram.png index 2dce3fcb..b1e881f5 100644 Binary files a/layers/water_name/mapping_diagram.png and b/layers/water_name/mapping_diagram.png differ diff --git a/layers/waterway/mapping.yaml b/layers/waterway/mapping.yaml index 2255b165..cda6d057 100644 --- a/layers/waterway/mapping.yaml +++ b/layers/waterway/mapping.yaml @@ -28,7 +28,7 @@ tables: # etldoc: imposm3 -> osm_waterway_linestring waterway_linestring: type: linestring - fields: + columns: - name: osm_id type: id - name: geometry