From 2e72698ed979985391818d34ef240209c66accca Mon Sep 17 00:00:00 2001 From: ImreSamu Date: Sun, 4 Dec 2016 01:37:28 +0100 Subject: [PATCH] refactor to `make generate-qareports` and `make generate-devdoc` --- Makefile | 48 +++++++++++--------- docker-compose.yml | 6 +-- qa/layer_freq.sh | 5 ++- qa/layer_numvar_analyze.sh | 6 ++- qa/layer_toplength.sh | 5 ++- qa/run.sh | 91 +++++++++++++++++++------------------- 6 files changed, 87 insertions(+), 74 deletions(-) diff --git a/Makefile b/Makefile index 969dc642..a6d7cbf2 100644 --- a/Makefile +++ b/Makefile @@ -14,10 +14,12 @@ help: @echo " make start-mapbox-studio # start Mapbox Studio" @echo " " @echo "Hints for developers:" - @echo " make # build source code " + @echo " make # build source code " @echo " make download-geofabrik area=albania # download OSM data from geofabrik, and create config file" @echo " make psql # start PostgreSQL console " @echo " make psql-list-tables # list all PostgreSQL tables " + @echo " make generate-qareports # generate reports [./build/qareports]" + @echo " make generate-devdoc # generate devdoc [./build/devdoc]" @echo " make import-sql-dev # start import-sql /bin/bash terminal " @echo " make import-osm-dev # start import-osm /bin/bash terminal (imposm3)" @echo " make clean-docker # remove docker containers, PG data volume " @@ -29,7 +31,7 @@ help: @echo " cat .env # list PG database and MIN_ZOOM and MAX_ZOOM informations" @echo " cat ./quickstart.log # backup of the last ./quickstart.sh " @echo " ....TODO.... # start lukasmartinelli/postgis-editor" - @echo " make help # help about avaialable commands" + @echo " make help # help about avaialable commands" @echo "==============================================================================" build/openmaptiles.tm2source/data.yml: @@ -53,7 +55,7 @@ list-docker-images: docker images | grep openmaptiles refresh-docker-images: - docker-compose pull + echo docker-compose pull remove-docker-images: docker rmi -f $(docker images | grep "openmaptiles" | awk "{print \$3}") @@ -93,9 +95,9 @@ download-geofabrik: docker-compose run --rm import-osm ./download-geofabrik.sh $(area) ls -la ./data/$(area).* @echo "Generated config file: ./data/docker-compose-config.yml" - @echo " " - cat ./data/docker-compose-config.yml - @echo " " + @echo " " + cat ./data/docker-compose-config.yml + @echo " " # the `download-geofabrik` error message mention `list`, if the area parameter is wrong. so I created a similar make command list: @@ -108,18 +110,24 @@ download-geofabrik-list: start-mapbox-studio: docker-compose up mapbox-studio +generate-qareports: + ./qa/run.sh + # work in progress - please don't remove -test_etlgraph: - generate-etlgraph layers/boundary/boundary.yaml - generate-etlgraph layers/highway/highway.yaml - generate-etlgraph layers/housenumber/housenumber.yaml - generate-etlgraph layers/landuse/landuse.yaml - generate-etlgraph layers/poi/poi.yaml - generate-etlgraph layers/water/water.yaml - generate-etlgraph layers/waterway/waterway.yaml - generate-etlgraph layers/building/building.yaml - generate-etlgraph layers/highway_name/highway_name.yaml - generate-etlgraph layers/landcover/landcover.yaml - generate-etlgraph layers/place/place.yaml - generate-etlgraph layers/railway/railway.yaml - generate-etlgraph layers/water_name/water_name.yaml +generate-devdoc: + mkdir -p ./build/devdoc + generate-etlgraph layers/aeroway/aeroway.yaml ./build/devdoc + generate-etlgraph layers/boundary/boundary.yaml ./build/devdoc + generate-etlgraph layers/building/building.yaml ./build/devdoc + generate-etlgraph layers/housenumber/housenumber.yaml ./build/devdoc + generate-etlgraph layers/landcover/landcover.yaml ./build/devdoc + generate-etlgraph layers/landuse/landuse.yaml ./build/devdoc + generate-etlgraph layers/park/park.yaml ./build/devdoc + generate-etlgraph layers/place/place.yaml ./build/devdoc + generate-etlgraph layers/poi/poi.yaml ./build/devdoc + generate-etlgraph layers/transportation/transportation.yaml ./build/devdoc + generate-etlgraph layers/transportation_name/transportation_name.yaml ./build/devdoc + generate-etlgraph layers/water/water.yaml ./build/devdoc + generate-etlgraph layers/water_name/water_name.yaml ./build/devdoc + generate-etlgraph layers/waterway/waterway.yaml ./build/devdoc + diff --git a/docker-compose.yml b/docker-compose.yml index 1f741cac..3574e729 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,11 +25,7 @@ services: links: - postgres import-osm: -<<<<<<< HEAD - image: "openmaptiles/import-osm:add_tools" -======= - image: "openmaptiles/import-osm:0.1" ->>>>>>> master + image: "openmaptiles/import-osm:0.2" env_file: .env environment: DIFF_MODE: "false" diff --git a/qa/layer_freq.sh b/qa/layer_freq.sh index df23cc19..b54d4bfa 100755 --- a/qa/layer_freq.sh +++ b/qa/layer_freq.sh @@ -1,5 +1,8 @@ #!/bin/bash +set -o errexit +set -o pipefail +set -o nounset layerid=$1 classvars=$2 @@ -11,7 +14,7 @@ do echo " " echo "## $layerid z$z - freq" -SQL=$(generate-qadoc layers/${layerid}/${layerid}.yaml $z ) +SQL=$(generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) read -r -d '' SQLCODE <<- EOMSQL select $classvars , count(*) as _count_ from diff --git a/qa/layer_numvar_analyze.sh b/qa/layer_numvar_analyze.sh index c18669de..cb5eb6d1 100755 --- a/qa/layer_numvar_analyze.sh +++ b/qa/layer_numvar_analyze.sh @@ -1,5 +1,8 @@ #!/bin/bash +set -o errexit +set -o pipefail +set -o nounset layerid=$1 var=$2 @@ -11,7 +14,7 @@ do echo " " echo "## $layerid z$z - $var " -SQL=$(generate-qadoc layers/${layerid}/${layerid}.yaml $z ) +SQL=$(generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) read -r -d '' SQLCODE <<- EOMSQL SELECT @@ -34,4 +37,3 @@ docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P borde | sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g' done - diff --git a/qa/layer_toplength.sh b/qa/layer_toplength.sh index 10a612c9..a4c71730 100755 --- a/qa/layer_toplength.sh +++ b/qa/layer_toplength.sh @@ -1,5 +1,8 @@ #!/bin/bash +set -o errexit +set -o pipefail +set -o nounset layerid=$1 @@ -12,7 +15,7 @@ do echo " " echo "## $layerid z$z max length ($classvar)" -SQL=$(generate-qadoc layers/${layerid}/${layerid}.yaml $z ) +SQL=$(generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) read -r -d '' SQLCODE <<- EOMSQL select $classvar , length( $classvar ) as _length_ from diff --git a/qa/run.sh b/qa/run.sh index 5d820e40..8909698d 100755 --- a/qa/run.sh +++ b/qa/run.sh @@ -1,81 +1,82 @@ #!/bin/bash +set -o errexit +set -o pipefail +set -o nounset # example call from the parent folder : ./qa/run.sh - - - +# or make generate-qareports # ---- freq --------------------------------------------------------------- mkdir -p ./build/qadoc -rm -f ./build/qadoc/*.md +rm -f ./build/qareports/*.md -./qa/layer_freq.sh aeroway "class" > ./build/qadoc/freq_aeroway__class.md +./qa/layer_freq.sh aeroway "class" > ./build/qareports/freq_aeroway__class.md -./qa/layer_freq.sh boundary "admin_level,disputed" > ./build/qadoc/freq_boundary__admin_level_disputed.md -./qa/layer_freq.sh boundary "admin_level" > ./build/qadoc/freq_boundary__admin_level.md -./qa/layer_freq.sh boundary "disputed" > ./build/qadoc/freq_boundary__disputed.md +./qa/layer_freq.sh boundary "admin_level,disputed" > ./build/qareports/freq_boundary__admin_level_disputed.md +./qa/layer_freq.sh boundary "admin_level" > ./build/qareports/freq_boundary__admin_level.md +./qa/layer_freq.sh boundary "disputed" > ./build/qareports/freq_boundary__disputed.md -./qa/layer_freq.sh building "render_min_height" > ./build/qadoc/freq_building__render_min_height.md +./qa/layer_freq.sh building "render_min_height" > ./build/qareports/freq_building__render_min_height.md #./qa/layer_freq.sh housenumber "housenumber " > ./build/qadocfreq_housenumber.md -./qa/layer_freq.sh landcover "class, subclass" > ./build/qadoc/freq_landcover__class_subclass.md +./qa/layer_freq.sh landcover "class, subclass" > ./build/qareports/freq_landcover__class_subclass.md -./qa/layer_freq.sh landuse "class " > ./build/qadoc/freq_landuse__class.md +./qa/layer_freq.sh landuse "class " > ./build/qareports/freq_landuse__class.md -./qa/layer_freq.sh park "class " > ./build/qadoc/freq_park__class.md +./qa/layer_freq.sh park "class " > ./build/qareports/freq_park__class.md -./qa/layer_freq.sh place "class " > ./build/qadoc/freq_place__class.md -./qa/layer_freq.sh place "class,capital " > ./build/qadoc/freq_place__class_capital.md -./qa/layer_freq.sh place "capital " > ./build/qadoc/freq_place__capital.md -./qa/layer_freq.sh place "class,capital,rank " > ./build/qadoc/freq_place__class_capital_rank.md -./qa/layer_freq.sh place "rank " > ./build/qadoc/freq_place__rank.md +./qa/layer_freq.sh place "class " > ./build/qareports/freq_place__class.md +./qa/layer_freq.sh place "class,capital " > ./build/qareports/freq_place__class_capital.md +./qa/layer_freq.sh place "capital " > ./build/qareports/freq_place__capital.md +./qa/layer_freq.sh place "class,capital,rank " > ./build/qareports/freq_place__class_capital_rank.md +./qa/layer_freq.sh place "rank " > ./build/qareports/freq_place__rank.md -./qa/layer_freq.sh poi "class " > ./build/qadoc/freq_poi__class.md -./qa/layer_freq.sh poi "subclass " > ./build/qadoc/freq_poi__subclass.md -./qa/layer_freq.sh poi "rank " > ./build/qadoc/freq_poi__rank.md -./qa/layer_freq.sh poi "class,subclass " > ./build/qadoc/freq_poi__class_subclass.md -./qa/layer_freq.sh poi "class,subclass,rank" > ./build/qadoc/freq_poi__class_subclass_rank.md -./qa/layer_freq.sh poi "class,rank " > ./build/qadoc/freq_poi__class_rank.md +./qa/layer_freq.sh poi "class " > ./build/qareports/freq_poi__class.md +./qa/layer_freq.sh poi "subclass " > ./build/qareports/freq_poi__subclass.md +./qa/layer_freq.sh poi "rank " > ./build/qareports/freq_poi__rank.md +./qa/layer_freq.sh poi "class,subclass " > ./build/qareports/freq_poi__class_subclass.md +./qa/layer_freq.sh poi "class,subclass,rank" > ./build/qareports/freq_poi__class_subclass_rank.md +./qa/layer_freq.sh poi "class,rank " > ./build/qareports/freq_poi__class_rank.md -./qa/layer_freq.sh transportation "class, oneway, ramp, brunnel, service" > ./build/qadoc/freq_transportation__class_oneway_ramp_brunnel_service.md -./qa/layer_freq.sh transportation "oneway, ramp, brunnel, service " > ./build/qadoc/freq_transportation__oneway_ramp_brunnel_service.md -./qa/layer_freq.sh transportation "class " > ./build/qadoc/freq_transportation__class.md +./qa/layer_freq.sh transportation "class, oneway, ramp, brunnel, service" > ./build/qareports/freq_transportation__class_oneway_ramp_brunnel_service.md +./qa/layer_freq.sh transportation "oneway, ramp, brunnel, service " > ./build/qareports/freq_transportation__oneway_ramp_brunnel_service.md +./qa/layer_freq.sh transportation "class " > ./build/qareports/freq_transportation__class.md -./qa/layer_freq.sh transportation_name "class " > ./build/qadoc/freq_transportation_name__class.md -./qa/layer_freq.sh transportation_name "ref_length" > ./build/qadoc/freq_transportation_name__ref_length.md +./qa/layer_freq.sh transportation_name "class " > ./build/qareports/freq_transportation_name__class.md +./qa/layer_freq.sh transportation_name "ref_length" > ./build/qareports/freq_transportation_name__ref_length.md -./qa/layer_freq.sh water "class " > ./build/qadoc/freq_water__class.md +./qa/layer_freq.sh water "class " > ./build/qareports/freq_water__class.md -./qa/layer_freq.sh water_name "class " > ./build/qadoc/freq_water_name__class.md +./qa/layer_freq.sh water_name "class " > ./build/qareports/freq_water_name__class.md -./qa/layer_freq.sh waterway "class " > ./build/qadoc/freq_waterway__class.md +./qa/layer_freq.sh waterway "class " > ./build/qareports/freq_waterway__class.md # ---- toplength ------------------------------------------- -./qa/layer_toplength.sh housenumber "housenumber" > ./build/qadoc/toplength_housenumber__housenumber.md +./qa/layer_toplength.sh housenumber "housenumber" > ./build/qareports/toplength_housenumber__housenumber.md -./qa/layer_toplength.sh place "name" > ./build/qadoc/toplength_place__name.md -./qa/layer_toplength.sh place "name_en" > ./build/qadoc/toplength_place__name_en.md +./qa/layer_toplength.sh place "name" > ./build/qareports/toplength_place__name.md +./qa/layer_toplength.sh place "name_en" > ./build/qareports/toplength_place__name_en.md -./qa/layer_toplength.sh poi "name" > ./build/qadoc/toplength_poi__name.md -./qa/layer_toplength.sh poi "name_en" > ./build/qadoc/toplength_poi__name_en.md +./qa/layer_toplength.sh poi "name" > ./build/qareports/toplength_poi__name.md +./qa/layer_toplength.sh poi "name_en" > ./build/qareports/toplength_poi__name_en.md -./qa/layer_toplength.sh transportation_name "name" > ./build/qadoc/toplength_transportation_name__name.md -./qa/layer_toplength.sh transportation_name "ref" > ./build/qadoc/toplength_transportation_name__ref.md -./qa/layer_toplength.sh transportation_name "network" > ./build/qadoc/toplength_transportation_name__network.md +./qa/layer_toplength.sh transportation_name "name" > ./build/qareports/toplength_transportation_name__name.md +./qa/layer_toplength.sh transportation_name "ref" > ./build/qareports/toplength_transportation_name__ref.md +./qa/layer_toplength.sh transportation_name "network" > ./build/qareports/toplength_transportation_name__network.md -./qa/layer_toplength.sh water_name "name" > ./build/qadoc/toplength_water_name__name.md -./qa/layer_toplength.sh water_name "name_en" > ./build/qadoc/toplength_water_name__name_en.md +./qa/layer_toplength.sh water_name "name" > ./build/qareports/toplength_water_name__name.md +./qa/layer_toplength.sh water_name "name_en" > ./build/qareports/toplength_water_name__name_en.md -./qa/layer_toplength.sh waterway "name" > ./build/qadoc/toplength_waterway__name.md +./qa/layer_toplength.sh waterway "name" > ./build/qareports/toplength_waterway__name.md # ---- numvar analyze ------------------------------------- -./qa/layer_numvar_analyze.sh building "render_min_height" > ./build/qadoc/numvara_building__render_min_height.md -./qa/layer_numvar_analyze.sh building "render_max_height" > ./build/qadoc/numvara_building__render_max_height.md +./qa/layer_numvar_analyze.sh building "render_min_height" > ./build/qareports/numvara_building__render_min_height.md +./qa/layer_numvar_analyze.sh building "render_height" > ./build/qareports/numvara_building__render_height.md -./qa/layer_numvar_analyze.sh transportation_name "ref_length" > ./build/qadoc/numvara_transportation_name__ref_length.md +./qa/layer_numvar_analyze.sh transportation_name "ref_length" > ./build/qareports/numvara_transportation_name__ref_length.md