From d180988f5f36b04bb9630383c0d2389cf8e25bc6 Mon Sep 17 00:00:00 2001 From: Tomas Pohanka Date: Thu, 12 Nov 2020 11:23:06 +0200 Subject: [PATCH] Move "qa" tools from OMT to OMT-T (#1031) With the new release of OMT-T (5.3) are available tools [`layer-stat`](https://github.com/openmaptiles/openmaptiles-tools/pull/293). With the new release of OMT-T can be replaced `make` target `generate-qareports` by `generate-qa` Used as: ``` make generate-qa STAT_FUNCTION=frequency LAYER=transportation ATTRIBUTE=class ``` --- Makefile | 13 ++-- QUICKSTART.md | 2 +- qa/layer_freq.sh | 35 ---------- qa/layer_numvar_analyze.sh | 40 ----------- qa/layer_toplength.sh | 34 --------- qa/run.sh | 83 ---------------------- qa/table_sizes.py | 140 ------------------------------------- 7 files changed, 10 insertions(+), 337 deletions(-) delete mode 100755 qa/layer_freq.sh delete mode 100755 qa/layer_numvar_analyze.sh delete mode 100755 qa/layer_toplength.sh delete mode 100755 qa/run.sh delete mode 100755 qa/table_sizes.py diff --git a/Makefile b/Makefile index 3220021a..a6f2edda 100644 --- a/Makefile +++ b/Makefile @@ -213,7 +213,7 @@ help: @echo " make psql-list-tables # list all PostgreSQL tables" @echo " make vacuum-db # PostgreSQL: VACUUM ANALYZE" @echo " make analyze-db # PostgreSQL: ANALYZE" - @echo " make generate-qareports # generate reports [./build/qareports]" + @echo " make generate-qa # statistics for a given layer's field" @echo " make generate-devdoc # generate devdoc including graphs for all layers [./layers/...]" @echo " make bash # start openmaptiles-tools /bin/bash terminal" @echo " make destroy-db # remove docker containers and PostgreSQL data volume" @@ -464,9 +464,14 @@ start-maputnik: stop-maputnik start-postserve stop-maputnik: -docker rm -f maputnik_editor -.PHONY: generate-qareports -generate-qareports: start-db - ./qa/run.sh +# STAT_FUNCTION=frequency|toplength|variance +.PHONY: generate-qa +generate-qa: all start-db-nowait + @echo " " + @echo "e.g. make generate-qa STAT_FUNCTION=frequency LAYER=transportation ATTRIBUTE=class" + @echo " " + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools \ + layer-stats $(STAT_FUNCTION) $(TILESET_FILE) $(LAYER) $(ATTRIBUTE) -m 0 -n 14 -v # generate all etl and mapping graphs .PHONY: generate-devdoc diff --git a/QUICKSTART.md b/QUICKSTART.md index 53851518..70bfaa7c 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -433,7 +433,7 @@ Hints for developers: make psql-list-tables # list all PostgreSQL tables make psql-vacuum-analyze # PostgreSQL: VACUUM ANALYZE make psql-analyze # PostgreSQL: ANALYZE - make generate-qareports # generate reports [./build/qareports] + make generate-qa # statistics for a given layer's field make generate-devdoc # generate devdoc [./build/devdoc] make tools-dev # start import-sql /bin/bash terminal make db-destroy # remove docker containers, PG data volume diff --git a/qa/layer_freq.sh b/qa/layer_freq.sh deleted file mode 100755 index 60c8a2c6..00000000 --- a/qa/layer_freq.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -set -o errexit -set -o pipefail -set -o nounset - -layerid=$1 -classvars=$2 - -echo "# FREQ - $layerid group by : $classvars " - -for z in {0..15} -do -echo " " -echo "## $layerid z$z - freq" - -SQL=$(docker-compose run --rm openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) - -SQLCODE=$(cat <<-END -select $classvars , count(*) as _count_ from -( $SQL ) as t -GROUP BY $classvars -ORDER BY $classvars -; -END -) - -#echo "\`\`\`sql" -#echo "$SQLCODE" -#echo "\`\`\`" - -docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \ - | sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g' - -done - diff --git a/qa/layer_numvar_analyze.sh b/qa/layer_numvar_analyze.sh deleted file mode 100755 index 614b4e14..00000000 --- a/qa/layer_numvar_analyze.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -set -o errexit -set -o pipefail -set -o nounset - -layerid=$1 -var=$2 - -echo "# NUMVAR-ANALYZE - $layerid - $var " - -for z in {0..15} -do -echo " " -echo "## $layerid z$z - $var " - -SQL=$(docker-compose run --rm openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) - -SQLCODE=$(cat <<-END -SELECT - count($var) as count - ,min($var) as min - ,max($var) as max - ,avg($var) as avg - ,stddev($var) as stddev - ,variance($var) as variance -FROM -( $SQL ) as t -; -END -) - - -#echo "\`\`\`sql" -#echo "$SQLCODE" -#echo "\`\`\`" - -docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \ - | sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g' - -done diff --git a/qa/layer_toplength.sh b/qa/layer_toplength.sh deleted file mode 100755 index 86a529eb..00000000 --- a/qa/layer_toplength.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -set -o errexit -set -o pipefail -set -o nounset - -layerid=$1 -classvar=$2 - -for z in {0..15} -do -echo " " -echo "## $layerid z$z max length ($classvar)" - -SQL=$(docker-compose run --rm openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z ) - -SQLCODE=$(cat <<-END -SELECT DISTINCT $classvar , length( $classvar ) AS _length_ from -( $SQL ) as t -WHERE length( $classvar ) > 0 -ORDER BY length( $classvar ) DESC NULLS LAST -LIMIT 30 -; -END -) - -#echo "\`\`\`sql" -#echo "$SQLCODE" -#echo "\`\`\`" - -docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \ - | sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g' - -done - diff --git a/qa/run.sh b/qa/run.sh deleted file mode 100755 index f0a1b961..00000000 --- a/qa/run.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash -set -o errexit -set -o pipefail -set -o nounset - -# example call from the parent folder : ./qa/run.sh -# or make generate-qareports - -# ---- freq --------------------------------------------------------------- - -mkdir -p ./build/qareports -rm -f ./build/qareports/*.md - -# ----- - -./qa/layer_freq.sh aeroway "class" > ./build/qareports/freq_aeroway__class.md - -###Todo:./qa/layer_freq.sh boundary "admin_level,disputed" > ./build/qareports/freq_boundary__admin_level_disputed.md -./qa/layer_freq.sh boundary "admin_level" > ./build/qareports/freq_boundary__admin_level.md -###Todo: ./qa/layer_freq.sh boundary "disputed" > ./build/qareports/freq_boundary__disputed.md - -./qa/layer_freq.sh building "render_min_height" > ./build/qareports/freq_building__render_min_height.md - -#./qa/layer_freq.sh housenumber "housenumber " > ./build/qadocfreq_housenumber.md -./qa/layer_freq.sh landcover "class, subclass" > ./build/qareports/freq_landcover__class_subclass.md - -./qa/layer_freq.sh landuse "class " > ./build/qareports/freq_landuse__class.md - -./qa/layer_freq.sh park "class " > ./build/qareports/freq_park__class.md - -./qa/layer_freq.sh place "class " > ./build/qareports/freq_place__class.md -./qa/layer_freq.sh place "class,capital " > ./build/qareports/freq_place__class_capital.md -./qa/layer_freq.sh place "capital " > ./build/qareports/freq_place__capital.md -./qa/layer_freq.sh place "class,capital,rank " > ./build/qareports/freq_place__class_capital_rank.md -./qa/layer_freq.sh place "rank " > ./build/qareports/freq_place__rank.md - -./qa/layer_freq.sh poi "class " > ./build/qareports/freq_poi__class.md -./qa/layer_freq.sh poi "subclass " > ./build/qareports/freq_poi__subclass.md -./qa/layer_freq.sh poi "rank " > ./build/qareports/freq_poi__rank.md -./qa/layer_freq.sh poi "class,subclass " > ./build/qareports/freq_poi__class_subclass.md -./qa/layer_freq.sh poi "class,subclass,rank" > ./build/qareports/freq_poi__class_subclass_rank.md -./qa/layer_freq.sh poi "class,rank " > ./build/qareports/freq_poi__class_rank.md - -./qa/layer_freq.sh transportation "class, oneway, ramp, brunnel, service" > ./build/qareports/freq_transportation__class_oneway_ramp_brunnel_service.md -./qa/layer_freq.sh transportation "oneway, ramp, brunnel, service " > ./build/qareports/freq_transportation__oneway_ramp_brunnel_service.md -./qa/layer_freq.sh transportation "class " > ./build/qareports/freq_transportation__class.md - -./qa/layer_freq.sh transportation_name "class " > ./build/qareports/freq_transportation_name__class.md -./qa/layer_freq.sh transportation_name "ref_length" > ./build/qareports/freq_transportation_name__ref_length.md - -./qa/layer_freq.sh water "class " > ./build/qareports/freq_water__class.md - -./qa/layer_freq.sh water_name "class " > ./build/qareports/freq_water_name__class.md - -./qa/layer_freq.sh waterway "class " > ./build/qareports/freq_waterway__class.md - -# ---- toplength ------------------------------------------- - -./qa/layer_toplength.sh housenumber "housenumber" > ./build/qareports/toplength_housenumber__housenumber.md - -./qa/layer_toplength.sh place "name" > ./build/qareports/toplength_place__name.md -./qa/layer_toplength.sh place "name_en" > ./build/qareports/toplength_place__name_en.md - -./qa/layer_toplength.sh poi "name" > ./build/qareports/toplength_poi__name.md -./qa/layer_toplength.sh poi "name_en" > ./build/qareports/toplength_poi__name_en.md - -./qa/layer_toplength.sh transportation_name "name" > ./build/qareports/toplength_transportation_name__name.md -./qa/layer_toplength.sh transportation_name "ref" > ./build/qareports/toplength_transportation_name__ref.md -###Todo: ./qa/layer_toplength.sh transportation_name "network" > ./build/qareports/toplength_transportation_name__network.md - -./qa/layer_toplength.sh water_name "name" > ./build/qareports/toplength_water_name__name.md -./qa/layer_toplength.sh water_name "name_en" > ./build/qareports/toplength_water_name__name_en.md - -./qa/layer_toplength.sh waterway "name" > ./build/qareports/toplength_waterway__name.md - - -# ---- numvar analyze ------------------------------------- - -./qa/layer_numvar_analyze.sh building "render_min_height" > ./build/qareports/numvara_building__render_min_height.md -./qa/layer_numvar_analyze.sh building "render_height" > ./build/qareports/numvara_building__render_height.md - -./qa/layer_numvar_analyze.sh transportation_name "ref_length" > ./build/qareports/numvara_transportation_name__ref_length.md - diff --git a/qa/table_sizes.py b/qa/table_sizes.py deleted file mode 100755 index e005b0e1..00000000 --- a/qa/table_sizes.py +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env python -import sys -import argparse -import subprocess - -parser = argparse.ArgumentParser() -parser.add_argument('--noan', action='store_true', help='Not to run make psql-analyze') - -TOTAL_SIZE_SQL = """SELECT - pg_size_pretty(sum(size)) AS size -FROM ( - SELECT - relname as "Table", - pg_total_relation_size(relid) as "size" - FROM pg_catalog.pg_statio_user_tables - WHERE schemaname='public' -) a -;""".replace('\"', '\\\"') - -TABLE_SIZES_SQL = """SELECT - a.relname as "table", - pg_table_size(a.relid) as "size", - b.n_live_tup as "rows" -FROM pg_catalog.pg_statio_user_tables a - LEFT JOIN pg_stat_user_tables b ON (a.relid = b.relid) -WHERE - a.schemaname='public' -ORDER BY a.relname; -""".replace('\"', '\\\"') - -TABLES_SQL = """SELECT - a.relname -FROM pg_catalog.pg_statio_user_tables a -WHERE - a.schemaname='public' -ORDER BY a.relname; -""" - -COLUMN_NAMES_SQL = """SELECT a.attname -FROM pg_class As c - INNER JOIN pg_attribute As a ON c.oid = a.attrelid - LEFT JOIN pg_namespace n ON n.oid = c.relnamespace - LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace -WHERE - c.relkind IN('r', 'v', 'm') AND - a.attnum > 0 AND - n.nspname = 'public' AND - c.relname = '{0}' AND - a.attisdropped = FALSE -ORDER BY a.attname; -""" - -COLUMNS_SQL = """select - sum(pg_column_size(t.*)) as "all", - {0} -from {1} t; -""".replace('\"', '\\\"') - - -def print_column_sizes(tables): - for table in tables: - print("Column sizes of table " + table) - cmds = [ - 'docker-compose run --rm import-osm', - '/usr/src/app/psql.sh -t -A -F\",\" -P pager=off', - '-c \"' + COLUMN_NAMES_SQL.format(table).replace('\n', ' ').replace('\r', - '') + '\"' - ] - # print " ".join(cmds) - output = subprocess.check_output(" ".join(cmds), shell=True) - columns = filter(lambda c: len(c) > 0, - map(lambda l: l.strip(), output.split('\n'))) - - # print columns - - col_sql = ",\n".join( - map(lambda c: "sum(pg_column_size(\\\"" + c + "\\\")) as \\\"" + c + "\\\"", - columns)) - - # print COLUMNS_SQL.format(col_sql, table); - - cmds = [ - 'docker-compose run --rm import-osm', - '/usr/src/app/psql.sh -F\",\" --no-align -P pager=off', - '-c \"' + COLUMNS_SQL.format(col_sql, table).replace('\n', ' ').replace( - '\r', '') + '\"' - ] - # print " ".join(cmds) - col_csv = subprocess.check_output(" ".join(cmds), shell=True) - print(col_csv) - - -if __name__ == "__main__": - args = parser.parse_args() - - try: - - if not args.noan: - print("Running make psql-analyze") - subprocess.check_output("make psql-analyze", shell=True) - - print("Total size of tables") - cmds = [ - 'docker-compose run --rm import-osm', - '/usr/src/app/psql.sh -F\",\" --no-align -P pager=off', - '-c \"' + TOTAL_SIZE_SQL.replace('\n', ' ').replace('\r', '') + '\"' - ] - # print " ".join(cmds) - TOTAL_SIZE_CSV = subprocess.check_output(" ".join(cmds), shell=True) - print(TOTAL_SIZE_CSV) - print("\n") - - print("Table sizes") - cmds = [ - 'docker-compose run --rm import-osm', - '/usr/src/app/psql.sh -F\",\" --no-align -P pager=off', - '-c \"' + TABLE_SIZES_SQL.replace('\n', ' ').replace('\r', '') + '\"' - ] - # print " ".join(cmds) - TABLE_SIZES_CSV = subprocess.check_output(" ".join(cmds), shell=True) - print(TABLE_SIZES_CSV) - print("\n") - - print("Column sizes") - cmds = [ - 'docker-compose run --rm import-osm', - '/usr/src/app/psql.sh -t -A -F\",\" -P pager=off', - '-c \"' + TABLES_SQL.replace('\n', ' ').replace('\r', '') + '\"' - ] - # print " ".join(cmds) - output = subprocess.check_output(" ".join(cmds), shell=True) - tables = filter(lambda t: len(t) > 0, - map(lambda l: l.strip(), output.split('\n'))) - - print_column_sizes(tables); - - # print tables - except subprocess.CalledProcessError as e: - print("Error:\n", e.output) - sys.exit(0)