mirror
/
openmaptiles
kopia lustrzana https://github.com/openmaptiles/openmaptiles
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Move "qa" tools from OMT to OMT-T (#1031) 

With the new release of OMT-T (5.3) are available tools [`layer-stat`](https://github.com/openmaptiles/openmaptiles-tools/pull/293). 

With the new release of OMT-T can be replaced `make` target `generate-qareports` by `generate-qa`

Used as:
```
make generate-qa STAT_FUNCTION=frequency LAYER=transportation ATTRIBUTE=class
```
pull/1039/head^2
Tomas Pohanka 2020-11-12 11:23:06 +02:00 zatwierdzone przez GitHub
rodzic 0aa6648bd9
commit d180988f5f
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
7 zmienionych plików z 10 dodań i 337 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

13
Makefile
Wyświetl plik

@ -213,7 +213,7 @@ help:
@echo " make psql-list-tables # list all PostgreSQL tables"
@echo " make vacuum-db # PostgreSQL: VACUUM ANALYZE"
@echo " make analyze-db # PostgreSQL: ANALYZE"
@echo " make generate-qareports # generate reports [./build/qareports]"
@echo " make generate-qa # statistics for a given layer's field"
@echo " make generate-devdoc # generate devdoc including graphs for all layers [./layers/...]"
@echo " make bash # start openmaptiles-tools /bin/bash terminal"
@echo " make destroy-db # remove docker containers and PostgreSQL data volume"
@ -464,9 +464,14 @@ start-maputnik: stop-maputnik start-postserve
stop-maputnik:
-docker rm -f maputnik_editor
.PHONY: generate-qareports
generate-qareports: start-db
./qa/run.sh
# STAT_FUNCTION=frequency|toplength|variance
.PHONY: generate-qa
generate-qa: all start-db-nowait
@echo " "
@echo "e.g. make generate-qa STAT_FUNCTION=frequency LAYER=transportation ATTRIBUTE=class"
@echo " "
$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools \
layer-stats $(STAT_FUNCTION) $(TILESET_FILE) $(LAYER) $(ATTRIBUTE) -m 0 -n 14 -v
# generate all etl and mapping graphs
.PHONY: generate-devdoc

2
QUICKSTART.md
Wyświetl plik

@ -433,7 +433,7 @@ Hints for developers:
make psql-list-tables # list all PostgreSQL tables
make psql-vacuum-analyze # PostgreSQL: VACUUM ANALYZE
make psql-analyze # PostgreSQL: ANALYZE
make generate-qareports # generate reports [./build/qareports]
make generate-qa # statistics for a given layer's field
make generate-devdoc # generate devdoc [./build/devdoc]
make tools-dev # start import-sql /bin/bash terminal
make db-destroy # remove docker containers, PG data volume

35
qa/layer_freq.sh
Wyświetl plik

@ -1,35 +0,0 @@
#!/bin/bash
set -o errexit
set -o pipefail
set -o nounset
layerid=$1
classvars=$2
echo "# FREQ - $layerid group by : $classvars "
for z in {0..15}
do
echo " "
echo "## $layerid z$z - freq"
SQL=$(docker-compose run --rm openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z )
SQLCODE=$(cat <<-END
select $classvars , count(*) as _count_ from
( $SQL ) as t
GROUP BY $classvars
ORDER BY $classvars
;
END
)
#echo "\`\`\`sql"
#echo "$SQLCODE"
#echo "\`\`\`"
docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \
| sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g'
done

40
qa/layer_numvar_analyze.sh
Wyświetl plik

@ -1,40 +0,0 @@
#!/bin/bash
set -o errexit
set -o pipefail
set -o nounset
layerid=$1
var=$2
echo "# NUMVAR-ANALYZE - $layerid - $var "
for z in {0..15}
do
echo " "
echo "## $layerid z$z - $var "
SQL=$(docker-compose run --rm openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z )
SQLCODE=$(cat <<-END
SELECT
count($var) as count
,min($var) as min
,max($var) as max
,avg($var) as avg
,stddev($var) as stddev
,variance($var) as variance
FROM
( $SQL ) as t
;
END
)
#echo "\`\`\`sql"
#echo "$SQLCODE"
#echo "\`\`\`"
docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \
| sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g'
done

34
qa/layer_toplength.sh
Wyświetl plik

@ -1,34 +0,0 @@
#!/bin/bash
set -o errexit
set -o pipefail
set -o nounset
layerid=$1
classvar=$2
for z in {0..15}
do
echo " "
echo "## $layerid z$z max length ($classvar)"
SQL=$(docker-compose run --rm openmaptiles-tools generate-sqlquery layers/${layerid}/${layerid}.yaml $z )
SQLCODE=$(cat <<-END
SELECT DISTINCT $classvar , length( $classvar ) AS _length_ from
( $SQL ) as t
WHERE length( $classvar ) > 0
ORDER BY length( $classvar ) DESC NULLS LAST
LIMIT 30
;
END
)
#echo "\`\`\`sql"
#echo "$SQLCODE"
#echo "\`\`\`"
docker-compose run --rm import-osm /usr/src/app/psql.sh -q -P pager=off -P border=2 -P footer=off -P null='(null)' -c "$SQLCODE" \
| sed '1d;$d' | sed '$d' | sed 's/+--/|--/g' | sed 's/--+/--|/g'
done

83
qa/run.sh
Wyświetl plik

@ -1,83 +0,0 @@
#!/bin/bash
set -o errexit
set -o pipefail
set -o nounset
# example call from the parent folder : ./qa/run.sh
# or make generate-qareports
# ---- freq ---------------------------------------------------------------
mkdir -p ./build/qareports
rm -f ./build/qareports/*.md
# -----
./qa/layer_freq.sh aeroway "class" > ./build/qareports/freq_aeroway__class.md
###Todo:./qa/layer_freq.sh boundary "admin_level,disputed" > ./build/qareports/freq_boundary__admin_level_disputed.md
./qa/layer_freq.sh boundary "admin_level" > ./build/qareports/freq_boundary__admin_level.md
###Todo: ./qa/layer_freq.sh boundary "disputed" > ./build/qareports/freq_boundary__disputed.md
./qa/layer_freq.sh building "render_min_height" > ./build/qareports/freq_building__render_min_height.md
#./qa/layer_freq.sh housenumber "housenumber " > ./build/qadocfreq_housenumber.md
./qa/layer_freq.sh landcover "class, subclass" > ./build/qareports/freq_landcover__class_subclass.md
./qa/layer_freq.sh landuse "class " > ./build/qareports/freq_landuse__class.md
./qa/layer_freq.sh park "class " > ./build/qareports/freq_park__class.md
./qa/layer_freq.sh place "class " > ./build/qareports/freq_place__class.md
./qa/layer_freq.sh place "class,capital " > ./build/qareports/freq_place__class_capital.md
./qa/layer_freq.sh place "capital " > ./build/qareports/freq_place__capital.md
./qa/layer_freq.sh place "class,capital,rank " > ./build/qareports/freq_place__class_capital_rank.md
./qa/layer_freq.sh place "rank " > ./build/qareports/freq_place__rank.md
./qa/layer_freq.sh poi "class " > ./build/qareports/freq_poi__class.md
./qa/layer_freq.sh poi "subclass " > ./build/qareports/freq_poi__subclass.md
./qa/layer_freq.sh poi "rank " > ./build/qareports/freq_poi__rank.md
./qa/layer_freq.sh poi "class,subclass " > ./build/qareports/freq_poi__class_subclass.md
./qa/layer_freq.sh poi "class,subclass,rank" > ./build/qareports/freq_poi__class_subclass_rank.md
./qa/layer_freq.sh poi "class,rank " > ./build/qareports/freq_poi__class_rank.md
./qa/layer_freq.sh transportation "class, oneway, ramp, brunnel, service" > ./build/qareports/freq_transportation__class_oneway_ramp_brunnel_service.md
./qa/layer_freq.sh transportation "oneway, ramp, brunnel, service " > ./build/qareports/freq_transportation__oneway_ramp_brunnel_service.md
./qa/layer_freq.sh transportation "class " > ./build/qareports/freq_transportation__class.md
./qa/layer_freq.sh transportation_name "class " > ./build/qareports/freq_transportation_name__class.md
./qa/layer_freq.sh transportation_name "ref_length" > ./build/qareports/freq_transportation_name__ref_length.md
./qa/layer_freq.sh water "class " > ./build/qareports/freq_water__class.md
./qa/layer_freq.sh water_name "class " > ./build/qareports/freq_water_name__class.md
./qa/layer_freq.sh waterway "class " > ./build/qareports/freq_waterway__class.md
# ---- toplength -------------------------------------------
./qa/layer_toplength.sh housenumber "housenumber" > ./build/qareports/toplength_housenumber__housenumber.md
./qa/layer_toplength.sh place "name" > ./build/qareports/toplength_place__name.md
./qa/layer_toplength.sh place "name_en" > ./build/qareports/toplength_place__name_en.md
./qa/layer_toplength.sh poi "name" > ./build/qareports/toplength_poi__name.md
./qa/layer_toplength.sh poi "name_en" > ./build/qareports/toplength_poi__name_en.md
./qa/layer_toplength.sh transportation_name "name" > ./build/qareports/toplength_transportation_name__name.md
./qa/layer_toplength.sh transportation_name "ref" > ./build/qareports/toplength_transportation_name__ref.md
###Todo: ./qa/layer_toplength.sh transportation_name "network" > ./build/qareports/toplength_transportation_name__network.md
./qa/layer_toplength.sh water_name "name" > ./build/qareports/toplength_water_name__name.md
./qa/layer_toplength.sh water_name "name_en" > ./build/qareports/toplength_water_name__name_en.md
./qa/layer_toplength.sh waterway "name" > ./build/qareports/toplength_waterway__name.md
# ---- numvar analyze -------------------------------------
./qa/layer_numvar_analyze.sh building "render_min_height" > ./build/qareports/numvara_building__render_min_height.md
./qa/layer_numvar_analyze.sh building "render_height" > ./build/qareports/numvara_building__render_height.md
./qa/layer_numvar_analyze.sh transportation_name "ref_length" > ./build/qareports/numvara_transportation_name__ref_length.md

140
qa/table_sizes.py
Wyświetl plik

@ -1,140 +0,0 @@
#!/usr/bin/env python
import sys
import argparse
import subprocess
parser = argparse.ArgumentParser()
parser.add_argument('--noan', action='store_true', help='Not to run make psql-analyze')
TOTAL_SIZE_SQL = """SELECT
pg_size_pretty(sum(size)) AS size
FROM (
SELECT
relname as "Table",
pg_total_relation_size(relid) as "size"
FROM pg_catalog.pg_statio_user_tables
WHERE schemaname='public'
) a
;""".replace('\"', '\\\"')
TABLE_SIZES_SQL = """SELECT
a.relname as "table",
pg_table_size(a.relid) as "size",
b.n_live_tup as "rows"
FROM pg_catalog.pg_statio_user_tables a
LEFT JOIN pg_stat_user_tables b ON (a.relid = b.relid)
WHERE
a.schemaname='public'
ORDER BY a.relname;
""".replace('\"', '\\\"')
TABLES_SQL = """SELECT
a.relname
FROM pg_catalog.pg_statio_user_tables a
WHERE
a.schemaname='public'
ORDER BY a.relname;
"""
COLUMN_NAMES_SQL = """SELECT a.attname
FROM pg_class As c
INNER JOIN pg_attribute As a ON c.oid = a.attrelid
LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
WHERE
c.relkind IN('r', 'v', 'm') AND
a.attnum > 0 AND
n.nspname = 'public' AND
c.relname = '{0}' AND
a.attisdropped = FALSE
ORDER BY a.attname;
"""
COLUMNS_SQL = """select
sum(pg_column_size(t.*)) as "all",
{0}
from {1} t;
""".replace('\"', '\\\"')
def print_column_sizes(tables):
for table in tables:
print("Column sizes of table " + table)
cmds = [
'docker-compose run --rm import-osm',
'/usr/src/app/psql.sh -t -A -F\",\" -P pager=off',
'-c \"' + COLUMN_NAMES_SQL.format(table).replace('\n', ' ').replace('\r',
'') + '\"'
]
# print " ".join(cmds)
output = subprocess.check_output(" ".join(cmds), shell=True)
columns = filter(lambda c: len(c) > 0,
map(lambda l: l.strip(), output.split('\n')))
# print columns
col_sql = ",\n".join(
map(lambda c: "sum(pg_column_size(\\\"" + c + "\\\")) as \\\"" + c + "\\\"",
columns))
# print COLUMNS_SQL.format(col_sql, table);
cmds = [
'docker-compose run --rm import-osm',
'/usr/src/app/psql.sh -F\",\" --no-align -P pager=off',
'-c \"' + COLUMNS_SQL.format(col_sql, table).replace('\n', ' ').replace(
'\r', '') + '\"'
]
# print " ".join(cmds)
col_csv = subprocess.check_output(" ".join(cmds), shell=True)
print(col_csv)
if __name__ == "__main__":
args = parser.parse_args()
try:
if not args.noan:
print("Running make psql-analyze")
subprocess.check_output("make psql-analyze", shell=True)
print("Total size of tables")
cmds = [
'docker-compose run --rm import-osm',
'/usr/src/app/psql.sh -F\",\" --no-align -P pager=off',
'-c \"' + TOTAL_SIZE_SQL.replace('\n', ' ').replace('\r', '') + '\"'
]
# print " ".join(cmds)
TOTAL_SIZE_CSV = subprocess.check_output(" ".join(cmds), shell=True)
print(TOTAL_SIZE_CSV)
print("\n")
print("Table sizes")
cmds = [
'docker-compose run --rm import-osm',
'/usr/src/app/psql.sh -F\",\" --no-align -P pager=off',
'-c \"' + TABLE_SIZES_SQL.replace('\n', ' ').replace('\r', '') + '\"'
]
# print " ".join(cmds)
TABLE_SIZES_CSV = subprocess.check_output(" ".join(cmds), shell=True)
print(TABLE_SIZES_CSV)
print("\n")
print("Column sizes")
cmds = [
'docker-compose run --rm import-osm',
'/usr/src/app/psql.sh -t -A -F\",\" -P pager=off',
'-c \"' + TABLES_SQL.replace('\n', ' ').replace('\r', '') + '\"'
]
# print " ".join(cmds)
output = subprocess.check_output(" ".join(cmds), shell=True)
tables = filter(lambda t: len(t) > 0,
map(lambda l: l.strip(), output.split('\n')))
print_column_sizes(tables);
# print tables
except subprocess.CalledProcessError as e:
print("Error:\n", e.output)
sys.exit(0)