-  R000 Environment
-  R001 Analyze Geometry tables
-  R002 System info - Geometry tables & indexes

RUN:
  ./qa_test.sh  443c7a5
  ./qa_test.sh  2904e5a
OUTPUT:
  -/wrk/*

  Changes to be committed:
	new file:   qa_test.sh
	new file:   tools/qa-reports/o2v_qa_r000.sql
	new file:   tools/qa-reports/o2v_qa_r001.sql
	new file:   tools/qa-reports/o2v_qa_r002.sql
	new file:   wrk/.gitkeep
qa_rip
ImreSamu 2016-06-29 19:08:02 +02:00
rodzic 80f7c18e09
commit 7cc3d06329
5 zmienionych plików z 406 dodań i 0 usunięć

145
qa_test.sh 100755
Wyświetl plik

@ -0,0 +1,145 @@
#!/bin/bash
set -o errexit
set -o pipefail
set -o nounset
set -o verbose
# Example call
# ./qa_test.sh 443c7a5
# ./qa_test.sh 806a5d9
# ./qa_test.sh 2904e5a
# Parameter1 = git revision number
#TO RUN ----------------------------------------------------------------------------------------------
#1. and check/replace the PBF in the ./import directory
ls ./import/*.pbf
#3. run this script with parameter : ./qa_test.sh 443c7a5
# ./qa_test.sh 443c7a5
# ./qa_test.sh 806a5d9
#4. check the report from the ./wrk/443c7a5*
#5. cleaninig ...
# cd ./wrk/${REVISION}_osm2vectortiles && docker-compose stop && docker-compose rm -f -v -a
#------------------------------------------------------------------------------------------------------
readonly REVISION=${1:-806a5d9}
REPORT_HASH=${REVISION}-$(md5sum ./import/*.osm.pbf | sed 's#./import/#-#g' | sed 's#.osm##g'| sed 's#.pbf##g' | sed 's# ##g' )
echo ${REPORT_HASH} "== ( REVISION - import md5sum - import osm-filename )"
# ---not modify from here -----------------------------------------------------
#DOCKER_CACHE=--no-cache
DOCKER_CACHE=
REPORTDIR=../../tools/qa-reports
#
# ---------------------------------------------------------------------
#
function runtest {
rm -f -r ./wrk/${REVISION}_osm2vectortiles
git clone https://github.com/osm2vectortiles/osm2vectortiles.git ./wrk/${REVISION}_osm2vectortiles
cd ./wrk/${REVISION}_osm2vectortiles
git reset --hard ${REVISION}
cp ../../import/* ./import/
# fix: build: "src/postgis" -> image: "osm2vectortiles/import-osm"
sed -i 's#build: "src/postgis"#image: "osm2vectortiles/postgis"#g' docker-compose.yml
# add a $REVISION prefix for the images
sed -i s#osm2vectortiles/#${REVISION}_osm2vectortiles/#g docker-compose.yml
cat docker-compose.yml | grep ${REVISION}
# sed -i 's#image: "osm2vectortiles/#image: "${REVISION}_osm2vectortiles/"#g' docker-compose.yml
# stop & clean docker environment.
docker-compose stop
docker-compose rm -f -v -a
# rebuild dockerimages from the local source
cd ./src/postgis && docker build ${DOCKER_CACHE} -t ${REVISION}_osm2vectortiles/postgis . && cd ../..
cd ./src/import-external && docker build ${DOCKER_CACHE} -t ${REVISION}_osm2vectortiles/import-external . && cd ../..
cd ./src/import-osm && docker build ${DOCKER_CACHE} -t ${REVISION}_osm2vectortiles/import-osm . && cd ../..
cd ./src/import-sql && docker build ${DOCKER_CACHE} -t ${REVISION}_osm2vectortiles/import-sql . && cd ../..
# Start images
docker-compose up -d postgis
# wait for postgis
postgis_cid=$(docker ps | grep ${REVISION} | grep osm2vectortiles | grep postgis | cut -d" " -f 1)
postgis_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${postgis_cid})
echo "postgis_ip = $postgis_ip"
while ! pg_isready -h $postgis_ip
do
echo "$(date) - waiting for PG database to start"
sleep 2
done
# run test
docker-compose up import-external
docker-compose up import-osm
docker-compose up import-sql
}
#
# ---------------------------------------------------------------------
#
function generate_reports {
postgis_cid=$(docker ps | grep ${REVISION} | grep osm2vectortiles | grep postgis | cut -d" " -f 1)
postgis_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${postgis_cid})
echo "postgis_ip = $postgis_ip"
# ----- Environment report
testfile=../${REPORT_HASH}--R000.txt
psql -P pager=off -h $postgis_ip -d osm -U osm -f ${REPORTDIR}/o2v_qa_r000.sql > ${testfile}
echo "#--------------------------------------------" >> ${testfile}
echo "# md5sum ./import/* " >> ${testfile}
md5sum ./import/* >> ${testfile}
echo "#--------------------------------------------" >> ${testfile}
echo "#" >> ${testfile}
echo "# git show --summary " >> ${testfile}
git show --summary >> ${testfile}
echo "#--------------------------------------------" >> ${testfile}
echo "#" >> ${testfile}
echo "# docker images " >> ${testfile}
docker images | grep ${REVISION} | grep osm2vectortiles | sort >> ${testfile}
psql -P pager=off -h $postgis_ip -d osm -U osm -f ${REPORTDIR}/o2v_qa_r001.sql > ../${REPORT_HASH}--R001.txt
psql -P pager=off -h $postgis_ip -d osm -U osm -f ${REPORTDIR}/o2v_qa_r002.sql > ../${REPORT_HASH}--R002.txt
# ---- export distinct osm_id ---for OSM_ID comparing
psql -P pager=off -h $postgis_ip -d osm -U osm -c "SELECT distinct osm_id FROM landuse_z5toz6 ORDER BY osm_id ; " > ../${REPORT_HASH}__id_landuse_z5toz6.txt
psql -P pager=off -h $postgis_ip -d osm -U osm -c "SELECT distinct osm_id FROM landuse_z7toz8 ORDER BY osm_id ; " > ../${REPORT_HASH}__id_landuse_z7toz8.txt
psql -P pager=off -h $postgis_ip -d osm -U osm -c "SELECT distinct osm_id FROM landuse_overlay_z5 ORDER BY osm_id ; " > ../${REPORT_HASH}__id_landuse_overlay_z5.txt
psql -P pager=off -h $postgis_ip -d osm -U osm -c "SELECT distinct osm_id FROM landuse_overlay_z6 ORDER BY osm_id ; " > ../${REPORT_HASH}__id_landuse_overlay_z6.txt
psql -P pager=off -h $postgis_ip -d osm -U osm -c "SELECT distinct osm_id FROM landuse_overlay_z7 ORDER BY osm_id ; " > ../${REPORT_HASH}__id_landuse_overlay_z7.txt
psql -P pager=off -h $postgis_ip -d osm -U osm -c "SELECT distinct osm_id FROM landuse_overlay_z8 ORDER BY osm_id ; " > ../${REPORT_HASH}__id_landuse_overlay_z8.txt
psql -P pager=off -h $postgis_ip -d osm -U osm -c "SELECT distinct osm_id FROM landuse_overlay_z9 ORDER BY osm_id ; " > ../${REPORT_HASH}__id_landuse_overlay_z9.txt
}
runtest
generate_reports
docker-compose stop
# docker-compose rm -f -v -a

Wyświetl plik

@ -0,0 +1,11 @@

-- postgis version info
SELECT 'ReportName :' as id, 'O2V-QA-R000 Environment '::text as ver
UNION ALL SELECT 'ReportVersion:' as id, 'v0.0.1/2016-jun-29'::text as ver
UNION ALL SELECT 'RunTime :' as id, Now()::text as ver
UNION ALL SELECT 'postgis :' as id, PostGIS_full_version()::text as ver
UNION ALL SELECT 'postgresql :' as id, version()::text as ver
;
-- orher part is generated from bash ...

Wyświetl plik

@ -0,0 +1,160 @@

-- postgis version info
SELECT 'ReportName :' as id, 'O2V-QA-R001 Analyze Geometry tables '::text as ver
UNION ALL SELECT 'ReportVersion:' as id, 'v0.0.1/2016-jun-29'::text as ver
UNION ALL SELECT 'RunTime :' as id, Now()::text as ver
UNION ALL SELECT 'postgis :' as id, PostGIS_full_version()::text as ver
UNION ALL SELECT 'postgresql :' as id, version()::text as ver
;
-- DROP FUNCTION o2v_analyze_area_layer(text,regclass,bool) ;
CREATE OR REPLACE FUNCTION o2v_analyze_area_layer( grp text, _tbl regclass ,typefield bool DEFAULT true )
RETURNS TABLE (
tgrp text
,tablename text
,count_rec numeric
,uniq_id numeric
,uniq_type numeric
,sum_area numeric
,avg_area numeric
,min_area numeric
,max_area numeric
,sum_length numeric
,avg_length numeric
,min_length numeric
,max_length numeric
,max_numgeom numeric
,avg_numgeom numeric
,n_point numeric
,n_linestring numeric
,n_multilinestring numeric
,n_polygon numeric
,n_multipolygon numeric
,n_other numeric
,n_polyarea_l10m2 numeric
,n_polyarea_l100m2 numeric
,n_polyarea_l1000m2 numeric
,n_linelen_l1m numeric
,n_linelen_l10m numeric
,n_linelen_l100m numeric
,n_linelen_l1000m numeric
,id_md5_1 numeric
,id_md5_2 numeric
,id_md5_3 numeric
,id_md5_4 numeric
) AS $func$
DECLARE
tid text;
type_qstr text;
BEGIN
IF SUBSTR( _tbl::text,1,3 ) = 'osm'
THEN tid='id';
ELSE tid='osm_id';
END IF;
IF typefield
THEN type_qstr=' count( distinct type)::numeric ';
ELSE type_qstr=' 0::numeric';
END IF;
RETURN QUERY EXECUTE format('
SELECT
%5$L::text AS tgrp
, %1$s::text AS tablename
, count( * )::numeric AS count_rec
, count( distinct %2$s)::numeric AS uniq_id
, %6$s AS uniq_type
, round(sum(st_area( geometry)::numeric),0)::numeric AS sum_area
, round(avg(st_area( geometry)::numeric),3)::numeric AS avg_area
, round(min(st_area( geometry)::numeric),8)::numeric AS min_area
, round(max(st_area( geometry)::numeric),0)::numeric AS max_area
, round(sum(st_length(geometry)::numeric),0)::numeric AS sum_length
, round(avg(st_length(geometry)::numeric),3)::numeric AS avg_length
, round(min(st_length(geometry)::numeric),8)::numeric AS min_length
, round(max(st_length(geometry)::numeric),0)::numeric AS max_length
, max( ST_NumGeometries( geometry ) )::numeric AS max_numgeom
, round(avg( ST_NumGeometries( geometry )),8)::numeric AS avg_numgeom
, sum( (ST_GeometryType(geometry)=$$ST_Point$$ )::int )::numeric AS n_point
, sum( (ST_GeometryType(geometry)=$$ST_LineString$$ )::int )::numeric AS n_linestring
, sum( (ST_GeometryType(geometry)=$$ST_MultiLineString$$)::int )::numeric AS n_multilinestring
, sum( (ST_GeometryType(geometry)=$$ST_Polygon$$ )::int )::numeric AS n_polygon
, sum( (ST_GeometryType(geometry)=$$ST_MultiPolygon$$ )::int )::numeric AS n_multipolygon
, sum( (ST_GeometryType(geometry) not in ( $$ST_Point$$
, $$ST_LineString$$
, $$ST_MultiLineString$$
, $$ST_Polygon$$
, $$ST_MultiPolygon$$) )::int )::numeric AS n_other
, sum( (ST_GeometryType(geometry) in ($$ST_Polygon$$,$$ST_MultiPolygon$$) )::int * (st_area(geometry)< 10 )::int )::numeric AS n_polyarea_l10m2
, sum( (ST_GeometryType(geometry) in ($$ST_Polygon$$,$$ST_MultiPolygon$$) )::int * (st_area(geometry)< 100 )::int )::numeric AS n_polyarea_l100m2
, sum( (ST_GeometryType(geometry) in ($$ST_Polygon$$,$$ST_MultiPolygon$$) )::int * (st_area(geometry)< 1000)::int )::numeric AS n_polyarea_l1000m2
, sum( (ST_GeometryType(geometry) in ($$ST_LineString$$,$$ST_MultiLineString$$) )::int * (st_length(geometry)< 1 )::int )::numeric AS n_linelen_l1m
, sum( (ST_GeometryType(geometry) in ($$ST_LineString$$,$$ST_MultiLineString$$) )::int * (st_length(geometry)< 10 )::int )::numeric AS n_linelen_l10m
, sum( (ST_GeometryType(geometry) in ($$ST_LineString$$,$$ST_MultiLineString$$) )::int * (st_length(geometry)< 100 )::int )::numeric AS n_linelen_l100m
, sum( (ST_GeometryType(geometry) in ($$ST_LineString$$,$$ST_MultiLineString$$) )::int * (st_length(geometry)< 1000)::int )::numeric AS n_linelen_l1000m
, sum(( %3$L || substring( md5( %2$s::text) , 1, 8))::bit(32)::bigint) AS id_md5_1
, sum(( %3$L || substring( md5( %2$s::text) , 9, 8))::bit(32)::bigint) AS id_md5_2
, sum(( %3$L || substring( md5( %2$s::text) , 17, 8))::bit(32)::bigint) AS id_md5_3
, sum(( %3$L || substring( md5( %2$s::text) , 25, 8))::bit(32)::bigint) AS id_md5_4
FROM %4$s ' , quote_literal(_tbl::text), tid, 'x', _tbl , grp , type_qstr )
;
END
$func$ LANGUAGE plpgsql;
-- query --
SELECT * from o2v_analyze_area_layer('road','osm_road_geometry', true )
UNION ALL SELECT * from o2v_analyze_area_layer('road','road_z5', true )
UNION ALL SELECT * from o2v_analyze_area_layer('road','road_z6toz7', true )
UNION ALL SELECT * from o2v_analyze_area_layer('road','road_z8toz9', true )
UNION ALL SELECT * from o2v_analyze_area_layer('road','road_z10', true )
UNION ALL SELECT * from o2v_analyze_area_layer('road','road_z11', true )
UNION ALL SELECT * from o2v_analyze_area_layer('road','road_z12', true )
UNION ALL SELECT * from o2v_analyze_area_layer('road','road_z13', true )
UNION ALL SELECT * from o2v_analyze_area_layer('road','road_z14', true )
UNION ALL SELECT * from o2v_analyze_area_layer('barrier','osm_barrier_polygon', true )
UNION ALL SELECT * from o2v_analyze_area_layer('barrier','barrier_line_z14', true )
UNION ALL SELECT * from o2v_analyze_area_layer('building','building_z13', false )
UNION ALL SELECT * from o2v_analyze_area_layer('building','building_z14', false )
UNION ALL SELECT * from o2v_analyze_area_layer('water','water_z0', false )
UNION ALL SELECT * from o2v_analyze_area_layer('water','water_z1', false )
UNION ALL SELECT * from o2v_analyze_area_layer('water','water_z2toz3', false )
UNION ALL SELECT * from o2v_analyze_area_layer('water','water_z4', false )
UNION ALL SELECT * from o2v_analyze_area_layer('water','water_z5toz7', false )
UNION ALL SELECT * from o2v_analyze_area_layer('water','water_z8toz10', false )
UNION ALL SELECT * from o2v_analyze_area_layer('water','water_z11toz12', false )
UNION ALL SELECT * from o2v_analyze_area_layer('water','water_z13toz14', false )
UNION ALL SELECT * from o2v_analyze_area_layer('land' ,'osm_landuse_polygon')
UNION ALL SELECT * from o2v_analyze_area_layer('land' ,'osm_landuse_polygon_gen0')
UNION ALL SELECT * from o2v_analyze_area_layer('land' ,'osm_landuse_polygon_gen1')
UNION ALL SELECT * from o2v_analyze_area_layer('land' ,'landuse_z5toz6')
UNION ALL SELECT * from o2v_analyze_area_layer('land' ,'landuse_z7toz8')
UNION ALL SELECT * from o2v_analyze_area_layer('land' ,'landuse_z9')
UNION ALL SELECT * from o2v_analyze_area_layer('land' ,'landuse_z10')
UNION ALL SELECT * from o2v_analyze_area_layer('land' ,'landuse_z11')
UNION ALL SELECT * from o2v_analyze_area_layer('land' ,'landuse_z12')
UNION ALL SELECT * from o2v_analyze_area_layer('land' ,'landuse_z13toz14')
UNION ALL SELECT * from o2v_analyze_area_layer('land' ,'landuse_overlay_z5')
UNION ALL SELECT * from o2v_analyze_area_layer('land' ,'landuse_overlay_z6')
UNION ALL SELECT * from o2v_analyze_area_layer('land' ,'landuse_overlay_z7')
UNION ALL SELECT * from o2v_analyze_area_layer('land' ,'landuse_overlay_z8')
UNION ALL SELECT * from o2v_analyze_area_layer('land' ,'landuse_overlay_z9')
UNION ALL SELECT * from o2v_analyze_area_layer('land' ,'landuse_overlay_z10')
UNION ALL SELECT * from o2v_analyze_area_layer('land' ,'landuse_overlay_z11toz12')
UNION ALL SELECT * from o2v_analyze_area_layer('land' ,'landuse_overlay_z13toz14')
;

Wyświetl plik

@ -0,0 +1,90 @@

-- postgis version info
SELECT 'ReportName :' as id, 'O2V-QA-R002 System info - Geometry tables & indexes'::text as ver
UNION ALL SELECT 'ReportVersion:' as id, 'v0.0.1/2016-jun-29'::text as ver
UNION ALL SELECT 'RunTime :' as id, Now()::text as ver
UNION ALL SELECT 'postgis :' as id, PostGIS_full_version()::text as ver
UNION ALL SELECT 'postgresql :' as id, version()::text as ver
;
-- Tables and Views ...
WITH o2v_vars AS (
SELECT table_name, column_name, udt_name
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name NOT IN ('geography_columns','geometry_columns','raster_columns'
,'raster_overviews' ,'spatial_ref_sys' )
ORDER BY table_name, column_name
)
,o2v_geom AS (
SELECT table_name, column_name as geomvar_name
FROM o2v_vars
WHERE udt_name='geometry'
)
,o2v_type AS (
SELECT table_name, column_name AS typevar_name
FROM o2v_vars
WHERE column_name='type'
)
,o2v_id AS (
SELECT table_name, column_name AS idvar_name
FROM o2v_vars
WHERE column_name IN ('id','osm_id','ogc_fid','gid')
)
,o2v_tables AS (
SELECT table_name, table_type
FROM information_schema.tables
WHERE table_schema = 'public'
)
,o2v_table_columns AS (
SELECT table_name, count(*) AS n_columns
FROM o2v_vars
GROUP BY table_name
ORDER BY table_name
)
,o2v_indexes AS (
SELECT relname as table_name, count(*) AS n_index
FROM pg_stat_all_indexes
WHERE schemaname = 'public'
AND relname NOT IN ('geography_columns','geometry_columns','raster_columns'
,'raster_overviews' ,'spatial_ref_sys' )
GROUP BY relname
ORDER BY relname
)
SELECT
o2v_geom.table_name
,o2v_tables.table_type
,o2v_table_columns.n_columns
,o2v_geom.geomvar_name
,o2v_type.typevar_name
,o2v_id.idvar_name
,o2v_indexes.n_index
,pg_size_pretty( pg_total_relation_size(o2v_geom.table_name)) AS full_total_size
,pg_total_relation_size(o2v_geom.table_name) AS total_relation_size
,pg_table_size(o2v_geom.table_name) AS table_size
,pg_indexes_size(o2v_geom.table_name) AS indexes_size
FROM o2v_geom
LEFT JOIN o2v_tables ON o2v_geom.table_name = o2v_tables.table_name
LEFT JOIN o2v_type ON o2v_geom.table_name = o2v_type.table_name
LEFT JOIN o2v_id ON o2v_geom.table_name = o2v_id.table_name
LEFT JOIN o2v_table_columns ON o2v_geom.table_name = o2v_table_columns.table_name
LEFT JOIN o2v_indexes ON o2v_geom.table_name = o2v_indexes.table_name
ORDER BY table_name
;
-- Indexes ...
SELECT relname, indexrelname, idx_scan, idx_tup_read,idx_tup_fetch, pg_size_pretty(pg_relation_size(indexrelname::text))
FROM pg_stat_all_indexes
WHERE schemaname = 'public'
AND relname NOT IN ('geography_columns','geometry_columns','raster_columns'
,'raster_overviews' ,'spatial_ref_sys' )
ORDER BY relname, indexrelname
;

0
wrk/.gitkeep 100644
Wyświetl plik