mirror
/
openmaptiles
kopia lustrzana https://github.com/openmaptiles/openmaptiles
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

refactor ETL graph

pull/17/head
ImreSamu 2016-11-11 17:26:53 +01:00
rodzic 34afba471f
commit d8cab7ad04
15 zmienionych plików z 68 dodań i 273 usunięć

23
sh/gen_doc.sh 100755
Wyświetl plik

@ -0,0 +1,23 @@
# Work in progress ...
#
# generate ETL graph from all layers
# start from the root :
# ./sh/gen_doc.sh
#
# outut -> ./doc/
mkdir -p ./doc/
rm -f ./doc/*
for f in ./layers/*
do
echo "Processing : $f"
layer_id=$(echo "$f" | rev | cut -d"/" -f1 | rev )
echo "layer_id = $layer_id"
./sh/gen_etl_graph.sh $layer_id
done

45
sh/gen_etl_graph.sh 100755
Wyświetl plik

@ -0,0 +1,45 @@
# Work in progress ...
#
# generate ETL graph from a single layer
# start from the root
# parameter1: "layer name"
# ./sh/gen_etl_graph.sh waterway
#
# outut -> ./doc/
ID=$1
## ID=waterway
mkdir -p ./doc
layer_dir=./layers/${ID}/
DOT=./doc/dot_${ID}.dot
echo "digraph G
{
rankdir=LR;
" > $DOT
if [ -f ${layer_dir}/mapping.yaml ]; then
echo "processing imposm3 mapping file ${layer_dir}/mapping.yaml "
cat ${layer_dir}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT
fi
for f in ${layer_dir}*.sql
do
echo "Processing : $f"
cat $f | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
done
echo "
}
" >> $DOT
##cat $DOT
dot -Tpng $DOT > ./doc/etl_layer_${ID}.png
dot -Tsvg $DOT > ./doc/etl_layer_${ID}.svg

21
sh/gen_etl_graph_boundary.sh
Wyświetl plik

@ -1,21 +0,0 @@
DOT=boundary.dot
echo "digraph G
{
rankdir=LR;
" > $DOT
cat ./layers/boundary/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT
cat ./layers/boundary/boundary.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
echo "
}
" >> $DOT
cat $DOT
dot -Tpng $DOT > layer_boundary_etl.png

21
sh/gen_etl_graph_building.sh
Wyświetl plik

@ -1,21 +0,0 @@
DOT=building.dot
echo "digraph G
{
rankdir=LR;
" > $DOT
cat ./layers/building/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT
cat ./layers/building/building.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
echo "
}
" >> $DOT
cat $DOT
dot -Tpng $DOT > layer_building_etl.png

21
sh/gen_etl_graph_highway.sh
Wyświetl plik

@ -1,21 +0,0 @@
DOT=highway.dot
echo "digraph G
{
rankdir=LR;
" > $DOT
cat ./layers/highway/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT
cat ./layers/highway/types.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
cat ./layers/highway/ne_global_roads.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
cat ./layers/highway/highway.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
echo "
}
" >> $DOT
cat $DOT
dot -Tpng $DOT > layer_highway_etl.png

19
sh/gen_etl_graph_highway_name.sh
Wyświetl plik

@ -1,19 +0,0 @@
DOT=highway_name.dot
echo "digraph G
{
rankdir=LR;
" > $DOT
cat ./layers/highway_name/layer.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
cat ./layers/highway_name/merge_highways.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
echo "
}
" >> $DOT
cat $DOT
dot -Tpng $DOT > layer_highway_names_etl.png

20
sh/gen_etl_graph_housenumber.sh
Wyświetl plik

@ -1,20 +0,0 @@
DOT=housenumber.dot
echo "digraph G
{
rankdir=LR;
" > $DOT
cat ./layers/housenumber/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT
cat ./layers/housenumber/housenumber_centroid.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
cat ./layers/housenumber/layer.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
echo "
}
" >> $DOT
cat $DOT
dot -Tpng $DOT > layer_housenumber_etl.png

19
sh/gen_etl_graph_landcover.sh
Wyświetl plik

@ -1,19 +0,0 @@
DOT=landcover.dot
echo "digraph G
{
rankdir=LR;
" > $DOT
cat ./layers/landcover/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT
cat ./layers/landcover/landcover.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
echo "
}
" >> $DOT
cat $DOT
dot -Tpng $DOT > layer_landcover_etl.png

19
sh/gen_etl_graph_landuse.sh
Wyświetl plik

@ -1,19 +0,0 @@
DOT=landuse.dot
echo "digraph G
{
rankdir=LR;
" > $DOT
cat ./layers/landuse/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT
cat ./layers/landuse/landuse.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
echo "
}
" >> $DOT
cat $DOT
dot -Tpng $DOT > layer_landuse_etl.png

27
sh/gen_etl_graph_place.sh
Wyświetl plik

@ -1,27 +0,0 @@
DOT=place.dot
echo "digraph G
{
rankdir=LR;
" > $DOT
cat ./layers/place/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT
cat ./layers/place/types.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
cat ./layers/place/city.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
cat ./layers/place/country.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
cat ./layers/place/state.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
cat ./layers/place/merge_country_rank.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
cat ./layers/place/merge_city_rank.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
cat ./layers/place/merge_state_rank.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
cat ./layers/place/place.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
echo "
}
" >> $DOT
cat $DOT
dot -Tpng $DOT > layer_place_etl.png

22
sh/gen_etl_graph_poi.sh
Wyświetl plik

@ -1,22 +0,0 @@
ID=poi
DOT=${ID}.dot
echo "digraph G
{
rankdir=LR;
" > $DOT
cat ./layers/${ID}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT
cat ./layers/${ID}/class.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
cat ./layers/${ID}/layer.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
echo "
}
" >> $DOT
cat $DOT
dot -Tpng $DOT > layer_${ID}_etl.png

22
sh/gen_etl_graph_railway.sh
Wyświetl plik

@ -1,22 +0,0 @@
ID=railway
DOT=${ID}.dot
echo "digraph G
{
rankdir=LR;
" > $DOT
cat ./layers/${ID}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT
cat ./layers/${ID}/types.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
cat ./layers/${ID}/railway.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
echo "
}
" >> $DOT
cat $DOT
dot -Tpng $DOT > layer_${ID}_etl.png

21
sh/gen_etl_graph_water.sh
Wyświetl plik

@ -1,21 +0,0 @@
ID=water
DOT=${ID}.dot
echo "digraph G
{
rankdir=LR;
" > $DOT
cat ./layers/${ID}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT
cat ./layers/${ID}/water.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
echo "
}
" >> $DOT
cat $DOT
dot -Tpng $DOT > layer_${ID}_etl.png

20
sh/gen_etl_graph_water_name.sh
Wyświetl plik

@ -1,20 +0,0 @@
ID=water_name
DOT=${ID}.dot
echo "digraph G
{
rankdir=LR;
" > $DOT
cat ./layers/${ID}/merge_lakelines.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
cat ./layers/${ID}/layer.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
echo "
}
" >> $DOT
cat $DOT
dot -Tpng $DOT > layer_${ID}_etl.png

21
sh/gen_etl_graph_waterway.sh
Wyświetl plik

@ -1,21 +0,0 @@
ID=waterway
DOT=${ID}.dot
echo "digraph G
{
rankdir=LR;
" > $DOT
cat ./layers/${ID}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT
cat ./layers/${ID}//waterway.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT
echo "
}
" >> $DOT
cat $DOT
dot -Tpng $DOT > layer_${ID}_etl.png