diff --git a/sh/gen_doc.sh b/sh/gen_doc.sh new file mode 100755 index 00000000..a8dae5da --- /dev/null +++ b/sh/gen_doc.sh @@ -0,0 +1,23 @@ + + +# Work in progress ... +# +# generate ETL graph from all layers +# start from the root : +# ./sh/gen_doc.sh +# +# outut -> ./doc/ + + +mkdir -p ./doc/ +rm -f ./doc/* +for f in ./layers/* +do + echo "Processing : $f" + layer_id=$(echo "$f" | rev | cut -d"/" -f1 | rev ) + echo "layer_id = $layer_id" + ./sh/gen_etl_graph.sh $layer_id + +done + + diff --git a/sh/gen_etl_graph.sh b/sh/gen_etl_graph.sh new file mode 100755 index 00000000..572fe83a --- /dev/null +++ b/sh/gen_etl_graph.sh @@ -0,0 +1,45 @@ + + + +# Work in progress ... +# +# generate ETL graph from a single layer +# start from the root +# parameter1: "layer name" +# ./sh/gen_etl_graph.sh waterway +# +# outut -> ./doc/ + +ID=$1 +## ID=waterway + +mkdir -p ./doc +layer_dir=./layers/${ID}/ +DOT=./doc/dot_${ID}.dot + + +echo "digraph G +{ +rankdir=LR; +" > $DOT + +if [ -f ${layer_dir}/mapping.yaml ]; then + echo "processing imposm3 mapping file ${layer_dir}/mapping.yaml " + cat ${layer_dir}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT +fi + + +for f in ${layer_dir}*.sql +do + echo "Processing : $f" + cat $f | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT +done + +echo " +} +" >> $DOT + +##cat $DOT + +dot -Tpng $DOT > ./doc/etl_layer_${ID}.png +dot -Tsvg $DOT > ./doc/etl_layer_${ID}.svg diff --git a/sh/gen_etl_graph_boundary.sh b/sh/gen_etl_graph_boundary.sh deleted file mode 100755 index 1430eee0..00000000 --- a/sh/gen_etl_graph_boundary.sh +++ /dev/null @@ -1,21 +0,0 @@ - - - - -DOT=boundary.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/boundary/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT -cat ./layers/boundary/boundary.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_boundary_etl.png diff --git a/sh/gen_etl_graph_building.sh b/sh/gen_etl_graph_building.sh deleted file mode 100755 index a74b0e63..00000000 --- a/sh/gen_etl_graph_building.sh +++ /dev/null @@ -1,21 +0,0 @@ - - - - -DOT=building.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/building/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT -cat ./layers/building/building.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_building_etl.png diff --git a/sh/gen_etl_graph_highway.sh b/sh/gen_etl_graph_highway.sh deleted file mode 100755 index ddd3520a..00000000 --- a/sh/gen_etl_graph_highway.sh +++ /dev/null @@ -1,21 +0,0 @@ - - -DOT=highway.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/highway/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT -cat ./layers/highway/types.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/highway/ne_global_roads.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/highway/highway.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_highway_etl.png diff --git a/sh/gen_etl_graph_highway_name.sh b/sh/gen_etl_graph_highway_name.sh deleted file mode 100755 index 39afd79f..00000000 --- a/sh/gen_etl_graph_highway_name.sh +++ /dev/null @@ -1,19 +0,0 @@ - - -DOT=highway_name.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/highway_name/layer.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/highway_name/merge_highways.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_highway_names_etl.png diff --git a/sh/gen_etl_graph_housenumber.sh b/sh/gen_etl_graph_housenumber.sh deleted file mode 100755 index 37c12a97..00000000 --- a/sh/gen_etl_graph_housenumber.sh +++ /dev/null @@ -1,20 +0,0 @@ - - -DOT=housenumber.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/housenumber/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT -cat ./layers/housenumber/housenumber_centroid.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/housenumber/layer.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_housenumber_etl.png diff --git a/sh/gen_etl_graph_landcover.sh b/sh/gen_etl_graph_landcover.sh deleted file mode 100755 index 008b0ce7..00000000 --- a/sh/gen_etl_graph_landcover.sh +++ /dev/null @@ -1,19 +0,0 @@ - - -DOT=landcover.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/landcover/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT -cat ./layers/landcover/landcover.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_landcover_etl.png diff --git a/sh/gen_etl_graph_landuse.sh b/sh/gen_etl_graph_landuse.sh deleted file mode 100755 index 25ff4f9a..00000000 --- a/sh/gen_etl_graph_landuse.sh +++ /dev/null @@ -1,19 +0,0 @@ - - -DOT=landuse.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/landuse/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT -cat ./layers/landuse/landuse.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_landuse_etl.png diff --git a/sh/gen_etl_graph_place.sh b/sh/gen_etl_graph_place.sh deleted file mode 100755 index 5056cfa1..00000000 --- a/sh/gen_etl_graph_place.sh +++ /dev/null @@ -1,27 +0,0 @@ - - -DOT=place.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/place/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT - -cat ./layers/place/types.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/place/city.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/place/country.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/place/state.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/place/merge_country_rank.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/place/merge_city_rank.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/place/merge_state_rank.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/place/place.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_place_etl.png diff --git a/sh/gen_etl_graph_poi.sh b/sh/gen_etl_graph_poi.sh deleted file mode 100755 index f565ba84..00000000 --- a/sh/gen_etl_graph_poi.sh +++ /dev/null @@ -1,22 +0,0 @@ - - -ID=poi -DOT=${ID}.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/${ID}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT - -cat ./layers/${ID}/class.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/${ID}/layer.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_${ID}_etl.png diff --git a/sh/gen_etl_graph_railway.sh b/sh/gen_etl_graph_railway.sh deleted file mode 100755 index 9d82d8bb..00000000 --- a/sh/gen_etl_graph_railway.sh +++ /dev/null @@ -1,22 +0,0 @@ - - -ID=railway -DOT=${ID}.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/${ID}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT - -cat ./layers/${ID}/types.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/${ID}/railway.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_${ID}_etl.png diff --git a/sh/gen_etl_graph_water.sh b/sh/gen_etl_graph_water.sh deleted file mode 100755 index 86547c76..00000000 --- a/sh/gen_etl_graph_water.sh +++ /dev/null @@ -1,21 +0,0 @@ - - -ID=water -DOT=${ID}.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/${ID}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT - -cat ./layers/${ID}/water.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_${ID}_etl.png diff --git a/sh/gen_etl_graph_water_name.sh b/sh/gen_etl_graph_water_name.sh deleted file mode 100755 index 4245c1a1..00000000 --- a/sh/gen_etl_graph_water_name.sh +++ /dev/null @@ -1,20 +0,0 @@ - - -ID=water_name -DOT=${ID}.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/${ID}/merge_lakelines.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT -cat ./layers/${ID}/layer.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_${ID}_etl.png diff --git a/sh/gen_etl_graph_waterway.sh b/sh/gen_etl_graph_waterway.sh deleted file mode 100755 index 4f7b4806..00000000 --- a/sh/gen_etl_graph_waterway.sh +++ /dev/null @@ -1,21 +0,0 @@ - - -ID=waterway -DOT=${ID}.dot - -echo "digraph G -{ -rankdir=LR; -" > $DOT - -cat ./layers/${ID}/mapping.yaml | grep "# etldoc:" | sed 's/# etldoc://g' >>$DOT - -cat ./layers/${ID}//waterway.sql | grep "\-\- etldoc:" | sed 's/-- etldoc://g' >>$DOT - -echo " -} -" >> $DOT - -cat $DOT - -dot -Tpng $DOT > layer_${ID}_etl.png