example graphviz based ETL graph

gen_etl_graph.sh

@@ -0,0 +1,21 @@
+
+
+DOT=highway.dot
+
+echo "digraph G
+{    
+rankdir=LR;
+" > $DOT
+
+cat ./layers/highway/mapping.yaml        | grep    "# etldoc:" |  sed 's/# etldoc://g'  >>$DOT
+cat ./layers/highway/types.sql           | grep "\-\- etldoc:" |  sed 's/-- etldoc://g' >>$DOT
+cat ./layers/highway/ne_global_roads.sql | grep "\-\- etldoc:" |  sed 's/-- etldoc://g' >>$DOT 
+cat ./layers/highway/highway.sql         | grep "\-\- etldoc:" |  sed 's/-- etldoc://g' >>$DOT
+
+echo "
+}
+" >> $DOT
+
+cat $DOT
+
+dot -Tpng $DOT > layer_highway_etl.png

layers/highway/highway.sql

@@ -1,7 +1,19 @@
+
+
+-- etldoc: highway_sql[label="highway.sql", shape=note ]
+-- etldoc: function_highway_is_link[label="FUNCTION highway_is_link"]
+-- etldoc: highway_sql-> function_highway_is_link
+-- etldoc: function_highway_is_link -> postgreSQL
+
 CREATE OR REPLACE FUNCTION highway_is_link(highway TEXT) RETURNS BOOLEAN AS $$
     SELECT highway LIKE '%_link';
 $$ LANGUAGE SQL IMMUTABLE STRICT;
 
+
+-- etldoc: postgreSQL -> layer_highway
+-- etldoc: highway_sql-> layer_highway
+-- etldoc: layer_highway[shape=record fillcolor=lightpink, style="rounded,filled",  
+-- etldoc:     label="<sql> layer_highway (highway.sql) |<z4z7> z4-z7 |<z8> z8 |<z9> z9 |<z10> z10 |<z11> z11 |<z12> z12|<z13> z13|<z14> z14" ] ;
 CREATE OR REPLACE FUNCTION layer_highway(bbox geometry, zoom_level int)
 RETURNS TABLE(osm_id bigint, geometry geometry, class highway_class, subclass text, properties highway_properties) AS $$
     SELECT
@@ -9,29 +21,41 @@ RETURNS TABLE(osm_id bigint, geometry geometry, class highway_class, subclass te
 		to_highway_class(highway) AS class, highway AS subclass,
 		to_highway_properties(is_bridge, is_tunnel, is_ford, is_ramp, is_oneway) AS properties
 	FROM (
-		SELECT
+
+		-- etldoc: ne_10m_global_roads ->  layer_highway:z4z7
+		SELECT 
 			NULL::bigint AS osm_id, geometry, highway,
 			FALSE AS is_bridge, FALSE AS is_tunnel, FALSE AS is_ford, FALSE AS is_ramp, FALSE AS is_oneway,
 			0 AS z_order
 		FROM ne_10m_global_roads
 		WHERE zoom_level BETWEEN 4 AND 7 AND scalerank <= 1 + zoom_level
 		UNION ALL
+
+		-- etldoc: osm_highway_linestring_gen4  ->  layer_highway:z8
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring_gen4
 		WHERE zoom_level = 8
 		UNION ALL
+
+		-- etldoc: osm_highway_linestring_gen3  ->  layer_highway:z9
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring_gen3
 		WHERE zoom_level = 9
 		UNION ALL
+
+		-- etldoc: osm_highway_linestring_gen2  ->  layer_highway:z10
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring_gen2
 		WHERE zoom_level = 10
 		UNION ALL
+
+		-- etldoc: osm_highway_linestring_gen1  ->  layer_highway:z11
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring_gen1
 		WHERE zoom_level = 11
 		UNION ALL
+
+		-- etldoc: osm_highway_linestring       ->  layer_highway:z12
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring
 		WHERE zoom_level = 12
@@ -39,18 +63,25 @@ RETURNS TABLE(osm_id bigint, geometry geometry, class highway_class, subclass te
             AND NOT highway_is_link(highway)
             AND NOT is_area
         UNION ALL
+
+		-- etldoc: osm_highway_linestring       ->  layer_highway:z13
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring
 		WHERE zoom_level = 13
             AND to_highway_class(highway) < 'path'::highway_class
             AND NOT is_area
         UNION ALL
+
+		-- etldoc: osm_highway_linestring       ->  layer_highway:z14
 		SELECT osm_id, geometry, highway, is_bridge, is_tunnel, is_ford, is_ramp, is_oneway, z_order
         FROM osm_highway_linestring
 		WHERE zoom_level >= 14 AND NOT is_area
         UNION ALL
+
         -- NOTE: We limit the selection of polys because we need to be careful to net get false positives here because
         -- it is possible that closed linestrings appear both as highway linestrings and as polygon
+        -- etldoc: osm_highway_polygon          ->  layer_highway:z13
+        -- etldoc: osm_highway_polygon          ->  layer_highway:z14
 		SELECT osm_id, geometry, highway, FALSE AS is_bridge, FALSE AS is_tunnel, FALSE AS is_ford, FALSE AS is_ramp, FALSE AS is_oneway, z_order
         FROM osm_highway_polygon
         -- We do not want underground pedestrian areas for now

layers/highway/mapping.yaml

@@ -1,21 +1,43 @@
+
+
+
+# etldoc: imposm3 [fillcolor=yellow, style="rounded,filled", shape=diamond ];
+# etldoc: osm [fillcolor=lightblue, 
+# etldoc:      style="rounded,filled", 
+# etldoc:      shape=box , label="OpenStreetMap", 
+# etldoc:      url="http://www.openstreetmap.org"];
+# etldoc: mapping_yml[label="mapping.yml", shape=note ]
+# etldoc: mapping_yml -> imposm3
+# etldoc: osm -> imposm3
+
 generalized_tables:
+
+# etldoc: imposm3 -> osm_highway_linestring_gen4
   highway_linestring_gen4:
     source: highway_linestring_gen3
     sql_filter: highway IN ('motorway','trunk') AND NOT is_area
     tolerance: 200.0
+
+# etldoc: imposm3 -> osm_highway_linestring_gen3    
   highway_linestring_gen3:
     source: highway_linestring_gen2
     sql_filter: highway IN ('motorway','trunk', 'primary') AND NOT is_area
     tolerance: 120.0
+
+# etldoc: imposm3 -> osm_highway_linestring_gen2    
   highway_linestring_gen2:
     source: highway_linestring_gen1
     sql_filter: highway IN ('motorway','trunk', 'primary', 'secondary') AND NOT is_area
     tolerance: 50.0
+
+# etldoc: imposm3 -> osm_highway_linestring_gen1    
   highway_linestring_gen1:
     source: highway_linestring
     sql_filter: highway IN ('motorway','trunk', 'primary', 'secondary', 'tertiary') AND NOT is_area
     tolerance: 20.0
+
 tables:
+# etldoc: imposm3 -> osm_highway_linestring
   highway_linestring:
     type: linestring
     fields:
@@ -85,6 +107,8 @@ tables:
       - corridor
       - crossing
       - pedestrian
+
+# etldoc: imposm3 -> osm_highway_polygon     
   highway_polygon:
     type: polygon
     fields:

layers/highway/ne_global_roads.sql

@@ -1,3 +1,10 @@
+
+
+-- etldoc: ne_global_roads_sql [label="ne_global_roads.sql", shape=note ]
+-- etldoc: function_ne_highway[label="FUNCTION ne_highway"]
+-- etldoc: ne_global_roads_sql -> function_ne_highway
+-- etldoc: function_ne_highway -> postgreSQL
+
 CREATE OR REPLACE FUNCTION ne_highway(type VARCHAR) RETURNS VARCHAR AS $$
   SELECT CASE type
     WHEN 'Major Highway' THEN 'motorway'
@@ -7,13 +14,24 @@ CREATE OR REPLACE FUNCTION ne_highway(type VARCHAR) RETURNS VARCHAR AS $$
   END;
 $$ LANGUAGE SQL IMMUTABLE;
 
+
+-- etldoc: natural_earth [fillcolor=lightblue, style="rounded,filled", shape=box , label="Natural Earth" ];
+-- etldoc: natural_earth -> ne_10m_roads 
+-- etldoc: natural_earth -> ne_10m_roads_north_america
+
+-- etldoc: ne_global_roads_sql -> ne_10m_global_roads  ;
 CREATE TABLE IF NOT EXISTS ne_10m_global_roads AS (
+
+    -- etldoc:  ne_10m_roads -> ne_10m_global_roads
     SELECT geom AS geometry, scalerank, ne_highway(type) AS highway
     FROM ne_10m_roads
     WHERE continent <> 'North America'
       AND featurecla = 'Road'
       AND type IN ('Major Highway', 'Secondary Highway', 'Road')
     UNION ALL
+
+    -- etldoc: function_ne_highway -> ne_10m_global_roads  
+    -- etldoc: ne_10m_roads_north_america ->  ne_10m_global_roads  
     SELECT geom AS geometry, scalerank, ne_highway(type) AS highway
     FROM ne_10m_roads_north_america
     WHERE type IN ('Major Highway', 'Secondary Highway', 'Road')

layers/highway/types.sql

@@ -1,4 +1,11 @@
+
+
+-- etldoc: types_sql[label="types.sql", shape=note ]
+
 DO $$
+-- etldoc: type_highway_class[label="TYPE highway_class"] 
+-- etldoc: types_sql-> type_highway_class
+-- etldoc: type_highway_class -> postgreSQL
 BEGIN
     IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'highway_class') THEN
 		CREATE TYPE highway_class AS ENUM ('motorway', 'major_road', 'minor_road', 'path');
@@ -7,6 +14,9 @@ END
 $$;
 
 DO $$
+-- etldoc: type_highway_properties[label="TYPE highway_properties"] 
+-- etldoc: types_sql-> type_highway_properties
+-- etldoc: type_highway_properties -> postgreSQL
 BEGIN
     IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'highway_properties') THEN
         CREATE TYPE highway_properties AS ENUM ('bridge:oneway', 'tunnel:oneway', 'ramp', 'ford', 'bridge', 'tunnel', 'oneway');
@@ -15,6 +25,9 @@ END
 $$;
 
 
+-- etldoc: function_to_highway_class[label="FUNCTION to_highway_class"]
+-- etldoc: types_sql-> function_to_highway_class 
+-- etldoc: function_to_highway_class -> postgreSQL
 CREATE OR REPLACE FUNCTION to_highway_class(highway TEXT) RETURNS highway_class AS $$
     SELECT CASE
         WHEN highway IN ('motorway', 'motorway_link') THEN 'motorway'::highway_class
@@ -29,6 +42,9 @@ CREATE OR REPLACE FUNCTION to_highway_class(highway TEXT) RETURNS highway_class
     END;
 $$ LANGUAGE SQL IMMUTABLE STRICT;
 
+-- etldoc: function_to_highway_properties[label="FUNCTION to_highway_properties"]
+-- etldoc: types_sql-> function_to_highway_properties
+-- etldoc: function_to_highway_properties -> postgreSQL
 CREATE OR REPLACE FUNCTION to_highway_properties(is_bridge boolean, is_tunnel boolean, is_ford boolean, is_ramp boolean, is_oneway boolean) RETURNS highway_properties AS $$
     SELECT CASE
          WHEN is_bridge AND is_oneway THEN 'bridge:oneway'::highway_properties