From 446d9fadf5623e6155fb81c81601641ac65661ed Mon Sep 17 00:00:00 2001 From: mazano Date: Sun, 16 Jun 2019 09:52:56 +0200 Subject: [PATCH 1/3] Add pgwatch to docker-compose (#83) * Fix import error due to differences in geom type * fix id missing in clip table and skipping importing shapefiles with geometry errors * fix clip id error in the sql * Change postgresql version from 9.6 to 11 * Added sql to validate geometry and other make commands to run them manually * added pgwatch to docker-compose --- docker-compose.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 5cbcef9..2e6f33b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,9 @@ volumes: import_done: import_queue: cache: + pg: + grafana: + pw2: services: db: @@ -105,3 +108,30 @@ services: # seconds between 2 executions of the script # if 0, then no update will be done, only the first initial import from the PBF - TIME=120 + pgwatch2: + image: cybertec/pgwatch2-postgres + hostname: pgwatch2 + volumes: + - pg:/var/lib/postgresql + - grafana:/var/lib/grafana + - pw2:/pgwatch2/persistent-config + environment: + - PW2_TESTDB=false + - PW2_PG_SCHEMA_TYPE=metric-dbname-time + - PW2_WEBNOANONYMOUS=true + - PW2_WEBUSER=admin + - PW2_WEBPASSWORD=pgwatch2 + - PW2_WEBNOCOMPONENTLOGS=false + - PW2_WEBHOST=0.0.0.0 + - PW2_WEBPORT=8080 + - PW2_GRAFANANOANONYMOUS=true + - PW2_GRAFANAUSER=admin + - PW2_GRAFANAPASSWORD=pgwatch2 + - PW2_GRAFANA_BASEURL=http://0.0.0.0:3000 + restart: on-failure + ports: + - 3000:3000 + - 8080:8080 + depends_on: + db: + condition: service_healthy \ No newline at end of file From f13395933aeff7639ed47cfa53abc5b0d6ee9760 Mon Sep 17 00:00:00 2001 From: Etienne Trimaille Date: Fri, 20 Sep 2019 22:53:05 +0200 Subject: [PATCH 2/3] fix typo and launching of docker-osm when we do not have clip shapefile by default --- Makefile | 6 ++---- docker-compose.yml | 2 +- docker-imposm3/importer.py | 8 +++----- settings/clip/clip.sql | 10 +++------- 4 files changed, 9 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 33320d8..c564cda 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,6 @@ status: @echo "------------------------------------------------------------------" @docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) ps - build: @echo @echo "------------------------------------------------------------------" @@ -129,15 +128,14 @@ validate_geom: validate_geom @echo "------------------------------------------------------------------" @echo "Validating geom for all tables" @echo "------------------------------------------------------------------" - @docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'Select validate_geom();'" - + @docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'SELECT validate_geom();'" clip_tables: clip_tables @echo @echo "------------------------------------------------------------------" @echo "Clip tables using the clip layer" @echo "------------------------------------------------------------------" - @docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'Select clean_tables();'" + @docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'SELECT clean_tables();'" diff --git a/docker-compose.yml b/docker-compose.yml index 2e6f33b..102e65a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,7 +73,7 @@ services: # Install some styles if you are using the default mapping. It can be 'yes' or 'no' - QGIS_STYLE=yes # Use clip in the database - To use this you should have run make import_clip to add your clip to the DB - - CLIP=yes + - CLIP=no osmupdate: build: docker-osmupdate diff --git a/docker-imposm3/importer.py b/docker-imposm3/importer.py index 6f4aaf9..8f7c4e0 100644 --- a/docker-imposm3/importer.py +++ b/docker-imposm3/importer.py @@ -101,7 +101,7 @@ class Importer(object): msg = 'QGIS_STYLE not supported : %s' % self.default['QGIS_STYLE'] self.error(msg) else: - self.info('Qgis style: ' + self.default['QGIS_STYLE']) + self.info('QGIS style: ' + self.default['QGIS_STYLE']) # Check folders. folders = ['IMPORT_QUEUE', 'IMPORT_DONE', 'SETTINGS', 'CACHE'] @@ -204,9 +204,7 @@ class Importer(object): self.default['POSTGRES_PASS'])) self.cursor = connection.cursor() except OperationalError as e: - print(stderr.write(e)) - - exit() + self.error(e) self.postgis_uri = 'postgis://%s:%s@%s/%s' % ( self.default['POSTGRES_USER'], @@ -246,7 +244,7 @@ class Importer(object): command += ['-d', self.default['POSTGRES_DBNAME']] command += ['-f', self.clip_sql_file] call(command) - self.info('!! Be sure to run \'make import_clip\' to import the shapefile into the DB !!') + self.info('!! Be sure to run \'make import_clip\' to import the Shapefile into the DB !!') def perform_clip_in_db(self): """Perform clipping if the clip table is here.""" diff --git a/settings/clip/clip.sql b/settings/clip/clip.sql index a85d79d..d834353 100644 --- a/settings/clip/clip.sql +++ b/settings/clip/clip.sql @@ -18,11 +18,8 @@ END; $BODY$ LANGUAGE plpgsql; - - - - --- Function to validate geometry of all tables. To run it after creating the function sinmply run Select validate_geom(); +-- Function to validate geometry of all tables. +-- To run it after creating the function simply run SELECT validate_geom(); CREATE OR REPLACE FUNCTION validate_geom() RETURNS void AS $BODY$ DECLARE osm_tables CURSOR FOR @@ -34,9 +31,8 @@ DECLARE osm_tables CURSOR FOR BEGIN FOR osm_table IN osm_tables LOOP EXECUTE 'UPDATE ' || quote_ident(osm_table.table_name) || ' SET - geometry = ST_MakeValid(geometry) where ST_isValidreason(geometry) = ''F'' ;'; + geometry = ST_MakeValid(geometry) where ST_IsValidReason(geometry) = ''F'' ;'; END LOOP; END; $BODY$ LANGUAGE plpgsql; - From e29ce8c08b0ff227eb939724760d7c38207a0bc8 Mon Sep 17 00:00:00 2001 From: Etienne Trimaille Date: Fri, 20 Sep 2019 23:05:51 +0200 Subject: [PATCH 3/3] update the QGIS project to 3.4.12 --- web/project.qgs | 1219 +++++++++++++++++++++++------------------------ 1 file changed, 586 insertions(+), 633 deletions(-) diff --git a/web/project.qgs b/web/project.qgs index 76ff7ec..a57e736 100644 --- a/web/project.qgs +++ b/web/project.qgs @@ -1,19 +1,43 @@ - + + - + + + + +proj=longlat +datum=WGS84 +no_defs + 3452 + 4326 + EPSG:4326 + WGS 84 + longlat + WGS84 + true + + + - + - + + + osm_buildings20160720014515980 + osm_roads20180302190105366 + + + + + + + - + degrees 47.52122050961614264 @@ -22,7 +46,6 @@ -18.90150075927243023 0 - 1 +proj=longlat +datum=WGS84 +no_defs @@ -36,47 +59,31 @@ 0 - - - - - - - - - - - - - - - - - - - - - osm_buildings20160720014515980 - osm_roads20180302190105366 - - + - + - + + - + + + -171.99783107077300315 + -46.96690365331490113 + 178.47205985402101192 + 71.23581296164789478 + osm_buildings20160720014515980 - dbname='gis' host=db port=5432 user='docker' password='docker' sslmode=disable key='id' srid=4326 type=MultiPolygon table="public"."osm_buildings" (geometry) sql= + dbname='gis' host=db port=5432 user='docker' password='docker' sslmode=disable key='id,osm_id' srid=4326 type=MultiPolygon checkPrimaryKeyUnicity='1' table="public"."osm_buildings" (geometry) sql= @@ -93,266 +100,190 @@ true - postgres + + + + + + + + + + + + + + 0 + 0 + + + + + false + + + + + postgres + + - - + + - - - - - - - - - - - - - - - + + + 1 + 1 + 1 + + - - - + + + - + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 0 0 - 0 - name - - - - - - - + 1 + + + + - - - - - - - - - - - - - - - - - - - - - - - - . + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + - + + + + + + + + + + + + + + + + + + + + - - . + + + + + . 0 . @@ -375,22 +306,21 @@ def my_form_open(dialog, layer, feature): ]]> 0 generatedlayout + + - - - - - - - - - - - + "name" + - + + + -172.03645261748201278 + -46.88596606843370296 + 178.46728317503999506 + 69.04282369931030416 + osm_roads20180302190105366 - dbname='gis' host=db port=5432 user='docker' password='docker' sslmode=disable key='id' srid=4326 type=LineString table="public"."osm_roads" (geometry) sql= + dbname='gis' host=db port=5432 user='docker' password='docker' sslmode=disable key='id,osm_id' srid=4326 type=LineString checkPrimaryKeyUnicity='1' table="public"."osm_roads" (geometry) sql= @@ -407,58 +337,52 @@ def my_form_open(dialog, layer, feature): true + + + + + + + + + + + + + + 0 + 0 + + + + + false + + + + postgres + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + 1 + 1 + 1 + + - - + + - + @@ -467,386 +391,415 @@ def my_form_open(dialog, layer, feature): - + + - + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 0 0 - 0 - name - - - - - - - + 1 + + + + - - - - - - - - - - - - - - - - - - - - - - - - . + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - . - - 0 - . - - 0 - generatedlayout - - - - - - - - - - - - - - - - + . + + 0 + . + + 0 + generatedlayout + + + + "name" + + + + + - - +proj=longlat +datum=WGS84 +no_defs EPSG:4326 - 3452 + +proj=longlat +datum=WGS84 +no_defs 1 + 3452 - - meters - m2 - + + + WGS84 + + + + + + conditions unknown + + 0 + 219 + 255 + 106 + 43 + 255 + 255 + + + + true + + 255 + + + + + MU + 2 + true + + false - - - - - true - 255 - - - None - false - - 90 - - - + true + true + + + + - + - - - conditions unknown + 90 - false - - - 1 + + + false + + + + + + EPSG:4326 EPSG:3857 + + 16 + true + 0 + 0 + false + 30 + false + false + 50 + + 1 + + + false + - 2 osm_buildings20160720014515980 - - disabled - - current_layer - - - 1 - to_vertex_and_segment + + disabled + + off + 2 + + 1 + + current_layer 0 0.000000 + + + meters + m2 + + + None - - - - - - - - - - - false - - - - 2 - true - MU - - - 0 - 106 - 43 - 255 - 255 - 255 - 219 - - - WGS84 - - - - - true - true + false + + + + + + + + + + + + + + +