From 1c1231e4309653979ac6d2fa86dd3ef15c531127 Mon Sep 17 00:00:00 2001 From: zstadler Date: Tue, 26 Jul 2022 19:59:34 +0300 Subject: [PATCH] Update UPDATE.md including Makefile improvements (#1408) - Add `make import-diff` and `make generate-changed-tiles` - Update `UPDATE.md` accordingly - Other updates and clarifications in `UPDATE.md` --- Makefile | 21 ++++++++++++-- UPDATE.md | 84 +++++++++++++++++++++++++++++++++++-------------------- 2 files changed, 73 insertions(+), 32 deletions(-) diff --git a/Makefile b/Makefile index 7be6017d..35169cd7 100644 --- a/Makefile +++ b/Makefile @@ -186,6 +186,7 @@ Hints for developers: make generate-qa # statistics for a given layer's field make generate-tiles-pg # generate vector tiles based on .env settings using PostGIS ST_MVT() make generate-tiles # generate vector tiles based on .env settings using Mapnik (obsolete) + make generate-changed-tiles # Generate tiles changed by import-diff make test-sql # run unit tests on the OpenMapTiles SQL schema cat .env # list PG database and MIN_ZOOM and MAX_ZOOM information cat quickstart.log # transcript of the last ./quickstart.sh run @@ -200,6 +201,7 @@ Hints for downloading & importing data: make download-bbbike area=Amsterdam # download OSM data from bbbike.org and create config file make import-data # Import data from OpenStreetMapData, Natural Earth and OSM Lake Labels. make import-osm # Import OSM data with the mapping rules from build/mapping.yaml + make import-diff # Import OSM updates from data/changes.osc.gz make import-wikidata # Import labels from Wikidata make import-sql # Import layers (run this after modifying layer SQL) @@ -397,7 +399,7 @@ import-osm: all start-db-nowait $(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools sh -c 'pgwait && import-osm $(PBF_FILE)' .PHONY: start-update-osm -start-update-osm: all start-db +start-update-osm: start-db @$(assert_area_is_given) $(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) up -d update-osm @@ -406,7 +408,7 @@ stop-update-osm: $(DOCKER_COMPOSE) stop update-osm .PHONY: import-diff -import-diff: all start-db-nowait +import-diff: start-db-nowait @$(assert_area_is_given) $(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools sh -c 'pgwait && import-diff' @@ -440,6 +442,21 @@ generate-tiles-pg: all start-db $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools \ mbtiles-tools meta-generate "$(MBTILES_LOCAL_FILE)" $(TILESET_FILE) --auto-minmax --show-ranges +.PHONY: data/tiles.txt +data/tiles.txt: + find ./data -name "*.tiles" -exec cat {} \; -exec rm {} \; | \ + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools \ + tile_multiplier $(MIN_ZOOM) $(MAX_ZOOM) >> data/tiles.txt + +.PHONY: generate-changed-tiles +generate-changed-tiles: data/tiles.txt + # Re-generating updated tiles, if needed + if [ -s data/tiles.txt ] ; then \ + $(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools refresh-views; \ + $(DOCKER_COMPOSE) run $(DC_OPTS) -e LIST_FILE=data/tiles.txt openmaptiles-tools generate-tiles; \ + rm data/tiles.txt; \ + fi + .PHONY: start-tileserver start-tileserver: init-dirs @echo " " diff --git a/UPDATE.md b/UPDATE.md index f3bb4cb7..7b29c1b0 100644 --- a/UPDATE.md +++ b/UPDATE.md @@ -10,44 +10,71 @@ or import specific change files. ### Choosing the Download Source -While GeoFabrik currently provides extracts of basically all countries, they provide only daily updates. +While GeoFabrik currently provides extracts of basically all countries, they provide only daily updates. If you need minutely updates you might want to try openstreetmap.fr, for example like this: `make download-osmfr area=africa/eritrea`, which configures minutely updates. ### Preparations -If you plan to keep data updated automatically, before importing any data, make sure to set +If you plan to keep data updated automatically, before importing any data, make sure to set ``` DIFF_MODE=true ``` - + in the `.env` Now download fresh data: -``` +``` make download area=your-area-of-choice ``` ### Keep Database Updated -You can use the new imposm3 feature to keep the database updated (thanks to the [work by @stirringhalo](https://github.com/openmaptiles/openmaptiles/pull/131)). This will automatically download -the OSM change feed and import it into the database. -After each run you should also have a list of tiles that have updated. +You can use imposm3 to keep the database updated (thanks to the [work by @stirringhalo](https://github.com/openmaptiles/openmaptiles/pull/131)). +This will repeatedly download the OSM change feed and import it into the database. +In order to be able to update the database, the initial download and import of the OSM data must be done when `DIFF_MODE=true` is set in the `.env` file. +In this mode the initial download also sets the update source and the update intervals. +To start the update process please use ``` -make update-osm +make start-update-osm ``` +To stop the update process please use +``` +make stop-update-osm +``` + +After each update activation, **imposm3** will store lists of updated tiles in text format in subfolders of the `diffdir`, +named for the date(s) on which the import took place (`YYYYMMDD`). + +See [Generate Changed Tiles](#generate-changed-tiles) below on how this file can be used. + +#### Note +When the update process is actively updating the DB it is impossible to successfully generate tiles, +as there will be conflicts and deadlocks related to the DB access. + +Unfortunately, there is no known way to execute an external command in-between rounds of the `update-osm` process. + #### Troubleshooting +The log file for osm update can be viewed using + +``` +docker-compose logs --tail 100 --follow update-osm +``` + +Use `Ctrl-C` to stop following the log. + The output will be similar to this: -``` -[info] Importing #4889572 including changes till ....... +0000 UTC (1m13s behind) -``` +``` +[info] Importing #4889572 including changes till ....... +0000 UTC (2h10m10s behind) +``` -It might take some time to catch up with the latest changes, but the "time behind" should always decrease. If it doesn't, you need to download a new extract our don't have enough system resources to keep-up with the changes. +It might take some time to catch up with the latest changes, but the "time behind" should decrease until it is a few minutes. +If it doesn't, you need to download a new extract or check that there are enough system resources to keep-up with the changes. Finally you will get an output like this - this indicates, that some 6 objects were changed: @@ -55,7 +82,7 @@ Finally you will get an output like this - this indicates, that some 6 objects w [progress] 3s C: 0/s (0) N: 0/s (0) W: 0/s (6) R: 0/s (0) ``` -The process will keep running forever and eventually print something like this - which just means that no changes were in the latest changeset: +The process will keep running foreverprint something like this - which just means that no changes were in the latest changeset: ``` [progress] 0s C: 0/s (0) N: 0/s (0) W: 0/s (0) R: 0/s (0) @@ -63,30 +90,27 @@ The process will keep running forever and eventually print something like this - ### Import Change File -Given you have a file `changes.osc.gz` in your import folder. Once you ran the import command you should also have a list of tiles that have updated. +You may perform a one-time import of OSM changes from the `changes.osc.gz` file in your import folder using ``` make import-diff ``` +Similar to[Keep Database Updated](#keep_database_updated) above, **imposm3** will store the list of updated tiles in text file in subfolders of the `diffdir`, +named for the date on which the import took place (`YYYYMMDD`). + +See [Generate Changed Tiles](#generate-changed-tiles) below. + +#### Note +There is no `make` command for downloading OSM changes into `changes.osc.gz`. +You may perform this task using [`osmupdate`](https://wiki.openstreetmap.org/wiki/Osmupdate), +[pyosmium-get-changes](https://docs.osmcode.org/pyosmium/latest/tools_get_changes.html), +or downloading the changefile directly from the replication server. + ## Generate Changed Tiles -After the import has finished **imposm3** will store lists of tiles in text format in subfolders of the `diffdir`, -named for the date(s) on which the import took place (`YYYYMMDD`). -Copy and merge the files to `tiles.txt` in the import folder (`data`), either manually or with the following command, which also removes duplicate tiles so they are only generated once: +To generate all changed tiles, based on the lists of all updated tiles, and update the existing MBtiles file, please use ``` -cd data && sort ./*/*.tiles | uniq > tiles.txt -``` - -After generating the tiles.txt you might and to delete the `*.tiles` files to not include them in the next run: - -``` -cd data && rm ./*/*.tiles -``` - -Finally run the command to read the tilelist and write the updated vector tiles in the existing MBtiles file. - -``` -docker-compose run generate-changed-vectortiles +make generate-changed-tiles ```