Update UPDATE.md including Makefile improvements (#1408)

- Add `make import-diff` and `make generate-changed-tiles`
- Update `UPDATE.md` accordingly
- Other updates and clarifications in `UPDATE.md`

Makefile

@@ -186,6 +186,7 @@ Hints for developers:
   make generate-qa                     # statistics for a given layer's field
   make generate-tiles-pg               # generate vector tiles based on .env settings using PostGIS ST_MVT()
   make generate-tiles                  # generate vector tiles based on .env settings using Mapnik (obsolete)
+  make generate-changed-tiles          # Generate tiles changed by import-diff
   make test-sql                        # run unit tests on the OpenMapTiles SQL schema
   cat  .env                            # list PG database and MIN_ZOOM and MAX_ZOOM information
   cat  quickstart.log                  # transcript of the last ./quickstart.sh run
@@ -200,6 +201,7 @@ Hints for downloading & importing data:
   make download-bbbike area=Amsterdam  # download OSM data from bbbike.org       and create config file
   make import-data                     # Import data from OpenStreetMapData, Natural Earth and OSM Lake Labels.
   make import-osm                      # Import OSM data with the mapping rules from build/mapping.yaml
+  make import-diff                     # Import OSM updates from data/changes.osc.gz
   make import-wikidata                 # Import labels from Wikidata
   make import-sql                      # Import layers (run this after modifying layer SQL)
 
@@ -397,7 +399,7 @@ import-osm: all start-db-nowait
 	$(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools sh -c 'pgwait && import-osm $(PBF_FILE)'
 
 .PHONY: start-update-osm
-start-update-osm: all start-db
+start-update-osm: start-db
 	@$(assert_area_is_given)
 	$(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) up -d update-osm
 
@@ -406,7 +408,7 @@ stop-update-osm:
 	$(DOCKER_COMPOSE) stop update-osm
 
 .PHONY: import-diff
-import-diff: all start-db-nowait
+import-diff: start-db-nowait
 	@$(assert_area_is_given)
 	$(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools sh -c 'pgwait && import-diff'
 
@@ -440,6 +442,21 @@ generate-tiles-pg: all start-db
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools \
 			mbtiles-tools meta-generate "$(MBTILES_LOCAL_FILE)" $(TILESET_FILE) --auto-minmax --show-ranges
 
+.PHONY: data/tiles.txt
+data/tiles.txt:
+	find ./data -name "*.tiles" -exec cat {} \; -exec rm {} \; | \
+	  $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools \
+	    tile_multiplier $(MIN_ZOOM) $(MAX_ZOOM) >> data/tiles.txt
+
+.PHONY: generate-changed-tiles
+generate-changed-tiles: data/tiles.txt
+	# Re-generating updated tiles, if needed
+	if [ -s data/tiles.txt ] ; then \
+	  $(DOCKER_COMPOSE) $(DC_CONFIG_CACHE) run $(DC_OPTS_CACHE) openmaptiles-tools refresh-views; \
+	  $(DOCKER_COMPOSE) run $(DC_OPTS) -e LIST_FILE=data/tiles.txt openmaptiles-tools generate-tiles; \
+	  rm data/tiles.txt; \
+	fi
+
 .PHONY: start-tileserver
 start-tileserver: init-dirs
 	@echo " "

UPDATE.md

@@ -31,23 +31,50 @@ make download area=your-area-of-choice
 
 ### Keep Database Updated
 
-You can use the new imposm3 feature to keep the database updated (thanks to the [work by @stirringhalo](https://github.com/openmaptiles/openmaptiles/pull/131)). This will automatically download
+You can use imposm3 to keep the database updated (thanks to the [work by @stirringhalo](https://github.com/openmaptiles/openmaptiles/pull/131)).
-the OSM change feed and import it into the database.
+This will repeatedly download the OSM change feed and import it into the database.
-After each run you should also have a list of tiles that have updated.
+In order to be able to update the database, the initial download and import of the OSM data must be done when `DIFF_MODE=true` is set in the `.env` file.
+In this mode the initial download also sets the update source and the update intervals.
 
+To start the update process please use
 ```
-make update-osm
+make start-update-osm
 ```
 
+To stop the update process please use
+```
+make stop-update-osm
+```
+
+After each update activation, **imposm3** will store lists of updated tiles in text format in subfolders of the `diffdir`,
+named for the date(s) on which the import took place (`YYYYMMDD`).
+
+See [Generate Changed Tiles](#generate-changed-tiles) below on how this file can be used.
+
+#### Note
+When the update process is actively updating the DB it is impossible to successfully generate tiles,
+as there will be conflicts and deadlocks related to the DB access.
+
+Unfortunately, there is no known way to execute an external command in-between rounds of the `update-osm` process.
+
 #### Troubleshooting
 
+The log file for osm update can be viewed using
+
+```
+docker-compose logs --tail 100 --follow update-osm
+```
+
+Use `Ctrl-C` to stop following the log.
+
 The output will be similar to this:
 
 ```
-[info] Importing #4889572 including changes till ....... +0000 UTC (1m13s behind)
+[info] Importing #4889572 including changes till ....... +0000 UTC (2h10m10s behind)
 ```
 
-It might take some time to catch up with the latest changes, but the "time behind" should always decrease. If it doesn't, you need to download a new extract our don't have enough system resources to keep-up with the changes.
+It might take some time to catch up with the latest changes, but the "time behind" should decrease until it is a few minutes.
+If it doesn't, you need to download a new extract or check that there are enough system resources to keep-up with the changes.
 
 Finally you will get an output like this - this indicates, that some 6 objects were changed:
 
@@ -55,7 +82,7 @@ Finally you will get an output like this - this indicates, that some 6 objects w
 [progress]     3s C:       0/s (0) N:       0/s (0) W:       0/s (6) R:      0/s (0)
 ```
 
-The process will keep running forever and eventually print something like this - which just means that no changes were in the latest changeset:
+The process will keep running foreverprint something like this - which just means that no changes were in the latest changeset:
 
 ```
 [progress]     0s C:       0/s (0) N:       0/s (0) W:       0/s (0) R:      0/s (0)
@@ -63,30 +90,27 @@ The process will keep running forever and eventually print something like this -
 
 ### Import Change File
 
-Given you have a file `changes.osc.gz` in your import folder. Once you ran the import command you should also have a list of tiles that have updated.
+You may perform a one-time import of OSM changes from the `changes.osc.gz` file in your import folder using
 
 ```
 make import-diff
 ```
 
+Similar to[Keep Database Updated](#keep_database_updated) above, **imposm3** will store the list of updated tiles in text file in subfolders of the `diffdir`,
+named for the date on which the import took place (`YYYYMMDD`).
+
+See [Generate Changed Tiles](#generate-changed-tiles) below.
+
+#### Note
+There is no `make` command for downloading OSM changes into `changes.osc.gz`.
+You may perform this task using [`osmupdate`](https://wiki.openstreetmap.org/wiki/Osmupdate),
+[pyosmium-get-changes](https://docs.osmcode.org/pyosmium/latest/tools_get_changes.html),
+or downloading the changefile directly from the replication server.
+
 ## Generate Changed Tiles
 
-After the import has finished **imposm3** will store lists of tiles in text format in subfolders of the `diffdir`,
+To generate all changed tiles, based on the lists of all updated tiles, and update the existing MBtiles file, please use
-named for the date(s) on which the import took place (`YYYYMMDD`).
-Copy and merge the files to `tiles.txt` in the import folder (`data`), either manually or with the following command, which also removes duplicate tiles so they are only generated once:  
 
 ```
-cd data && sort ./*/*.tiles | uniq > tiles.txt
+make generate-changed-tiles
-```
-
-After generating the tiles.txt you might and to delete the `*.tiles` files to not include them in the next run:
-
-```
-cd data && rm ./*/*.tiles
-```
-
-Finally run the command to read the tilelist and write the updated vector tiles in the existing MBtiles file.
-
-```
-docker-compose run generate-changed-vectortiles
 ```