From 746a2dc7b93386768f08dc9524d5fce99a3b5d37 Mon Sep 17 00:00:00 2001 From: zstadler Date: Thu, 9 Apr 2020 19:05:01 +0300 Subject: [PATCH] Support multiple openmaptiles instances by specifying a Docker Compose project (#806) The implementation adds the `DC_PROJECT` parameter. It can also be set by an environment variable in the hosting shell. The environment variable can be overwritten by a make parameter, including `DC_PROJECT=` which restores the automatic project name. #### _NOTE:_ It may be worthwhile to review the following make targets that are currently not impacted by the PR: - `make start-postserve` - `make list-docker-images` - `make remove-docker-images` - `make docker-unnecessary-clean` This PR was suggested by @nyurik: > zstadler 3:30 PM Is it possible to run two instances of openmaptiles, and the postgis container in particular, on the same Linux host? nyurik 4:17 PM use docker-compose --project-name -- that should allow you to run everything in parallel 4:17 might need to update make file 4:17 btw, that would be a good PR for makefile -- to specify --project-name based on the current DIR name, but so that it can be overwritten by a makefile param Co-authored-by: Yuri Astrakhan --- Makefile | 91 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 50 insertions(+), 41 deletions(-) diff --git a/Makefile b/Makefile index 77df96f9..d31ae8ee 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,15 @@ # Containers run as the current user rather than root (so that created files are not root-owned) DC_OPTS?=--rm -u $$(id -u $${USER}):$$(id -g $${USER}) +# Allow a custom docker-compose project name +ifeq ($(strip $(DC_PROJECT)),) + override DC_PROJECT:=$(notdir $(shell pwd)) +endif +DOCKER_COMPOSE:= docker-compose --project-name $(DC_PROJECT) + +# Use `xargs --no-run-if-empty` flag, if supported +XARGS:=xargs $(shell xargs --no-run-if-empty /dev/null && echo --no-run-if-empty) + # If running in the test mode, compare files rather than copy them TEST_MODE?=no ifeq ($(TEST_MODE),yes) @@ -57,13 +66,13 @@ init-dirs: build/openmaptiles.tm2source/data.yml: init-dirs mkdir -p build/openmaptiles.tm2source - docker-compose run $(DC_OPTS) openmaptiles-tools generate-tm2source openmaptiles.yaml --host="postgres" --port=5432 --database="openmaptiles" --user="openmaptiles" --password="openmaptiles" > $@ + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-tm2source openmaptiles.yaml --host="postgres" --port=5432 --database="openmaptiles" --user="openmaptiles" --password="openmaptiles" > $@ build/mapping.yaml: init-dirs - docker-compose run $(DC_OPTS) openmaptiles-tools generate-imposm3 openmaptiles.yaml > $@ + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-imposm3 openmaptiles.yaml > $@ build/tileset.sql: init-dirs - docker-compose run $(DC_OPTS) openmaptiles-tools generate-sql openmaptiles.yaml > $@ + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-sql openmaptiles.yaml > $@ .PHONY: clean clean: @@ -71,20 +80,20 @@ clean: .PHONY: clean-docker clean-docker: - docker-compose down -v --remove-orphans - docker-compose rm -fv - docker volume ls -q | grep openmaptiles | xargs -r docker volume rm || true + $(DOCKER_COMPOSE) down -v --remove-orphans + $(DOCKER_COMPOSE) rm -fv + docker volume ls -q -f "name=^$${DC_PROJECT,,*}_" | $(XARGS) docker volume rm rm -rf cache .PHONY: db-start db-start: - docker-compose up -d postgres + $(DOCKER_COMPOSE) up -d postgres @echo "Wait for PostgreSQL to start..." - docker-compose run $(DC_OPTS) import-osm ./pgwait.sh + $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./pgwait.sh .PHONY: db-stop db-stop: - docker-compose stop postgres + $(DOCKER_COMPOSE) stop postgres .PHONY: download-geofabrik download-geofabrik: init-dirs @@ -100,7 +109,7 @@ download-geofabrik: init-dirs else \ echo "=============== download-geofabrik =======================" ;\ echo "Download area: $(area)" ;\ - docker-compose run $(DC_OPTS) openmaptiles-tools bash -c \ + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash -c \ 'download-osm geofabrik $(area) \ --minzoom $$QUICKSTART_MIN_ZOOM \ --maxzoom $$QUICKSTART_MAX_ZOOM \ @@ -111,45 +120,45 @@ download-geofabrik: init-dirs .PHONY: psql psql: db-start - docker-compose run $(DC_OPTS) import-osm ./psql.sh + $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh .PHONY: import-osm import-osm: db-start all - docker-compose run $(DC_OPTS) import-osm + $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm .PHONY: import-sql import-sql: db-start all - docker-compose run $(DC_OPTS) openmaptiles-tools import-sql + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-sql .PHONY: import-osmsql import-osmsql: db-start all import-osm import-sql .PHONY: import-borders import-borders: db-start - docker-compose run $(DC_OPTS) openmaptiles-tools import-borders + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-borders .PHONY: import-water import-water: db-start - docker-compose run $(DC_OPTS) import-water + $(DOCKER_COMPOSE) run $(DC_OPTS) import-water .PHONY: import-natural-earth import-natural-earth: db-start - docker-compose run $(DC_OPTS) import-natural-earth + $(DOCKER_COMPOSE) run $(DC_OPTS) import-natural-earth .PHONY: import-lakelines import-lakelines: db-start - docker-compose run $(DC_OPTS) import-lakelines + $(DOCKER_COMPOSE) run $(DC_OPTS) import-lakelines .PHONY: generate-tiles generate-tiles: init-dirs db-start all rm -rf data/tiles.mbtiles if [ -f ./data/docker-compose-config.yml ]; then \ - docker-compose -f docker-compose.yml -f ./data/docker-compose-config.yml \ + $(DOCKER_COMPOSE) -f docker-compose.yml -f ./data/docker-compose-config.yml \ run $(DC_OPTS) generate-vectortiles; \ else \ - docker-compose run $(DC_OPTS) generate-vectortiles; \ + $(DOCKER_COMPOSE) run $(DC_OPTS) generate-vectortiles; \ fi - docker-compose run $(DC_OPTS) openmaptiles-tools generate-metadata ./data/tiles.mbtiles + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-metadata ./data/tiles.mbtiles .PHONY: start-tileserver start-tileserver: init-dirs @@ -181,7 +190,7 @@ start-postserve: db-start @echo "* " @echo "***********************************************************" @echo " " - docker-compose up -d postserve + $(DOCKER_COMPOSE) up -d postserve docker pull maputnik/editor @echo " " @echo "***********************************************************" @@ -192,7 +201,7 @@ start-postserve: db-start @echo "* " @echo "***********************************************************" @echo " " - docker rm -f maputnik_editor || true + -docker rm -f maputnik_editor docker run $(DC_OPTS) --name maputnik_editor -d -p 8088:8888 maputnik/editor .PHONY: generate-qareports @@ -203,29 +212,29 @@ generate-qareports: .PHONY: generate-devdoc generate-devdoc: init-dirs mkdir -p ./build/devdoc && \ - docker-compose run $(DC_OPTS) openmaptiles-tools-latest sh -c \ + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools-latest sh -c \ 'generate-etlgraph openmaptiles.yaml $(GRAPH_PARAMS) && \ generate-mapping-graph openmaptiles.yaml $(GRAPH_PARAMS)' .PHONY: import-sql-dev import-sql-dev: - docker-compose run $(DC_OPTS) openmaptiles-tools bash + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash .PHONY: import-osm-dev import-osm-dev: - docker-compose run $(DC_OPTS) import-osm /bin/bash + $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm /bin/bash .PHONY: import-wikidata import-wikidata: - docker-compose run $(DC_OPTS) openmaptiles-tools import-wikidata openmaptiles.yaml + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-wikidata openmaptiles.yaml .PHONY: psql-pg-stat-reset psql-pg-stat-reset: - docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'SELECT pg_stat_statements_reset();' + $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'SELECT pg_stat_statements_reset();' .PHONY: forced-clean-sql forced-clean-sql: - docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 \ + $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 \ -c "DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA IF NOT EXISTS public;" \ -c "CREATE EXTENSION hstore; CREATE EXTENSION postgis; CREATE EXTENSION unaccent;" \ -c "CREATE EXTENSION fuzzystrmatch; CREATE EXTENSION osml10n; CREATE EXTENSION pg_stat_statements;" \ @@ -233,27 +242,27 @@ forced-clean-sql: .PHONY: list-views list-views: - @docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \ + $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \ -c "select schemaname, viewname from pg_views where schemaname='public' order by viewname;" .PHONY: list-tables list-tables: - @docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \ + $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -A -F"," -P pager=off -P footer=off \ -c "select schemaname, tablename from pg_tables where schemaname='public' order by tablename;" .PHONY: psql-list-tables psql-list-tables: - docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c "\d+" + $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c "\d+" .PHONY: psql-vacuum-analyze psql-vacuum-analyze: @echo "Start - postgresql: VACUUM ANALYZE VERBOSE;" - docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'VACUUM ANALYZE VERBOSE;' + $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'VACUUM ANALYZE VERBOSE;' .PHONY: psql-analyze psql-analyze: @echo "Start - postgresql: ANALYZE VERBOSE;" - docker-compose run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'ANALYZE VERBOSE;' + $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./psql.sh -v ON_ERROR_STOP=1 -P pager=off -c 'ANALYZE VERBOSE;' .PHONY: list-docker-images list-docker-images: @@ -261,23 +270,23 @@ list-docker-images: .PHONY: refresh-docker-images refresh-docker-images: - docker-compose pull --ignore-pull-failures + $(DOCKER_COMPOSE) pull --ignore-pull-failures .PHONY: remove-docker-images remove-docker-images: @echo "Deleting all openmaptiles related docker image(s)..." - @docker-compose down - @docker images | grep "openmaptiles" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi -f - @docker images | grep "osm2vectortiles/mapbox-studio" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi -f - @docker images | grep "klokantech/tileserver-gl" | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi -f + @$(DOCKER_COMPOSE) down + @docker images "openmaptiles/*" | $(XARGS) docker rmi -f + @docker images "osm2vectortiles/mapbox-studio" | $(XARGS) docker rmi -f + @docker images "klokantech/tileserver-gl" | $(XARGS) docker rmi -f .PHONY: docker-unnecessary-clean docker-unnecessary-clean: @echo "Deleting unnecessary container(s)..." - @docker ps -a | grep Exited | awk -F" " '{print $$1}' | xargs --no-run-if-empty docker rm + @docker ps -a --filter "status=exited" | $(XARGS) docker rm @echo "Deleting unnecessary image(s)..." - @docker images | grep \ | awk -F" " '{print $$3}' | xargs --no-run-if-empty docker rmi + @docker images | grep \ | awk -F" " '{print $$3}' | $(XARGS) docker rmi .PHONY: test-perf-null test-perf-null: - docker-compose run $(DC_OPTS) openmaptiles-tools test-perf openmaptiles.yaml --test null --no-color + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools test-perf openmaptiles.yaml --test null --no-color