diff --git a/.env b/.env index aa66d0c5..70d706a0 100644 --- a/.env +++ b/.env @@ -1,12 +1,16 @@ +# This file defines default environment variables for all images + +TOOLS_VERSION=4.1.0 + POSTGRES_DB=openmaptiles POSTGRES_USER=openmaptiles POSTGRES_PASSWORD=openmaptiles POSTGRES_HOST=postgres POSTGRES_PORT=5432 + QUICKSTART_MIN_ZOOM=0 QUICKSTART_MAX_ZOOM=7 DIFF_MODE=false -TOOLS_VERSION=4.1.0 BBOX=-180.0,-85.0511,180.0,85.0511 MIN_ZOOM=0 diff --git a/.github/workflows/omt_ci.yml b/.github/workflows/omt_ci.yml index a2230d5f..8728bf0a 100644 --- a/.github/workflows/omt_ci.yml +++ b/.github/workflows/omt_ci.yml @@ -1,4 +1,4 @@ -# MapTiler OpenMapTiles +# MapTiler OpenMapTiles ####################### # Workflow to validate OMT`s new Pull Requests and commits pushed into OMT repo @@ -28,16 +28,16 @@ jobs: # Named steps - name: generate all zooms run: sed -i 's/QUICKSTART_MAX_ZOOM=7/QUICKSTART_MAX_ZOOM=14/g' .env - - # Runs quickstart + + # Runs quickstart - name: quickstart env: area: northamptonshire run: bash ./quickstart.sh $area - + - name: generate devdoc run: TEST_MODE=yes make generate-devdoc - + # todo: use artifact to store result of tests #- uses: actions/upload-artifact@v1 # with: diff --git a/Makefile b/Makefile index 34d323ca..5462580e 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,9 @@ help: .PHONY: init-dirs init-dirs: - mkdir -p build && mkdir -p data && mkdir -p cache + @mkdir -p build + @mkdir -p data + @mkdir -p cache build/openmaptiles.tm2source/data.yml: init-dirs mkdir -p build/openmaptiles.tm2source @@ -94,12 +96,13 @@ clean-docker: .PHONY: db-start db-start: - $(DOCKER_COMPOSE) up -d postgres + $(DOCKER_COMPOSE) up --no-recreate -d postgres @echo "Wait for PostgreSQL to start..." $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./pgwait.sh .PHONY: db-stop db-stop: + @echo "Stopping PostgreSQL..." $(DOCKER_COMPOSE) stop postgres OSM_SERVERS:=geofabrik osmfr bbbike @@ -142,22 +145,14 @@ import-sql: db-start all .PHONY: import-osmsql import-osmsql: db-start all import-osm import-sql +.PHONY: import-data +import-data: db-start + $(DOCKER_COMPOSE) run $(DC_OPTS) import-data + .PHONY: import-borders import-borders: db-start $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-borders -.PHONY: import-water -import-water: db-start - $(DOCKER_COMPOSE) run $(DC_OPTS) import-water - -.PHONY: import-natural-earth -import-natural-earth: db-start - $(DOCKER_COMPOSE) run $(DC_OPTS) import-natural-earth - -.PHONY: import-lakelines -import-lakelines: db-start - $(DOCKER_COMPOSE) run $(DC_OPTS) import-lakelines - .PHONY: generate-tiles ifneq ($(wildcard data/docker-compose-config.yml),) DC_CONFIG_TILES:=-f docker-compose.yml -f ./data/docker-compose-config.yml @@ -219,7 +214,7 @@ generate-qareports: .PHONY: generate-devdoc generate-devdoc: init-dirs mkdir -p ./build/devdoc && \ - $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools-latest sh -c \ + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c \ 'generate-etlgraph openmaptiles.yaml $(GRAPH_PARAMS) && \ generate-mapping-graph openmaptiles.yaml $(GRAPH_PARAMS)' diff --git a/README.md b/README.md index 1da2a97a..5e34b48e 100644 --- a/README.md +++ b/README.md @@ -100,9 +100,7 @@ make db-start Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/), [Natural Earth](http://www.naturalearthdata.com/) and [OpenStreetMap Lake Labels](https://github.com/lukasmartinelli/osm-lakelines). ```bash -make import-water -make import-natural-earth -make import-lakelines +make import-data ``` [Download OpenStreetMap data extracts](http://download.geofabrik.de/) and store the PBF file in the `./data` directory. @@ -119,7 +117,7 @@ make download-geofabrik area=albania ``` [Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm) with the mapping rules from -`build/mapping.yaml` (which has been created by `make`). Run after any change in layers definiton. Also create borders table using extra processing with [osmborder](https://github.com/pnorman/osmborder) tool. +`build/mapping.yaml` (which has been created by `make`). Run after any change in layers definition. Also create borders table using extra processing with [osmborder](https://github.com/pnorman/osmborder) tool. ```bash make import-osm diff --git a/docker-compose.yml b/docker-compose.yml index 41087d41..5e0552cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,15 @@ -version: "2" +# This version must match the MAKE_DC_VERSION value below +version: "2.3" + volumes: pgdata: + +networks: + postgres_conn: + driver: bridge + services: + postgres: image: "openmaptiles/postgis:${TOOLS_VERSION}" volumes: @@ -11,21 +19,13 @@ services: ports: - "5432" env_file: .env - import-natural-earth: - image: "openmaptiles/import-natural-earth:${TOOLS_VERSION}" - env_file: .env - networks: - - postgres_conn - import-water: - image: "openmaptiles/import-water:${TOOLS_VERSION}" - env_file: .env - networks: - - postgres_conn - import-lakelines: - image: "openmaptiles/import-lakelines:${TOOLS_VERSION}" + + import-data: + image: "openmaptiles/import-data:${TOOLS_VERSION}" env_file: .env networks: - postgres_conn + import-osm: image: "openmaptiles/import-osm:${TOOLS_VERSION}" env_file: .env @@ -37,6 +37,7 @@ services: - ./data:/import - ./build:/mapping - ./cache:/cache + import-osm-diff: image: "openmaptiles/import-osm:${TOOLS_VERSION}" env_file: .env @@ -49,6 +50,7 @@ services: - ./data:/import - ./build:/mapping - ./cache:/cache + update-osm: image: "openmaptiles/import-osm:${TOOLS_VERSION}" env_file: .env @@ -61,26 +63,21 @@ services: - ./data:/import - ./build:/mapping - ./cache:/cache + openmaptiles-tools: image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}" env_file: .env + environment: + # Must match the version of this file (first line) + # download-osm will use it when generating a composer file + MAKE_DC_VERSION: "2.3" networks: - postgres_conn volumes: - .:/tileset - ./data:/import - ./build:/sql - openmaptiles-tools-latest: - # This target exists for experimental tools that have not yet been published. - # Do not use this for production. - image: "openmaptiles/openmaptiles-tools:latest" - env_file: .env - networks: - - postgres_conn - volumes: - - .:/tileset - - ./data:/import - - ./build:/sql + generate-changed-vectortiles: image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}" command: ./export-list.sh @@ -90,6 +87,7 @@ services: networks: - postgres_conn env_file: .env + generate-vectortiles: image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}" volumes: @@ -102,6 +100,7 @@ services: BBOX: ${BBOX} MIN_ZOOM: ${MIN_ZOOM} MAX_ZOOM: ${MAX_ZOOM} + postserve: image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}" command: postserve openmaptiles.yaml --verbose @@ -112,7 +111,3 @@ services: - "8090:8090" volumes: - .:/tileset - -networks: - postgres_conn: - driver: bridge diff --git a/quickstart.sh b/quickstart.sh index 4af930b6..4e158769 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -105,8 +105,8 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc ) echo "system memory (GB): ${mem}" grep SwapTotal /proc/meminfo - echo "cpu number: $(grep -c processor /proc/cpuinfo) x $(cat /proc/cpuinfo | grep "bogomips" | head -1)" - cat /proc/meminfo | grep Free + echo "cpu number: $(grep -c processor /proc/cpuinfo) x $(grep "bogomips" /proc/cpuinfo | head -1)" + grep Free /proc/meminfo else echo " " echo "Warning : Platforms other than Linux are less tested" @@ -120,7 +120,7 @@ make clean-docker echo " " echo "-------------------------------------------------------------------------------------" -echo "====> : Checking OpenMapTiles docker images " +echo "====> : Existing OpenMapTiles docker images. Will use version $(source .env && echo "$TOOLS_VERSION")" docker images | grep openmaptiles echo " " @@ -162,7 +162,7 @@ echo "-------------------------------------------------------------------------- echo "====> : Code generating from the layer definitions ( ./build/mapping.yaml; ./build/tileset.sql )" echo " : The tool source code: https://github.com/openmaptiles/openmaptiles-tools " echo " : But we generate the tm2source, Imposm mappings and SQL functions from the layer definitions! " -make +make all echo " " echo "-------------------------------------------------------------------------------------" @@ -178,29 +178,18 @@ echo "====> : Drop and Recreate PostgreSQL public schema " # This adds an extra safety belt if the user modifies the docker volume settings make forced-clean-sql -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : Start importing water data from http://osmdata.openstreetmap.de/ into PostgreSQL " -echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-water " -echo " : Data license: https://osmdata.openstreetmap.de/info/license.html " -echo " : Thank you: https://osmdata.openstreetmap.de/info/ " -make import-water - -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : Start importing http://www.naturalearthdata.com into PostgreSQL " -echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-natural-earth " -echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use " -echo " : Thank you: Natural Earth Contributors! " -make import-natural-earth - -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : Start importing OpenStreetMap Lakelines data " -echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-lakelines " -echo " : https://github.com/lukasmartinelli/osm-lakelines " -echo " : Data license: .. " -make import-lakelines +echo "====> : Importing all the data:" +echo " : * Water data from http://osmdata.openstreetmap.de" +echo " : Data license: https://osmdata.openstreetmap.de/info/license.html" +echo " : * Natural Earth from http://www.naturalearthdata.com" +echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use" +echo " : * OpenStreetMap Lakelines data https://github.com/lukasmartinelli/osm-lakelines" +echo " :" +echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-data" +echo " : includes all data from the import-data image" +echo " :" +echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !" +make import-data echo " " echo "-------------------------------------------------------------------------------------" @@ -269,16 +258,13 @@ echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Inputs - Outputs md5sum for debugging " rm -f ./data/quickstart_checklist.chk -md5sum build/mapping.yaml >> ./data/quickstart_checklist.chk -md5sum build/tileset.sql >> ./data/quickstart_checklist.chk -md5sum build/openmaptiles.tm2source/data.yml >> ./data/quickstart_checklist.chk -md5sum "./data/${testdata}" >> ./data/quickstart_checklist.chk -md5sum ./data/tiles.mbtiles >> ./data/quickstart_checklist.chk -md5sum ./data/docker-compose-config.yml >> ./data/quickstart_checklist.chk +{ + find build -type f | sort | xargs md5sum ; + find data -type f | sort | xargs md5sum ; +} >> ./data/quickstart_checklist.chk cat ./data/quickstart_checklist.chk ENDTIME=$(date +%s) -ENDDATE=$(date +"%Y-%m-%dT%H:%M%z") if stat --help >/dev/null 2>&1; then MODDATE=$(stat -c %y "./data/${testdata}" ) else