Use new import-data image (#818)

This is a partial migration of https://github.com/openmaptiles/openmaptiles/pull/785 * Use `import-data` instead of `import-lakelines`, `import-water`, and `import-natural-earth` * Upgrade docker-compose.yml to version 2.3 (allows some extra env var usage in yaml file itself) * Remove `openmaptiles-tools:latest` usage -- no longer needed, can use current version 4.1 * `db-start` does not do a container recreation in case docker-compose.yml definition has changed. * a few minor cleanups in quickstart.sh
2020-04-22 02:48:57 -04:00 · 2020-04-22 02:48:57 -04:00 · 2ca55abb7d
commit 2ca55abb7d
--- a/.env
+++ b/.env
@ -1,12 +1,16 @@
+# This file defines default environment variables for all images
+
+TOOLS_VERSION=4.1.0
+
 POSTGRES_DB=openmaptiles
 POSTGRES_USER=openmaptiles
 POSTGRES_PASSWORD=openmaptiles
 POSTGRES_HOST=postgres
 POSTGRES_PORT=5432
+
 QUICKSTART_MIN_ZOOM=0
 QUICKSTART_MAX_ZOOM=7
 DIFF_MODE=false
-TOOLS_VERSION=4.1.0

 BBOX=-180.0,-85.0511,180.0,85.0511
 MIN_ZOOM=0
--- a/.github/workflows/omt_ci.yml
+++ b/.github/workflows/omt_ci.yml
@ -1,4 +1,4 @@
-# MapTiler OpenMapTiles 
+# MapTiler OpenMapTiles
 #######################

 # Workflow to validate OMT`s new Pull Requests and commits pushed into OMT repo
@ -28,16 +28,16 @@ jobs:
    # Named steps
    - name: generate all zooms
      run: sed -i 's/QUICKSTART_MAX_ZOOM=7/QUICKSTART_MAX_ZOOM=14/g' .env
-    
-    # Runs quickstart  
+
+    # Runs quickstart
    - name: quickstart
      env:
        area: northamptonshire
      run:  bash ./quickstart.sh $area
-      
+
    - name: generate devdoc
      run: TEST_MODE=yes make generate-devdoc
-    
+
    # todo: use artifact to store result of tests
    #- uses: actions/upload-artifact@v1
    #  with:
--- a/25
+++ b/25
@ -69,7 +69,9 @@ help:

 .PHONY: init-dirs
 init-dirs:
-	mkdir -p build && mkdir -p data && mkdir -p cache
+	@mkdir -p build
+	@mkdir -p data
+	@mkdir -p cache

 build/openmaptiles.tm2source/data.yml: init-dirs
 	mkdir -p build/openmaptiles.tm2source
@ -94,12 +96,13 @@ clean-docker:

 .PHONY: db-start
 db-start:
-	$(DOCKER_COMPOSE) up -d postgres
+	$(DOCKER_COMPOSE) up --no-recreate -d postgres
 	@echo "Wait for PostgreSQL to start..."
 	$(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./pgwait.sh

 .PHONY: db-stop
 db-stop:
+	@echo "Stopping PostgreSQL..."
 	$(DOCKER_COMPOSE) stop postgres

 OSM_SERVERS:=geofabrik osmfr bbbike
@ -142,22 +145,14 @@ import-sql: db-start all
 .PHONY: import-osmsql
 import-osmsql: db-start all import-osm import-sql

+.PHONY: import-data
+import-data: db-start
+	$(DOCKER_COMPOSE) run $(DC_OPTS) import-data
+
 .PHONY: import-borders
 import-borders: db-start
 	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-borders

-.PHONY: import-water
-import-water: db-start
-	$(DOCKER_COMPOSE) run $(DC_OPTS) import-water
-
-.PHONY: import-natural-earth
-import-natural-earth: db-start
-	$(DOCKER_COMPOSE) run $(DC_OPTS) import-natural-earth
-
-.PHONY: import-lakelines
-import-lakelines: db-start
-	$(DOCKER_COMPOSE) run $(DC_OPTS) import-lakelines
-
 .PHONY: generate-tiles
 ifneq ($(wildcard data/docker-compose-config.yml),)
  DC_CONFIG_TILES:=-f docker-compose.yml -f ./data/docker-compose-config.yml
@ -219,7 +214,7 @@ generate-qareports:
 .PHONY: generate-devdoc
 generate-devdoc: init-dirs
 	mkdir -p ./build/devdoc && \
-	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools-latest sh -c \
+	$(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c \
 			'generate-etlgraph openmaptiles.yaml $(GRAPH_PARAMS) && \
 			 generate-mapping-graph openmaptiles.yaml $(GRAPH_PARAMS)'

--- a/README.md
+++ b/README.md
@ -100,9 +100,7 @@ make db-start
 Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/), [Natural Earth](http://www.naturalearthdata.com/) and [OpenStreetMap Lake Labels](https://github.com/lukasmartinelli/osm-lakelines).

 ```bash
-make import-water
-make import-natural-earth
-make import-lakelines
+make import-data
 ```

 [Download OpenStreetMap data extracts](http://download.geofabrik.de/) and store the PBF file in the `./data` directory.
@ -119,7 +117,7 @@ make download-geofabrik area=albania
 ```

 [Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm) with the mapping rules from
-`build/mapping.yaml` (which has been created by `make`). Run after any change in layers definiton.  Also create borders table using extra processing with [osmborder](https://github.com/pnorman/osmborder) tool.
+`build/mapping.yaml` (which has been created by `make`). Run after any change in layers definition.  Also create borders table using extra processing with [osmborder](https://github.com/pnorman/osmborder) tool.

 ```bash
 make import-osm
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,7 +1,15 @@
-version: "2"
+# This version must match the MAKE_DC_VERSION value below
+version: "2.3"
+
 volumes:
  pgdata:
+
+networks:
+  postgres_conn:
+    driver: bridge
+
 services:
+
  postgres:
    image: "openmaptiles/postgis:${TOOLS_VERSION}"
    volumes:
@ -11,21 +19,13 @@ services:
    ports:
      - "5432"
    env_file: .env
-  import-natural-earth:
-    image: "openmaptiles/import-natural-earth:${TOOLS_VERSION}"
-    env_file: .env
-    networks:
-      - postgres_conn
-  import-water:
-    image: "openmaptiles/import-water:${TOOLS_VERSION}"
-    env_file: .env
-    networks:
-      - postgres_conn
-  import-lakelines:
-    image: "openmaptiles/import-lakelines:${TOOLS_VERSION}"
+
+  import-data:
+    image: "openmaptiles/import-data:${TOOLS_VERSION}"
    env_file: .env
    networks:
      - postgres_conn
+
  import-osm:
    image: "openmaptiles/import-osm:${TOOLS_VERSION}"
    env_file: .env
@ -37,6 +37,7 @@ services:
      - ./data:/import
      - ./build:/mapping
      - ./cache:/cache
+
  import-osm-diff:
    image: "openmaptiles/import-osm:${TOOLS_VERSION}"
    env_file: .env
@ -49,6 +50,7 @@ services:
      - ./data:/import
      - ./build:/mapping
      - ./cache:/cache
+
  update-osm:
    image: "openmaptiles/import-osm:${TOOLS_VERSION}"
    env_file: .env
@ -61,26 +63,21 @@ services:
      - ./data:/import
      - ./build:/mapping
      - ./cache:/cache
+
  openmaptiles-tools:
    image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
    env_file: .env
+    environment:
+      # Must match the version of this file (first line)
+      # download-osm will use it when generating a composer file
+      MAKE_DC_VERSION: "2.3"
    networks:
      - postgres_conn
    volumes:
      - .:/tileset
      - ./data:/import
      - ./build:/sql
-  openmaptiles-tools-latest:
-    # This target exists for experimental tools that have not yet been published.
-    # Do not use this for production.
-    image: "openmaptiles/openmaptiles-tools:latest"
-    env_file: .env
-    networks:
-      - postgres_conn
-    volumes:
-      - .:/tileset
-      - ./data:/import
-      - ./build:/sql
+
  generate-changed-vectortiles:
    image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
    command: ./export-list.sh
@ -90,6 +87,7 @@ services:
    networks:
      - postgres_conn
    env_file: .env
+
  generate-vectortiles:
    image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}"
    volumes:
@ -102,6 +100,7 @@ services:
      BBOX: ${BBOX}
      MIN_ZOOM: ${MIN_ZOOM}
      MAX_ZOOM: ${MAX_ZOOM}
+
  postserve:
    image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}"
    command: postserve openmaptiles.yaml --verbose
@ -112,7 +111,3 @@ services:
      - "8090:8090"
    volumes:
      - .:/tileset
-
-networks:
-  postgres_conn:
-    driver: bridge
--- a/quickstart.sh
+++ b/quickstart.sh
@ -105,8 +105,8 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then
    mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc  )
    echo "system memory (GB): ${mem}"
    grep SwapTotal /proc/meminfo
-    echo "cpu number: $(grep -c processor /proc/cpuinfo) x $(cat /proc/cpuinfo | grep "bogomips" | head -1)"
-    cat /proc/meminfo  | grep Free
+    echo "cpu number: $(grep -c processor /proc/cpuinfo) x $(grep "bogomips" /proc/cpuinfo | head -1)"
+    grep Free /proc/meminfo
 else
    echo " "
    echo "Warning : Platforms other than Linux are less tested"
@ -120,7 +120,7 @@ make clean-docker

 echo " "
 echo "-------------------------------------------------------------------------------------"
-echo "====> : Checking OpenMapTiles docker images "
+echo "====> : Existing OpenMapTiles docker images. Will use version $(source .env && echo "$TOOLS_VERSION")"
 docker images | grep openmaptiles

 echo " "
@ -162,7 +162,7 @@ echo "--------------------------------------------------------------------------
 echo "====> : Code generating from the layer definitions ( ./build/mapping.yaml; ./build/tileset.sql )"
 echo "      : The tool source code: https://github.com/openmaptiles/openmaptiles-tools "
 echo "      : But we generate the tm2source, Imposm mappings and SQL functions from the layer definitions! "
-make
+make all

 echo " "
 echo "-------------------------------------------------------------------------------------"
@ -178,29 +178,18 @@ echo "====> : Drop and Recreate PostgreSQL  public schema "
 # This adds an extra safety belt if the user modifies the docker volume settings
 make forced-clean-sql

-echo " "
-echo "-------------------------------------------------------------------------------------"
-echo "====> : Start importing water data from http://osmdata.openstreetmap.de/ into PostgreSQL "
-echo "      : Source code:  https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-water "
-echo "      : Data license: https://osmdata.openstreetmap.de/info/license.html "
-echo "      : Thank you: https://osmdata.openstreetmap.de/info/ "
-make import-water
-
-echo " "
-echo "-------------------------------------------------------------------------------------"
-echo "====> : Start importing  http://www.naturalearthdata.com  into PostgreSQL "
-echo "      : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-natural-earth "
-echo "      : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use  "
-echo "      : Thank you: Natural Earth Contributors! "
-make import-natural-earth
-
-echo " "
-echo "-------------------------------------------------------------------------------------"
-echo "====> : Start importing OpenStreetMap Lakelines data "
-echo "      : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-lakelines "
-echo "      :              https://github.com/lukasmartinelli/osm-lakelines "
-echo "      : Data license: .. "
-make import-lakelines
+echo "====> : Importing all the data:"
+echo "      : * Water data from http://osmdata.openstreetmap.de"
+echo "      :   Data license: https://osmdata.openstreetmap.de/info/license.html"
+echo "      : * Natural Earth from http://www.naturalearthdata.com"
+echo "      :   Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use"
+echo "      : * OpenStreetMap Lakelines data https://github.com/lukasmartinelli/osm-lakelines"
+echo "      :"
+echo "      : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-data"
+echo "      :   includes all data from the import-data image"
+echo "      :"
+echo "      : Thank you: https://www.postgresql.org !  Thank you http://postgis.org !"
+make import-data

 echo " "
 echo "-------------------------------------------------------------------------------------"
@ -269,16 +258,13 @@ echo " "
 echo "-------------------------------------------------------------------------------------"
 echo "====> : Inputs - Outputs md5sum for debugging "
 rm -f ./data/quickstart_checklist.chk
-md5sum build/mapping.yaml                     >> ./data/quickstart_checklist.chk
-md5sum build/tileset.sql                      >> ./data/quickstart_checklist.chk
-md5sum build/openmaptiles.tm2source/data.yml  >> ./data/quickstart_checklist.chk
-md5sum "./data/${testdata}"                   >> ./data/quickstart_checklist.chk
-md5sum ./data/tiles.mbtiles                   >> ./data/quickstart_checklist.chk
-md5sum ./data/docker-compose-config.yml       >> ./data/quickstart_checklist.chk
+{
+  find build -type f | sort | xargs md5sum ;
+  find data -type f | sort | xargs md5sum ;
+} >> ./data/quickstart_checklist.chk
 cat ./data/quickstart_checklist.chk

 ENDTIME=$(date +%s)
-ENDDATE=$(date +"%Y-%m-%dT%H:%M%z")
 if stat --help >/dev/null 2>&1; then
  MODDATE=$(stat -c %y "./data/${testdata}" )
 else