diff --git a/.env b/.env index aa66d0c5..70d706a0 100644 --- a/.env +++ b/.env @@ -1,12 +1,16 @@ +# This file defines default environment variables for all images + +TOOLS_VERSION=4.1.0 + POSTGRES_DB=openmaptiles POSTGRES_USER=openmaptiles POSTGRES_PASSWORD=openmaptiles POSTGRES_HOST=postgres POSTGRES_PORT=5432 + QUICKSTART_MIN_ZOOM=0 QUICKSTART_MAX_ZOOM=7 DIFF_MODE=false -TOOLS_VERSION=4.1.0 BBOX=-180.0,-85.0511,180.0,85.0511 MIN_ZOOM=0 diff --git a/.github/workflows/omt_ci.yml b/.github/workflows/omt_ci.yml deleted file mode 100644 index a2230d5f..00000000 --- a/.github/workflows/omt_ci.yml +++ /dev/null @@ -1,45 +0,0 @@ -# MapTiler OpenMapTiles -####################### - -# Workflow to validate OMT`s new Pull Requests and commits pushed into OMT repo - -name: OMT_CI - -# Controls when the action will run. Triggers the workflow on push and pull request -# events but only for the master branch -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -# jobs can run parallel -jobs: - # This workflow contains a single job called "build" - build: - # runs on ubuntu (can run on windows and mac os) - runs-on: ubuntu-latest - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 - - # Named steps - - name: generate all zooms - run: sed -i 's/QUICKSTART_MAX_ZOOM=7/QUICKSTART_MAX_ZOOM=14/g' .env - - # Runs quickstart - - name: quickstart - env: - area: northamptonshire - run: bash ./quickstart.sh $area - - - name: generate devdoc - run: TEST_MODE=yes make generate-devdoc - - # todo: use artifact to store result of tests - #- uses: actions/upload-artifact@v1 - # with: - # name: quickstart log file - # path: quickstart.log diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..886ec472 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,200 @@ +# Workflow to validate OMT`s new Pull Requests and commits pushed into OMT repo + +name: OpenMapTiles CI + +on: + push: + branches: [ master ] + pull_request: + +jobs: + + integrity_test: + name: Run integrity test + runs-on: ubuntu-latest + steps: + + - name: Checkout the changes + uses: actions/checkout@v2 + + - name: Run quickstart for a small area + env: + area: monaco + run: | + # For now, change the quickstart values directly in the .env file + # TODO: We should probably use env vars instead + sed -i 's/QUICKSTART_MAX_ZOOM=7/QUICKSTART_MAX_ZOOM=14/g' .env + ./quickstart.sh $area + + - name: Save quickstart.log + uses: actions/upload-artifact@v1 + with: + name: quickstart.log + path: quickstart.log + + - name: Test etldoc images + run: | + export TEST_MODE=yes + make generate-devdoc + + performance: + name: Evaluate performance + runs-on: ubuntu-latest + # Even though we technically don't need to wait for integrity test to finish, + # there is no point to run long perf test until we know the code is OK + needs: integrity_test + env: + # Smaller tests (runs everything in about 30 minutes) + TEST_PERF_PARAMS: "--minzoom 0 --maxzoom 14 --test equatorial-guinea --test liechtenstein" + TEST_DATA_URL: "https://drive.google.com/uc?export=download&id=12vw07f9W0MiAHIqMztRiIMwahJfqTi21" + + ## Large test data -- we should switch to it after everything is working ok + # TEST_PERF_PARAMS: "--minzoom 0 --maxzoom 14 --test hungary --test isle-of-man" + # TEST_DATA_URL: "https://drive.google.com/uc?export=download&id=1kw7XPDPd1Rc-Zi2XxGLTXdinUSq-S4pT" + steps: + - name: Cache test data download + id: cache-testdata + uses: actions/cache@v1 + with: + path: ci_cache + key: "${{ env.TEST_DATA_URL }}" + + - name: Download test data on cache miss + if: steps.cache-testdata.outputs.cache-hit != 'true' + run: | + echo "Data file does not exist, downloading $TEST_DATA_URL" + mkdir -p ci_cache + curl --silent --show-error --location --output ci_cache/perf-test-areas-latest.osm.pbf "$TEST_DATA_URL" + + - name: Get code + uses: actions/checkout@v2 + with: + # Fetch the last two commits in case this is a PR, + # and we need to profile the base branch first + fetch-depth: 2 + path: code + + - name: Compute git revision hash to cache + id: calc + run: | + # If this is a pull request, we should cache the parent (base) revision + # Otherwise cache the current one + cd code + REV_HASH=$(git log -1 --format="%H") + if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + # Take the first parent of the grafted commit (cannot use HEAD^1 with shallow clones) + REV_HASH=$(git cat-file -p $REV_HASH | awk 'NR > 1 {if(/^parent/){print $2; exit}}') + fi + echo "::set-output name=hash::$REV_HASH" + + - name: Set up caching for the performance results + uses: actions/cache@v1 + with: + path: perf_cache + key: "${{ steps.calc.outputs.hash }}-${{ env.TEST_DATA_URL }}" + + - name: Load test data into DB and run performance test + id: main + env: + CACHE_SHA: "${{ steps.calc.outputs.hash }}" + run: | + create_db() { + make clean + make init-dirs + cp ../ci_cache/perf-test-areas-latest.osm.pbf data/perf-test-areas-latest.osm.pbf + make db-destroy + make all + make db-start + time make import-data + time make import-osm + time make import-borders + time make import-wikidata + time make import-sql + } + + mkdir -p perf_cache + mkdir -p artifacts + cd code + + CURRENT_SHA=$(git log -1 --format="%H") + + if [ ! -f ../perf_cache/results.json ]; then + echo "We do not have cached performance results, create them..." + if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + git reset --hard ${CURRENT_SHA}^1 + fi + + create_db + + # Use latest tools version because these specific tests do not yet exist in the 4.1 tools version + # Custom TOOLS_VERSION can be removed once OMT master is migrated to the next tools version + TOOLS_VERSION=latest docker-compose run --rm openmaptiles-tools \ + test-perf openmaptiles.yaml $TEST_PERF_PARAMS \ + --record /tileset/results.json + mv results.json ../perf_cache + + if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + # For pull requests, restore to the PR version before continuing + git reset --hard ${CURRENT_SHA} + fi + else + echo "Found cached performance results" + fi + + if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + cp ../perf_cache/results.json ../artifacts/base-results.json + else + cp ../perf_cache/results.json ../artifacts/results.json + fi + + if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + echo "Comparing pull request results with the base..." + + create_db + + # Use latest tools version because these specific tests do not yet exist in the 4.1 tools version + # Custom TOOLS_VERSION can be removed once OMT master is migrated to the next tools version + cp ../perf_cache/results.json . + OUTPUT="$(TOOLS_VERSION=latest docker-compose run --rm openmaptiles-tools \ + test-perf openmaptiles.yaml $TEST_PERF_PARAMS \ + --compare /tileset/results.json --record /tileset/pr-results.json)" + rm results.json + mv pr-results.json ../artifacts/ + + # Convert multiline output into a single long string. + # See https://github.community/t5/GitHub-Actions/set-output-Truncates-Multiline-Strings/td-p/37870 + OUTPUT="${OUTPUT//'%'/'%25'}" + OUTPUT="${OUTPUT//$'\n'/'%0A'}" + OUTPUT="${OUTPUT//$'\r'/'%0D'}" + + # Split into two parts -- before and after the ===== SUMMARY ===== + echo "::set-output name=summary::${OUTPUT##*========}" + echo "::set-output name=details::${OUTPUT%%========*}" + fi + + - name: Save artifacts + uses: actions/upload-artifact@v1 + with: + name: performance_results + path: artifacts + + - name: Post a comment on Pull Request + if: "github.event_name == 'pull_request'" + uses: marocchino/sticky-pull-request-comment@v1 + timeout-minutes: 1 + continue-on-error: true + with: + message: |- + ``` + ${{ steps.main.outputs.summary }} + ``` + + <details> + <summary>expand for details...</summary> + + ``` + ${{ steps.main.outputs.details }} + ``` + + </details> + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Makefile b/Makefile index 775f6705..19acae21 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ else endif .PHONY: all -all: build/openmaptiles.tm2source/data.yml build/mapping.yaml build/tileset.sql +all: build/openmaptiles.tm2source/data.yml build/mapping.yaml build-sql # Set OpenMapTiles host OMT_HOST:=http://$(firstword $(subst :, ,$(subst tcp://,,$(DOCKER_HOST))) localhost) @@ -38,8 +38,9 @@ help: @echo " ./quickstart.sh <<your-area>> # example: ./quickstart.sh madagascar " @echo " " @echo "Hints for designers:" - @echo " make start-postserve # start Postserver + Maputnik Editor [ see $(OMT_HOST):8088 ] " - @echo " make start-tileserver # start maptiler/tileserver-gl [ see $(OMT_HOST):8080 ] " + @echo " make maputnik-start # start Maputnik Editor + dynamic tile server [ see $(OMT_HOST):8088 ]" + @echo " make postserve-start # start dynamic tile server [ see $(OMT_HOST):8090 ]" + @echo " make tileserver-start # start maptiler/tileserver-gl [ see $(OMT_HOST):8080 ]" @echo " " @echo "Hints for developers:" @echo " make # build source code" @@ -53,10 +54,12 @@ help: @echo " make psql-analyze # PostgreSQL: ANALYZE" @echo " make generate-qareports # generate reports [./build/qareports]" @echo " make generate-devdoc # generate devdoc including graphs for all layers [./layers/...]" + @echo " make tools-dev # start openmaptiles-tools /bin/bash terminal" + @echo " make db-destroy # remove docker containers and PostgreSQL data volume" + @echo " make db-start # start PostgreSQL, creating it if it doesn't exist" + @echo " make db-stop # stop PostgreSQL database without destroying the data" @echo " make import-sql-dev # start import-sql /bin/bash terminal" @echo " make import-osm-dev # start import-osm /bin/bash terminal (imposm3)" - @echo " make clean-docker # remove docker containers, PG data volume" - @echo " make forced-clean-sql # drop all PostgreSQL tables for clean environment" @echo " make docker-unnecessary-clean # clean unnecessary docker image(s) and container(s)" @echo " make refresh-docker-images # refresh openmaptiles docker images from Docker HUB" @echo " make remove-docker-images # remove openmaptiles docker images" @@ -69,7 +72,9 @@ help: .PHONY: init-dirs init-dirs: - mkdir -p build && mkdir -p data && mkdir -p cache + @mkdir -p build + @mkdir -p data + @mkdir -p cache build/openmaptiles.tm2source/data.yml: init-dirs mkdir -p build/openmaptiles.tm2source @@ -78,15 +83,16 @@ build/openmaptiles.tm2source/data.yml: init-dirs build/mapping.yaml: init-dirs $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-imposm3 openmaptiles.yaml > $@ -build/tileset.sql: init-dirs - $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-sql openmaptiles.yaml > $@ +.PHONY: build-sql +build-sql: init-dirs + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-sql openmaptiles.yaml > build/tileset.sql .PHONY: clean clean: rm -rf build -.PHONY: clean-docker -clean-docker: +.PHONY: db-destroy +db-destroy: $(DOCKER_COMPOSE) down -v --remove-orphans $(DOCKER_COMPOSE) rm -fv docker volume ls -q -f "name=^$${DC_PROJECT,,*}_" | $(XARGS) docker volume rm @@ -94,14 +100,19 @@ clean-docker: .PHONY: db-start db-start: - $(DOCKER_COMPOSE) up -d postgres + $(DOCKER_COMPOSE) up --no-recreate -d postgres @echo "Wait for PostgreSQL to start..." $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm ./pgwait.sh .PHONY: db-stop db-stop: + @echo "Stopping PostgreSQL..." $(DOCKER_COMPOSE) stop postgres +.PHONY: list-geofabrik +list-geofabrik: + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools download-osm list geofabrik + OSM_SERVERS:=geofabrik osmfr bbbike ALL_DOWNLOADS:=$(addprefix download-,$(OSM_SERVERS)) OSM_SERVER=$(patsubst download-%,%,$@) @@ -135,28 +146,20 @@ psql: db-start import-osm: db-start all $(DOCKER_COMPOSE) run $(DC_OPTS) import-osm -.PHONY: import-sql -import-sql: db-start all - $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-sql - .PHONY: import-osmsql import-osmsql: db-start all import-osm import-sql +.PHONY: import-data +import-data: db-start + $(DOCKER_COMPOSE) run $(DC_OPTS) import-data + .PHONY: import-borders import-borders: db-start $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-borders -.PHONY: import-water -import-water: db-start - $(DOCKER_COMPOSE) run $(DC_OPTS) import-water - -.PHONY: import-natural-earth -import-natural-earth: db-start - $(DOCKER_COMPOSE) run $(DC_OPTS) import-natural-earth - -.PHONY: import-lakelines -import-lakelines: db-start - $(DOCKER_COMPOSE) run $(DC_OPTS) import-lakelines +.PHONY: import-sql +import-sql: db-start all + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-sql .PHONY: generate-tiles ifneq ($(wildcard data/docker-compose-config.yml),) @@ -164,11 +167,13 @@ ifneq ($(wildcard data/docker-compose-config.yml),) endif generate-tiles: init-dirs db-start all rm -rf data/tiles.mbtiles + echo "Generating tiles ..."; \ $(DOCKER_COMPOSE) $(DC_CONFIG_TILES) run $(DC_OPTS) generate-vectortiles + @echo "Updating generated tile metadata ..." $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools generate-metadata ./data/tiles.mbtiles -.PHONY: start-tileserver -start-tileserver: init-dirs +.PHONY: tileserver-start +tileserver-start: init-dirs @echo " " @echo "***********************************************************" @echo "* " @@ -188,29 +193,42 @@ start-tileserver: init-dirs @echo " " docker run $(DC_OPTS) -it --name tileserver-gl -v $$(pwd)/data:/data -p 8080:8080 maptiler/tileserver-gl --port 8080 -.PHONY: start-postserve -start-postserve: db-start +.PHONY: postserve-start +postserve-start: db-start @echo " " @echo "***********************************************************" @echo "* " @echo "* Bring up postserve at $(OMT_HOST):8090" + @echo "* --> can view it locally (use make maputnik-start)" + @echo "* --> or can use https://maputnik.github.io/editor" + @echo "* " + @echo "* set data source / TileJSON URL to http://$(OMT_HOST):8090" @echo "* " @echo "***********************************************************" @echo " " $(DOCKER_COMPOSE) up -d postserve - docker pull maputnik/editor + +.PHONY: postserve-stop +postserve-stop: + $(DOCKER_COMPOSE) stop postserve + +.PHONY: maputnik-start +maputnik-start: maputnik-stop postserve-start @echo " " @echo "***********************************************************" @echo "* " @echo "* Start maputnik/editor " @echo "* ---> go to http://$(OMT_HOST):8088 " - @echo "* ---> set 'data source' to http://$(OMT_HOST):8090" + @echo "* ---> set data source / TileJSON URL to http://$(OMT_HOST):8090" @echo "* " @echo "***********************************************************" @echo " " - -docker rm -f maputnik_editor docker run $(DC_OPTS) --name maputnik_editor -d -p 8088:8888 maputnik/editor +.PHONY: maputnik-stop +maputnik-stop: + -docker rm -f maputnik_editor + .PHONY: generate-qareports generate-qareports: ./qa/run.sh @@ -219,12 +237,12 @@ generate-qareports: .PHONY: generate-devdoc generate-devdoc: init-dirs mkdir -p ./build/devdoc && \ - $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools-latest sh -c \ + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools sh -c \ 'generate-etlgraph openmaptiles.yaml $(GRAPH_PARAMS) && \ generate-mapping-graph openmaptiles.yaml $(GRAPH_PARAMS)' -.PHONY: import-sql-dev -import-sql-dev: +.PHONY: tools-dev +tools-dev: $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools bash .PHONY: import-osm-dev @@ -233,7 +251,7 @@ import-osm-dev: .PHONY: import-wikidata import-wikidata: - $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-wikidata openmaptiles.yaml + $(DOCKER_COMPOSE) run $(DC_OPTS) openmaptiles-tools import-wikidata --cache /cache/wikidata-cache.json openmaptiles.yaml .PHONY: psql-pg-stat-reset psql-pg-stat-reset: @@ -277,7 +295,13 @@ list-docker-images: .PHONY: refresh-docker-images refresh-docker-images: +ifneq ($(strip $(NO_REFRESH)),) + @echo "Skipping docker image refresh" +else + @echo "" + @echo "Refreshing docker images... Use NO_REFRESH=1 to skip." $(DOCKER_COMPOSE) pull --ignore-pull-failures +endif .PHONY: remove-docker-images remove-docker-images: diff --git a/README.md b/README.md index 1da2a97a..5e34b48e 100644 --- a/README.md +++ b/README.md @@ -100,9 +100,7 @@ make db-start Import external data from [OpenStreetMapData](http://osmdata.openstreetmap.de/), [Natural Earth](http://www.naturalearthdata.com/) and [OpenStreetMap Lake Labels](https://github.com/lukasmartinelli/osm-lakelines). ```bash -make import-water -make import-natural-earth -make import-lakelines +make import-data ``` [Download OpenStreetMap data extracts](http://download.geofabrik.de/) and store the PBF file in the `./data` directory. @@ -119,7 +117,7 @@ make download-geofabrik area=albania ``` [Import OpenStreetMap data](https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-osm) with the mapping rules from -`build/mapping.yaml` (which has been created by `make`). Run after any change in layers definiton. Also create borders table using extra processing with [osmborder](https://github.com/pnorman/osmborder) tool. +`build/mapping.yaml` (which has been created by `make`). Run after any change in layers definition. Also create borders table using extra processing with [osmborder](https://github.com/pnorman/osmborder) tool. ```bash make import-osm diff --git a/docker-compose.yml b/docker-compose.yml index 41087d41..75c479f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,9 +1,17 @@ -version: "2" +# This version must match the MAKE_DC_VERSION value below +version: "2.3" + volumes: pgdata: + +networks: + postgres_conn: + driver: bridge + services: + postgres: - image: "openmaptiles/postgis:${TOOLS_VERSION}" + image: "${POSTGIS_IMAGE:-openmaptiles/postgis}:${TOOLS_VERSION}" volumes: - pgdata:/var/lib/postgresql/data networks: @@ -11,21 +19,13 @@ services: ports: - "5432" env_file: .env - import-natural-earth: - image: "openmaptiles/import-natural-earth:${TOOLS_VERSION}" - env_file: .env - networks: - - postgres_conn - import-water: - image: "openmaptiles/import-water:${TOOLS_VERSION}" - env_file: .env - networks: - - postgres_conn - import-lakelines: - image: "openmaptiles/import-lakelines:${TOOLS_VERSION}" + + import-data: + image: "openmaptiles/import-data:${TOOLS_VERSION}" env_file: .env networks: - postgres_conn + import-osm: image: "openmaptiles/import-osm:${TOOLS_VERSION}" env_file: .env @@ -37,6 +37,7 @@ services: - ./data:/import - ./build:/mapping - ./cache:/cache + import-osm-diff: image: "openmaptiles/import-osm:${TOOLS_VERSION}" env_file: .env @@ -49,6 +50,7 @@ services: - ./data:/import - ./build:/mapping - ./cache:/cache + update-osm: image: "openmaptiles/import-osm:${TOOLS_VERSION}" env_file: .env @@ -61,26 +63,25 @@ services: - ./data:/import - ./build:/mapping - ./cache:/cache + openmaptiles-tools: image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}" env_file: .env + environment: + # Must match the version of this file (first line) + # download-osm will use it when generating a composer file + MAKE_DC_VERSION: "2.3" + # Allow DIFF_MODE to be overwritten from shell + DIFF_MODE: ${DIFF_MODE} networks: - postgres_conn volumes: - .:/tileset - ./data:/import - ./build:/sql - openmaptiles-tools-latest: - # This target exists for experimental tools that have not yet been published. - # Do not use this for production. - image: "openmaptiles/openmaptiles-tools:latest" - env_file: .env - networks: - - postgres_conn - volumes: - - .:/tileset - - ./data:/import - - ./build:/sql + - ./build:/mapping + - ./cache:/cache + generate-changed-vectortiles: image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}" command: ./export-list.sh @@ -90,6 +91,7 @@ services: networks: - postgres_conn env_file: .env + generate-vectortiles: image: "openmaptiles/generate-vectortiles:${TOOLS_VERSION}" volumes: @@ -102,6 +104,7 @@ services: BBOX: ${BBOX} MIN_ZOOM: ${MIN_ZOOM} MAX_ZOOM: ${MAX_ZOOM} + postserve: image: "openmaptiles/openmaptiles-tools:${TOOLS_VERSION}" command: postserve openmaptiles.yaml --verbose @@ -112,7 +115,3 @@ services: - "8090:8090" volumes: - .:/tileset - -networks: - postgres_conn: - driver: bridge diff --git a/layers/building/building.sql b/layers/building/building.sql index 95f729c6..81f39fdb 100644 --- a/layers/building/building.sql +++ b/layers/building/building.sql @@ -72,7 +72,8 @@ CREATE OR REPLACE VIEW osm_all_buildings AS ( FALSE as hide_3d FROM osm_building_polygon obp - WHERE osm_id < 0 + -- OSM mulipolygons once imported can give unique postgis polygons with holes, or multi parts polygons + WHERE osm_id < 0 AND ST_GeometryType(geometry) IN ('ST_Polygon', 'ST_MultiPolygon') UNION ALL -- etldoc: osm_building_polygon -> layer_building:z14_ @@ -88,7 +89,8 @@ CREATE OR REPLACE VIEW osm_all_buildings AS ( FROM osm_building_polygon obp LEFT JOIN osm_building_relation obr ON (obr.member = obp.osm_id) - WHERE obp.osm_id >= 0 + -- Only check for ST_Polygon as we exclude buildings from relations keeping only positive ids + WHERE obp.osm_id >= 0 AND ST_GeometryType(obp.geometry) = 'ST_Polygon' ); CREATE OR REPLACE FUNCTION layer_building(bbox geometry, zoom_level int) diff --git a/layers/poi/poi.yaml b/layers/poi/poi.yaml index d678bd65..ed49fea7 100644 --- a/layers/poi/poi.yaml +++ b/layers/poi/poi.yaml @@ -146,10 +146,10 @@ layer: query: (SELECT osm_id, geometry, name, name_en, name_de, {name_languages}, class, subclass, agg_stop, layer, level, indoor, rank FROM layer_poi(!bbox!, z(!scale_denominator!), !pixel_width!)) AS t schema: - ./public_transport_stop_type.sql - - ./update_poi_polygon.sql - - ./update_poi_point.sql - ./class.sql - ./poi_stop_agg.sql + - ./update_poi_polygon.sql + - ./update_poi_point.sql - ./layer.sql datasources: - type: imposm3 diff --git a/layers/poi/poi_stop_agg.sql b/layers/poi/poi_stop_agg.sql index 788bcda6..4e4f40ac 100644 --- a/layers/poi/poi_stop_agg.sql +++ b/layers/poi/poi_stop_agg.sql @@ -33,6 +33,3 @@ CREATE MATERIALIZED VIEW osm_poi_stop_rank AS ( subclass IN ('bus_stop', 'bus_station', 'tram_stop', 'subway') ORDER BY p.uic_ref, rk ) /* DELAY_MATERIALIZED_VIEW_CREATION */; - -ALTER TABLE osm_poi_point ADD COLUMN IF NOT EXISTS agg_stop INTEGER DEFAULT NULL; -SELECT update_osm_poi_point_agg(); diff --git a/layers/poi/update_poi_point.sql b/layers/poi/update_poi_point.sql index dddec1d8..bd6aeb41 100644 --- a/layers/poi/update_poi_point.sql +++ b/layers/poi/update_poi_point.sql @@ -43,6 +43,9 @@ BEGIN END; $$ LANGUAGE plpgsql; +ALTER TABLE osm_poi_point ADD COLUMN IF NOT EXISTS agg_stop INTEGER DEFAULT NULL; +SELECT update_osm_poi_point_agg(); + -- Handle updates CREATE SCHEMA IF NOT EXISTS poi_point; diff --git a/quickstart.sh b/quickstart.sh index 4af930b6..01083438 100755 --- a/quickstart.sh +++ b/quickstart.sh @@ -105,8 +105,8 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then mem=$( grep MemTotal /proc/meminfo | awk '{print $2}' | xargs -I {} echo "scale=4; {}/1024^2" | bc ) echo "system memory (GB): ${mem}" grep SwapTotal /proc/meminfo - echo "cpu number: $(grep -c processor /proc/cpuinfo) x $(cat /proc/cpuinfo | grep "bogomips" | head -1)" - cat /proc/meminfo | grep Free + echo "cpu number: $(grep -c processor /proc/cpuinfo) x $(grep "bogomips" /proc/cpuinfo | head -1)" + grep Free /proc/meminfo else echo " " echo "Warning : Platforms other than Linux are less tested" @@ -116,11 +116,11 @@ fi echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Stopping running services & removing old containers" -make clean-docker +make db-destroy echo " " echo "-------------------------------------------------------------------------------------" -echo "====> : Checking OpenMapTiles docker images " +echo "====> : Existing OpenMapTiles docker images. Will use version $(source .env && echo "$TOOLS_VERSION")" docker images | grep openmaptiles echo " " @@ -162,7 +162,7 @@ echo "-------------------------------------------------------------------------- echo "====> : Code generating from the layer definitions ( ./build/mapping.yaml; ./build/tileset.sql )" echo " : The tool source code: https://github.com/openmaptiles/openmaptiles-tools " echo " : But we generate the tm2source, Imposm mappings and SQL functions from the layer definitions! " -make +make all echo " " echo "-------------------------------------------------------------------------------------" @@ -178,29 +178,18 @@ echo "====> : Drop and Recreate PostgreSQL public schema " # This adds an extra safety belt if the user modifies the docker volume settings make forced-clean-sql -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : Start importing water data from http://osmdata.openstreetmap.de/ into PostgreSQL " -echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-water " -echo " : Data license: https://osmdata.openstreetmap.de/info/license.html " -echo " : Thank you: https://osmdata.openstreetmap.de/info/ " -make import-water - -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : Start importing http://www.naturalearthdata.com into PostgreSQL " -echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-natural-earth " -echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use " -echo " : Thank you: Natural Earth Contributors! " -make import-natural-earth - -echo " " -echo "-------------------------------------------------------------------------------------" -echo "====> : Start importing OpenStreetMap Lakelines data " -echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-lakelines " -echo " : https://github.com/lukasmartinelli/osm-lakelines " -echo " : Data license: .. " -make import-lakelines +echo "====> : Importing all the data:" +echo " : * Water data from http://osmdata.openstreetmap.de" +echo " : Data license: https://osmdata.openstreetmap.de/info/license.html" +echo " : * Natural Earth from http://www.naturalearthdata.com" +echo " : Terms-of-use: http://www.naturalearthdata.com/about/terms-of-use" +echo " : * OpenStreetMap Lakelines data https://github.com/lukasmartinelli/osm-lakelines" +echo " :" +echo " : Source code: https://github.com/openmaptiles/openmaptiles-tools/tree/master/docker/import-data" +echo " : includes all data from the import-data image" +echo " :" +echo " : Thank you: https://www.postgresql.org ! Thank you http://postgis.org !" +make import-data echo " " echo "-------------------------------------------------------------------------------------" @@ -269,16 +258,13 @@ echo " " echo "-------------------------------------------------------------------------------------" echo "====> : Inputs - Outputs md5sum for debugging " rm -f ./data/quickstart_checklist.chk -md5sum build/mapping.yaml >> ./data/quickstart_checklist.chk -md5sum build/tileset.sql >> ./data/quickstart_checklist.chk -md5sum build/openmaptiles.tm2source/data.yml >> ./data/quickstart_checklist.chk -md5sum "./data/${testdata}" >> ./data/quickstart_checklist.chk -md5sum ./data/tiles.mbtiles >> ./data/quickstart_checklist.chk -md5sum ./data/docker-compose-config.yml >> ./data/quickstart_checklist.chk +{ + find build -type f | sort | xargs md5sum ; + find data -type f | sort | xargs md5sum ; +} >> ./data/quickstart_checklist.chk cat ./data/quickstart_checklist.chk ENDTIME=$(date +%s) -ENDDATE=$(date +"%Y-%m-%dT%H:%M%z") if stat --help >/dev/null 2>&1; then MODDATE=$(stat -c %y "./data/${testdata}" ) else @@ -312,8 +298,8 @@ echo "We saved the log file to $log_file ( for debugging ) You can compare with echo " " echo "Start experimenting! And check the QUICKSTART.MD file!" echo " " -echo "* Use make start-postserve to explore tile generation on request" -echo "* Use make start-tileserver to view pre-generated tiles" +echo "* Use make maputnik-start to explore tile generation on request" +echo "* Use make tileserver-start to view pre-generated tiles" echo " " echo "Available help commands (make help) " make help