refactor for GitHub Action

2021-08-31 10:16:33 +00:00 · 2021-08-31 10:16:33 +00:00 · 9d1656e78c
commit 9d1656e78c
--- a/.example.env
+++ b/.example.env
@ -1,4 +1,8 @@
-COMPOSE_PROJECT_NAME=kartozadockerosm
+COMPOSE_PROJECT_NAME=dockerosm
+# Change the following variable if you want to merge multiple compose files
+# docker-compose will automatically merged docker-compose.yml and docker-compose.override.yml if exists.
+# COMPOSE_FILE=docker-compose.yml:docker-compose.develop.yml:docker-compose-web.yml
+COMPOSE_FILE=docker-compose.yml:docker-compose.develop.yml
 POSTGRES_USER=docker
 POSTGRES_PASS=docker
 POSTGRES_DBNAME=gis
--- a/.github/workflows/build-image-latest.yaml
+++ b/.github/workflows/build-image-latest.yaml
@ -0,0 +1,56 @@
+name: build-image-latest
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - develop
+jobs:
+  docker-builder:
+    strategy:
+      matrix:
+        component:
+          - imposm
+          - osmupdate
+          - osmenrich
+    # Only push if PR happens in the same repo
+    # if: github.event.pull_request.base.repo.url == github.event.pull_request.head.repo.url
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      # We can't use Docker meta GitHub Action because
+      # our tag name is unconventional
+      # - name: Docker meta
+      #   id: docker_meta
+      #   uses: docker/metadata-action@v3
+      #   with:
+      #     images: ${{ secrets.DOCKERHUB_REPO }}/docker-osm
+      #     tags: |
+      #       type=semver,pattern=\d.\d.\d
+      #       type=ref,event=branch
+      #       type=ref,event=pr
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          file: docker-${{ matrix.component }}
+          push: true
+          target: prod
+          tags: ${{ secrets.DOCKERHUB_REPO }}/docker-osm:${{ matrix.component }}-latest
+          cache-from: |
+            type=gha,scope=prod
+          cache-to: type=gha,scope=prod
+
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}
--- a/.gitignore
+++ b/.gitignore
@ -16,3 +16,6 @@ settings/clip/clip.prj
 settings/clip/clip.qpj
 settings/clip/clip.shp
 settings/clip/clip.shx
+
+.venv
+.env
--- a/40
+++ b/40
@ -1,8 +1,3 @@
-PROJECT_ID := dockerosm
-COMPOSE_FILE := docker-compose.yml
-# Uncomment the next line if you want to display data with Leaflet.
-# COMPOSE_FILE := docker-compose-web.yml
-
 .PHONY: logs

 ###
@ -14,70 +9,77 @@ status:
 	@echo "------------------------------------------------------------------"
 	@echo "Status in production mode"
 	@echo "------------------------------------------------------------------"
-	@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) ps
+	@docker-compose ps

 build:
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Building in production mode"
 	@echo "------------------------------------------------------------------"
-	@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) build
+	@docker-compose build
+
+redeploy:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Recreate containers"
+	@echo "------------------------------------------------------------------"
+	@docker-compose up -d

 run:
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Running in production mode"
 	@echo "------------------------------------------------------------------"
-	@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) up -d --no-recreate
+	@docker-compose up -d --no-recreate

 rundev:
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Running in DEVELOPMENT mode"
 	@echo "------------------------------------------------------------------"
-	@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) up
+	@docker-compose up

 stop:
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Stopping in production mode"
 	@echo "------------------------------------------------------------------"
-	@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) stop
+	@docker-compose stop

 kill:
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Killing in production mode"
 	@echo "------------------------------------------------------------------"
-	@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) kill
+	@docker-compose kill

 rm: kill rm-volumes
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Removing production instance and all volumes!!! "
 	@echo "------------------------------------------------------------------"
-	@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) rm
+	@docker-compose rm

 rm-volumes:
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Removing all volumes!!!! "
 	@echo "------------------------------------------------------------------"
-	@docker volume rm $(PROJECT_ID)_osm-postgis-data $(PROJECT_ID)_import_queue $(PROJECT_ID)_import_done $(PROJECT_ID)_cache
+	@docker-compose down --volumes

 logs:
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Logs"
 	@echo "------------------------------------------------------------------"
-	@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) logs
+	@docker-compose logs

 live_logs:
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Live Logs"
 	@echo "------------------------------------------------------------------"
-	@docker-compose -f $(COMPOSE_FILE) -p $(PROJECT_ID) logs -f
+	@docker-compose logs -f



@ -91,7 +93,7 @@ timestamp:
 	@echo "------------------------------------------------------------------"
 	@echo "Timestamp"
 	@echo "------------------------------------------------------------------"
-	@docker exec -t -i $(PROJECT_ID)_imposm cat /home/settings/timestamp.txt
+	@docker-compose exec -T imposm cat /home/settings/timestamp.txt


 ###
@ -104,14 +106,14 @@ import_styles: import_styles
 	@echo "------------------------------------------------------------------"
 	@echo "Importing QGIS styles"
 	@echo "------------------------------------------------------------------"
-	@docker exec -i $(PROJECT_ID)_db su - postgres -c "psql -f /home/settings/qgis_style.sql gis"
+	@docker-compose exec -T db su - postgres -c "psql -f /home/settings/qgis_style.sql gis"

 remove_styles:
 	@echo
 	@echo "------------------------------------------------------------------"
 	@echo "Removing QGIS styles"
 	@echo "------------------------------------------------------------------"
-	@docker exec -t -i $(PROJECT_ID)_db /bin/su - postgres -c "psql gis -c 'DROP TABLE IF EXISTS layer_styles;'"
+	@docker-compose exec -T db /bin/su - postgres -c "psql gis -c 'DROP TABLE IF EXISTS layer_styles;'"

 backup_styles:
 	@echo
@ -119,4 +121,4 @@ backup_styles:
 	@echo "Backup QGIS styles to BACKUP.sql"
 	@echo "------------------------------------------------------------------"
 	@echo "SET XML OPTION DOCUMENT;" > BACKUP-STYLES.sql
-	@ docker exec -t $(PROJECT_ID)_db su - postgres -c "/usr/bin/pg_dump --format plain --inserts --table public.layer_styles gis" >> BACKUP-STYLES.sql
+	@docker-compose exec -T db su - postgres -c "/usr/bin/pg_dump --format plain --inserts --table public.layer_styles gis" >> BACKUP-STYLES.sql
--- a/docker-compose-web.yml
+++ b/docker-compose-web.yml
@ -1,135 +0,0 @@
-version: '2.1'
-
-volumes:
-  osm-postgis-data:
-  import_done:
-  import_queue:
-  cache:
-  pgadmin_data:
-
-services:
-  db:
-    image: kartoza/postgis:${POSTGRES_VERSION}
-    hostname: db
-    container_name: dockerosm_db
-    environment:
-      - POSTGRES_USER=${POSTGRES_USER}
-      - POSTGRES_PASS=${POSTGRES_PASS}
-      - POSTGRES_DBNAME=${POSTGRES_DBNAME}
-      - ALLOW_IP_RANGE=${ALLOW_IP_RANGE}
-    volumes:
-      - osm-postgis-data:/var/lib/postgresql
-    ports:
-      - ${DB_PORT}:5432
-    healthcheck:
-      test: "exit 0"
-
-  imposm:
-    image: kartoza/docker-osm:imposm-latest
-    build: docker-imposm3
-    container_name: dockerosm_imposm
-    volumes:
-      - ./settings:/home/settings
-      - import_done:/home/import_done
-      - import_queue:/home/import_queue
-      - cache:/home/cache
-    depends_on:
-      db:
-        condition: service_healthy
-    environment:
-      - POSTGRES_USER=${POSTGRES_USER}
-      - POSTGRES_PASS=${POSTGRES_PASS}
-      - POSTGRES_DBNAME=${POSTGRES_DBNAME}
-      - POSTGRES_PORT=${POSTGRES_PORT}
-      - POSTGRES_HOST=${POSTGRES_HOST}
-      - TIME=${TIME}
-      - SETTINGS=${SETTINGS}
-      - CACHE=${CACHE}
-      - IMPORT_DONE=${IMPORT_DONE}
-      - IMPORT_QUEUE=${IMPORT_QUEUE}
-      - SRID=${SRID}
-      - OPTIMIZE=${OPTIMIZE}
-      - DBSCHEMA_PRODUCTION=${DBSCHEMA_PRODUCTION}
-      - DBSCHEMA_IMPORT=${DBSCHEMA_IMPORT}
-      - DBSCHEMA_BACKUP=${DBSCHEMA_BACKUP}
-      - QGIS_STYLE=${QGIS_STYLE}
-      - CLIP=${CLIP}
-    command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/importer.py"
-
-  osmupdate:
-    build: docker-osmupdate
-    image: kartoza/docker-osm:osmupdate-latest
-    container_name: dockerosm_osmupdate
-    volumes_from:
-      - imposm
-    depends_on:
-      db:
-        condition: service_healthy
-    environment:
-      - MAX_DAYS=${MAX_DAYS}
-      - DIFF=${DIFF}
-      - MAX_MERGE=${MAX_MERGE}
-      - COMPRESSION_LEVEL=${COMPRESSION_LEVEL}
-      - BASE_URL=${BASE_URL}
-      - IMPORT_QUEUE=${IMPORT_QUEUE}
-      - IMPORT_DONE=${IMPORT_DONE}
-      - TIME=${TIME}
-    command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/download.py"
-
-  pgadmin4:
-    image: dpage/pgadmin4:4.16
-    hostname: pgadmin4
-    volumes:
-      - pgadmin_data:/var/lib/pgadmin
-    environment:
-      - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL}
-      - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_DEFAULT_PASSWORD}
-    ports:
-      - ${PGDB_PORT}:80
-    restart: on-failure
-    depends_on:
-      db:
-        condition: service_healthy
-
-  osmenrich:
-    build: docker-osmenrich
-    volumes_from:
-      - imposm
-    depends_on:
-      db:
-        condition: service_healthy
-    environment:
-      - IMPORT_QUEUE=${IMPORT_QUEUE}
-      - IMPORT_DONE=${IMPORT_DONE}
-      - TIME=${TIME}
-      - DBSCHEMA_PRODUCTION=${DBSCHEMA_PRODUCTION}
-
-  martin:
-    image: urbica/martin
-    restart: on-failure
-    ports:
-      - ${MARTIN_PORT}:3000
-    environment:
-      - WATCH_MODE=${WATCH_MODE}
-      - DATABASE_URL=${DATABASE_URL}
-    depends_on:
-      db:
-        condition: service_healthy
-
-  qgisserver:
-    image: openquake/qgis-server:ltr
-    
-    environment:
-      - QGIS_PROJECT_FILE=/project/project.qgs
-      - QGIS_SERVER_LOG_LEVEL=DEBUG
-      - QGIS_SERVER_PARALLEL_RENDERING=1
-    volumes:
-      - ./logs:/var/log/apache2
-      - ./web:/project
-      - ./settings:/web/settings
-    depends_on:
-      db:
-        condition: service_healthy
-    ports:
-      - 8198:80
-    restart: on-failure
--- a/docker-compose.build.yml
+++ b/docker-compose.build.yml
@ -0,0 +1,10 @@
+version: '3.4'
+services:
+  imposm:
+    build: docker-imposm3
+
+  osmupdate:
+    build: docker-osmupdate
+
+  osmenrich:
+    build: docker-osmenrich
--- a/docker-compose.develop.yml
+++ b/docker-compose.develop.yml
@ -0,0 +1,19 @@
+version: '3.4'
+services:
+  db:
+    volumes:
+      - ./settings:/home/settings
+    ports:
+      - ${DB_PORT}:5432
+
+  imposm:
+    volumes:
+      - ./settings:/home/settings
+  
+  osmupdate:
+    volumes:
+      - ./settings:/home/settings
+  
+  osmenrich:
+    volumes:
+      - ./settings:/home/settings
--- a/docker-compose.pgadmin.yml
+++ b/docker-compose.pgadmin.yml
@ -0,0 +1,17 @@
+version: '3.4'
+volumes: 
+  pgadmin_data:
+services: 
+  pgadmin4:
+    image: dpage/pgadmin4:latest
+    volumes:
+      - pgadmin_data:/var/lib/pgadmin
+    environment:
+      - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL}
+      - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_DEFAULT_PASSWORD}
+    ports:
+      - ${PGDB_PORT}:80
+    restart: on-failure
+    depends_on:
+      db:
+        condition: service_healthy
--- a/docker-compose.web.yml
+++ b/docker-compose.web.yml
@ -0,0 +1,30 @@
+version: '3.4'
+services:
+  martin:
+    image: urbica/martin
+    restart: on-failure
+    ports:
+      - ${MARTIN_PORT}:3000
+    environment:
+      - WATCH_MODE=${WATCH_MODE}
+      - DATABASE_URL=${DATABASE_URL}
+    depends_on:
+      db:
+        condition: service_healthy
+
+  qgisserver:
+    image: openquake/qgis-server:ltr
+    environment:
+      - QGIS_PROJECT_FILE=/project/project.qgs
+      - QGIS_SERVER_LOG_LEVEL=DEBUG
+      - QGIS_SERVER_PARALLEL_RENDERING=1
+    volumes:
+      - ./logs:/var/log/apache2
+      - ./web:/project
+      - ./settings:/web/settings
+    depends_on:
+      db:
+        condition: service_healthy
+    ports:
+      - 8198:80
+    restart: on-failure
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,17 +1,15 @@
-version: '2.1'
+version: '3.4'

 volumes:
  osm-postgis-data:
+  settings-data:
  import_done:
  import_queue:
  cache:
-  pgadmin_data:

 services:
  db:
    image: kartoza/postgis:${POSTGRES_VERSION}
-    hostname: db
-    container_name: dockerosm_db
    environment:
      - POSTGRES_USER=${POSTGRES_USER}
      - POSTGRES_PASS=${POSTGRES_PASS}
@ -20,18 +18,14 @@ services:
      - FORCE_SSL=${FORCE_SSL}
    volumes:
      - osm-postgis-data:/var/lib/postgresql
-    ports:
-      - ${DB_PORT}:5432
+      - settings-data:/home/settings
    healthcheck:
-      test: "exit 0"
-
+      test: "pg_isready -d ${POSTGRES_DBNAME}"

  imposm:
    image: kartoza/docker-osm:imposm-latest
-    build: docker-imposm3
-    container_name: dockerosm_imposm
    volumes:
-      - ./settings:/home/settings
+      - settings-data:/home/settings
      - import_done:/home/import_done
      - import_queue:/home/import_queue
      - cache:/home/cache
@ -57,14 +51,14 @@ services:
      - QGIS_STYLE=${QGIS_STYLE}
      - CLIP=${CLIP}
      - SSL_MODE=${SSL_MODE}
-    command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/importer.py"

  osmupdate:
-    build: docker-osmupdate
    image: kartoza/docker-osm:osmupdate-latest
-    container_name: dockerosm_osmupdate
-    volumes_from:
-      - imposm
+    volumes:
+      - settings-data:/home/settings
+      - import_done:/home/import_done
+      - import_queue:/home/import_queue
+      - cache:/home/cache
    depends_on:
      db:
        condition: service_healthy
@ -78,26 +72,13 @@ services:
      - IMPORT_DONE=${IMPORT_DONE}
      - TIME=${TIME}

-  pgadmin4:
-    image: dpage/pgadmin4:latest
-    hostname: pgadmin4
-    volumes:
-      - pgadmin_data:/var/lib/pgadmin
-    environment:
-      - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL}
-      - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_DEFAULT_PASSWORD}
-    ports:
-      - ${PGDB_PORT}:80
-    restart: on-failure
-    depends_on:
-      db:
-        condition: service_healthy
-
  osmenrich:
-    build: docker-osmenrich
-    container_name: dockerosm_osmenrich
-    volumes_from:
-      - imposm
+    image: kartoza/docker-osm:osmenrich-latest
+    volumes:
+      - settings-data:/home/settings
+      - import_done:/home/import_done
+      - import_queue:/home/import_queue
+      - cache:/home/cache
    depends_on:
      db:
        condition: service_healthy
--- a/docker-imposm3/Dockerfile
+++ b/docker-imposm3/Dockerfile
--- a/docker-imposm3/importer.py
+++ b/docker-imposm3/importer.py
--- a/docker-imposm3/requirements.txt
+++ b/docker-imposm3/requirements.txt
--- a/readme.md
+++ b/readme.md
@ -17,13 +17,56 @@ As a quick example, we are going to setup Docker-OSM with default values everywh

 Alternatively you can execute the `settings_downloader.sh` script to download the pbf and the clip file
 ```bash
-./settings_downloader.sh GEOJSON_URL CONTINENT COUNTRY ie
-./settings_downloader.sh https://github.com/kartoza/docker-osm/raw/develop/settings/clip.geojson africa south-africa
+bash ./settings_downloader.sh GEOJSON_URL CONTINENT COUNTRY ie
+bash ./settings_downloader.sh https://github.com/kartoza/docker-osm/raw/develop/settings/clip.geojson africa south-africa
 ```
 For a full list of allowed file names read json file `countries.json`

 Alternatively you can use the python script `pbf_downloader.py`

+For local usage, the containers are set up using docker-compose configuration. The configuration files consists of 
+two sets of config file. The first one is `.env` which contains lists of environment variables.
+Copy the `.env` file from the `.example.env` in this repo.
+
+```bash
+cp .example.env .env
+```
+
+For subsequent configuration, edit the `.env` files to tweak your options.
+
+The second set of configuration is using a `docker-compose.yml` files, which is a compose files docker-compose is using.
+
+For minimum set of production environment, the configuration file is described in `docker-compose.yml` file.
+The other YAML files with prefix `docker-compose` is a configuration file that you can merge with the basic `docker-compose.yml` file. To use more than one configuration file, you edit `.env` file and change the `COMPOSE_FILE` variable to include all 
+the compose file you desired, separated by a colon `:` for each file.
+
+For example, by default, the example file is using both `docker-compose.yml` and `docker-compose.develop.yml` because
+we expect you to provide the necessary settings in `settings` folder. Thus the `COMPOSE_FILE` variable looks like this:
+
+```
+COMPOSE_FILE=docker-compose.yml:docker-compose.develop.yml
+```
+
+In production environment, normally all the persistent data is stored in a volume, instead of bind mounted.
+In this case only the `docker-compose.yml` is enough, and you need to provide the settings inside the volume itself.
+
+To build the image yourself, include the `docker-compose.build.yml` file.
+
+To use helper services such as pgadmin, include `docker-compose.pgadmin.yml`, and `docker-compose.web.yml` for web demo.
+
+If you are familiar with how docker-compose work, you can also use a standard convention by putting `docker-compose.override.yml`
+file and include it in the `COMPOSE_FILE` variable.
+
+To store the configuration for long-term use (for archiving or diffing of different configuration). You can interpolate the 
+current variables in `.env` and generate a full config files:
+
+```bash
+docker-compose config > docker-compose.production.yml
+```
+
+The above command will include and merged all your config files specified in `COMPOSE_FILE` variable and also fill out the 
+variables parameterized in the docker-compose file.
+
 * If you want to connect from your local QGIS Desktop:
  * In the file `docker-compose.yml`, uncomment the block: