From 77af65a54c74d48b91a5b87153c3c503896a5d23 Mon Sep 17 00:00:00 2001 From: Nicola Ferruzzi Date: Fri, 13 Sep 2019 17:38:52 +0200 Subject: [PATCH] Add docker support --- deployment/docker/Makefile | 4 + deployment/docker/cups/README.txt | 1 + deployment/docker/data/README.txt | 2 + deployment/docker/docker-compose.yml | 121 ++++++++++++++++++ deployment/docker/ogn-pg-importer/Dockerfile | 13 ++ .../ogn-pg-importer/docker-entrypoint.sh | 16 +++ deployment/docker/ogn-python/Dockerfile | 12 ++ deployment/docker/ogn-python/prestart.sh | 7 + deployment/docker/ogn-python/wait.sh | 5 + 9 files changed, 181 insertions(+) create mode 100644 deployment/docker/Makefile create mode 100644 deployment/docker/cups/README.txt create mode 100644 deployment/docker/data/README.txt create mode 100644 deployment/docker/docker-compose.yml create mode 100644 deployment/docker/ogn-pg-importer/Dockerfile create mode 100755 deployment/docker/ogn-pg-importer/docker-entrypoint.sh create mode 100644 deployment/docker/ogn-python/Dockerfile create mode 100755 deployment/docker/ogn-python/prestart.sh create mode 100755 deployment/docker/ogn-python/wait.sh diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile new file mode 100644 index 0000000..f00e273 --- /dev/null +++ b/deployment/docker/Makefile @@ -0,0 +1,4 @@ +build: + @cd ../../; docker build -t ogn -f deployment/docker/ogn-python/Dockerfile . + @cd ../../; docker build -t ogn-pg-importer -f deployment/docker/ogn-pg-importer/Dockerfile . + diff --git a/deployment/docker/cups/README.txt b/deployment/docker/cups/README.txt new file mode 100644 index 0000000..4ea5dd3 --- /dev/null +++ b/deployment/docker/cups/README.txt @@ -0,0 +1 @@ +All cups file in this directory are imported when the backed service starts. diff --git a/deployment/docker/data/README.txt b/deployment/docker/data/README.txt new file mode 100644 index 0000000..eb60170 --- /dev/null +++ b/deployment/docker/data/README.txt @@ -0,0 +1,2 @@ +Copy your `public.elevation` file here. +When docker-compose is run the `ogn-pg-importer` service will take care of it. diff --git a/deployment/docker/docker-compose.yml b/deployment/docker/docker-compose.yml new file mode 100644 index 0000000..f4504c6 --- /dev/null +++ b/deployment/docker/docker-compose.yml @@ -0,0 +1,121 @@ +version: "3" +services: + db: + restart: always + image: timescale/timescaledb-postgis:latest-pg11 + networks: + - ogn + # volumes: + # - ./postgres-data:/var/lib/postgresql/data + environment: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: ogn + ports: + - "127.0.0.1:5432:5432" + + redis: + restart: always + image: redis + networks: + - ogn + ports: + - "127.0.0.1:6379:6379" + + ogn-pg-importer: + image: ogn-pg-importer + networks: + - ogn + depends_on: + - db + - backend + volumes: + - ./data:/data + environment: + PGHOST: db + PGDATABASE: ogn + PGPASSWORD: postgres + PGUSER: postgres + BACKENDHOST: backend + BACKENDPORT: 80 + + backend: + restart: always + image: ogn:latest + networks: + - ogn + depends_on: + - db + - redis + ports: + - "0.0.0.0:8080:80" + volumes: + - ./cups:/cups + environment: + PGHOST: db + SQLALCHEMY_DATABASE_URI: "postgresql://postgres:postgres@db:5432/ogn" + CELERY_BROKER_URL: "redis://redis:6379/0" + CELERY_RESULT_BACKEND: "redis://redis:6379/0" + MODULE_NAME: "app" + + gateway: + restart: always + image: ogn:latest + networks: + - ogn + depends_on: + - db + - redis + - backend + command: "./wait.sh flask gateway run" + environment: + SQLALCHEMY_DATABASE_URI: "postgresql://postgres:postgres@db:5432/ogn" + CELERY_BROKER_URL: "redis://redis:6379/0" + CELERY_RESULT_BACKEND: "redis://redis:6379/0" + BACKENDHOST: backend + BACKENDPORT: 80 + worker: + restart: always + image: ogn:latest + networks: + - ogn + depends_on: + - db + - redis + - backend + command: "./wait.sh celery -A app.collect worker -l info" + environment: + SQLALCHEMY_DATABASE_URI: "postgresql://postgres:postgres@db:5432/ogn" + CELERY_BROKER_URL: "redis://redis:6379/0" + CELERY_RESULT_BACKEND: "redis://redis:6379/0" + BACKENDHOST: backend + BACKENDPORT: 80 + beat: + restart: always + image: ogn:latest + networks: + - ogn + depends_on: + - db + - redis + - backend + command: "./wait.sh celery -A app.collect beat -l info" + environment: + SQLALCHEMY_DATABASE_URI: "postgresql://postgres:postgres@db:5432/ogn" + CELERY_BROKER_URL: "redis://redis:6379/0" + CELERY_RESULT_BACKEND: "redis://redis:6379/0" + BACKENDHOST: backend + BACKENDPORT: 80 + flower: + restart: always + image: mher/flower + networks: + - ogn + depends_on: + - redis + ports: + - "0.0.0.0:5555:5555" + command: "flower --port=5555 -l info --broker=redis://redis:6379/0" + +networks: + ogn: + driver: bridge diff --git a/deployment/docker/ogn-pg-importer/Dockerfile b/deployment/docker/ogn-pg-importer/Dockerfile new file mode 100644 index 0000000..3215465 --- /dev/null +++ b/deployment/docker/ogn-pg-importer/Dockerfile @@ -0,0 +1,13 @@ +# We build from the postgis image because it contains all required gdal scripts +FROM alpine as builder +RUN apk update && apk add wget && rm -rf /var/cache/apk/* +RUN wget --no-check-certificate -O borders.zip http://thematicmapping.org/downloads/TM_WORLD_BORDERS-0.3.zip +RUN unzip borders.zip + +FROM mdillon/postgis +RUN mkdir /extra +WORKDIR /extra +COPY --from=builder /TM_* /extra/ +RUN shp2pgsql -s 4326 TM_WORLD_BORDERS-0.3.shp world_borders_temp > world_borders_temp +COPY deployment/docker/ogn-pg-importer/docker-entrypoint.sh . +ENTRYPOINT [ "./docker-entrypoint.sh" ] \ No newline at end of file diff --git a/deployment/docker/ogn-pg-importer/docker-entrypoint.sh b/deployment/docker/ogn-pg-importer/docker-entrypoint.sh new file mode 100755 index 0000000..ecd9a74 --- /dev/null +++ b/deployment/docker/ogn-pg-importer/docker-entrypoint.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# wait a bit for db and backend to properly start +echo Waiting backend to start... +bash -c 'while ! /dev/null +psql -t -d ogn -c "SELECT EXISTS ( SELECT 1 FROM information_schema.tables WHERE table_name = 'elevation' )" | grep -q t +if [ $? -eq 1 ] +then + echo Importing elevation... + cat /data/public.elevation | psql -d ogn > /dev/null + echo Importing borders... + cat /extra/world_borders_temp | psql -d ogn > /dev/null + psql -d ogn -c "INSERT INTO countries SELECT * FROM world_borders_temp;" + psql -d ogn -c "DROP TABLE world_borders_temp;" +fi +echo Elevation and borders added +while true; do sleep 10000; done; diff --git a/deployment/docker/ogn-python/Dockerfile b/deployment/docker/ogn-python/Dockerfile new file mode 100644 index 0000000..3dc5b7f --- /dev/null +++ b/deployment/docker/ogn-python/Dockerfile @@ -0,0 +1,12 @@ +FROM tiangolo/meinheld-gunicorn-flask:python3.7 +RUN rm /app/main.py +WORKDIR /app +ADD app app +ADD migrations migrations +COPY *.py /app/ +COPY requirements.txt . +COPY README.md . +COPY setup.cfg . +RUN pip install -r requirements.txt +COPY deployment/docker/ogn-python/prestart.sh . +COPY deployment/docker/ogn-python/wait.sh . diff --git a/deployment/docker/ogn-python/prestart.sh b/deployment/docker/ogn-python/prestart.sh new file mode 100755 index 0000000..1cf59a0 --- /dev/null +++ b/deployment/docker/ogn-python/prestart.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +echo Waiting db to start... +bash -c 'while ! /dev/null +flask database init +flask database init_timescaledb +find /cups -name "*.cup" -exec flask database import_airports {} \; +flask database import_ddb diff --git a/deployment/docker/ogn-python/wait.sh b/deployment/docker/ogn-python/wait.sh new file mode 100755 index 0000000..45a7e6c --- /dev/null +++ b/deployment/docker/ogn-python/wait.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# wait a bit for db and backend to properly start before running parms +echo Waiting for backend to start... +bash -c 'while ! /dev/null +$1 $2 $3 $4 $5 $6 $7 $8 $9 \ No newline at end of file