From 77578f4a5ceac15725eb33fbfa1ec2f2e6b9b64b Mon Sep 17 00:00:00 2001 From: admire Date: Wed, 4 Nov 2020 21:25:18 +0200 Subject: [PATCH 01/29] add pg-cron back with the image --- Dockerfile | 2 +- scripts/env-data.sh | 2 +- scripts/setup-conf.sh | 1 + scripts/setup-database.sh | 5 +++++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e0db067..eb39050 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,7 +36,7 @@ RUN set -eux \ netcat postgresql-${POSTGRES_MAJOR_VERSION}-ogr-fdw \ postgresql-${POSTGRES_MAJOR_VERSION}-postgis-${POSTGIS_MAJOR}-scripts \ postgresql-plpython3-${POSTGRES_MAJOR_VERSION} postgresql-${POSTGRES_MAJOR_VERSION}-pgrouting \ - postgresql-server-dev-${POSTGRES_MAJOR_VERSION} + postgresql-server-dev-${POSTGRES_MAJOR_VERSION} postgresql-${POSTGRES_MAJOR_VERSION}-cron RUN echo $POSTGRES_MAJOR_VERSION >/tmp/pg_version.txt diff --git a/scripts/env-data.sh b/scripts/env-data.sh index fdc7069..f288f83 100644 --- a/scripts/env-data.sh +++ b/scripts/env-data.sh @@ -249,7 +249,7 @@ if [ -z "$EXTRA_CONF" ]; then fi if [ -z "${SHARED_PRELOAD_LIBRARIES}" ]; then - SHARED_PRELOAD_LIBRARIES='' + SHARED_PRELOAD_LIBRARIES='pg_cron' fi if [ -z "$PASSWORD_AUTHENTICATION" ]; then diff --git a/scripts/setup-conf.sh b/scripts/setup-conf.sh index 900487d..5c214c8 100644 --- a/scripts/setup-conf.sh +++ b/scripts/setup-conf.sh @@ -36,6 +36,7 @@ shared_preload_libraries = '${SHARED_PRELOAD_LIBRARIES}' cron.database_name = '${SINGLE_DB}' password_encryption= '${PASSWORD_AUTHENTICATION}' timezone='${TIMEZONE}' +cron.use_background_workers = on EOF # This script will setup necessary replication settings diff --git a/scripts/setup-database.sh b/scripts/setup-database.sh index 7bd2861..c255f6b 100644 --- a/scripts/setup-database.sh +++ b/scripts/setup-database.sh @@ -79,6 +79,11 @@ for db in $(echo ${POSTGRES_DBNAME} | tr ',' ' '); do fi done +CRON_LOCKFILE="${ROOT_CONF}/.cron_ext.lock" +if [ ! -f "${CRON_LOCKFILE}" ]; then + su - postgres -c "psql -c 'CREATE EXTENSION IF NOT EXISTS pg_cron cascade;' ${SINGLE_DB}" + touch ${CRON_LOCKFILE} +fi # This should show up in docker logs afterwards su - postgres -c "psql -l 2>&1" From 2f6cf6a071621d9dacdd316fc63f58bd767869cb Mon Sep 17 00:00:00 2001 From: admire Date: Sat, 2 Jan 2021 23:34:42 +0200 Subject: [PATCH 02/29] added base build folder into main branch and various optimisations - revert commit for fixing passwords with strange characters - not worth the hassle --- Dockerfile | 15 +- README.md | 29 +- base_build/Dockerfile | 47 +++ base_build/README.md | 44 +++ base_build/build.sh | 4 + base_build/gdal_install.sh | 7 + base_build/scripts/locale.gen | 484 ++++++++++++++++++++++++++++ build.sh | 7 +- scenario_tests/replications/test.sh | 6 +- scripts/env-data.sh | 26 +- scripts/setup-replication.sh | 26 -- scripts/setup-user.sh | 36 +-- 12 files changed, 652 insertions(+), 79 deletions(-) create mode 100644 base_build/Dockerfile create mode 100644 base_build/README.md create mode 100644 base_build/build.sh create mode 100644 base_build/gdal_install.sh create mode 100644 base_build/scripts/locale.gen diff --git a/Dockerfile b/Dockerfile index eb39050..3b7b55d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,22 @@ #--------- Generic stuff all our Dockerfiles should start with so we get caching ------------ ARG DISTRO=debian -ARG IMAGE_VERSION=buster +ARG IMAGE_VERSION=bullseye ARG IMAGE_VARIANT=slim -FROM kartoza/postgis:$DISTRO-$IMAGE_VERSION-$IMAGE_VARIANT +FROM kartoza/postgis:$DISTRO-$IMAGE_VERSION-$IMAGE_VARIANT-base + MAINTAINER Tim Sutton # Reset ARG for version ARG IMAGE_VERSION ARG POSTGRES_MAJOR_VERSION=13 ARG POSTGIS_MAJOR=3 +ARG POSTGIS_MINOR_RELEASE=3.1 + RUN set -eux \ && export DEBIAN_FRONTEND=noninteractive \ - && apt-get upgrade;apt-get update \ + && apt-get update \ && sh -c "echo \"deb http://apt.postgresql.org/pub/repos/apt/ ${IMAGE_VERSION}-pgdg main\" > /etc/apt/sources.list.d/pgdg.list" \ && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc -O- | apt-key add - \ && apt-get -y --purge autoremove \ @@ -39,8 +42,10 @@ RUN set -eux \ postgresql-server-dev-${POSTGRES_MAJOR_VERSION} postgresql-${POSTGRES_MAJOR_VERSION}-cron -RUN echo $POSTGRES_MAJOR_VERSION >/tmp/pg_version.txt - +RUN echo $POSTGRES_MAJOR_VERSION >/tmp/pg_version.txt +RUN echo $POSTGIS_MINOR_RELEASE >/tmp/pg_minor_version.txt +ENV \ + PATH="$PATH:/usr/lib/postgresql/${POSTGRES_MAJOR_VERSION}/bin" # Compile pointcloud extension RUN wget -O- https://github.com/pgpointcloud/pointcloud/archive/master.tar.gz | tar xz && \ diff --git a/README.md b/README.md index 5a9503d..9e58c04 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,7 @@ docker pull kartoza/postgis:image_version ## Building the image -To build the image yourself without apt-cacher (also consumes more bandwidth -since deb packages need to be fetched each time you build) do: +To build the image yourself do: ``` docker build -t kartoza/postgis git://github.com/kartoza/docker-postgis @@ -79,7 +78,7 @@ Then do: docker build -t kartoza/postgis . ``` -Or +Or build against a specific PostgreSQL version ``` docker build --build-arg POSTGRES_MAJOR_VERSION=13 --build-arg POSTGIS_MAJOR=3 -t kartoza/postgis:POSTGRES_MAJOR_VERSION . @@ -91,9 +90,10 @@ and `IMAGE_VARIANT` (=slim) which can be used to control the base image used (but it still needs to be Debian based and have PostgreSQL official apt repo). For example making Ubuntu 20.04 based build (for better arm64 support) -First build the base image using the branch `postgres-base` following instructions from [Kartoza base image builds](https://github.com/kartoza/docker-postgis/tree/postgres-base#alternative-base-distributions-builds) +First build the base image using instructions in the folder `base_build` using the +build script from [Kartoza base image builds](https://github.com/kartoza/docker-postgis/blob/develop/base_build/build.sh) -And then build the `PostGIS Image` using +Then build the `PostGIS Image` to match the base build ``` docker build --build-arg DISTRO=ubuntu --build-arg IMAGE_VERSION=focal --build-arg IMAGE_VARIANT="" -t kartoza/postgis . @@ -105,14 +105,13 @@ By default, the image build will include **all** `locales` to cover any value fo You can safely delete all `locales` except for the ones you need in `scripts/locale.gen`. This will speed up the build considerably. -You can also run the container using the environment variables. ### Environment variables #### Cluster Initializations -With minimum setup, our image will use initial cluster located in the +With a minimum setup, our image will use an initial cluster located in the `DATADIR` environment variable. If you want to use persistence, mount these -location into your volume/host. By default, `DATADIR` will point to `/var/lib/postgresql/{major-version}`. +locations into your volume/host. By default, `DATADIR` will point to `/var/lib/postgresql/{major-version}`. You can instead mount the parent location like this: * `-v data-volume:/var/lib/postgresql` @@ -134,8 +133,8 @@ You need to specify different empty directory, like this The containers will use above parameters to initialize a new db cluster in the specified directory. If the directory is not empty, then initialization parameter will be ignored. -These are some initialization parameter that will only be used to initialize new cluster. -If the container uses existing cluster, it will be ignored (for example, when the container restarts). +These are some initialization parameters that will only get used to initialize a new cluster. +If the container uses an existing cluster, it is ignored (for example, when the container restarts). * `DEFAULT_ENCODING`: cluster encoding * `DEFAULT_COLLATION`: cluster collation @@ -150,7 +149,7 @@ new db cluster. * `RECREATE_DATADIR`: Force database reinitialization in the location `DATADIR` -If you used `RECREATE_DATADIR` and successfully created new cluster. Remember +If you used `RECREATE_DATADIR` and successfully created a new cluster. Remember that you should remove this parameter afterwards. Because, if it was not omitted, it will always recreate new db cluster after every container restarts. @@ -170,7 +169,7 @@ or If you use default `DATADIR` location. -If you need to setup a database cluster with other encoding parameters you need +If you need to set up a database cluster with other encoding parameters you need to pass the environment variables when you initialize the cluster. * -e DEFAULT_ENCODING="UTF8" @@ -186,7 +185,7 @@ delete the current cluster and create a new one. Make sure to remove parameter #### Basic configuration You can use the following environment variables to pass a -user name, password and/or default database name(or multiple databases comma separated). +username, password and/or default database name(or multiple databases comma separated). * `-e POSTGRES_USER=` * `-e POSTGRES_PASS=` @@ -236,12 +235,12 @@ Maximum size to let the WAL grow to between automatic WAL checkpoints. * `-e MAINTAINANCE_WORK_MEM=128MB` #### Configure networking -You can open up the PG port by using the following environment variable. By default +You can open up the PG port by using the following environment variable. By default, the container will allow connections only from the docker private subnet. * `-e ALLOW_IP_RANGE=<0.0.0.0/0> By default` -Postgres conf is setup to listen to all connections and if a user needs to restrict which IP address +Postgres conf is set up to listen to all connections and if a user needs to restrict which IP address PostgreSQL listens to you can define it with the following environment variable. The default is set to listen to all connections. * `-e IP_LIST=<*>` diff --git a/base_build/Dockerfile b/base_build/Dockerfile new file mode 100644 index 0000000..1ffa5c2 --- /dev/null +++ b/base_build/Dockerfile @@ -0,0 +1,47 @@ +#--------- Generic stuff all our Dockerfiles should start with so we get caching ------------ +ARG DISTRO=debian +ARG IMAGE_VERSION=bullseye +ARG IMAGE_VARIANT=slim +FROM $DISTRO:$IMAGE_VERSION-$IMAGE_VARIANT +MAINTAINER Tim Sutton + +# Reset ARG for version +ARG IMAGE_VERSION + +RUN apt-get -qq update --fix-missing && apt-get -qq --yes upgrade + +RUN set -eux \ + && export DEBIAN_FRONTEND=noninteractive \ + && apt-get update \ + && apt-get -y --no-install-recommends install \ + locales gnupg2 wget ca-certificates rpl pwgen software-properties-common iputils-ping \ + apt-transport-https curl \ + && dpkg-divert --local --rename --add /sbin/initctl + + + +RUN apt-get -y update; apt-get -y install build-essential autoconf libxml2-dev zlib1g-dev netcat gdal-bin + + + +# Temporary - PostgreSQL requires this which is not available in bullseye +ADD gdal_install.sh /gdal_install.sh +RUN chmod 0755 /gdal_install.sh;/bin/bash /gdal_install.sh + +# Generating locales takes a long time. Utilize caching by runnig it by itself +# early in the build process. + +COPY scripts/locale.gen /etc/locale.gen +RUN set -eux \ + && /usr/sbin/locale-gen + +ENV LANG=en_US.UTF-8 \ + LANGUAGE=en_US:en \ + LC_ALL=en_US.UTF-8 +RUN update-locale ${LANG} + +# Cleanup resources +RUN apt-get -y --purge autoremove \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + diff --git a/base_build/README.md b/base_build/README.md new file mode 100644 index 0000000..9979565 --- /dev/null +++ b/base_build/README.md @@ -0,0 +1,44 @@ +[![Build Status](https://travis-ci.org/kartoza/docker-postgis.svg?branch=develop)](https://travis-ci.org/kartoza/docker-postgis) + +# docker-postgis + +A docker image that installs all the dependency for building `kartoza/postgis` image variants. + +Visit our page on the docker hub at: https://hub.docker.com/r/kartoza/postgis/ + + +#### Alternative base distributions builds + +There are build args for `DISTRO` (=debian), `IMAGE_VERSION` (=buster) +and `IMAGE_VARIANT` (=slim) which can be used to control the base image used +(but it still needs to be Debian based and have PostgreSQL official apt repo). + +For example making Ubuntu 20.04 based build (for better arm64 support) +First build the base image using the in the folder `base_build` using the +build script +following instructions from [Kartoza base image builds](https://github.com/kartoza/docker-postgis/tree/postgres-base#alternative-base-distributions-builds) + +Then build the `PostGIS base Image` using + +``` +docker build --build-arg DISTRO=ubuntu --build-arg IMAGE_VERSION=focal --build-arg IMAGE_VARIANT="" -t kartoza/postgis:$DISTRO-$IMAGE_VERSION-$IMAGE_VARIANT-base . +``` + +#### Locales + +By default, the image build will include **all** `locales` to cover any value for `locale` settings such as `DEFAULT_COLLATION`, `DEFAULT_CTYPE` or `DEFAULT_ENCODING`. + +You can safely delete all `locales` except for the ones you need in `scripts/locale.gen`. This will speed up the build considerably. + +### Support + +If you require more substantial assistance from [kartoza](https://kartoza.com) (because our work and interaction on docker-postgis is pro bono), +please consider taking out a [Support Level Agreeement](https://kartoza.com/en/shop/product/support) + +## Credits + +Tim Sutton (tim@kartoza.com) +Gavin Fleming (gavin@kartoza.com) +Rizky Maulana (rizky@kartoza.com) +Admire Nyakudya (admire@kartoza.com) +December 2020 diff --git a/base_build/build.sh b/base_build/build.sh new file mode 100644 index 0000000..a15bace --- /dev/null +++ b/base_build/build.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Building an debian base image + +docker build --build-arg DISTRO=debian --build-arg IMAGE_VERSION=bullseye --build-arg IMAGE_VARIANT=slim -t kartoza/postgis:debian-bullseye-slim-base . diff --git a/base_build/gdal_install.sh b/base_build/gdal_install.sh new file mode 100644 index 0000000..c2551f7 --- /dev/null +++ b/base_build/gdal_install.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +if [[ ${IMAGE_VERSION} =~ [Bb][Uu][Ll][Ll][Ss][Ee][Yy][Ee] ]]; then + wget http://ftp.br.debian.org/debian/pool/main/g/gdal/libgdal27_3.1.4+dfsg-1+b1_amd64.deb + dpkg -i libgdal27_3.1.4+dfsg-1+b1_amd64.deb +fi + diff --git a/base_build/scripts/locale.gen b/base_build/scripts/locale.gen new file mode 100644 index 0000000..3995286 --- /dev/null +++ b/base_build/scripts/locale.gen @@ -0,0 +1,484 @@ +# This file lists locales that you wish to have built. You can find a list +# of valid supported locales at /usr/share/i18n/SUPPORTED, and you can add +# user defined locales to /usr/local/share/i18n/SUPPORTED. If you change +# this file, you need to rerun locale-gen. + + +aa_DJ ISO-8859-1 +aa_DJ.UTF-8 UTF-8 +aa_ER UTF-8 +aa_ER@saaho UTF-8 +aa_ET UTF-8 +af_ZA ISO-8859-1 +af_ZA.UTF-8 UTF-8 +ak_GH UTF-8 +am_ET UTF-8 +an_ES ISO-8859-15 +an_ES.UTF-8 UTF-8 +anp_IN UTF-8 +ar_AE ISO-8859-6 +ar_AE.UTF-8 UTF-8 +ar_BH ISO-8859-6 +ar_BH.UTF-8 UTF-8 +ar_DZ ISO-8859-6 +ar_DZ.UTF-8 UTF-8 +ar_EG ISO-8859-6 +ar_EG.UTF-8 UTF-8 +ar_IN UTF-8 +ar_IQ ISO-8859-6 +ar_IQ.UTF-8 UTF-8 +ar_JO ISO-8859-6 +ar_JO.UTF-8 UTF-8 +ar_KW ISO-8859-6 +ar_KW.UTF-8 UTF-8 +ar_LB ISO-8859-6 +ar_LB.UTF-8 UTF-8 +ar_LY ISO-8859-6 +ar_LY.UTF-8 UTF-8 +ar_MA ISO-8859-6 +ar_MA.UTF-8 UTF-8 +ar_OM ISO-8859-6 +ar_OM.UTF-8 UTF-8 +ar_QA ISO-8859-6 +ar_QA.UTF-8 UTF-8 +ar_SA ISO-8859-6 +ar_SA.UTF-8 UTF-8 +ar_SD ISO-8859-6 +ar_SD.UTF-8 UTF-8 +ar_SS UTF-8 +ar_SY ISO-8859-6 +ar_SY.UTF-8 UTF-8 +ar_TN ISO-8859-6 +ar_TN.UTF-8 UTF-8 +ar_YE ISO-8859-6 +ar_YE.UTF-8 UTF-8 +as_IN UTF-8 +ast_ES ISO-8859-15 +ast_ES.UTF-8 UTF-8 +ayc_PE UTF-8 +az_AZ UTF-8 +be_BY CP1251 +be_BY.UTF-8 UTF-8 +be_BY@latin UTF-8 +bem_ZM UTF-8 +ber_DZ UTF-8 +ber_MA UTF-8 +bg_BG CP1251 +bg_BG.UTF-8 UTF-8 +bhb_IN.UTF-8 UTF-8 +bho_IN UTF-8 +bn_BD UTF-8 +bn_IN UTF-8 +bo_CN UTF-8 +bo_IN UTF-8 +br_FR ISO-8859-1 +br_FR.UTF-8 UTF-8 +br_FR@euro ISO-8859-15 +brx_IN UTF-8 +bs_BA ISO-8859-2 +bs_BA.UTF-8 UTF-8 +byn_ER UTF-8 +ca_AD ISO-8859-15 +ca_AD.UTF-8 UTF-8 +ca_ES ISO-8859-1 +ca_ES.UTF-8 UTF-8 +ca_ES.UTF-8@valencia UTF-8 +ca_ES@euro ISO-8859-15 +ca_ES@valencia ISO-8859-15 +ca_FR ISO-8859-15 +ca_FR.UTF-8 UTF-8 +ca_IT ISO-8859-15 +ca_IT.UTF-8 UTF-8 +ce_RU UTF-8 +chr_US UTF-8 +cmn_TW UTF-8 +crh_UA UTF-8 +cs_CZ ISO-8859-2 +cs_CZ.UTF-8 UTF-8 +csb_PL UTF-8 +cv_RU UTF-8 +cy_GB ISO-8859-14 +cy_GB.UTF-8 UTF-8 +da_DK ISO-8859-1 +da_DK.UTF-8 UTF-8 +de_AT ISO-8859-1 +de_AT.UTF-8 UTF-8 +de_AT@euro ISO-8859-15 +de_BE ISO-8859-1 +de_BE.UTF-8 UTF-8 +de_BE@euro ISO-8859-15 +de_CH ISO-8859-1 +de_CH.UTF-8 UTF-8 +de_DE ISO-8859-1 +de_DE.UTF-8 UTF-8 +de_DE@euro ISO-8859-15 +de_IT ISO-8859-1 +de_IT.UTF-8 UTF-8 +de_LI.UTF-8 UTF-8 +de_LU ISO-8859-1 +de_LU.UTF-8 UTF-8 +de_LU@euro ISO-8859-15 +doi_IN UTF-8 +dv_MV UTF-8 +dz_BT UTF-8 +el_CY ISO-8859-7 +el_CY.UTF-8 UTF-8 +el_GR ISO-8859-7 +el_GR.UTF-8 UTF-8 +en_AG UTF-8 +en_AU ISO-8859-1 +en_AU.UTF-8 UTF-8 +en_BW ISO-8859-1 +en_BW.UTF-8 UTF-8 +en_CA ISO-8859-1 +en_CA.UTF-8 UTF-8 +en_DK ISO-8859-1 +en_DK.ISO-8859-15 ISO-8859-15 +en_DK.UTF-8 UTF-8 +en_GB ISO-8859-1 +en_GB.ISO-8859-15 ISO-8859-15 +en_GB.UTF-8 UTF-8 +en_HK ISO-8859-1 +en_HK.UTF-8 UTF-8 +en_IE ISO-8859-1 +en_IE.UTF-8 UTF-8 +en_IE@euro ISO-8859-15 +en_IL UTF-8 +en_IN UTF-8 +en_NG UTF-8 +en_NZ ISO-8859-1 +en_NZ.UTF-8 UTF-8 +en_PH ISO-8859-1 +en_PH.UTF-8 UTF-8 +en_SG ISO-8859-1 +en_SG.UTF-8 UTF-8 +en_US ISO-8859-1 +en_US.ISO-8859-15 ISO-8859-15 +en_US.UTF-8 UTF-8 +en_ZA ISO-8859-1 +en_ZA.UTF-8 UTF-8 +en_ZM UTF-8 +en_ZW ISO-8859-1 +en_ZW.UTF-8 UTF-8 +eo UTF-8 +es_AR ISO-8859-1 +es_AR.UTF-8 UTF-8 +es_BO ISO-8859-1 +es_BO.UTF-8 UTF-8 +es_CL ISO-8859-1 +es_CL.UTF-8 UTF-8 +es_CO ISO-8859-1 +es_CO.UTF-8 UTF-8 +es_CR ISO-8859-1 +es_CR.UTF-8 UTF-8 +es_CU UTF-8 +es_DO ISO-8859-1 +es_DO.UTF-8 UTF-8 +es_EC ISO-8859-1 +es_EC.UTF-8 UTF-8 +es_ES ISO-8859-1 +es_ES.UTF-8 UTF-8 +es_ES@euro ISO-8859-15 +es_GT ISO-8859-1 +es_GT.UTF-8 UTF-8 +es_HN ISO-8859-1 +es_HN.UTF-8 UTF-8 +es_MX ISO-8859-1 +es_MX.UTF-8 UTF-8 +es_NI ISO-8859-1 +es_NI.UTF-8 UTF-8 +es_PA ISO-8859-1 +es_PA.UTF-8 UTF-8 +es_PE ISO-8859-1 +es_PE.UTF-8 UTF-8 +es_PR ISO-8859-1 +es_PR.UTF-8 UTF-8 +es_PY ISO-8859-1 +es_PY.UTF-8 UTF-8 +es_SV ISO-8859-1 +es_SV.UTF-8 UTF-8 +es_US ISO-8859-1 +es_US.UTF-8 UTF-8 +es_UY ISO-8859-1 +es_UY.UTF-8 UTF-8 +es_VE ISO-8859-1 +es_VE.UTF-8 UTF-8 +et_EE ISO-8859-1 +et_EE.ISO-8859-15 ISO-8859-15 +et_EE.UTF-8 UTF-8 +eu_ES ISO-8859-1 +eu_ES.UTF-8 UTF-8 +eu_ES@euro ISO-8859-15 +eu_FR ISO-8859-1 +eu_FR.UTF-8 UTF-8 +eu_FR@euro ISO-8859-15 +fa_IR UTF-8 +ff_SN UTF-8 +fi_FI ISO-8859-1 +fi_FI.UTF-8 UTF-8 +fi_FI@euro ISO-8859-15 +fil_PH UTF-8 +fo_FO ISO-8859-1 +fo_FO.UTF-8 UTF-8 +fr_BE ISO-8859-1 +fr_BE.UTF-8 UTF-8 +fr_BE@euro ISO-8859-15 +fr_CA ISO-8859-1 +fr_CA.UTF-8 UTF-8 +fr_CH ISO-8859-1 +fr_CH.UTF-8 UTF-8 +fr_FR ISO-8859-1 +fr_FR.UTF-8 UTF-8 +fr_FR@euro ISO-8859-15 +fr_LU ISO-8859-1 +fr_LU.UTF-8 UTF-8 +fr_LU@euro ISO-8859-15 +fur_IT UTF-8 +fy_DE UTF-8 +fy_NL UTF-8 +ga_IE ISO-8859-1 +ga_IE.UTF-8 UTF-8 +ga_IE@euro ISO-8859-15 +gd_GB ISO-8859-15 +gd_GB.UTF-8 UTF-8 +gez_ER UTF-8 +gez_ER@abegede UTF-8 +gez_ET UTF-8 +gez_ET@abegede UTF-8 +gl_ES ISO-8859-1 +gl_ES.UTF-8 UTF-8 +gl_ES@euro ISO-8859-15 +gu_IN UTF-8 +gv_GB ISO-8859-1 +gv_GB.UTF-8 UTF-8 +ha_NG UTF-8 +hak_TW UTF-8 +he_IL ISO-8859-8 +he_IL.UTF-8 UTF-8 +hi_IN UTF-8 +hne_IN UTF-8 +hr_HR ISO-8859-2 +hr_HR.UTF-8 UTF-8 +hsb_DE ISO-8859-2 +hsb_DE.UTF-8 UTF-8 +ht_HT UTF-8 +hu_HU ISO-8859-2 +hu_HU.UTF-8 UTF-8 +hy_AM UTF-8 +hy_AM.ARMSCII-8 ARMSCII-8 +ia_FR UTF-8 +id_ID ISO-8859-1 +id_ID.UTF-8 UTF-8 +ig_NG UTF-8 +ik_CA UTF-8 +is_IS ISO-8859-1 +is_IS.UTF-8 UTF-8 +it_CH ISO-8859-1 +it_CH.UTF-8 UTF-8 +it_IT ISO-8859-1 +it_IT.UTF-8 UTF-8 +it_IT@euro ISO-8859-15 +iu_CA UTF-8 +ja_JP.EUC-JP EUC-JP +ja_JP.UTF-8 UTF-8 +ka_GE GEORGIAN-PS +ka_GE.UTF-8 UTF-8 +kk_KZ PT154 +kk_KZ.RK1048 RK1048 +kk_KZ.UTF-8 UTF-8 +kl_GL ISO-8859-1 +kl_GL.UTF-8 UTF-8 +km_KH UTF-8 +kn_IN UTF-8 +ko_KR.EUC-KR EUC-KR +ko_KR.UTF-8 UTF-8 +kok_IN UTF-8 +ks_IN UTF-8 +ks_IN@devanagari UTF-8 +ku_TR ISO-8859-9 +ku_TR.UTF-8 UTF-8 +kw_GB ISO-8859-1 +kw_GB.UTF-8 UTF-8 +ky_KG UTF-8 +lb_LU UTF-8 +lg_UG ISO-8859-10 +lg_UG.UTF-8 UTF-8 +li_BE UTF-8 +li_NL UTF-8 +lij_IT UTF-8 +ln_CD UTF-8 +lo_LA UTF-8 +lt_LT ISO-8859-13 +lt_LT.UTF-8 UTF-8 +lv_LV ISO-8859-13 +lv_LV.UTF-8 UTF-8 +lzh_TW UTF-8 +mag_IN UTF-8 +mai_IN UTF-8 +mg_MG ISO-8859-15 +mg_MG.UTF-8 UTF-8 +mhr_RU UTF-8 +mi_NZ ISO-8859-13 +mi_NZ.UTF-8 UTF-8 +mk_MK ISO-8859-5 +mk_MK.UTF-8 UTF-8 +ml_IN UTF-8 +mn_MN UTF-8 +mni_IN UTF-8 +mr_IN UTF-8 +ms_MY ISO-8859-1 +ms_MY.UTF-8 UTF-8 +mt_MT ISO-8859-3 +mt_MT.UTF-8 UTF-8 +my_MM UTF-8 +nan_TW UTF-8 +nan_TW@latin UTF-8 +nb_NO ISO-8859-1 +nb_NO.UTF-8 UTF-8 +nds_DE UTF-8 +nds_NL UTF-8 +ne_NP UTF-8 +nhn_MX UTF-8 +niu_NU UTF-8 +niu_NZ UTF-8 +nl_AW UTF-8 +nl_BE ISO-8859-1 +nl_BE.UTF-8 UTF-8 +nl_BE@euro ISO-8859-15 +nl_NL ISO-8859-1 +nl_NL.UTF-8 UTF-8 +nl_NL@euro ISO-8859-15 +nn_NO ISO-8859-1 +nn_NO.UTF-8 UTF-8 +nr_ZA UTF-8 +nso_ZA UTF-8 +oc_FR ISO-8859-1 +oc_FR.UTF-8 UTF-8 +om_ET UTF-8 +om_KE ISO-8859-1 +om_KE.UTF-8 UTF-8 +or_IN UTF-8 +os_RU UTF-8 +pa_IN UTF-8 +pa_PK UTF-8 +pap_AW UTF-8 +pap_CW UTF-8 +pl_PL ISO-8859-2 +pl_PL.UTF-8 UTF-8 +ps_AF UTF-8 +pt_BR ISO-8859-1 +pt_BR.UTF-8 UTF-8 +pt_PT ISO-8859-1 +pt_PT.UTF-8 UTF-8 +pt_PT@euro ISO-8859-15 +quz_PE UTF-8 +raj_IN UTF-8 +ro_RO ISO-8859-2 +ro_RO.UTF-8 UTF-8 +ru_RU ISO-8859-5 +ru_RU.CP1251 CP1251 +ru_RU.KOI8-R KOI8-R +ru_RU.UTF-8 UTF-8 +ru_UA KOI8-U +ru_UA.UTF-8 UTF-8 +rw_RW UTF-8 +sa_IN UTF-8 +sat_IN UTF-8 +sc_IT UTF-8 +sd_IN UTF-8 +sd_IN@devanagari UTF-8 +se_NO UTF-8 +sgs_LT UTF-8 +shs_CA UTF-8 +si_LK UTF-8 +sid_ET UTF-8 +sk_SK ISO-8859-2 +sk_SK.UTF-8 UTF-8 +sl_SI ISO-8859-2 +sl_SI.UTF-8 UTF-8 +so_DJ ISO-8859-1 +so_DJ.UTF-8 UTF-8 +so_ET UTF-8 +so_KE ISO-8859-1 +so_KE.UTF-8 UTF-8 +so_SO ISO-8859-1 +so_SO.UTF-8 UTF-8 +sq_AL ISO-8859-1 +sq_AL.UTF-8 UTF-8 +sq_MK UTF-8 +sr_ME UTF-8 +sr_RS UTF-8 +sr_RS@latin UTF-8 +ss_ZA UTF-8 +st_ZA ISO-8859-1 +st_ZA.UTF-8 UTF-8 +sv_FI ISO-8859-1 +sv_FI.UTF-8 UTF-8 +sv_FI@euro ISO-8859-15 +sv_SE ISO-8859-1 +sv_SE.ISO-8859-15 ISO-8859-15 +sv_SE.UTF-8 UTF-8 +sw_KE UTF-8 +sw_TZ UTF-8 +szl_PL UTF-8 +ta_IN UTF-8 +ta_LK UTF-8 +tcy_IN.UTF-8 UTF-8 +te_IN UTF-8 +tg_TJ KOI8-T +tg_TJ.UTF-8 UTF-8 +th_TH TIS-620 +th_TH.UTF-8 UTF-8 +the_NP UTF-8 +ti_ER UTF-8 +ti_ET UTF-8 +tig_ER UTF-8 +tk_TM UTF-8 +tl_PH ISO-8859-1 +tl_PH.UTF-8 UTF-8 +tn_ZA UTF-8 +tr_CY ISO-8859-9 +tr_CY.UTF-8 UTF-8 +tr_TR ISO-8859-9 +tr_TR.UTF-8 UTF-8 +ts_ZA UTF-8 +tt_RU UTF-8 +tt_RU@iqtelif UTF-8 +ug_CN UTF-8 +uk_UA KOI8-U +uk_UA.UTF-8 UTF-8 +unm_US UTF-8 +ur_IN UTF-8 +ur_PK UTF-8 +uz_UZ ISO-8859-1 +uz_UZ.UTF-8 UTF-8 +uz_UZ@cyrillic UTF-8 +ve_ZA UTF-8 +vi_VN UTF-8 +wa_BE ISO-8859-1 +wa_BE.UTF-8 UTF-8 +wa_BE@euro ISO-8859-15 +wae_CH UTF-8 +wal_ET UTF-8 +wo_SN UTF-8 +xh_ZA ISO-8859-1 +xh_ZA.UTF-8 UTF-8 +yi_US CP1255 +yi_US.UTF-8 UTF-8 +yo_NG UTF-8 +yue_HK UTF-8 +zh_CN GB2312 +zh_CN.GB18030 GB18030 +zh_CN.GBK GBK +zh_CN.UTF-8 UTF-8 +zh_HK BIG5-HKSCS +zh_HK.UTF-8 UTF-8 +zh_SG GB2312 +zh_SG.GBK GBK +zh_SG.UTF-8 UTF-8 +zh_TW BIG5 +zh_TW.EUC-TW EUC-TW +zh_TW.UTF-8 UTF-8 +zu_ZA ISO-8859-1 +zu_ZA.UTF-8 UTF-8 +zu_ZA.UTF-8 UTF-8 diff --git a/build.sh b/build.sh index 2d0215e..71ac500 100755 --- a/build.sh +++ b/build.sh @@ -1,3 +1,8 @@ #!/usr/bin/env bash +POSTGRES_MAJOR_VERSION=13 + +cd base_build +./build.sh +cd .. docker build -t kartoza/postgis:manual-build . -docker build -t kartoza/postgis:13.0 . +docker build --build-arg DISTRO=debian --build-arg IMAGE_VERSION=bullseye --build-arg IMAGE_VARIANT=slim -t kartoza/postgis:${POSTGRES_MAJOR_VERSION}.0 . diff --git a/scenario_tests/replications/test.sh b/scenario_tests/replications/test.sh index cd7b07c..9395fc9 100755 --- a/scenario_tests/replications/test.sh +++ b/scenario_tests/replications/test.sh @@ -8,11 +8,11 @@ source ../test-env.sh # Run service docker-compose up -d -sleep 5 +sleep 30 # Preparing master cluster until docker-compose exec pg-master pg_isready; do - sleep 1 + sleep 30 done; # Execute tests @@ -20,7 +20,7 @@ docker-compose exec pg-master /bin/bash /tests/test_master.sh # Preparing node cluster until docker-compose exec pg-node pg_isready; do - sleep 1 + sleep 30 done; # Execute tests diff --git a/scripts/env-data.sh b/scripts/env-data.sh index f288f83..ab562cc 100644 --- a/scripts/env-data.sh +++ b/scripts/env-data.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash POSTGRES_MAJOR_VERSION=$(cat /tmp/pg_version.txt) +POSTGIS_MINOR_RELEASE=$(cat /tmp/pg_minor_version.txt) DEFAULT_DATADIR="/var/lib/postgresql/${POSTGRES_MAJOR_VERSION}/main" ROOT_CONF="/etc/postgresql/${POSTGRES_MAJOR_VERSION}/main" PG_ENV="$ROOT_CONF/environment" @@ -8,7 +9,7 @@ WAL_ARCHIVE="/opt/archivedir" RECOVERY_CONF="$ROOT_CONF/recovery.conf" POSTGRES="/usr/lib/postgresql/${POSTGRES_MAJOR_VERSION}/bin/postgres" INITDB="/usr/lib/postgresql/${POSTGRES_MAJOR_VERSION}/bin/initdb" -SQLDIR="/usr/share/postgresql/${POSTGRES_MAJOR_VERSION}/contrib/postgis-3.0/" +SQLDIR="/usr/share/postgresql/${POSTGRES_MAJOR_VERSION}/contrib/postgis-$POSTGIS_MINOR_RELEASE/" SETVARS="POSTGIS_ENABLE_OUTDB_RASTERS=1 POSTGIS_GDAL_ENABLED_DRIVERS=ENABLE_ALL" LOCALONLY="-c listen_addresses='127.0.0.1'" PG_BASEBACKUP="/usr/bin/pg_basebackup" @@ -337,3 +338,26 @@ function entry_point_script { return 0 } + +function configure_replication_permissions { + + echo "Setup data permissions" + echo "----------------------" + chown -R postgres:postgres $(getent passwd postgres | cut -d: -f6) + su - postgres -c "echo \"${REPLICATE_FROM}:${REPLICATE_PORT}:*:${REPLICATION_USER}:${REPLICATION_PASS}\" > ~/.pgpass" + su - postgres -c "chmod 0600 ~/.pgpass" +} + +function streaming_replication { +until su - postgres -c "${PG_BASEBACKUP} -X stream -h ${REPLICATE_FROM} -p ${REPLICATE_PORT} -D ${DATADIR} -U ${REPLICATION_USER} -R -vP -w --label=gis_pg_custer" + do + echo "Waiting for master to connect..." + sleep 1s + if [[ "$(ls -A ${DATADIR})" ]]; then + echo "Need empty folder. Cleaning directory..." + rm -rf ${DATADIR}/* + fi + done + +} + diff --git a/scripts/setup-replication.sh b/scripts/setup-replication.sh index 0269295..cb5a09a 100755 --- a/scripts/setup-replication.sh +++ b/scripts/setup-replication.sh @@ -7,37 +7,11 @@ source /scripts/env-data.sh # Adapted from https://github.com/DanielDent/docker-postgres-replication # To set up replication - - create_dir ${WAL_ARCHIVE} chown -R postgres:postgres ${DATADIR} ${WAL_ARCHIVE} chmod -R 750 ${DATADIR} ${WAL_ARCHIVE} - -function configure_replication_permissions { - - echo "Setup data permissions" - echo "----------------------" - chown -R postgres:postgres $(getent passwd postgres | cut -d: -f6) - su - postgres -c "echo \"${REPLICATE_FROM}:${REPLICATE_PORT}:*:${REPLICATION_USER}:${REPLICATION_PASS}\" > ~/.pgpass" - su - postgres -c "chmod 0600 ~/.pgpass" -} - -function streaming_replication { -until su - postgres -c "${PG_BASEBACKUP} -X stream -h ${REPLICATE_FROM} -p ${REPLICATE_PORT} -D ${DATADIR} -U ${REPLICATION_USER} -R -vP -w --label=gis_pg_custer" - do - echo "Waiting for master to connect..." - sleep 1s - if [[ "$(ls -A ${DATADIR})" ]]; then - echo "Need empty folder. Cleaning directory..." - rm -rf ${DATADIR}/* - fi - done - -} - - if [[ "$WAL_LEVEL" == 'replica' && "${REPLICATION}" =~ [Tt][Rr][Uu][Ee] ]]; then # No content yet - but this is a slave database if [ -z "${REPLICATE_FROM}" ]; then diff --git a/scripts/setup-user.sh b/scripts/setup-user.sh index 3f4d6e9..35b4d1c 100644 --- a/scripts/setup-user.sh +++ b/scripts/setup-user.sh @@ -14,44 +14,24 @@ source /scripts/env-data.sh # Only create credentials if this is a master database # Slave database will just mirror from master users +echo "Setup postgres User:Password" +echo "postgresql user: $POSTGRES_USER" > /tmp/PGPASSWORD.txt +echo "postgresql password: $POSTGRES_PASS" >> /tmp/PGPASSWORD.txt -echo "$POSTGRES_PASS" >> /tmp/PGPASSWORD.txt -# Check super user already exists +# Check user already exists +echo "Creating superuser $POSTGRES_USER" RESULT=`su - postgres -c "psql postgres -t -c \"SELECT 1 FROM pg_roles WHERE rolname = '$POSTGRES_USER'\""` COMMAND="ALTER" if [ -z "$RESULT" ]; then COMMAND="CREATE" fi +su - postgres -c "psql postgres -c \"$COMMAND USER $POSTGRES_USER WITH SUPERUSER ENCRYPTED PASSWORD '$POSTGRES_PASS';\"" -echo "Creating superuser user $POSTGRES_USER using $PASSWORD_AUTHENTICATION authentication " -if [ PASSWORD_AUTHENTICATION="md5" ]; then - PG_PASS=$(U=$POSTGRES_USER; P=$(cat /tmp/PGPASSWORD.txt); echo -n md5; echo -n $P$U | md5sum | cut -d' ' -f1) - su - postgres -c "psql postgres -c \"$COMMAND USER $POSTGRES_USER WITH SUPERUSER PASSWORD '$PG_PASS';\"" -elif [ PASSWORD_AUTHENTICATION="scram-sha-256" ]; then - PG_PASS=$(U=$POSTGRES_USER; P=$(cat /tmp/PGPASSWORD.txt); echo -n sha256; echo -n $P$U | sha256sum | cut -d' ' -f1) - su - postgres -c "psql postgres -c \"$COMMAND USER $POSTGRES_USER WITH SUPERUSER PASSWORD '$PG_PASS';\"" -fi - -echo "$REPLICATION_PASS" >> /tmp/REPLICATION_PASS.txt - -# Check replication user already exists +echo "Creating replication user $REPLICATION_USER" RESULT_REPLICATION=`su - postgres -c "psql postgres -t -c \"SELECT 1 FROM pg_roles WHERE rolname = '$REPLICATION_USER'\""` COMMANDS="ALTER" if [ -z "$RESULT_REPLICATION" ]; then COMMANDS="CREATE" fi +su - postgres -c "psql postgres -c \"$COMMANDS USER $REPLICATION_USER WITH REPLICATION ENCRYPTED PASSWORD '$REPLICATION_PASS';\"" -if [ -z "$RESULT" ]; then - COMMAND="CREATE" -fi - -echo "Creating replication user $REPLICATION_USER using $PASSWORD_AUTHENTICATION authentication " -if [ PASSWORD_AUTHENTICATION="md5" ]; then - REP_PASS=$(U=$REPLICATION_USER; P=$(cat /tmp/REPLICATION_PASS.txt); echo -n md5; echo -n $P$U | md5sum | cut -d' ' -f1) - su - postgres -c "psql postgres -c \"$COMMANDS USER $REPLICATION_USER WITH REPLICATION PASSWORD '$REP_PASS';\"" -elif [ PASSWORD_AUTHENTICATION="scram-sha-256" ]; then - REP_PASS=$(U=$REPLICATION_USER; P=$(cat /tmp/REPLICATION_PASS.txt); echo -n sha256; echo -n $P$U | sha256sum | cut -d' ' -f1) - su - postgres -c "psql postgres -c \"$COMMANDS USER $REPLICATION_USER WITH REPLICATION PASSWORD '$REP_PASS';\"" -fi - -rm /tmp/PGPASSWORD.txt /tmp/REPLICATION_PASS.txt \ No newline at end of file From a80b240397796110aacea893ca51ec5d0ab7cffa Mon Sep 17 00:00:00 2001 From: admire Date: Sun, 3 Jan 2021 13:18:32 +0200 Subject: [PATCH 03/29] split conf into seperate files --- scripts/setup-conf.sh | 19 ++++++++++++++++--- scripts/setup-ssl.sh | 20 ++++++++++++-------- scripts/setup.sh | 10 +++------- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/scripts/setup-conf.sh b/scripts/setup-conf.sh index 5c214c8..f087b82 100644 --- a/scripts/setup-conf.sh +++ b/scripts/setup-conf.sh @@ -21,7 +21,11 @@ sed -i '/data_directory/d' $CONF echo "data_directory = '${DATADIR}'" >> $CONF # This script will setup necessary configuration to optimise for PostGIS and to enable replications -cat >> $CONF <> ${ROOT_CONF}/postgis.conf <> $CONF # This script will setup necessary replication settings if [[ "${REPLICATION}" =~ [Tt][Rr][Uu][Ee] && "$WAL_LEVEL" == 'logical' ]]; then -cat >> "$CONF" <> ${ROOT_CONF}/streaming_replication.conf <> $CONF fi if [[ "${REPLICATION}" =~ [Tt][Rr][Uu][Ee] && "$WAL_LEVEL" == 'replica' ]]; then -cat >> "$CONF" <> ${ROOT_CONF}/streaming_replication.conf <> $CONF fi echo -e $EXTRA_CONF >> $CONF diff --git a/scripts/setup-ssl.sh b/scripts/setup-ssl.sh index 8ce2a6f..0fa12a9 100644 --- a/scripts/setup-ssl.sh +++ b/scripts/setup-ssl.sh @@ -22,15 +22,19 @@ mkdir -p ${PGSTAT_TMP} chmod 0777 ${PGSTAT_TMP} # moved from setup.sh -echo "ssl = true" >> $CONF -#echo "ssl_ciphers = 'DEFAULT:!LOW:!EXP:!MD5:@STRENGTH' " >> $CONF -#echo "ssl_renegotiation_limit = 512MB " >> $CONF -echo "ssl_cert_file = '${SSL_CERT_FILE}'" >> $CONF -echo "ssl_key_file = '${SSL_KEY_FILE}'" >> $CONF -if [ ! -z "${SSL_CA_FILE}" ]; then - echo "ssl_ca_file = '${SSL_CA_FILE}' # (change requires restart)" >> $CONF +if [[ -f ${ROOT_CONF}/ssl.conf ]];then + rm $CONF/ssl.conf fi -#echo "ssl_crl_file = ''" >> $CONF +cat >> ${ROOT_CONF}/ssl.conf <> ${ROOT_CONF}/ssl.conf +fi +echo "include 'ssl.conf'" >> $CONF # Put lock file to make sure conf was not reinitialized touch ${SETUP_LOCKFILE} diff --git a/scripts/setup.sh b/scripts/setup.sh index d64c2fd..f2f78c5 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -7,13 +7,9 @@ source /scripts/env-data.sh # Restrict subnet to docker private network -echo "host all all 172.0.0.0/8 md5" >> $ROOT_CONF/pg_hba.conf -# And allow access from DockerToolbox / Boottodocker on OSX -echo "host all all 192.168.0.0/16 md5" >> $ROOT_CONF/pg_hba.conf -# Listen on all ip addresses -echo "listen_addresses = '*'" >> $CONF -echo "port = 5432" >> $CONF - +echo "host all all 172.0.0.0/8 ${PASSWORD_AUTHENTICATION}" >> $ROOT_CONF/pg_hba.conf +# And allow access from DockerToolbox / Boot to docker on OSX +echo "host all all 192.168.0.0/16 ${PASSWORD_AUTHENTICATION}" >> $ROOT_CONF/pg_hba.conf # Create backup template for conf cat $CONF > $CONF.template From c3182d69e9033d91c064119f522e92ec4521fa43 Mon Sep 17 00:00:00 2001 From: admire Date: Sun, 3 Jan 2021 13:35:33 +0200 Subject: [PATCH 04/29] DRY --- scripts/setup-conf.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/setup-conf.sh b/scripts/setup-conf.sh index f087b82..d22a688 100644 --- a/scripts/setup-conf.sh +++ b/scripts/setup-conf.sh @@ -7,9 +7,6 @@ if [ -f "${SETUP_LOCKFILE}" ]; then return 0 fi -list=(`echo ${POSTGRES_DBNAME} | tr ',' ' '`) -arr=(${list}) -SINGLE_DB=${arr[0]} # This script will setup necessary configuration to enable replications # Refresh configuration in case environment settings changed. From 0da01b8092e3454b85fcf822c21327d5730f03c1 Mon Sep 17 00:00:00 2001 From: admire Date: Tue, 5 Jan 2021 11:32:26 +0200 Subject: [PATCH 05/29] Change replication to streaming replication --- .travis.yml | 2 +- .../docker-compose.yml | 0 .../{replications => streaming_replication}/test.sh | 0 .../tests/__init__.py | 0 .../tests/test_master.sh | 0 .../tests/test_node.sh | 0 .../tests/test_replication.py | 0 scripts/setup-conf.sh | 9 +++------ 8 files changed, 4 insertions(+), 7 deletions(-) rename scenario_tests/{replications => streaming_replication}/docker-compose.yml (100%) rename scenario_tests/{replications => streaming_replication}/test.sh (100%) rename scenario_tests/{replications => streaming_replication}/tests/__init__.py (100%) rename scenario_tests/{replications => streaming_replication}/tests/test_master.sh (100%) rename scenario_tests/{replications => streaming_replication}/tests/test_node.sh (100%) rename scenario_tests/{replications => streaming_replication}/tests/test_replication.py (100%) diff --git a/.travis.yml b/.travis.yml index aad4656..658c33c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ python: env: - SCENARIO=datadir_init - - SCENARIO=replications + - SCENARIO=streaming_replication - SCENARIO=collations - SCENARIO=extensions - SCENARIO=logical_replication diff --git a/scenario_tests/replications/docker-compose.yml b/scenario_tests/streaming_replication/docker-compose.yml similarity index 100% rename from scenario_tests/replications/docker-compose.yml rename to scenario_tests/streaming_replication/docker-compose.yml diff --git a/scenario_tests/replications/test.sh b/scenario_tests/streaming_replication/test.sh similarity index 100% rename from scenario_tests/replications/test.sh rename to scenario_tests/streaming_replication/test.sh diff --git a/scenario_tests/replications/tests/__init__.py b/scenario_tests/streaming_replication/tests/__init__.py similarity index 100% rename from scenario_tests/replications/tests/__init__.py rename to scenario_tests/streaming_replication/tests/__init__.py diff --git a/scenario_tests/replications/tests/test_master.sh b/scenario_tests/streaming_replication/tests/test_master.sh similarity index 100% rename from scenario_tests/replications/tests/test_master.sh rename to scenario_tests/streaming_replication/tests/test_master.sh diff --git a/scenario_tests/replications/tests/test_node.sh b/scenario_tests/streaming_replication/tests/test_node.sh similarity index 100% rename from scenario_tests/replications/tests/test_node.sh rename to scenario_tests/streaming_replication/tests/test_node.sh diff --git a/scenario_tests/replications/tests/test_replication.py b/scenario_tests/streaming_replication/tests/test_replication.py similarity index 100% rename from scenario_tests/replications/tests/test_replication.py rename to scenario_tests/streaming_replication/tests/test_replication.py diff --git a/scripts/setup-conf.sh b/scripts/setup-conf.sh index d22a688..81d5a17 100644 --- a/scripts/setup-conf.sh +++ b/scripts/setup-conf.sh @@ -7,8 +7,6 @@ if [ -f "${SETUP_LOCKFILE}" ]; then return 0 fi -# This script will setup necessary configuration to enable replications - # Refresh configuration in case environment settings changed. cat $CONF.template > $CONF @@ -17,7 +15,7 @@ cat $CONF.template > $CONF sed -i '/data_directory/d' $CONF echo "data_directory = '${DATADIR}'" >> $CONF -# This script will setup necessary configuration to optimise for PostGIS and to enable replications +# Create a config to optimise postgis if [[ -f ${ROOT_CONF}/postgis.conf ]];then rm $CONF/postgis.conf fi @@ -41,10 +39,8 @@ cron.use_background_workers = on EOF echo "include 'postgis.conf'" >> $CONF -# This script will setup necessary replication settings - - +# Create a config for logical replication if [[ "${REPLICATION}" =~ [Tt][Rr][Uu][Ee] && "$WAL_LEVEL" == 'logical' ]]; then if [[ -f ${ROOT_CONF}/logical_replication.conf ]];then rm $CONF/logical_replication.conf @@ -61,6 +57,7 @@ EOF echo "include 'logical_replication.conf'" >> $CONF fi +# Create a config for streaming replication if [[ "${REPLICATION}" =~ [Tt][Rr][Uu][Ee] && "$WAL_LEVEL" == 'replica' ]]; then if [[ -f ${ROOT_CONF}/streaming_replication.conf ]];then rm $CONF/streaming_replication.conf From ea053db2dd62dc385e0595d1dc83b4fcbe046d54 Mon Sep 17 00:00:00 2001 From: admire Date: Tue, 5 Jan 2021 11:39:02 +0200 Subject: [PATCH 06/29] Fix build script to reflect building correct version --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 71ac500..c07743e 100755 --- a/build.sh +++ b/build.sh @@ -5,4 +5,4 @@ cd base_build ./build.sh cd .. docker build -t kartoza/postgis:manual-build . -docker build --build-arg DISTRO=debian --build-arg IMAGE_VERSION=bullseye --build-arg IMAGE_VARIANT=slim -t kartoza/postgis:${POSTGRES_MAJOR_VERSION}.0 . +docker build --build-arg DISTRO=debian --build-arg IMAGE_VERSION=bullseye --build-arg IMAGE_VARIANT=slim -t kartoza/postgis:${POSTGRES_MAJOR_VERSION}.1 . From 99f0c016f3a5e3a3f901d41a016c46e5a4a45786 Mon Sep 17 00:00:00 2001 From: admire Date: Tue, 5 Jan 2021 13:40:28 +0200 Subject: [PATCH 07/29] make script executable --- base_build/build.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 base_build/build.sh diff --git a/base_build/build.sh b/base_build/build.sh old mode 100644 new mode 100755 From ce862ea55d555becd908a66a6330b97dac1c6409 Mon Sep 17 00:00:00 2001 From: admire Date: Tue, 5 Jan 2021 15:46:47 +0200 Subject: [PATCH 08/29] Include extra config and fix wget in gdal install --- base_build/gdal_install.sh | 2 +- scripts/setup-conf.sh | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/base_build/gdal_install.sh b/base_build/gdal_install.sh index c2551f7..50f9590 100644 --- a/base_build/gdal_install.sh +++ b/base_build/gdal_install.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash if [[ ${IMAGE_VERSION} =~ [Bb][Uu][Ll][Ll][Ss][Ee][Yy][Ee] ]]; then - wget http://ftp.br.debian.org/debian/pool/main/g/gdal/libgdal27_3.1.4+dfsg-1+b1_amd64.deb + wget --progress=bar:force:noscroll -c --no-check-certificate http://ftp.br.debian.org/debian/pool/main/g/gdal/libgdal27_3.1.4+dfsg-1+b1_amd64.deb dpkg -i libgdal27_3.1.4+dfsg-1+b1_amd64.deb fi diff --git a/scripts/setup-conf.sh b/scripts/setup-conf.sh index 81d5a17..d8e8f3a 100644 --- a/scripts/setup-conf.sh +++ b/scripts/setup-conf.sh @@ -82,7 +82,14 @@ EOF echo "include 'streaming_replication.conf'" >> $CONF fi -echo -e $EXTRA_CONF >> $CONF +if [[ -f ${ROOT_CONF}/extra.conf ]];then + rm $CONF/extra.conf +fi + +if [[ ! -z $EXTRA_CONF ]]; then + echo -e $EXTRA_CONF >> ${ROOT_CONF}/extra.conf + echo "include 'extra.conf'" >> $CONF +fi # Optimise PostgreSQL shared memory for PostGIS # shmall units are pages and shmmax units are bytes(?) equivalent to the desired shared_buffer size set in setup_conf.sh - in this case 500MB From 09c57349ee350f630821f03c49e36406e61beaed Mon Sep 17 00:00:00 2001 From: admire Date: Tue, 5 Jan 2021 19:09:48 +0200 Subject: [PATCH 09/29] Some minor tweaks --- Dockerfile | 3 ++- build.sh | 3 ++- docker-compose.yml | 4 ++-- scripts/env-data.sh | 3 ++- scripts/setup-conf.sh | 4 ++-- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3b7b55d..a9e5b0a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ MAINTAINER Tim Sutton ARG IMAGE_VERSION ARG POSTGRES_MAJOR_VERSION=13 ARG POSTGIS_MAJOR=3 -ARG POSTGIS_MINOR_RELEASE=3.1 +ARG POSTGIS_MINOR_RELEASE=1 @@ -43,6 +43,7 @@ RUN set -eux \ RUN echo $POSTGRES_MAJOR_VERSION >/tmp/pg_version.txt +RUN echo $POSTGIS_MAJOR >/tmp/pg_major_version.txt RUN echo $POSTGIS_MINOR_RELEASE >/tmp/pg_minor_version.txt ENV \ PATH="$PATH:/usr/lib/postgresql/${POSTGRES_MAJOR_VERSION}/bin" diff --git a/build.sh b/build.sh index c07743e..72b1a0a 100755 --- a/build.sh +++ b/build.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash POSTGRES_MAJOR_VERSION=13 +POSTGIS_MINOR_RELEASE=1 cd base_build ./build.sh cd .. docker build -t kartoza/postgis:manual-build . -docker build --build-arg DISTRO=debian --build-arg IMAGE_VERSION=bullseye --build-arg IMAGE_VARIANT=slim -t kartoza/postgis:${POSTGRES_MAJOR_VERSION}.1 . +docker build --build-arg DISTRO=debian --build-arg IMAGE_VERSION=bullseye --build-arg IMAGE_VARIANT=slim -t kartoza/postgis:${POSTGRES_MAJOR_VERSION}.${POSTGIS_MINOR_RELEASE} . diff --git a/docker-compose.yml b/docker-compose.yml index 481a9f7..b2d2546 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ volumes: services: db: - image: kartoza/postgis:12.0 + image: kartoza/postgis:13.1 volumes: - postgis-data:/var/lib/postgresql - dbbackups:/backups @@ -26,7 +26,7 @@ services: test: "exit 0" dbbackups: - image: kartoza/pg-backup:12.0 + image: kartoza/pg-backup:13.0 hostname: pg-backups volumes: - dbbackups:/backups diff --git a/scripts/env-data.sh b/scripts/env-data.sh index ab562cc..c308ed8 100644 --- a/scripts/env-data.sh +++ b/scripts/env-data.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash POSTGRES_MAJOR_VERSION=$(cat /tmp/pg_version.txt) +POSTGIS_MAJOR=$(cat /tmp/pg_major_version.txt) POSTGIS_MINOR_RELEASE=$(cat /tmp/pg_minor_version.txt) DEFAULT_DATADIR="/var/lib/postgresql/${POSTGRES_MAJOR_VERSION}/main" ROOT_CONF="/etc/postgresql/${POSTGRES_MAJOR_VERSION}/main" @@ -9,7 +10,7 @@ WAL_ARCHIVE="/opt/archivedir" RECOVERY_CONF="$ROOT_CONF/recovery.conf" POSTGRES="/usr/lib/postgresql/${POSTGRES_MAJOR_VERSION}/bin/postgres" INITDB="/usr/lib/postgresql/${POSTGRES_MAJOR_VERSION}/bin/initdb" -SQLDIR="/usr/share/postgresql/${POSTGRES_MAJOR_VERSION}/contrib/postgis-$POSTGIS_MINOR_RELEASE/" +SQLDIR="/usr/share/postgresql/${POSTGRES_MAJOR_VERSION}/contrib/postgis-${POSTGIS_MAJOR}.${POSTGIS_MINOR_RELEASE}/" SETVARS="POSTGIS_ENABLE_OUTDB_RASTERS=1 POSTGIS_GDAL_ENABLED_DRIVERS=ENABLE_ALL" LOCALONLY="-c listen_addresses='127.0.0.1'" PG_BASEBACKUP="/usr/bin/pg_basebackup" diff --git a/scripts/setup-conf.sh b/scripts/setup-conf.sh index d8e8f3a..1850cc4 100644 --- a/scripts/setup-conf.sh +++ b/scripts/setup-conf.sh @@ -10,16 +10,16 @@ fi # Refresh configuration in case environment settings changed. cat $CONF.template > $CONF -# Reflect DATADIR loaction +# Reflect DATA DIR location # Delete any data_dir declarations sed -i '/data_directory/d' $CONF -echo "data_directory = '${DATADIR}'" >> $CONF # Create a config to optimise postgis if [[ -f ${ROOT_CONF}/postgis.conf ]];then rm $CONF/postgis.conf fi cat >> ${ROOT_CONF}/postgis.conf < Date: Wed, 27 Jan 2021 17:36:24 +0700 Subject: [PATCH 10/29] Implement Github Action for base image --- .travis.yml => .disabled.travis.yml | 0 .github/workflows/build-base.yaml | 40 +++++++++++++++++++++++++++++ build.sh | 4 +-- 3 files changed, 42 insertions(+), 2 deletions(-) rename .travis.yml => .disabled.travis.yml (100%) create mode 100644 .github/workflows/build-base.yaml diff --git a/.travis.yml b/.disabled.travis.yml similarity index 100% rename from .travis.yml rename to .disabled.travis.yml diff --git a/.github/workflows/build-base.yaml b/.github/workflows/build-base.yaml new file mode 100644 index 0000000..eb6590b --- /dev/null +++ b/.github/workflows/build-base.yaml @@ -0,0 +1,40 @@ +name: build-latest +on: + workflow_dispatch: + inputs: + distro: + description: Base image distro + required: true + default: debian + imageVersion: + description: Base distro image version/release + required: true + default: bullseye + imageVariant: + description: Base image variant + required: true + default: slim + push: +jobs: + build-base: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build base image + id: docker_build + uses: docker/build-push-action@v2 + with: + context: base_build + file: Dockerfile + push: true + tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ github.event.inputs.distro }}-${{ github.event.inputs.imageVersion }}-${{ github.event.inputs.imageVariant }} diff --git a/build.sh b/build.sh index 72b1a0a..1cf2ea1 100755 --- a/build.sh +++ b/build.sh @@ -2,8 +2,8 @@ POSTGRES_MAJOR_VERSION=13 POSTGIS_MINOR_RELEASE=1 -cd base_build +pushd base_build ./build.sh -cd .. +popd docker build -t kartoza/postgis:manual-build . docker build --build-arg DISTRO=debian --build-arg IMAGE_VERSION=bullseye --build-arg IMAGE_VARIANT=slim -t kartoza/postgis:${POSTGRES_MAJOR_VERSION}.${POSTGIS_MINOR_RELEASE} . From 3773f1f25ad9c026ea496285fce903904987f54a Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Wed, 27 Jan 2021 19:50:49 +0700 Subject: [PATCH 11/29] Modify GH Action for caching --- .github/workflows/build-base.yaml | 209 +++++++++++++++++++++++++++--- 1 file changed, 191 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-base.yaml b/.github/workflows/build-base.yaml index eb6590b..cdc9a03 100644 --- a/.github/workflows/build-base.yaml +++ b/.github/workflows/build-base.yaml @@ -1,29 +1,49 @@ name: build-latest on: - workflow_dispatch: - inputs: - distro: - description: Base image distro - required: true - default: debian - imageVersion: - description: Base distro image version/release - required: true - default: bullseye - imageVariant: - description: Base image variant - required: true - default: slim +# workflow_dispatch: +# inputs: +# distro: +# description: Base image distro +# required: true +# default: debian +# imageVersion: +# description: Base distro image version/release +# required: true +# default: bullseye +# imageVariant: +# description: Base image variant +# required: true +# default: slim push: + pull_request: jobs: - build-base: + build-image: runs-on: ubuntu-latest + env: + latest-ref: refs/heads/develop + strategy: + matrix: + postgresMajorVersion: + - 13 + postgresMinorRelease: + - 1 + include: + - distro: debian + imageVersion: bullseye + imageVariant: slim steps: - uses: actions/checkout@v2 - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 + - name: Get build cache + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: buildx-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }}-${{ github.sha }} + restore-keys: | + buildx-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }}- - name: Login to DockerHub uses: docker/login-action@v1 with: @@ -31,10 +51,163 @@ jobs: password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Build base image - id: docker_build + id: docker_build_base uses: docker/build-push-action@v2 with: context: base_build file: Dockerfile - push: true - tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ github.event.inputs.distro }}-${{ github.event.inputs.imageVersion }}-${{ github.event.inputs.imageVariant }} + push: false + tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:base-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }} + build-args: | + DISTRO=${{ matrix.distro }} + IMAGE_VERSION=${{ matrix.imageVersion }} + IMAGE_VARIANT=${{ matrix.imageVariant }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + + - name: Build prod image + id: docker_build_prod + uses: docker/build-push-action@v2 + with: + context: . + file: Dockerfile + push: false + tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ matrix.postgresMajorVersion }}.${{ matrix.postgresMinorRelease }} + build-args: | + DISTRO=${{ matrix.distro }} + IMAGE_VERSION=${{ matrix.imageVersion }} + IMAGE_VARIANT=${{ matrix.imageVariant }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + + run-scenario-tests: + runs-on: ubuntu-latest + needs: [ build-image ] + strategy: + matrix: + postgresMajorVersion: + - 13 + postgresMinorRelease: + - 1 + scenario: + - datadir_init + - streaming_replication + - collations + - extensions + - logical_replication + include: + - distro: debian + imageVersion: bullseye + imageVariant: slim + steps: + - uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Get build cache + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: buildx-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }}-${{ github.sha }} + restore-keys: | + buildx-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }}- + + - name: Build base image for testing + id: docker_build_base + uses: docker/build-push-action@v2 + with: + context: . + file: Dockerfile + push: false + tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:manual-build + build-args: | + DISTRO=${{ matrix.distro }} + IMAGE_VERSION=${{ matrix.imageVersion }} + IMAGE_VARIANT=${{ matrix.imageVariant }} + cache-from: type=local,src=/tmp/.buildx-cache + + - name: Build image for testing + id: docker_build_testing_image + uses: docker/build-push-action@v2 + with: + context: . + file: Dockerfile.test + push: false + tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:manual-build + load: true + - name: Run scenario test + run: | + pushd scenario_tests/${{ matrix.scenario }} + ./test.sh + popd + + + + + + + deploy-image: + # Identical to the build-image step, but now push to registry + runs-on: ubuntu-latest + env: + latest-ref: refs/heads/develop + strategy: + matrix: + postgresMajorVersion: + - 13 + postgresMinorRelease: + - 1 + include: + - distro: debian + imageVersion: bullseye + imageVariant: slim + needs: [ build-image, run-scenario-tests ] + steps: + - uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Get build cache + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: buildx-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }}-${{ github.sha }} + restore-keys: | + buildx-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }}- + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build base image + id: docker_build_base + uses: docker/build-push-action@v2 + with: + context: base_build + file: Dockerfile + push: ${{ github.ref == env.latest-ref }} + tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:base-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }} + build-args: | + DISTRO=${{ matrix.distro }} + IMAGE_VERSION=${{ matrix.imageVersion }} + IMAGE_VARIANT=${{ matrix.imageVariant }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + + - name: Build prod image + id: docker_build_prod + uses: docker/build-push-action@v2 + with: + context: . + file: Dockerfile + push: ${{ github.ref == env.latest-ref }} + tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ matrix.postgresMajorVersion }}.${{ matrix.postgresMinorRelease }} + build-args: | + DISTRO=${{ matrix.distro }} + IMAGE_VERSION=${{ matrix.imageVersion }} + IMAGE_VARIANT=${{ matrix.imageVariant }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache From 0342801ab26dbfe643926074b981d63a8f5d1db3 Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Wed, 27 Jan 2021 20:04:19 +0700 Subject: [PATCH 12/29] Fix recipe and build matrix --- .github/workflows/build-base.yaml | 34 ++++++++++++++++++++----------- Dockerfile | 4 ++-- base_build/build.sh | 2 +- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-base.yaml b/.github/workflows/build-base.yaml index cdc9a03..67a8298 100644 --- a/.github/workflows/build-base.yaml +++ b/.github/workflows/build-base.yaml @@ -25,7 +25,9 @@ jobs: matrix: postgresMajorVersion: - 13 - postgresMinorRelease: + postgisMajorVersion: + - 3 + postgisMinorRelease: - 1 include: - distro: debian @@ -55,7 +57,7 @@ jobs: uses: docker/build-push-action@v2 with: context: base_build - file: Dockerfile + file: base_build/Dockerfile push: false tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:base-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }} build-args: | @@ -72,11 +74,14 @@ jobs: context: . file: Dockerfile push: false - tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ matrix.postgresMajorVersion }}.${{ matrix.postgresMinorRelease }} + tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ matrix.postgresMajorVersion }}-${{ matrix.postgisMajorRelease }}.${{ matrix.postgisMinorRelease }} build-args: | DISTRO=${{ matrix.distro }} IMAGE_VERSION=${{ matrix.imageVersion }} IMAGE_VARIANT=${{ matrix.imageVariant }} + POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }} + POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }} + POSTGIS_MINOR_VERSION=${{ matrix.postgresMinorVersion }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache @@ -87,7 +92,9 @@ jobs: matrix: postgresMajorVersion: - 13 - postgresMinorRelease: + postgisMajorVersion: + - 3 + postgisMinorRelease: - 1 scenario: - datadir_init @@ -125,6 +132,9 @@ jobs: DISTRO=${{ matrix.distro }} IMAGE_VERSION=${{ matrix.imageVersion }} IMAGE_VARIANT=${{ matrix.imageVariant }} + POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }} + POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }} + POSTGIS_MINOR_VERSION=${{ matrix.postgresMinorVersion }} cache-from: type=local,src=/tmp/.buildx-cache - name: Build image for testing @@ -142,11 +152,6 @@ jobs: ./test.sh popd - - - - - deploy-image: # Identical to the build-image step, but now push to registry runs-on: ubuntu-latest @@ -156,7 +161,9 @@ jobs: matrix: postgresMajorVersion: - 13 - postgresMinorRelease: + postgisMajorVersion: + - 3 + postgisMinorRelease: - 1 include: - distro: debian @@ -187,7 +194,7 @@ jobs: uses: docker/build-push-action@v2 with: context: base_build - file: Dockerfile + file: base_build/Dockerfile push: ${{ github.ref == env.latest-ref }} tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:base-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }} build-args: | @@ -204,10 +211,13 @@ jobs: context: . file: Dockerfile push: ${{ github.ref == env.latest-ref }} - tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ matrix.postgresMajorVersion }}.${{ matrix.postgresMinorRelease }} + tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ matrix.postgresMajorVersion }}-${{ matrix.postgisMajorRelease }}.${{ matrix.postgisMinorRelease }} build-args: | DISTRO=${{ matrix.distro }} IMAGE_VERSION=${{ matrix.imageVersion }} IMAGE_VARIANT=${{ matrix.imageVariant }} + POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }} + POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }} + POSTGIS_MINOR_VERSION=${{ matrix.postgresMinorVersion }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/Dockerfile b/Dockerfile index a9e5b0a..ac76399 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG DISTRO=debian ARG IMAGE_VERSION=bullseye ARG IMAGE_VARIANT=slim -FROM kartoza/postgis:$DISTRO-$IMAGE_VERSION-$IMAGE_VARIANT-base +FROM kartoza/postgis:base-$DISTRO-$IMAGE_VERSION-$IMAGE_VARIANT MAINTAINER Tim Sutton @@ -74,4 +74,4 @@ RUN set -eux \ VOLUME /var/lib/postgresql -ENTRYPOINT /scripts/docker-entrypoint.sh \ No newline at end of file +ENTRYPOINT /scripts/docker-entrypoint.sh diff --git a/base_build/build.sh b/base_build/build.sh index a15bace..53c3c7f 100755 --- a/base_build/build.sh +++ b/base_build/build.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash # Building an debian base image -docker build --build-arg DISTRO=debian --build-arg IMAGE_VERSION=bullseye --build-arg IMAGE_VARIANT=slim -t kartoza/postgis:debian-bullseye-slim-base . +docker build --build-arg DISTRO=debian --build-arg IMAGE_VERSION=bullseye --build-arg IMAGE_VARIANT=slim -t kartoza/postgis:base-debian-bullseye-slim . From e703e864315acf8d7e246aad144f0b2b865b974f Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Wed, 27 Jan 2021 20:42:40 +0700 Subject: [PATCH 13/29] Generate all locale only in deloyment mode --- .github/workflows/build-base.yaml | 12 ++++++++++++ base_build/Dockerfile | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/.github/workflows/build-base.yaml b/.github/workflows/build-base.yaml index 67a8298..6124302 100644 --- a/.github/workflows/build-base.yaml +++ b/.github/workflows/build-base.yaml @@ -52,6 +52,11 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Do we need to generate locale? + id: is_generate_locale + if: github.ref == env.latest-ref + run: echo "::set-output name=is_generate::" + - name: Build base image id: docker_build_base uses: docker/build-push-action@v2 @@ -61,6 +66,7 @@ jobs: push: false tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:base-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }} build-args: | + GENERATE_ALL_LOCALE=${{ steps.is_generate_locale.outputs.is_generate }} DISTRO=${{ matrix.distro }} IMAGE_VERSION=${{ matrix.imageVersion }} IMAGE_VARIANT=${{ matrix.imageVariant }} @@ -189,6 +195,11 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Do we need to generate locale? + id: is_generate_locale + if: github.ref == env.latest-ref + run: echo "::set-output name=is_generate::" + - name: Build base image id: docker_build_base uses: docker/build-push-action@v2 @@ -198,6 +209,7 @@ jobs: push: ${{ github.ref == env.latest-ref }} tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:base-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }} build-args: | + GENERATE_ALL_LOCALE=${{ steps.is_generate_locale.outputs.is_generate }} DISTRO=${{ matrix.distro }} IMAGE_VERSION=${{ matrix.imageVersion }} IMAGE_VARIANT=${{ matrix.imageVariant }} diff --git a/base_build/Dockerfile b/base_build/Dockerfile index 1ffa5c2..3e10698 100644 --- a/base_build/Dockerfile +++ b/base_build/Dockerfile @@ -31,8 +31,13 @@ RUN chmod 0755 /gdal_install.sh;/bin/bash /gdal_install.sh # Generating locales takes a long time. Utilize caching by runnig it by itself # early in the build process. +# Generate all locale only on deployment mode build +# Set to empty string to generate only default locale +ARG GENERATE_ALL_LOCALE=1 + COPY scripts/locale.gen /etc/locale.gen RUN set -eux \ + && [[ (! -z "${GENERATE_ALL_LOCALE}") && (! $GENERATE_ALL_LOCALE -eq 0) ]] \ && /usr/sbin/locale-gen ENV LANG=en_US.UTF-8 \ From 8cca0c80078106213376e7f1a99cb5655e0b85b2 Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Wed, 27 Jan 2021 20:45:28 +0700 Subject: [PATCH 14/29] Fix tag name --- .github/workflows/build-base.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-base.yaml b/.github/workflows/build-base.yaml index 6124302..9f3c4d2 100644 --- a/.github/workflows/build-base.yaml +++ b/.github/workflows/build-base.yaml @@ -80,7 +80,7 @@ jobs: context: . file: Dockerfile push: false - tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ matrix.postgresMajorVersion }}-${{ matrix.postgisMajorRelease }}.${{ matrix.postgisMinorRelease }} + tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ matrix.postgresMajorVersion }}-${{ matrix.postgisMajorVersion }}.${{ matrix.postgisMinorRelease }} build-args: | DISTRO=${{ matrix.distro }} IMAGE_VERSION=${{ matrix.imageVersion }} @@ -223,7 +223,7 @@ jobs: context: . file: Dockerfile push: ${{ github.ref == env.latest-ref }} - tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ matrix.postgresMajorVersion }}-${{ matrix.postgisMajorRelease }}.${{ matrix.postgisMinorRelease }} + tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ matrix.postgresMajorVersion }}-${{ matrix.postgisMajorVersion }}.${{ matrix.postgisMinorRelease }} build-args: | DISTRO=${{ matrix.distro }} IMAGE_VERSION=${{ matrix.imageVersion }} From d38b05d57ca5264d63857f66b160a3799ee498df Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Wed, 27 Jan 2021 20:52:43 +0700 Subject: [PATCH 15/29] Separate GH Action of build and deploy --- .../{build-base.yaml => build-latest.yaml} | 83 +---------------- .github/workflows/deploy-image.yaml | 91 +++++++++++++++++++ 2 files changed, 92 insertions(+), 82 deletions(-) rename .github/workflows/{build-base.yaml => build-latest.yaml} (61%) create mode 100644 .github/workflows/deploy-image.yaml diff --git a/.github/workflows/build-base.yaml b/.github/workflows/build-latest.yaml similarity index 61% rename from .github/workflows/build-base.yaml rename to .github/workflows/build-latest.yaml index 9f3c4d2..ca9e03c 100644 --- a/.github/workflows/build-base.yaml +++ b/.github/workflows/build-latest.yaml @@ -52,11 +52,6 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Do we need to generate locale? - id: is_generate_locale - if: github.ref == env.latest-ref - run: echo "::set-output name=is_generate::" - - name: Build base image id: docker_build_base uses: docker/build-push-action@v2 @@ -66,7 +61,7 @@ jobs: push: false tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:base-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }} build-args: | - GENERATE_ALL_LOCALE=${{ steps.is_generate_locale.outputs.is_generate }} + GENERATE_ALL_LOCALE=0 DISTRO=${{ matrix.distro }} IMAGE_VERSION=${{ matrix.imageVersion }} IMAGE_VARIANT=${{ matrix.imageVariant }} @@ -157,79 +152,3 @@ jobs: pushd scenario_tests/${{ matrix.scenario }} ./test.sh popd - - deploy-image: - # Identical to the build-image step, but now push to registry - runs-on: ubuntu-latest - env: - latest-ref: refs/heads/develop - strategy: - matrix: - postgresMajorVersion: - - 13 - postgisMajorVersion: - - 3 - postgisMinorRelease: - - 1 - include: - - distro: debian - imageVersion: bullseye - imageVariant: slim - needs: [ build-image, run-scenario-tests ] - steps: - - uses: actions/checkout@v2 - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Get build cache - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: buildx-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }}-${{ github.sha }} - restore-keys: | - buildx-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }}- - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Do we need to generate locale? - id: is_generate_locale - if: github.ref == env.latest-ref - run: echo "::set-output name=is_generate::" - - - name: Build base image - id: docker_build_base - uses: docker/build-push-action@v2 - with: - context: base_build - file: base_build/Dockerfile - push: ${{ github.ref == env.latest-ref }} - tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:base-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }} - build-args: | - GENERATE_ALL_LOCALE=${{ steps.is_generate_locale.outputs.is_generate }} - DISTRO=${{ matrix.distro }} - IMAGE_VERSION=${{ matrix.imageVersion }} - IMAGE_VARIANT=${{ matrix.imageVariant }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - - name: Build prod image - id: docker_build_prod - uses: docker/build-push-action@v2 - with: - context: . - file: Dockerfile - push: ${{ github.ref == env.latest-ref }} - tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ matrix.postgresMajorVersion }}-${{ matrix.postgisMajorVersion }}.${{ matrix.postgisMinorRelease }} - build-args: | - DISTRO=${{ matrix.distro }} - IMAGE_VERSION=${{ matrix.imageVersion }} - IMAGE_VARIANT=${{ matrix.imageVariant }} - POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }} - POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }} - POSTGIS_MINOR_VERSION=${{ matrix.postgresMinorVersion }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/.github/workflows/deploy-image.yaml b/.github/workflows/deploy-image.yaml new file mode 100644 index 0000000..2ac7e92 --- /dev/null +++ b/.github/workflows/deploy-image.yaml @@ -0,0 +1,91 @@ +name: deploy-image +on: + workflow_dispatch: +# inputs: +# distro: +# description: Base image distro +# required: true +# default: debian +# imageVersion: +# description: Base distro image version/release +# required: true +# default: bullseye +# imageVariant: +# description: Base image variant +# required: true +# default: slim + workflow_run: + workflows: + - build-latest + branches: + - develop + types: + - completed +jobs: + deploy-image: + runs-on: ubuntu-latest + env: + latest-ref: refs/heads/develop + strategy: + matrix: + postgresMajorVersion: + - 13 + postgisMajorVersion: + - 3 + postgisMinorRelease: + - 1 + include: + - distro: debian + imageVersion: bullseye + imageVariant: slim + steps: + - uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Get build cache + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: buildx-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }}-${{ github.sha }} + restore-keys: | + buildx-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }}- + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build base image + id: docker_build_base + uses: docker/build-push-action@v2 + with: + context: base_build + file: base_build/Dockerfile + push: true + tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:base-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }} + build-args: | + DISTRO=${{ matrix.distro }} + IMAGE_VERSION=${{ matrix.imageVersion }} + IMAGE_VARIANT=${{ matrix.imageVariant }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + + - name: Build prod image + id: docker_build_prod + uses: docker/build-push-action@v2 + with: + context: . + file: Dockerfile + push: true + tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ matrix.postgresMajorVersion }}-${{ matrix.postgisMajorVersion }}.${{ matrix.postgisMinorRelease }} + build-args: | + DISTRO=${{ matrix.distro }} + IMAGE_VERSION=${{ matrix.imageVersion }} + IMAGE_VARIANT=${{ matrix.imageVariant }} + POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }} + POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }} + POSTGIS_MINOR_VERSION=${{ matrix.postgresMinorVersion }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache From d1ca873b24e405fdace1805811a18afbfea1972c Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Wed, 27 Jan 2021 22:09:29 +0700 Subject: [PATCH 16/29] Update locale generator logic --- base_build/Dockerfile | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/base_build/Dockerfile b/base_build/Dockerfile index 3e10698..fcbb2d3 100644 --- a/base_build/Dockerfile +++ b/base_build/Dockerfile @@ -34,15 +34,20 @@ RUN chmod 0755 /gdal_install.sh;/bin/bash /gdal_install.sh # Generate all locale only on deployment mode build # Set to empty string to generate only default locale ARG GENERATE_ALL_LOCALE=1 - -COPY scripts/locale.gen /etc/locale.gen -RUN set -eux \ - && [[ (! -z "${GENERATE_ALL_LOCALE}") && (! $GENERATE_ALL_LOCALE -eq 0) ]] \ - && /usr/sbin/locale-gen - ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ LC_ALL=en_US.UTF-8 + +COPY scripts/locale.gen /etc/all.locale.gen +RUN if [ -z "${GENERATE_ALL_LOCALE}" ] || [ $GENERATE_ALL_LOCALE -eq 0 ]; \ + then \ + cat /etc/all.locale.gen | grep "${LANG}" > /etc/locale.gen; \ + else \ + cp -f /etc/all.locale.gen /etc/locale.gen; \ + fi; \ + set -eux \ + && /usr/sbin/locale-gen + RUN update-locale ${LANG} # Cleanup resources From 6ecfb52d982ec89b47d6d7e07b78743bf3b8fac1 Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Wed, 27 Jan 2021 22:48:24 +0700 Subject: [PATCH 17/29] Set GH Action buildx to load images --- .github/workflows/build-latest.yaml | 3 +++ .github/workflows/deploy-image.yaml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/build-latest.yaml b/.github/workflows/build-latest.yaml index ca9e03c..e284271 100644 --- a/.github/workflows/build-latest.yaml +++ b/.github/workflows/build-latest.yaml @@ -59,6 +59,7 @@ jobs: context: base_build file: base_build/Dockerfile push: false + load: true tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:base-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }} build-args: | GENERATE_ALL_LOCALE=0 @@ -75,6 +76,7 @@ jobs: context: . file: Dockerfile push: false + load: true tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ matrix.postgresMajorVersion }}-${{ matrix.postgisMajorVersion }}.${{ matrix.postgisMinorRelease }} build-args: | DISTRO=${{ matrix.distro }} @@ -128,6 +130,7 @@ jobs: context: . file: Dockerfile push: false + load: true tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:manual-build build-args: | DISTRO=${{ matrix.distro }} diff --git a/.github/workflows/deploy-image.yaml b/.github/workflows/deploy-image.yaml index 2ac7e92..338e94f 100644 --- a/.github/workflows/deploy-image.yaml +++ b/.github/workflows/deploy-image.yaml @@ -64,6 +64,7 @@ jobs: context: base_build file: base_build/Dockerfile push: true + load: true tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:base-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }} build-args: | DISTRO=${{ matrix.distro }} @@ -79,6 +80,7 @@ jobs: context: . file: Dockerfile push: true + load: true tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ matrix.postgresMajorVersion }}-${{ matrix.postgisMajorVersion }}.${{ matrix.postgisMinorRelease }} build-args: | DISTRO=${{ matrix.distro }} From 879099acc4a392cb31f8b7cd47e55ad75aa92ea7 Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Wed, 27 Jan 2021 23:10:25 +0700 Subject: [PATCH 18/29] Allow setting image owner --- .github/workflows/build-latest.yaml | 6 ++++-- .github/workflows/deploy-image.yaml | 1 + Dockerfile | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-latest.yaml b/.github/workflows/build-latest.yaml index e284271..aa6a97d 100644 --- a/.github/workflows/build-latest.yaml +++ b/.github/workflows/build-latest.yaml @@ -80,6 +80,7 @@ jobs: tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ matrix.postgresMajorVersion }}-${{ matrix.postgisMajorVersion }}.${{ matrix.postgisMinorRelease }} build-args: | DISTRO=${{ matrix.distro }} + IMAGE_OWNER=${{ secrets.DOCKERHUB_REPO }} IMAGE_VERSION=${{ matrix.imageVersion }} IMAGE_VARIANT=${{ matrix.imageVariant }} POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }} @@ -131,9 +132,10 @@ jobs: file: Dockerfile push: false load: true - tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:manual-build + tags: kartoza/postgis:manual-build build-args: | DISTRO=${{ matrix.distro }} + IMAGE_OWNER=kartoza IMAGE_VERSION=${{ matrix.imageVersion }} IMAGE_VARIANT=${{ matrix.imageVariant }} POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }} @@ -148,7 +150,7 @@ jobs: context: . file: Dockerfile.test push: false - tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:manual-build + tags: kartoza/postgis:manual-build load: true - name: Run scenario test run: | diff --git a/.github/workflows/deploy-image.yaml b/.github/workflows/deploy-image.yaml index 338e94f..3335047 100644 --- a/.github/workflows/deploy-image.yaml +++ b/.github/workflows/deploy-image.yaml @@ -84,6 +84,7 @@ jobs: tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ matrix.postgresMajorVersion }}-${{ matrix.postgisMajorVersion }}.${{ matrix.postgisMinorRelease }} build-args: | DISTRO=${{ matrix.distro }} + IMAGE_OWNER=${{ secrets.DOCKERHUB_REPO }} IMAGE_VERSION=${{ matrix.imageVersion }} IMAGE_VARIANT=${{ matrix.imageVariant }} POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }} diff --git a/Dockerfile b/Dockerfile index ac76399..54416ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,9 @@ #--------- Generic stuff all our Dockerfiles should start with so we get caching ------------ ARG DISTRO=debian +ARG IMAGE_OWNER=kartoza ARG IMAGE_VERSION=bullseye ARG IMAGE_VARIANT=slim -FROM kartoza/postgis:base-$DISTRO-$IMAGE_VERSION-$IMAGE_VARIANT +FROM $IMAGE_OWNER/postgis:base-$DISTRO-$IMAGE_VERSION-$IMAGE_VARIANT MAINTAINER Tim Sutton From 00268d980213426aea33e81458f67f2cdbeb244c Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Wed, 27 Jan 2021 23:15:03 +0700 Subject: [PATCH 19/29] Test printing image list --- .github/workflows/build-latest.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-latest.yaml b/.github/workflows/build-latest.yaml index aa6a97d..92759ca 100644 --- a/.github/workflows/build-latest.yaml +++ b/.github/workflows/build-latest.yaml @@ -69,6 +69,8 @@ jobs: cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache + - run: docker images -a + - name: Build prod image id: docker_build_prod uses: docker/build-push-action@v2 From 8c9f610e4e8e0cee05a9bea01a9b61bb6a7caabe Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Thu, 28 Jan 2021 01:06:59 +0700 Subject: [PATCH 20/29] Use multi-stage build --- .example.env | 16 +++++ .github/workflows/build-latest.yaml | 33 +++++------ .github/workflows/deploy-image.yaml | 13 +++-- .gitignore | 2 + Dockerfile | 91 ++++++++++++++++++++++++++--- Dockerfile.test | 16 ----- base_build/Dockerfile | 57 ------------------ base_build/build.sh | 4 -- build-base.sh | 5 ++ build-test.sh | 5 +- build.sh | 8 +-- docker-compose.build.yml | 33 +++++++++++ 12 files changed, 165 insertions(+), 118 deletions(-) create mode 100644 .example.env delete mode 100644 Dockerfile.test delete mode 100644 base_build/Dockerfile delete mode 100755 base_build/build.sh create mode 100755 build-base.sh create mode 100644 docker-compose.build.yml diff --git a/.example.env b/.example.env new file mode 100644 index 0000000..14f0d9c --- /dev/null +++ b/.example.env @@ -0,0 +1,16 @@ +## Example environment file for docker-compose and builders + +COMPOSE_PROJECT_NAME=postgis + +## For build arguments +DISTRO=debian +IMAGE_VERSION=bullseye +IMAGE_VARIANT=slim +# Set GENERATE_ALL_LOCALE to empty value or 0 to build just default LOCALE: en_US.UTF-8 +GENERATE_ALL_LOCALE=1 +# Set the language if you need to specify LANG locale at build time +LANG=en_US.UTF-8 + +POSTGRES_MAJOR_VERSION=13 +POSTGIS_MAJOR_VERSION=3 +POSTGIS_MINOR_RELEASE=1 diff --git a/.github/workflows/build-latest.yaml b/.github/workflows/build-latest.yaml index 92759ca..fc76b3b 100644 --- a/.github/workflows/build-latest.yaml +++ b/.github/workflows/build-latest.yaml @@ -56,20 +56,21 @@ jobs: id: docker_build_base uses: docker/build-push-action@v2 with: - context: base_build - file: base_build/Dockerfile + context: . + file: Dockerfile push: false - load: true tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:base-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }} build-args: | - GENERATE_ALL_LOCALE=0 DISTRO=${{ matrix.distro }} IMAGE_VERSION=${{ matrix.imageVersion }} IMAGE_VARIANT=${{ matrix.imageVariant }} + GENERATE_ALL_LOCALE=0 + POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }} + POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }} + POSTGIS_MINOR_VERSION=${{ matrix.postgresMinorVersion }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache - - - run: docker images -a + target: postgis-base - name: Build prod image id: docker_build_prod @@ -82,14 +83,15 @@ jobs: tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ matrix.postgresMajorVersion }}-${{ matrix.postgisMajorVersion }}.${{ matrix.postgisMinorRelease }} build-args: | DISTRO=${{ matrix.distro }} - IMAGE_OWNER=${{ secrets.DOCKERHUB_REPO }} IMAGE_VERSION=${{ matrix.imageVersion }} IMAGE_VARIANT=${{ matrix.imageVariant }} + GENERATE_ALL_LOCALE=0 POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }} POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }} POSTGIS_MINOR_VERSION=${{ matrix.postgresMinorVersion }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache + target: postgis-prod run-scenario-tests: runs-on: ubuntu-latest @@ -126,8 +128,8 @@ jobs: restore-keys: | buildx-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }}- - - name: Build base image for testing - id: docker_build_base + - name: Build image for testing + id: docker_build_testing_image uses: docker/build-push-action@v2 with: context: . @@ -137,23 +139,14 @@ jobs: tags: kartoza/postgis:manual-build build-args: | DISTRO=${{ matrix.distro }} - IMAGE_OWNER=kartoza IMAGE_VERSION=${{ matrix.imageVersion }} IMAGE_VARIANT=${{ matrix.imageVariant }} + GENERATE_ALL_LOCALE=0 POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }} POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }} POSTGIS_MINOR_VERSION=${{ matrix.postgresMinorVersion }} - cache-from: type=local,src=/tmp/.buildx-cache + target: postgis-test - - name: Build image for testing - id: docker_build_testing_image - uses: docker/build-push-action@v2 - with: - context: . - file: Dockerfile.test - push: false - tags: kartoza/postgis:manual-build - load: true - name: Run scenario test run: | pushd scenario_tests/${{ matrix.scenario }} diff --git a/.github/workflows/deploy-image.yaml b/.github/workflows/deploy-image.yaml index 3335047..5f5d83d 100644 --- a/.github/workflows/deploy-image.yaml +++ b/.github/workflows/deploy-image.yaml @@ -61,17 +61,21 @@ jobs: id: docker_build_base uses: docker/build-push-action@v2 with: - context: base_build - file: base_build/Dockerfile + context: . + file: Dockerfile push: true - load: true tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:base-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }} build-args: | DISTRO=${{ matrix.distro }} IMAGE_VERSION=${{ matrix.imageVersion }} IMAGE_VARIANT=${{ matrix.imageVariant }} + GENERATE_ALL_LOCALE=1 + POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }} + POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }} + POSTGIS_MINOR_VERSION=${{ matrix.postgresMinorVersion }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache + target: postgis-base - name: Build prod image id: docker_build_prod @@ -84,11 +88,12 @@ jobs: tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:${{ matrix.postgresMajorVersion }}-${{ matrix.postgisMajorVersion }}.${{ matrix.postgisMinorRelease }} build-args: | DISTRO=${{ matrix.distro }} - IMAGE_OWNER=${{ secrets.DOCKERHUB_REPO }} IMAGE_VERSION=${{ matrix.imageVersion }} IMAGE_VARIANT=${{ matrix.imageVariant }} + GENERATE_ALL_LOCALE=1 POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }} POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }} POSTGIS_MINOR_VERSION=${{ matrix.postgresMinorVersion }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache + target: postgis-prod diff --git a/.gitignore b/.gitignore index 05e944f..bcf9e43 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ .python-version venv __pycache__ + +.env diff --git a/Dockerfile b/Dockerfile index 54416ac..8a01f5f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,75 @@ -#--------- Generic stuff all our Dockerfiles should start with so we get caching ------------ +############################################################################## +# Base stage # +############################################################################## ARG DISTRO=debian -ARG IMAGE_OWNER=kartoza ARG IMAGE_VERSION=bullseye ARG IMAGE_VARIANT=slim -FROM $IMAGE_OWNER/postgis:base-$DISTRO-$IMAGE_VERSION-$IMAGE_VARIANT +FROM $DISTRO:$IMAGE_VERSION-$IMAGE_VARIANT AS postgis-base +MAINTAINER Tim Sutton + +# Reset ARG for version +ARG IMAGE_VERSION + +RUN apt-get -qq update --fix-missing && apt-get -qq --yes upgrade + +RUN set -eux \ + && export DEBIAN_FRONTEND=noninteractive \ + && apt-get update \ + && apt-get -y --no-install-recommends install \ + locales gnupg2 wget ca-certificates rpl pwgen software-properties-common iputils-ping \ + apt-transport-https curl \ + && dpkg-divert --local --rename --add /sbin/initctl + + + +RUN apt-get -y update; apt-get -y install build-essential autoconf libxml2-dev zlib1g-dev netcat gdal-bin + + + +# Temporary - PostgreSQL requires this which is not available in bullseye +ADD base_build/gdal_install.sh /gdal_install.sh +RUN chmod 0755 /gdal_install.sh;/bin/bash /gdal_install.sh + +# Generating locales takes a long time. Utilize caching by runnig it by itself +# early in the build process. + +# Generate all locale only on deployment mode build +# Set to empty string to generate only default locale +ARG GENERATE_ALL_LOCALE=1 +ARG LANG=en_US.UTF-8 +ENV LANG=en_US.UTF-8 \ + LANGUAGE=en_US:en \ + LC_ALL=en_US.UTF-8 + +COPY base_build/scripts/locale.gen /etc/all.locale.gen +RUN if [ -z "${GENERATE_ALL_LOCALE}" ] || [ $GENERATE_ALL_LOCALE -eq 0 ]; \ + then \ + cat /etc/all.locale.gen | grep "${LANG}" > /etc/locale.gen; \ + else \ + cp -f /etc/all.locale.gen /etc/locale.gen; \ + fi; \ + set -eux \ + && /usr/sbin/locale-gen + +RUN update-locale ${LANG} + +# Cleanup resources +RUN apt-get -y --purge autoremove \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + + +############################################################################## +# Production Stage # +############################################################################## +FROM postgis-base AS postgis-prod MAINTAINER Tim Sutton # Reset ARG for version ARG IMAGE_VERSION ARG POSTGRES_MAJOR_VERSION=13 -ARG POSTGIS_MAJOR=3 +ARG POSTGIS_MAJOR_VERSION=3 ARG POSTGIS_MINOR_RELEASE=1 @@ -36,15 +95,15 @@ RUN set -eux \ && apt-get update \ && apt-get -y --no-install-recommends install postgresql-client-${POSTGRES_MAJOR_VERSION} \ postgresql-common postgresql-${POSTGRES_MAJOR_VERSION} \ - postgresql-${POSTGRES_MAJOR_VERSION}-postgis-${POSTGIS_MAJOR} \ + postgresql-${POSTGRES_MAJOR_VERSION}-postgis-${POSTGIS_MAJOR_VERSION} \ netcat postgresql-${POSTGRES_MAJOR_VERSION}-ogr-fdw \ - postgresql-${POSTGRES_MAJOR_VERSION}-postgis-${POSTGIS_MAJOR}-scripts \ + postgresql-${POSTGRES_MAJOR_VERSION}-postgis-${POSTGIS_MAJOR_VERSION}-scripts \ postgresql-plpython3-${POSTGRES_MAJOR_VERSION} postgresql-${POSTGRES_MAJOR_VERSION}-pgrouting \ postgresql-server-dev-${POSTGRES_MAJOR_VERSION} postgresql-${POSTGRES_MAJOR_VERSION}-cron RUN echo $POSTGRES_MAJOR_VERSION >/tmp/pg_version.txt -RUN echo $POSTGIS_MAJOR >/tmp/pg_major_version.txt +RUN echo $POSTGIS_MAJOR_VERSION >/tmp/pg_major_version.txt RUN echo $POSTGIS_MINOR_RELEASE >/tmp/pg_minor_version.txt ENV \ PATH="$PATH:/usr/lib/postgresql/${POSTGRES_MAJOR_VERSION}/bin" @@ -76,3 +135,21 @@ RUN set -eux \ VOLUME /var/lib/postgresql ENTRYPOINT /scripts/docker-entrypoint.sh + + +############################################################################## +# Testing Stage # +############################################################################## +FROM postgis-prod AS postgis-test + +COPY scenario_tests/utils/requirements.txt /lib/utils/requirements.txt + +RUN set -eux \ + && export DEBIAN_FRONTEND=noninteractive \ + && apt-get update \ + && apt-get -y --no-install-recommends install python3-pip \ + && apt-get -y --purge autoremove \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN pip3 install -r /lib/utils/requirements.txt diff --git a/Dockerfile.test b/Dockerfile.test deleted file mode 100644 index c897705..0000000 --- a/Dockerfile.test +++ /dev/null @@ -1,16 +0,0 @@ -#--------- Generic stuff all our Dockerfiles should start with so we get caching ------------ -FROM kartoza/postgis:manual-build - -# For testing - -COPY scenario_tests/utils/requirements.txt /lib/utils/requirements.txt - -RUN set -eux \ - && export DEBIAN_FRONTEND=noninteractive \ - && apt-get update \ - && apt-get -y --no-install-recommends install python3-pip \ - && apt-get -y --purge autoremove \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -RUN pip3 install -r /lib/utils/requirements.txt diff --git a/base_build/Dockerfile b/base_build/Dockerfile deleted file mode 100644 index fcbb2d3..0000000 --- a/base_build/Dockerfile +++ /dev/null @@ -1,57 +0,0 @@ -#--------- Generic stuff all our Dockerfiles should start with so we get caching ------------ -ARG DISTRO=debian -ARG IMAGE_VERSION=bullseye -ARG IMAGE_VARIANT=slim -FROM $DISTRO:$IMAGE_VERSION-$IMAGE_VARIANT -MAINTAINER Tim Sutton - -# Reset ARG for version -ARG IMAGE_VERSION - -RUN apt-get -qq update --fix-missing && apt-get -qq --yes upgrade - -RUN set -eux \ - && export DEBIAN_FRONTEND=noninteractive \ - && apt-get update \ - && apt-get -y --no-install-recommends install \ - locales gnupg2 wget ca-certificates rpl pwgen software-properties-common iputils-ping \ - apt-transport-https curl \ - && dpkg-divert --local --rename --add /sbin/initctl - - - -RUN apt-get -y update; apt-get -y install build-essential autoconf libxml2-dev zlib1g-dev netcat gdal-bin - - - -# Temporary - PostgreSQL requires this which is not available in bullseye -ADD gdal_install.sh /gdal_install.sh -RUN chmod 0755 /gdal_install.sh;/bin/bash /gdal_install.sh - -# Generating locales takes a long time. Utilize caching by runnig it by itself -# early in the build process. - -# Generate all locale only on deployment mode build -# Set to empty string to generate only default locale -ARG GENERATE_ALL_LOCALE=1 -ENV LANG=en_US.UTF-8 \ - LANGUAGE=en_US:en \ - LC_ALL=en_US.UTF-8 - -COPY scripts/locale.gen /etc/all.locale.gen -RUN if [ -z "${GENERATE_ALL_LOCALE}" ] || [ $GENERATE_ALL_LOCALE -eq 0 ]; \ - then \ - cat /etc/all.locale.gen | grep "${LANG}" > /etc/locale.gen; \ - else \ - cp -f /etc/all.locale.gen /etc/locale.gen; \ - fi; \ - set -eux \ - && /usr/sbin/locale-gen - -RUN update-locale ${LANG} - -# Cleanup resources -RUN apt-get -y --purge autoremove \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - diff --git a/base_build/build.sh b/base_build/build.sh deleted file mode 100755 index 53c3c7f..0000000 --- a/base_build/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -# Building an debian base image - -docker build --build-arg DISTRO=debian --build-arg IMAGE_VERSION=bullseye --build-arg IMAGE_VARIANT=slim -t kartoza/postgis:base-debian-bullseye-slim . diff --git a/build-base.sh b/build-base.sh new file mode 100755 index 0000000..670b3cb --- /dev/null +++ b/build-base.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Building an debian base image + +docker-compose -f docker-compose.build.yml build postgis-base + diff --git a/build-test.sh b/build-test.sh index f01d44c..cba9093 100755 --- a/build-test.sh +++ b/build-test.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash +# For scenario testing purposes -./build.sh - -docker build -t kartoza/postgis:manual-build -f Dockerfile.test . +docker-compose -f docker-compose.build.yml build postgis-test diff --git a/build.sh b/build.sh index 1cf2ea1..da239a6 100755 --- a/build.sh +++ b/build.sh @@ -1,9 +1,3 @@ #!/usr/bin/env bash -POSTGRES_MAJOR_VERSION=13 -POSTGIS_MINOR_RELEASE=1 -pushd base_build -./build.sh -popd -docker build -t kartoza/postgis:manual-build . -docker build --build-arg DISTRO=debian --build-arg IMAGE_VERSION=bullseye --build-arg IMAGE_VARIANT=slim -t kartoza/postgis:${POSTGRES_MAJOR_VERSION}.${POSTGIS_MINOR_RELEASE} . +docker-compose -f docker-compose.build.yml build postgis-prod diff --git a/docker-compose.build.yml b/docker-compose.build.yml new file mode 100644 index 0000000..5bbea86 --- /dev/null +++ b/docker-compose.build.yml @@ -0,0 +1,33 @@ +# Used solely for docker-compose build +version: '3' +services: + postgis-base: + image: kartoza/postgis:base-${DISTRO}-${IMAGE_VERSION}-${IMAGE_VARIANT} + build: + context: . + # Use yaml anchor for reusability + args: &build-args-anchor + DISTRO: ${DISTRO} + IMAGE_VERSION: ${IMAGE_VERSION} + IMAGE_VARIANT: ${IMAGE_VARIANT} + GENERATE_ALL_LOCALE: ${GENERATE_ALL_LOCALE} + POSTGRES_MAJOR_VERSION: ${POSTGRES_MAJOR_VERSION} + POSTGIS_MAJOR_VERSION: ${POSTGIS_MAJOR_VERSION} + POSTGIS_MINOR_RELEASE: ${POSTGIS_MINOR_RELEASE} + target: postgis-base + + postgis-prod: + image: kartoza/postgis:${POSTGRES_MAJOR_VERSION}-${POSTGIS_MAJOR_VERSION}.${POSTGIS_MINOR_RELEASE} + build: + context: . + args: + <<: *build-args-anchor + target: postgis-prod + + postgis-test: + image: kartoza/postgis:manual-build + build: + context: . + args: + <<: *build-args-anchor + target: postgis-test From bd42b7632f94dab30d194cbf7e6eced9d869db93 Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Thu, 28 Jan 2021 01:26:41 +0700 Subject: [PATCH 21/29] Optimize docker buildx cache --- .github/workflows/build-latest.yaml | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-latest.yaml b/.github/workflows/build-latest.yaml index fc76b3b..5777dcc 100644 --- a/.github/workflows/build-latest.yaml +++ b/.github/workflows/build-latest.yaml @@ -59,6 +59,7 @@ jobs: context: . file: Dockerfile push: false + load: true tags: ${{ secrets.DOCKERHUB_REPO }}/postgis:base-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }} build-args: | DISTRO=${{ matrix.distro }} @@ -69,7 +70,7 @@ jobs: POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }} POSTGIS_MINOR_VERSION=${{ matrix.postgresMinorVersion }} cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache target: postgis-base - name: Build prod image @@ -90,9 +91,30 @@ jobs: POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }} POSTGIS_MINOR_VERSION=${{ matrix.postgresMinorVersion }} cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache target: postgis-prod + - name: Build image for testing + id: docker_build_testing_image + uses: docker/build-push-action@v2 + with: + context: . + file: Dockerfile + push: false + load: true + tags: kartoza/postgis:manual-build + build-args: | + DISTRO=${{ matrix.distro }} + IMAGE_VERSION=${{ matrix.imageVersion }} + IMAGE_VARIANT=${{ matrix.imageVariant }} + GENERATE_ALL_LOCALE=0 + POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }} + POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }} + POSTGIS_MINOR_VERSION=${{ matrix.postgresMinorVersion }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,mode=max,dest=/tmp/.buildx-cache + target: postgis-test + run-scenario-tests: runs-on: ubuntu-latest needs: [ build-image ] @@ -145,6 +167,7 @@ jobs: POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }} POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }} POSTGIS_MINOR_VERSION=${{ matrix.postgresMinorVersion }} + cache-from: type=local,src=/tmp/.buildx-cache target: postgis-test - name: Run scenario test From 0b66c28097723fcd73aa01c0f6005e56d72f841f Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Thu, 28 Jan 2021 01:59:12 +0700 Subject: [PATCH 22/29] Add several default locale for testing --- .example.env | 2 ++ .github/workflows/build-latest.yaml | 4 ++++ Dockerfile | 3 +++ base_build/scripts/locale-filter.sh | 9 +++++++++ 4 files changed, 18 insertions(+) create mode 100644 base_build/scripts/locale-filter.sh diff --git a/.example.env b/.example.env index 14f0d9c..7bc4714 100644 --- a/.example.env +++ b/.example.env @@ -10,6 +10,8 @@ IMAGE_VARIANT=slim GENERATE_ALL_LOCALE=1 # Set the language if you need to specify LANG locale at build time LANG=en_US.UTF-8 +# locale filter to include in the locale generator +LANGS="en_US.UTF-8,id_ID.UTF-8" POSTGRES_MAJOR_VERSION=13 POSTGIS_MAJOR_VERSION=3 diff --git a/.github/workflows/build-latest.yaml b/.github/workflows/build-latest.yaml index 5777dcc..6e55bd3 100644 --- a/.github/workflows/build-latest.yaml +++ b/.github/workflows/build-latest.yaml @@ -65,6 +65,7 @@ jobs: DISTRO=${{ matrix.distro }} IMAGE_VERSION=${{ matrix.imageVersion }} IMAGE_VARIANT=${{ matrix.imageVariant }} + LANGS=en_US.UTF-8,id_ID.UTF-8 GENERATE_ALL_LOCALE=0 POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }} POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }} @@ -86,6 +87,7 @@ jobs: DISTRO=${{ matrix.distro }} IMAGE_VERSION=${{ matrix.imageVersion }} IMAGE_VARIANT=${{ matrix.imageVariant }} + LANGS=en_US.UTF-8,id_ID.UTF-8 GENERATE_ALL_LOCALE=0 POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }} POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }} @@ -107,6 +109,7 @@ jobs: DISTRO=${{ matrix.distro }} IMAGE_VERSION=${{ matrix.imageVersion }} IMAGE_VARIANT=${{ matrix.imageVariant }} + LANGS=en_US.UTF-8,id_ID.UTF-8 GENERATE_ALL_LOCALE=0 POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }} POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }} @@ -163,6 +166,7 @@ jobs: DISTRO=${{ matrix.distro }} IMAGE_VERSION=${{ matrix.imageVersion }} IMAGE_VARIANT=${{ matrix.imageVariant }} + LANGS=en_US.UTF-8,id_ID.UTF-8 GENERATE_ALL_LOCALE=0 POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }} POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }} diff --git a/Dockerfile b/Dockerfile index 8a01f5f..b63c9ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,15 +36,18 @@ RUN chmod 0755 /gdal_install.sh;/bin/bash /gdal_install.sh # Generate all locale only on deployment mode build # Set to empty string to generate only default locale ARG GENERATE_ALL_LOCALE=1 +ARG LANGS="en_US.UTF-8,id_ID.UTF-8" ARG LANG=en_US.UTF-8 ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ LC_ALL=en_US.UTF-8 COPY base_build/scripts/locale.gen /etc/all.locale.gen +COPY base_build/scripts/locale-filter.sh /etc/locale-filter.sh RUN if [ -z "${GENERATE_ALL_LOCALE}" ] || [ $GENERATE_ALL_LOCALE -eq 0 ]; \ then \ cat /etc/all.locale.gen | grep "${LANG}" > /etc/locale.gen; \ + /bin/bash /etc/locale-filter.sh; \ else \ cp -f /etc/all.locale.gen /etc/locale.gen; \ fi; \ diff --git a/base_build/scripts/locale-filter.sh b/base_build/scripts/locale-filter.sh new file mode 100644 index 0000000..8b37747 --- /dev/null +++ b/base_build/scripts/locale-filter.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +## Filter list of locales from a given filter args +## Parse into array +LANG_ARR=(${LANGS//,/ }) +echo "" > /etc/locale.gen +for i in "${LANG_ARR[@]}"; do + cat /etc/all.locale.gen | grep "$i" >> /etc/locale.gen +done From 1f708f9312d55905734e9317cdcfe06f581c1491 Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Thu, 28 Jan 2021 02:19:12 +0700 Subject: [PATCH 23/29] Fix docker-compose tty mode --- .github/workflows/build-latest.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-latest.yaml b/.github/workflows/build-latest.yaml index 6e55bd3..0bd2a77 100644 --- a/.github/workflows/build-latest.yaml +++ b/.github/workflows/build-latest.yaml @@ -175,6 +175,8 @@ jobs: target: postgis-test - name: Run scenario test + env: + COMPOSE_INTERACTIVE_NO_CLI: 1 run: | pushd scenario_tests/${{ matrix.scenario }} ./test.sh From 4a42a370693fd27c13b77024e39e6626d9758677 Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Thu, 28 Jan 2021 02:42:40 +0700 Subject: [PATCH 24/29] Improve scenario tests logging --- .github/workflows/build-latest.yaml | 2 + scenario_tests/collations/test.sh | 10 +- scenario_tests/datadir_init/test.sh | 12 +- scenario_tests/extensions/test.sh | 10 +- .../logical_replications/docker-compose.yml | 104 ++++++++++++++++++ .../scripts/setup-publisher.sql | 22 ++++ .../scripts/setup-subscriber.sql | 20 ++++ scenario_tests/logical_replications/test.sh | 33 ++++++ .../logical_replications/tests/__init__.py | 0 .../logical_replications/tests/test_master.sh | 13 +++ .../logical_replications/tests/test_node.sh | 13 +++ .../tests/test_replication.py | 75 +++++++++++++ scenario_tests/streaming_replication/test.sh | 12 +- 13 files changed, 315 insertions(+), 11 deletions(-) create mode 100644 scenario_tests/logical_replications/docker-compose.yml create mode 100644 scenario_tests/logical_replications/scripts/setup-publisher.sql create mode 100644 scenario_tests/logical_replications/scripts/setup-subscriber.sql create mode 100755 scenario_tests/logical_replications/test.sh create mode 100644 scenario_tests/logical_replications/tests/__init__.py create mode 100644 scenario_tests/logical_replications/tests/test_master.sh create mode 100644 scenario_tests/logical_replications/tests/test_node.sh create mode 100644 scenario_tests/logical_replications/tests/test_replication.py diff --git a/.github/workflows/build-latest.yaml b/.github/workflows/build-latest.yaml index 0bd2a77..8d24c4c 100644 --- a/.github/workflows/build-latest.yaml +++ b/.github/workflows/build-latest.yaml @@ -135,6 +135,7 @@ jobs: - collations - extensions - logical_replication + - logical_replications include: - distro: debian imageVersion: bullseye @@ -177,6 +178,7 @@ jobs: - name: Run scenario test env: COMPOSE_INTERACTIVE_NO_CLI: 1 + PRINT_TEST_LOGS: 1 run: | pushd scenario_tests/${{ matrix.scenario }} ./test.sh diff --git a/scenario_tests/collations/test.sh b/scenario_tests/collations/test.sh index 28a3953..703fb2e 100755 --- a/scenario_tests/collations/test.sh +++ b/scenario_tests/collations/test.sh @@ -8,6 +8,10 @@ source ../test-env.sh # Run service docker-compose up -d +if [[ -n "${PRINT_TEST_LOGS}" ]]; then + docker-compose logs -f & +fi + sleep 30 services=("pg" "pg-new") @@ -15,10 +19,12 @@ services=("pg" "pg-new") for service in "${services[@]}"; do # Execute tests - until docker-compose exec $service pg_isready; do + until docker-compose exec -T $service pg_isready; do sleep 30 + echo "Wait service to be ready" done; - docker-compose exec $service /bin/bash /tests/test.sh + echo "Execute test for $service" + docker-compose exec -T $service /bin/bash /tests/test.sh done diff --git a/scenario_tests/datadir_init/test.sh b/scenario_tests/datadir_init/test.sh index 4c041c5..78b212c 100755 --- a/scenario_tests/datadir_init/test.sh +++ b/scenario_tests/datadir_init/test.sh @@ -8,6 +8,10 @@ source ../test-env.sh # Run service docker-compose up -d +if [[ -n "${PRINT_TEST_LOGS}" ]]; then + docker-compose logs -f & +fi + sleep 60 services=("pg-default" "pg-new" "pg-recreate") @@ -15,10 +19,12 @@ services=("pg-default" "pg-new" "pg-recreate") for service in "${services[@]}"; do # Execute tests - until docker-compose exec $service pg_isready; do - sleep 1 + until docker-compose exec -T $service pg_isready; do + sleep 5 + echo "Wait service to be ready" done; - docker-compose exec $service /bin/bash /tests/test.sh + echo "Execute test for $service" + docker-compose exec -T $service /bin/bash /tests/test.sh done diff --git a/scenario_tests/extensions/test.sh b/scenario_tests/extensions/test.sh index d1e4dd7..06131df 100755 --- a/scenario_tests/extensions/test.sh +++ b/scenario_tests/extensions/test.sh @@ -8,6 +8,10 @@ source ../test-env.sh # Run service docker-compose up -d +if [[ -n "${PRINT_TEST_LOGS}" ]]; then + docker-compose logs -f & +fi + sleep 30 services=("pg" "pg-two-extensions") @@ -15,10 +19,12 @@ services=("pg" "pg-two-extensions") for service in "${services[@]}"; do # Execute tests - until docker-compose exec $service pg_isready; do + until docker-compose exec -T $service pg_isready; do sleep 30 + echo "Wait service to be ready" done; - docker-compose exec $service /bin/bash /tests/test.sh + echo "Execute test for $service" + docker-compose exec -T $service /bin/bash /tests/test.sh done diff --git a/scenario_tests/logical_replications/docker-compose.yml b/scenario_tests/logical_replications/docker-compose.yml new file mode 100644 index 0000000..7681b66 --- /dev/null +++ b/scenario_tests/logical_replications/docker-compose.yml @@ -0,0 +1,104 @@ + +version: '2.1' + +volumes: + pg-master-data-dir: + pg-node-data-dir: + +services: + pg-publisher: + image: 'kartoza/postgis:${TAG:-manual-build}' + restart: 'always' + # You can optionally mount to volume, to play with the persistence and + # observe how the node will behave after restarts. + volumes: + - pg-master-data-dir:/var/lib/postgresql + - ./scripts/setup-publisher.sql:/docker-entrypoint-initdb.d/setup-publisher.sql + - ./tests:/tests + - ../utils:/lib/utils + environment: + # ALLOW_IP_RANGE option is used to specify additionals allowed domains + # in pg_hba. + # This range should allow nodes to connect to master + ALLOW_IP_RANGE: '0.0.0.0/0' + + # We can specify optional credentials + REPLICATION_USER: 'replicator' + REPLICATION_PASS: 'replicator' + WAL_LEVEL: 'logical' + # Setup master replication variables + #PG_MAX_WAL_SENDERS: 8 + #PG_WAL_KEEP_SEGMENTS: 100 + # You can expose the port to observe it in your local machine + ports: + - "7777:5432" + healthcheck: + interval: 60s + timeout: 30s + retries: 3 + test: "pg_isready" + + pg-subscriber: + image: 'kartoza/postgis:${TAG:-manual-build}' + restart: 'always' + # You can optionally mount to volume, but we're not able to scale it + # in that case. + # The node will always destroy its database and copy from master at + # runtime + volumes: + - pg-node-data-dir:/var/lib/postgresql + - ./scripts/setup-subscriber.sql:/docker-entrypoint-initdb.d/setup-subscriber.sql + - ./tests:/tests + - ../utils:/lib/utils + + environment: + # ALLOW_IP_RANGE option is used to specify additionals allowed domains + # in pg_hba. + # Not really needed in nodes for the replication, but optionally can + # be put when nodes are needed to be a failover server when master + # is down. The IP Range are generally needed if other services wants to + # connect to this node + ALLOW_IP_RANGE: '0.0.0.0/0' + + # REPLICATE_FROM options accepts domain-name or IP address + # with this in mind, you can also put docker service name, because it + # will be resolved as host name. + #REPLICATE_FROM: 'pg-publisher' + + # REPLICATE_PORT will default to 5432 if not specified. + # REPLICATE_PORT: '5432' + # In the case where you need to replicate from outside service, + # you can put the server address and port here, as long as the target + # where configured as master, and replicable. +# REPLICATE_FROM: '192.168.1.8' +# REPLICATE_PORT: '7777' + + # DESTROY_DATABASE_ON_RESTART will default to True if not specified. + # If specified other than True, it will prevent node from destroying + # database on restart + DESTROY_DATABASE_ON_RESTART: 'True' + + # PROMOTE_MASTER Default empty. + # If specified with any value, then it will convert current node into + # a writable state. Useful if master is down and the current node needs + # to be promoted until manual recovery. +# PROMOTE_MASTER: 'True' + + # For now we don't support different credentials for replication + # so we use the same credentials as master's superuser, or anything that + # have replication role. + REPLICATION_USER: 'replicator' + REPLICATION_PASS: 'replicator' + WAL_LEVEL: 'logical' + depends_on: + pg-publisher: + condition: service_healthy + # You can expose the port to observe it in your local machine + # For this sample, it was disabled by default to allow scaling test + ports: + - "7776:5432" +# healthcheck: +# interval: 60s +# timeout: 30s +# retries: 3 +# test: "pg_isready" diff --git a/scenario_tests/logical_replications/scripts/setup-publisher.sql b/scenario_tests/logical_replications/scripts/setup-publisher.sql new file mode 100644 index 0000000..ffac7ba --- /dev/null +++ b/scenario_tests/logical_replications/scripts/setup-publisher.sql @@ -0,0 +1,22 @@ +-- Create a table +CREATE TABLE sweets + ( + id SERIAL, + name TEXT, + price DECIMAL, + CONSTRAINT sweets_pkey PRIMARY KEY (id) + ); + +CREATE TABLE public.block ( + id serial NOT NULL, + geom public.geometry(Polygon,4326), + fid bigint, + tile_name character varying, + location character varying +); + +-- Add table to publication called logical_replication which is created by the scripts +ALTER PUBLICATION logical_replication ADD TABLE sweets; +ALTER PUBLICATION logical_replication ADD TABLE block; +-- Inserts records into the table +INSERT INTO sweets (name, price) VALUES ('strawberry', 4.50), ('Coffee', 6.20), ('lollipop', 3.80); diff --git a/scenario_tests/logical_replications/scripts/setup-subscriber.sql b/scenario_tests/logical_replications/scripts/setup-subscriber.sql new file mode 100644 index 0000000..0d0e456 --- /dev/null +++ b/scenario_tests/logical_replications/scripts/setup-subscriber.sql @@ -0,0 +1,20 @@ +-- Create a table +CREATE TABLE sweets + ( + id SERIAL, + name TEXT, + price DECIMAL, + CONSTRAINT sweets_pkey PRIMARY KEY (id) + ); + +CREATE TABLE public.block ( + id serial NOT NULL, + geom public.geometry(Polygon,4326), + fid bigint, + tile_name character varying, + location character varying +); +-- Create a publication +CREATE SUBSCRIPTION logical_subscription + CONNECTION 'host=pg-publisher port=5432 password=docker user=docker dbname=gis' + PUBLICATION logical_replication; diff --git a/scenario_tests/logical_replications/test.sh b/scenario_tests/logical_replications/test.sh new file mode 100755 index 0000000..4d4105a --- /dev/null +++ b/scenario_tests/logical_replications/test.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# exit immediately if test fails +set -e + +source ../test-env.sh + +# Run service +docker-compose up -d + +if [[ -n "${PRINT_TEST_LOGS}" ]]; then + docker-compose logs -f & +fi + +sleep 30 + +# Preparing master cluster +until docker-compose exec -T pg-publisher pg_isready; do + sleep 1 +done; + +# Execute tests +docker-compose exec -T pg-publisher /bin/bash /tests/test_master.sh + +# Preparing node cluster +until docker-compose exec -T pg-subscriber pg_isready; do + sleep 1 +done; + +# Execute tests +docker-compose exec -T pg-node /bin/bash /tests/test_node.sh + +docker-compose down -v diff --git a/scenario_tests/logical_replications/tests/__init__.py b/scenario_tests/logical_replications/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scenario_tests/logical_replications/tests/test_master.sh b/scenario_tests/logical_replications/tests/test_master.sh new file mode 100644 index 0000000..fdc4042 --- /dev/null +++ b/scenario_tests/logical_replications/tests/test_master.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -e + +source /scripts/env-data.sh + +# execute tests +pushd /tests + +PGHOST=localhost \ +PGDATABASE=gis \ +PYTHONPATH=/lib \ + python3 -m unittest -v test_replication.TestReplicationMaster diff --git a/scenario_tests/logical_replications/tests/test_node.sh b/scenario_tests/logical_replications/tests/test_node.sh new file mode 100644 index 0000000..b21e3d5 --- /dev/null +++ b/scenario_tests/logical_replications/tests/test_node.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -e + +source /scripts/env-data.sh + +# execute tests +pushd /tests + +PGHOST=localhost \ +PGDATABASE=gis \ +PYTHONPATH=/lib \ + python3 -m unittest -v test_replication.TestReplicationNode diff --git a/scenario_tests/logical_replications/tests/test_replication.py b/scenario_tests/logical_replications/tests/test_replication.py new file mode 100644 index 0000000..b0e46aa --- /dev/null +++ b/scenario_tests/logical_replications/tests/test_replication.py @@ -0,0 +1,75 @@ +import unittest +from utils.utils import DBConnection + + +class TestReplicationMaster(unittest.TestCase): + + def setUp(self): + self.db = DBConnection() + + def test_create_new_data(self): + # create new table + self.db.conn.autocommit = True + with self.db.cursor() as c: + c.execute( + """ + CREATE TABLE IF NOT EXISTS test_replication_table ( + id integer not null + constraint pkey primary key, + geom geometry(Point, 4326), + name varchar(30), + alias varchar(30), + description varchar(255) + ); + """ + ) + + c.execute( + """ + ALTER PUBLICATION logical_replication + ADD TABLE test_replication_table; + """ + ) + + c.execute( + """ + INSERT INTO test_replication_table (id, geom, name, alias, description) + VALUES + ( + 1, + st_setsrid(st_point(107.6097, 6.9120), 4326), + 'Bandung', + 'Paris van Java', + 'Asia-Africa conference was held here' + ) ON CONFLICT DO NOTHING; + """ + ) + + +class TestReplicationNode(unittest.TestCase): + + def setUp(self): + self.db = DBConnection() + + def test_read_data(self): + # create new table + self.db.conn.autocommit = True + with self.db.cursor() as c: + c.execute( + """ + CREATE SUBSCRIPTION logical_subscription + CONNECTION 'host=${PG_PUBLISHER} port=5432 + password=${PG_PASSWORD} user=${PG_USER} dbname=gis' + PUBLICATION logical_replication; + """ + ) + + c.execute( + """ + SELECT * FROM test_replication_table; + """ + ) + + rows = c.fetchall() + self.assertEqual(len(rows), 1) + diff --git a/scenario_tests/streaming_replication/test.sh b/scenario_tests/streaming_replication/test.sh index 9395fc9..5ddb0e8 100755 --- a/scenario_tests/streaming_replication/test.sh +++ b/scenario_tests/streaming_replication/test.sh @@ -8,22 +8,26 @@ source ../test-env.sh # Run service docker-compose up -d +if [[ -n "${PRINT_TEST_LOGS}" ]]; then + docker-compose logs -f & +fi + sleep 30 # Preparing master cluster -until docker-compose exec pg-master pg_isready; do +until docker-compose exec -T pg-master pg_isready; do sleep 30 done; # Execute tests -docker-compose exec pg-master /bin/bash /tests/test_master.sh +docker-compose exec -T pg-master /bin/bash /tests/test_master.sh # Preparing node cluster -until docker-compose exec pg-node pg_isready; do +until docker-compose exec -T pg-node pg_isready; do sleep 30 done; # Execute tests -docker-compose exec pg-node /bin/bash /tests/test_node.sh +docker-compose exec -T pg-node /bin/bash /tests/test_node.sh docker-compose down -v From 14c16edd95202d2a8a7ddf4f75bb3522ff62a2cd Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Thu, 28 Jan 2021 02:54:59 +0700 Subject: [PATCH 25/29] Improve build cache keys --- .github/workflows/build-latest.yaml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-latest.yaml b/.github/workflows/build-latest.yaml index 8d24c4c..ed8aca9 100644 --- a/.github/workflows/build-latest.yaml +++ b/.github/workflows/build-latest.yaml @@ -43,9 +43,13 @@ jobs: uses: actions/cache@v2 with: path: /tmp/.buildx-cache - key: buildx-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }}-${{ github.sha }} + # Build inputs are: + # - Dockerfile recipe + # - docker-compose.build.yml + # - build args (.example.env) + key: buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env') }}-${{ github.sha }} restore-keys: | - buildx-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }}- + buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env') }}- - name: Login to DockerHub uses: docker/login-action@v1 with: @@ -150,9 +154,13 @@ jobs: uses: actions/cache@v2 with: path: /tmp/.buildx-cache - key: buildx-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }}-${{ github.sha }} + # Build inputs are: + # - Dockerfile recipe + # - docker-compose.build.yml + # - build args (.example.env) + key: buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env') }}-${{ github.sha }} restore-keys: | - buildx-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }}- + buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env') }}- - name: Build image for testing id: docker_build_testing_image From 3e5c3717201d6b5f09d450987b1528787d0e2aa4 Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Thu, 28 Jan 2021 02:55:22 +0700 Subject: [PATCH 26/29] Enable travis for comparison --- .disabled.travis.yml => .travis.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .disabled.travis.yml => .travis.yml (100%) diff --git a/.disabled.travis.yml b/.travis.yml similarity index 100% rename from .disabled.travis.yml rename to .travis.yml From 0df962e1f83c15ba8d4e1460764b9ab8e46ba270 Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Wed, 27 Jan 2021 22:00:19 +0000 Subject: [PATCH 27/29] Fix scenario tests - Fix cache keys for GH Action - Disable logical_replication test health-check By disabling health-check, we can immediately see the logs --- .github/workflows/build-latest.yaml | 14 ++- .github/workflows/deploy-image.yaml | 11 +- .gitignore | 1 + .../logical_replication/docker-compose.yml | 3 +- scenario_tests/logical_replication/test.sh | 14 ++- .../tests/test_logical_replication.py | 2 +- .../logical_replications/docker-compose.yml | 104 ------------------ .../scripts/setup-publisher.sql | 22 ---- .../scripts/setup-subscriber.sql | 20 ---- scenario_tests/logical_replications/test.sh | 33 ------ .../logical_replications/tests/__init__.py | 0 .../logical_replications/tests/test_master.sh | 13 --- .../logical_replications/tests/test_node.sh | 13 --- .../tests/test_replication.py | 75 ------------- scripts/setup-conf.sh | 2 +- 15 files changed, 31 insertions(+), 296 deletions(-) delete mode 100644 scenario_tests/logical_replications/docker-compose.yml delete mode 100644 scenario_tests/logical_replications/scripts/setup-publisher.sql delete mode 100644 scenario_tests/logical_replications/scripts/setup-subscriber.sql delete mode 100755 scenario_tests/logical_replications/test.sh delete mode 100644 scenario_tests/logical_replications/tests/__init__.py delete mode 100644 scenario_tests/logical_replications/tests/test_master.sh delete mode 100644 scenario_tests/logical_replications/tests/test_node.sh delete mode 100644 scenario_tests/logical_replications/tests/test_replication.py diff --git a/.github/workflows/build-latest.yaml b/.github/workflows/build-latest.yaml index ed8aca9..15d0384 100644 --- a/.github/workflows/build-latest.yaml +++ b/.github/workflows/build-latest.yaml @@ -47,9 +47,12 @@ jobs: # - Dockerfile recipe # - docker-compose.build.yml # - build args (.example.env) - key: buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env') }}-${{ github.sha }} + # - base_build directory + # - scripts directory + key: buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env', 'base_build', 'scripts') }}-${{ github.sha }} restore-keys: | - buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env') }}- + buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env', 'base_build', 'scripts') }}- + - name: Login to DockerHub uses: docker/login-action@v1 with: @@ -139,7 +142,6 @@ jobs: - collations - extensions - logical_replication - - logical_replications include: - distro: debian imageVersion: bullseye @@ -158,9 +160,11 @@ jobs: # - Dockerfile recipe # - docker-compose.build.yml # - build args (.example.env) - key: buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env') }}-${{ github.sha }} + # - base_build directory + # - scripts directory + key: buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env', 'base_build', 'scripts') }}-${{ github.sha }} restore-keys: | - buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env') }}- + buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env', 'base_build', 'scripts') }}- - name: Build image for testing id: docker_build_testing_image diff --git a/.github/workflows/deploy-image.yaml b/.github/workflows/deploy-image.yaml index 5f5d83d..2398514 100644 --- a/.github/workflows/deploy-image.yaml +++ b/.github/workflows/deploy-image.yaml @@ -48,9 +48,16 @@ jobs: uses: actions/cache@v2 with: path: /tmp/.buildx-cache - key: buildx-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }}-${{ github.sha }} + # Build inputs are: + # - Dockerfile recipe + # - docker-compose.build.yml + # - build args (.example.env) + # - base_build directory + # - scripts directory + key: buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env', 'base_build', 'scripts') }}-${{ github.sha }} restore-keys: | - buildx-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }}- + buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env', 'base_build', 'scripts') }}- + - name: Login to DockerHub uses: docker/login-action@v1 with: diff --git a/.gitignore b/.gitignore index bcf9e43..781602a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ .DS_Store .python-version venv +.venv __pycache__ .env diff --git a/scenario_tests/logical_replication/docker-compose.yml b/scenario_tests/logical_replication/docker-compose.yml index b5f27df..9cb6050 100644 --- a/scenario_tests/logical_replication/docker-compose.yml +++ b/scenario_tests/logical_replication/docker-compose.yml @@ -47,8 +47,7 @@ services: REPLICATION_PASS: 'replicator' REPLICATION: 'true' depends_on: - pg-publisher: - condition: service_healthy + - pg-publisher # You can expose the port to observe it in your local machine # For this sample, it was disabled by default to allow scaling test ports: diff --git a/scenario_tests/logical_replication/test.sh b/scenario_tests/logical_replication/test.sh index b0945c8..36eaf4e 100755 --- a/scenario_tests/logical_replication/test.sh +++ b/scenario_tests/logical_replication/test.sh @@ -8,22 +8,26 @@ source ../test-env.sh # Run service docker-compose up -d -sleep 5 +if [[ -n "${PRINT_TEST_LOGS}" ]]; then + docker-compose logs -f & +fi + +sleep 60 # Preparing publisher cluster -until docker-compose exec pg-publisher pg_isready; do +until docker-compose exec -T pg-publisher pg_isready; do sleep 1 done; # Execute tests -docker-compose exec pg-publisher /bin/bash /tests/test_publisher.sh +docker-compose exec -T pg-publisher /bin/bash /tests/test_publisher.sh # Preparing node cluster -until docker-compose exec pg-subscriber pg_isready; do +until docker-compose exec -T pg-subscriber pg_isready; do sleep 1 done; # Execute tests -docker-compose exec pg-subscriber /bin/bash /tests/test_subscriber.sh +docker-compose exec -T pg-subscriber /bin/bash /tests/test_subscriber.sh docker-compose down -v diff --git a/scenario_tests/logical_replication/tests/test_logical_replication.py b/scenario_tests/logical_replication/tests/test_logical_replication.py index 7286752..49638c4 100644 --- a/scenario_tests/logical_replication/tests/test_logical_replication.py +++ b/scenario_tests/logical_replication/tests/test_logical_replication.py @@ -72,7 +72,7 @@ class TestReplicationSubscriber(unittest.TestCase): try: output = func_action() func_assert(output) - print('Assertion succes') + print('Assertion success') return except Exception as e: last_error = e diff --git a/scenario_tests/logical_replications/docker-compose.yml b/scenario_tests/logical_replications/docker-compose.yml deleted file mode 100644 index 7681b66..0000000 --- a/scenario_tests/logical_replications/docker-compose.yml +++ /dev/null @@ -1,104 +0,0 @@ - -version: '2.1' - -volumes: - pg-master-data-dir: - pg-node-data-dir: - -services: - pg-publisher: - image: 'kartoza/postgis:${TAG:-manual-build}' - restart: 'always' - # You can optionally mount to volume, to play with the persistence and - # observe how the node will behave after restarts. - volumes: - - pg-master-data-dir:/var/lib/postgresql - - ./scripts/setup-publisher.sql:/docker-entrypoint-initdb.d/setup-publisher.sql - - ./tests:/tests - - ../utils:/lib/utils - environment: - # ALLOW_IP_RANGE option is used to specify additionals allowed domains - # in pg_hba. - # This range should allow nodes to connect to master - ALLOW_IP_RANGE: '0.0.0.0/0' - - # We can specify optional credentials - REPLICATION_USER: 'replicator' - REPLICATION_PASS: 'replicator' - WAL_LEVEL: 'logical' - # Setup master replication variables - #PG_MAX_WAL_SENDERS: 8 - #PG_WAL_KEEP_SEGMENTS: 100 - # You can expose the port to observe it in your local machine - ports: - - "7777:5432" - healthcheck: - interval: 60s - timeout: 30s - retries: 3 - test: "pg_isready" - - pg-subscriber: - image: 'kartoza/postgis:${TAG:-manual-build}' - restart: 'always' - # You can optionally mount to volume, but we're not able to scale it - # in that case. - # The node will always destroy its database and copy from master at - # runtime - volumes: - - pg-node-data-dir:/var/lib/postgresql - - ./scripts/setup-subscriber.sql:/docker-entrypoint-initdb.d/setup-subscriber.sql - - ./tests:/tests - - ../utils:/lib/utils - - environment: - # ALLOW_IP_RANGE option is used to specify additionals allowed domains - # in pg_hba. - # Not really needed in nodes for the replication, but optionally can - # be put when nodes are needed to be a failover server when master - # is down. The IP Range are generally needed if other services wants to - # connect to this node - ALLOW_IP_RANGE: '0.0.0.0/0' - - # REPLICATE_FROM options accepts domain-name or IP address - # with this in mind, you can also put docker service name, because it - # will be resolved as host name. - #REPLICATE_FROM: 'pg-publisher' - - # REPLICATE_PORT will default to 5432 if not specified. - # REPLICATE_PORT: '5432' - # In the case where you need to replicate from outside service, - # you can put the server address and port here, as long as the target - # where configured as master, and replicable. -# REPLICATE_FROM: '192.168.1.8' -# REPLICATE_PORT: '7777' - - # DESTROY_DATABASE_ON_RESTART will default to True if not specified. - # If specified other than True, it will prevent node from destroying - # database on restart - DESTROY_DATABASE_ON_RESTART: 'True' - - # PROMOTE_MASTER Default empty. - # If specified with any value, then it will convert current node into - # a writable state. Useful if master is down and the current node needs - # to be promoted until manual recovery. -# PROMOTE_MASTER: 'True' - - # For now we don't support different credentials for replication - # so we use the same credentials as master's superuser, or anything that - # have replication role. - REPLICATION_USER: 'replicator' - REPLICATION_PASS: 'replicator' - WAL_LEVEL: 'logical' - depends_on: - pg-publisher: - condition: service_healthy - # You can expose the port to observe it in your local machine - # For this sample, it was disabled by default to allow scaling test - ports: - - "7776:5432" -# healthcheck: -# interval: 60s -# timeout: 30s -# retries: 3 -# test: "pg_isready" diff --git a/scenario_tests/logical_replications/scripts/setup-publisher.sql b/scenario_tests/logical_replications/scripts/setup-publisher.sql deleted file mode 100644 index ffac7ba..0000000 --- a/scenario_tests/logical_replications/scripts/setup-publisher.sql +++ /dev/null @@ -1,22 +0,0 @@ --- Create a table -CREATE TABLE sweets - ( - id SERIAL, - name TEXT, - price DECIMAL, - CONSTRAINT sweets_pkey PRIMARY KEY (id) - ); - -CREATE TABLE public.block ( - id serial NOT NULL, - geom public.geometry(Polygon,4326), - fid bigint, - tile_name character varying, - location character varying -); - --- Add table to publication called logical_replication which is created by the scripts -ALTER PUBLICATION logical_replication ADD TABLE sweets; -ALTER PUBLICATION logical_replication ADD TABLE block; --- Inserts records into the table -INSERT INTO sweets (name, price) VALUES ('strawberry', 4.50), ('Coffee', 6.20), ('lollipop', 3.80); diff --git a/scenario_tests/logical_replications/scripts/setup-subscriber.sql b/scenario_tests/logical_replications/scripts/setup-subscriber.sql deleted file mode 100644 index 0d0e456..0000000 --- a/scenario_tests/logical_replications/scripts/setup-subscriber.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Create a table -CREATE TABLE sweets - ( - id SERIAL, - name TEXT, - price DECIMAL, - CONSTRAINT sweets_pkey PRIMARY KEY (id) - ); - -CREATE TABLE public.block ( - id serial NOT NULL, - geom public.geometry(Polygon,4326), - fid bigint, - tile_name character varying, - location character varying -); --- Create a publication -CREATE SUBSCRIPTION logical_subscription - CONNECTION 'host=pg-publisher port=5432 password=docker user=docker dbname=gis' - PUBLICATION logical_replication; diff --git a/scenario_tests/logical_replications/test.sh b/scenario_tests/logical_replications/test.sh deleted file mode 100755 index 4d4105a..0000000 --- a/scenario_tests/logical_replications/test.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -# exit immediately if test fails -set -e - -source ../test-env.sh - -# Run service -docker-compose up -d - -if [[ -n "${PRINT_TEST_LOGS}" ]]; then - docker-compose logs -f & -fi - -sleep 30 - -# Preparing master cluster -until docker-compose exec -T pg-publisher pg_isready; do - sleep 1 -done; - -# Execute tests -docker-compose exec -T pg-publisher /bin/bash /tests/test_master.sh - -# Preparing node cluster -until docker-compose exec -T pg-subscriber pg_isready; do - sleep 1 -done; - -# Execute tests -docker-compose exec -T pg-node /bin/bash /tests/test_node.sh - -docker-compose down -v diff --git a/scenario_tests/logical_replications/tests/__init__.py b/scenario_tests/logical_replications/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/scenario_tests/logical_replications/tests/test_master.sh b/scenario_tests/logical_replications/tests/test_master.sh deleted file mode 100644 index fdc4042..0000000 --- a/scenario_tests/logical_replications/tests/test_master.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source /scripts/env-data.sh - -# execute tests -pushd /tests - -PGHOST=localhost \ -PGDATABASE=gis \ -PYTHONPATH=/lib \ - python3 -m unittest -v test_replication.TestReplicationMaster diff --git a/scenario_tests/logical_replications/tests/test_node.sh b/scenario_tests/logical_replications/tests/test_node.sh deleted file mode 100644 index b21e3d5..0000000 --- a/scenario_tests/logical_replications/tests/test_node.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source /scripts/env-data.sh - -# execute tests -pushd /tests - -PGHOST=localhost \ -PGDATABASE=gis \ -PYTHONPATH=/lib \ - python3 -m unittest -v test_replication.TestReplicationNode diff --git a/scenario_tests/logical_replications/tests/test_replication.py b/scenario_tests/logical_replications/tests/test_replication.py deleted file mode 100644 index b0e46aa..0000000 --- a/scenario_tests/logical_replications/tests/test_replication.py +++ /dev/null @@ -1,75 +0,0 @@ -import unittest -from utils.utils import DBConnection - - -class TestReplicationMaster(unittest.TestCase): - - def setUp(self): - self.db = DBConnection() - - def test_create_new_data(self): - # create new table - self.db.conn.autocommit = True - with self.db.cursor() as c: - c.execute( - """ - CREATE TABLE IF NOT EXISTS test_replication_table ( - id integer not null - constraint pkey primary key, - geom geometry(Point, 4326), - name varchar(30), - alias varchar(30), - description varchar(255) - ); - """ - ) - - c.execute( - """ - ALTER PUBLICATION logical_replication - ADD TABLE test_replication_table; - """ - ) - - c.execute( - """ - INSERT INTO test_replication_table (id, geom, name, alias, description) - VALUES - ( - 1, - st_setsrid(st_point(107.6097, 6.9120), 4326), - 'Bandung', - 'Paris van Java', - 'Asia-Africa conference was held here' - ) ON CONFLICT DO NOTHING; - """ - ) - - -class TestReplicationNode(unittest.TestCase): - - def setUp(self): - self.db = DBConnection() - - def test_read_data(self): - # create new table - self.db.conn.autocommit = True - with self.db.cursor() as c: - c.execute( - """ - CREATE SUBSCRIPTION logical_subscription - CONNECTION 'host=${PG_PUBLISHER} port=5432 - password=${PG_PASSWORD} user=${PG_USER} dbname=gis' - PUBLICATION logical_replication; - """ - ) - - c.execute( - """ - SELECT * FROM test_replication_table; - """ - ) - - rows = c.fetchall() - self.assertEqual(len(rows), 1) - diff --git a/scripts/setup-conf.sh b/scripts/setup-conf.sh index 1850cc4..d8e709a 100644 --- a/scripts/setup-conf.sh +++ b/scripts/setup-conf.sh @@ -45,7 +45,7 @@ if [[ "${REPLICATION}" =~ [Tt][Rr][Uu][Ee] && "$WAL_LEVEL" == 'logical' ]]; the if [[ -f ${ROOT_CONF}/logical_replication.conf ]];then rm $CONF/logical_replication.conf fi -cat >> ${ROOT_CONF}/streaming_replication.conf <> ${ROOT_CONF}/logical_replication.conf < Date: Thu, 28 Jan 2021 00:04:40 +0000 Subject: [PATCH 28/29] Fix travis --- .travis.yml | 1 + build-base.sh | 5 +++++ build-test.sh | 5 +++++ build.sh | 5 +++++ requirements.txt | 1 + 5 files changed, 17 insertions(+) create mode 100644 requirements.txt diff --git a/.travis.yml b/.travis.yml index 658c33c..a6b766d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ env: - SCENARIO=logical_replication before_script: + - pip install -U docker-compose - ./build-test.sh script: diff --git a/build-base.sh b/build-base.sh index 670b3cb..ab8be6d 100755 --- a/build-base.sh +++ b/build-base.sh @@ -1,5 +1,10 @@ #!/usr/bin/env bash # Building an debian base image +if [[ ! -f .env ]]; then + echo "Default build arguments don't exists. Creating one from default value." + cp .example.env .env +fi + docker-compose -f docker-compose.build.yml build postgis-base diff --git a/build-test.sh b/build-test.sh index cba9093..08aff95 100755 --- a/build-test.sh +++ b/build-test.sh @@ -1,4 +1,9 @@ #!/usr/bin/env bash # For scenario testing purposes +if [[ ! -f .env ]]; then + echo "Default build arguments don't exists. Creating one from default value." + cp .example.env .env +fi + docker-compose -f docker-compose.build.yml build postgis-test diff --git a/build.sh b/build.sh index da239a6..3d5f058 100755 --- a/build.sh +++ b/build.sh @@ -1,3 +1,8 @@ #!/usr/bin/env bash +if [[ ! -f .env ]]; then + echo "Default build arguments don't exists. Creating one from default value." + cp .example.env .env +fi + docker-compose -f docker-compose.build.yml build postgis-prod diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d765ccc --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +docker-compose==1.28 \ No newline at end of file From 6d3c1c1af47f39c29b016fda29bf0e6143a2b72f Mon Sep 17 00:00:00 2001 From: Rizky Maulana Nugraha Date: Thu, 28 Jan 2021 00:31:53 +0000 Subject: [PATCH 29/29] Goodbye Travis I don't really know why the build failed in Travis, even though it is using the same scripts. Thank you for all the hardwork. --- .travis.yml | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index a6b766d..0000000 --- a/.travis.yml +++ /dev/null @@ -1,25 +0,0 @@ -sudo: required - -language: python - -services: - - docker - -python: - - '3.7' - -env: - - SCENARIO=datadir_init - - SCENARIO=streaming_replication - - SCENARIO=collations - - SCENARIO=extensions - - SCENARIO=logical_replication - -before_script: - - pip install -U docker-compose - - ./build-test.sh - -script: - - pushd scenario_tests/${SCENARIO} - - ./test.sh - - popd