Upgrade version to 13.0 (#276)

* Upgrade version to 13.0

* Fix failing tests and add more env variable customisations

* Use args for building versions

* remove unneeded line

Dockerfile

@@ -7,10 +7,13 @@ MAINTAINER Tim Sutton<tim@kartoza.com>
 
 # Reset ARG for version
 ARG IMAGE_VERSION
+ARG POSTGRES_MAJOR_VERSION=13
+ARG POSTGIS_MAJOR=3
+
 
 RUN set -eux \
     && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get update \
+    && apt-get upgrade;apt-get update \
     && sh -c "echo \"deb http://apt.postgresql.org/pub/repos/apt/ ${IMAGE_VERSION}-pgdg main\" > /etc/apt/sources.list.d/pgdg.list" \
     && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc -O- | apt-key add - \
     && apt-get -y --purge autoremove \
@@ -26,11 +29,17 @@ RUN set -eux \
 # The following packages have unmet dependencies:
 RUN set -eux \
     && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get update \
-    && apt-get -y --no-install-recommends install postgresql-client-12 \
-        postgresql-common postgresql-12 postgresql-12-postgis-3 \
-        netcat postgresql-12-ogr-fdw postgresql-12-postgis-3-scripts \
-        postgresql-12-cron postgresql-plpython3-12 postgresql-12-pgrouting postgresql-server-dev-12
+    &&  apt-get update \
+    && apt-get -y --no-install-recommends install postgresql-client-${POSTGRES_MAJOR_VERSION} \
+        postgresql-common postgresql-${POSTGRES_MAJOR_VERSION} \
+        postgresql-${POSTGRES_MAJOR_VERSION}-postgis-${POSTGIS_MAJOR} \
+        netcat postgresql-${POSTGRES_MAJOR_VERSION}-ogr-fdw \
+        postgresql-${POSTGRES_MAJOR_VERSION}-postgis-${POSTGIS_MAJOR}-scripts \
+        postgresql-plpython3-${POSTGRES_MAJOR_VERSION} postgresql-${POSTGRES_MAJOR_VERSION}-pgrouting \
+        postgresql-server-dev-${POSTGRES_MAJOR_VERSION}
+
+
+RUN echo $POSTGRES_MAJOR_VERSION >/tmp/pg_version.txt
 
 # Compile pointcloud extension
 

README.md

@@ -35,7 +35,7 @@ kartoza/postgis:[postgres_major_version]-[postgis-point-releases]
 
 So for example:
 
-``kartoza/postgis:12.0`` Provides PostgreSQL 12.0, PostGIS 3.0
+``kartoza/postgis:13.0`` Provides PostgreSQL 13.0, PostGIS 3.0
 
 **Note:** We highly recommend that you use tagged versions because
 successive minor versions of PostgreSQL write their database clusters
@@ -60,7 +60,7 @@ docker pull kartoza/postgis:image_version
 ## Building the image
 
 To build the image yourself without apt-cacher (also consumes more bandwidth
-since deb packages need to be refetched each time you build) do:
+since deb packages need to be fetched each time you build) do:
 
 ```
 docker build -t kartoza/postgis git://github.com/kartoza/docker-postgis
@@ -79,6 +79,11 @@ Then do:
 docker build -t kartoza/postgis .
 ```
 
+Or
+
+```
+docker build --build-arg POSTGRES_MAJOR_VERSION=13 --build-arg POSTGIS_MAJOR=3 -t kartoza/postgis:POSTGRES_MAJOR_VERSION .
+```
 #### Alternative base distributions builds
 
 There are build args for `DISTRO` (=debian), `IMAGE_VERSION` (=buster)
@@ -382,17 +387,13 @@ See [the postgres documentation about encoding](https://www.postgresql.org/docs/
 
 ## Postgres Replication Setup
 
-The image supports replication out of the box. The two mains replication methods allowed are
+The image supports replication out of the box. By default replication is turned off.
+The two mains replication methods allowed are
 * Streaming replication
 * Logical replication
 
-You can also use the environment variable `-e REPLICATION=false` that disables replication.
-This can be useful in situation you need to have a database for running example Unit tests.
-
-`docker run --name "repl" -e REPLICATION=false -it  kartoza/postgis:12.0`
-
 ### Streaming replication
-By default a running container will support streaming replication.
+
 Replication allows you to maintain two or more synchronised copies of a database, with a
 single **master** copy and one or more **replicant** copies. The animation below illustrates
 this - the layer with the red boundary is accessed from the master database and the layer
@@ -400,6 +401,11 @@ with the green fill is accessed from the replicant database. When edits to the m
 layer are saved, they are automatically propagated to the replicant. Note also that the
 replicant is read-only.
 
+```
+docker run --name "streaming-replication" -e REPLICATION=true -e WAL_LEVEL='replica' -d -p 25432:5432  kartoza/postgis:13.0
+```
+
+
 ![qgis](https://user-images.githubusercontent.com/178003/37755610-dd3b774a-2dae-11e8-9fa1-4877e2034675.gif)
 
 This image is provided with replication abilities. We can
@@ -533,7 +539,7 @@ To activate the following you need to use the environment variable
 `WAL_LEVEL=logical` to get a running instance like
 
 ```
-docker run --name "logical-replication" -e WAL_LEVEL=logical -d  kartoza/postgis:12.0
+docker run --name "logical-replication" -e WAL_LEVEL=logical -d  kartoza/postgis:13.0
 ```
 For a detailed example see the docker-compose in the folder `sample/logical_replication`.
 
@@ -549,4 +555,4 @@ Tim Sutton (tim@kartoza.com)
 Gavin Fleming (gavin@kartoza.com)
 Rizky Maulana (rizky@kartoza.com)
 Admire Nyakudya (admire@kartoza.com)
-December 2018
+October 2020

build.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 docker build -t kartoza/postgis:manual-build .
-docker build -t kartoza/postgis:12.0 .
+docker build -t kartoza/postgis:13.0 .

sample/logical_replication/docker-compose.yml

@@ -8,7 +8,7 @@ volumes:
 
 services:
   pg-publisher:
-    image: kartoza/postgis:12.0
+    image: kartoza/postgis:13.0
     restart: 'always'
     volumes:
       - pg-publisher-data-dir:/var/lib/postgresql
@@ -17,6 +17,7 @@ services:
       ALLOW_IP_RANGE: '0.0.0.0/0'
       REPLICATION_USER: 'replicator'
       REPLICATION_PASS: 'replicator'
+      REPLICATION: 'true'
       WAL_LEVEL: 'logical'
     ports:
       - "7777:5432"
@@ -24,7 +25,7 @@ services:
       test: "exit 0"
 
   pg-subscriber:
-    image: kartoza/postgis:12.0
+    image: kartoza/postgis:13.0
     restart: 'always'
     volumes:
       - pg-subscriber-data-dir:/var/lib/postgresql
@@ -33,6 +34,7 @@ services:
       ALLOW_IP_RANGE: '0.0.0.0/0'
       REPLICATION_USER: 'replicator'
       REPLICATION_PASS: 'replicator'
+      REPLICATION: 'true'
       WAL_LEVEL: 'logical'
     depends_on:
         pg-publisher:

sample/replication/docker-compose.yml

@@ -8,7 +8,7 @@ volumes:
 
 services:
   pg-master:
-    image: 'kartoza/postgis:manual-build'
+    image: 'kartoza/postgis:13.0'
     restart: 'always'
     # You can optionally mount to volume, to play with the persistence and
     # observe how the slave will behave after restarts.
@@ -24,6 +24,7 @@ services:
       # We can specify optional credentials
       REPLICATION_USER: 'replicator'
       REPLICATION_PASS: 'replicator'
+      REPLICATION: 'true'
       # Setup master replication variables
       #PG_MAX_WAL_SENDERS: 8
       #PG_WAL_KEEP_SEGMENTS: 100
@@ -34,7 +35,7 @@ services:
       test: "exit 0"
 
   pg-slave:
-    image: 'kartoza/postgis:manual-build'
+    image: 'kartoza/postgis:13.0'
     restart: 'always'
     # You can optionally mount to volume, but we're not able to scale it
     # in that case.
@@ -57,6 +58,7 @@ services:
       # with this in mind, you can also put docker service name, because it
       # will be resolved as host name.
       REPLICATE_FROM: 'pg-master'
+      REPLICATION: 'true'
 
       # REPLICATE_PORT will default to 5432 if not specified.
       # REPLICATE_PORT: '5432'

scenario_tests/logical_replication/docker-compose.yml

@@ -19,6 +19,7 @@ services:
       ALLOW_IP_RANGE: '0.0.0.0/0'
       REPLICATION_USER: 'replicator'
       REPLICATION_PASS: 'replicator'
+      REPLICATION: 'true'
       WAL_LEVEL: 'logical'
     ports:
       - "7777:5432"
@@ -44,6 +45,7 @@ services:
       WAL_LEVEL: 'logical'
       REPLICATION_USER: 'replicator'
       REPLICATION_PASS: 'replicator'
+      REPLICATION: 'true'
     depends_on:
         pg-publisher:
           condition: service_healthy

scenario_tests/replications/docker-compose.yml

@@ -24,6 +24,7 @@ services:
       # We can specify optional credentials
       REPLICATION_USER: 'replicator'
       REPLICATION_PASS: 'replicator'
+      REPLICATION: 'true'
       # Setup master replication variables
       #PG_MAX_WAL_SENDERS: 8
       #PG_WAL_KEEP_SEGMENTS: 100
@@ -61,6 +62,7 @@ services:
       # with this in mind, you can also put docker service name, because it
       # will be resolved as host name.
       REPLICATE_FROM: 'pg-master'
+      REPLICATION: 'true'
 
       # REPLICATE_PORT will default to 5432 if not specified.
       # REPLICATE_PORT: '5432'

scripts/env-data.sh

@@ -1,20 +1,20 @@
 #!/usr/bin/env bash
-
-DEFAULT_DATADIR="/var/lib/postgresql/12/main"
-ROOT_CONF="/etc/postgresql/12/main"
+POSTGRES_MAJOR_VERSION=$(cat /tmp/pg_version.txt)
+DEFAULT_DATADIR="/var/lib/postgresql/${POSTGRES_MAJOR_VERSION}/main"
+ROOT_CONF="/etc/postgresql/${POSTGRES_MAJOR_VERSION}/main"
 PG_ENV="$ROOT_CONF/environment"
 CONF="$ROOT_CONF/postgresql.conf"
 WAL_ARCHIVE="/opt/archivedir"
 RECOVERY_CONF="$ROOT_CONF/recovery.conf"
-POSTGRES="/usr/lib/postgresql/12/bin/postgres"
-INITDB="/usr/lib/postgresql/12/bin/initdb"
-SQLDIR="/usr/share/postgresql/12/contrib/postgis-3.0/"
+POSTGRES="/usr/lib/postgresql/${POSTGRES_MAJOR_VERSION}/bin/postgres"
+INITDB="/usr/lib/postgresql/${POSTGRES_MAJOR_VERSION}/bin/initdb"
+SQLDIR="/usr/share/postgresql/${POSTGRES_MAJOR_VERSION}/contrib/postgis-3.0/"
 SETVARS="POSTGIS_ENABLE_OUTDB_RASTERS=1 POSTGIS_GDAL_ENABLED_DRIVERS=ENABLE_ALL"
 LOCALONLY="-c listen_addresses='127.0.0.1'"
 PG_BASEBACKUP="/usr/bin/pg_basebackup"
 PROMOTE_FILE="/tmp/pg_promote_master"
 PGSTAT_TMP="/var/run/postgresql/"
-PG_PID="/var/run/postgresql/12-main.pid"
+PG_PID="/var/run/postgresql/${POSTGRES_MAJOR_VERSION}-main.pid"
 
 
 # Read data from secrets into env variables.
@@ -101,7 +101,7 @@ fi
 # Replication settings
 
 if [ -z "${REPLICATION}" ]; then
-	REPLICATION=true
+	REPLICATION=false
 fi
 if [ -z "${REPLICATE_PORT}" ]; then
 	REPLICATE_PORT=5432
@@ -112,8 +112,8 @@ fi
 if [ -z "${PG_MAX_WAL_SENDERS}" ]; then
 	PG_MAX_WAL_SENDERS=10
 fi
-if [ -z "${PG_WAL_KEEP_SEGMENTS}" ]; then
-	PG_WAL_KEEP_SEGMENTS=20
+if [ -z "${PG_WAL_KEEP_SIZE}" ]; then
+	PG_WAL_KEEP_SIZE=20
 fi
 
 
@@ -164,11 +164,23 @@ if [ -z "${WAL_SIZE}" ]; then
 fi
 
 if [ -z "${MIN_WAL_SIZE}" ]; then
-	MIN_WAL_SIZE=2048MB
+	MIN_WAL_SIZE=1024MB
 fi
 
 if [ -z "${WAL_SEGSIZE}" ]; then
-	WAL_SEGSIZE=256
+	WAL_SEGSIZE=32
+fi
+
+if [ -z "${SHARED_BUFFERS}" ]; then
+	SHARED_BUFFERS=256MB
+fi
+
+if [ -z "${WORK_MEM}" ]; then
+	WORK_MEM=16MB
+fi
+
+if [ -z "${WAL_BUFFERS}" ]; then
+	WAL_BUFFERS=1MB
 fi
 
 if [ -z "${CHECK_POINT_TIMEOUT}" ]; then
@@ -237,11 +249,11 @@ if [ -z "$EXTRA_CONF" ]; then
 fi
 
 if [ -z "${SHARED_PRELOAD_LIBRARIES}" ]; then
-    SHARED_PRELOAD_LIBRARIES='pg_cron'
+    SHARED_PRELOAD_LIBRARIES=''
 fi
 
 if [ -z "$PASSWORD_AUTHENTICATION" ]; then
-    PASSWORD_AUTHENTICATION="md5"
+    PASSWORD_AUTHENTICATION="scram-sha-256"
 fi
 
 # Compatibility with official postgres variable

scripts/setup-conf.sh

@@ -24,10 +24,10 @@ echo "data_directory = '${DATADIR}'" >> $CONF
 cat >> $CONF <<EOF
 superuser_reserved_connections= 10
 listen_addresses = '${IP_LIST}'
-shared_buffers = 500MB
-work_mem = 16MB
+shared_buffers = ${SHARED_BUFFERS}
+work_mem = ${WORK_MEM}
 maintenance_work_mem = ${MAINTAINANCE_WORK_MEM}
-wal_buffers = 1MB
+wal_buffers = ${WAL_BUFFERS}
 random_page_cost = 2.0
 xmloption = 'document'
 max_parallel_maintenance_workers = ${MAINTAINANCE_WORKERS}
@@ -46,7 +46,7 @@ if [[  "${REPLICATION}" =~ [Tt][Rr][Uu][Ee] && "$WAL_LEVEL" == 'logical' ]]; the
 cat >> "$CONF" <<EOF
 wal_level = ${WAL_LEVEL}
 max_wal_senders = ${PG_MAX_WAL_SENDERS}
-wal_keep_segments = ${PG_WAL_KEEP_SEGMENTS}
+wal_keep_size = ${PG_WAL_KEEP_SIZE}
 min_wal_size = ${MIN_WAL_SIZE}
 max_wal_size = ${WAL_SIZE}
 max_logical_replication_workers = ${MAX_LOGICAL_REPLICATION_WORKERS}
@@ -62,7 +62,7 @@ archive_command = '${ARCHIVE_COMMAND}'
 restore_command = '${RESTORE_COMMAND}'
 archive_cleanup_command = '${ARCHIVE_CLEANUP_COMMAND}'
 max_wal_senders = ${PG_MAX_WAL_SENDERS}
-wal_keep_segments = ${PG_WAL_KEEP_SEGMENTS}
+wal_keep_size = ${PG_WAL_KEEP_SIZE}
 min_wal_size = ${MIN_WAL_SIZE}
 max_wal_size = ${WAL_SIZE}
 hot_standby = on

scripts/setup-database.sh

@@ -78,11 +78,7 @@ for db in $(echo ${POSTGRES_DBNAME} | tr ',' ' '); do
          echo "${db} db already exists"
         fi
 done
-CRON_LOCKFILE="${ROOT_CONF}/.cron_ext.lock"
-if [ ! -f "${CRON_LOCKFILE}" ]; then
-	su - postgres -c "psql -c 'CREATE EXTENSION IF NOT EXISTS pg_cron cascade;' ${SINGLE_DB}"
-	touch ${CRON_LOCKFILE}
-fi
+
 
 # This should show up in docker logs afterwards
 su - postgres -c "psql -l 2>&1"

scripts/setup-replication.sh

@@ -38,10 +38,7 @@ until su - postgres -c "${PG_BASEBACKUP} -X stream -h ${REPLICATE_FROM} -p ${REP
 }
 
 
-
-if [[ "$WAL_LEVEL" == 'logical' ]]; then
-	echo "We have setup logical replication"
-elif [[ "$WAL_LEVEL" == 'replica' ]]; then
+if [[ "$WAL_LEVEL" == 'replica' && "${REPLICATION}" =~ [Tt][Rr][Uu][Ee] ]]; then
   # No content yet - but this is a slave database
   if [ -z "${REPLICATE_FROM}" ]; then
     echo "You have not set REPLICATE_FROM variable."
@@ -62,10 +59,11 @@ elif [[ "$WAL_LEVEL" == 'replica' ]]; then
       streaming_replication
     fi
  fi
-
-fi
-
-# Promote to master if desired
+ # Promote to master if desired
 if [[ ! -z "${PROMOTE_MASTER}" ]]; then
 	touch ${PROMOTE_FILE}
 fi
+
+fi
+
+