Port changes for Postgres version 11 (#114)

* Fix typo in helper script

* Port work in develop to 10 branch (#90)

* Port 9.6 to develop (#89)

* Part one of porting work from 9.6 to 10

* Backported more scripts from 9.6 branch

* Added missing apt update in dockerfile

* Updates to entrypoint to reference image and update docker-compose to reference 10 pg

* Added sample and docs from 9.6 branch

* Removed my diagram as Rizky had already added one

* Fix env paths for pg 10

* Fixes for backporting work from 9.6 to 10 - dbb now spins up and accepts connections properly

* Update README.md

* Backport from branch: 9.6-2.4

Fix default datadir

- Change into default datadir
- Add small unittest

* Optimise PostgreSQL performance and align with the changes done in 9.6
version

* Fix version numbers

* Minor change to correct env file

* Merged 10 branch into develop

* Remove reduntant conf file from dockerfile

* Remove reduntant conf file directive from setup script

* Remove reduntant conf file directive from  script

* Migrate settings to be compatible with PostgreSQL version 11

* fix ssl setup

Dockerfile

@@ -7,6 +7,7 @@ ENV  DEBIAN_FRONTEND noninteractive
 RUN  dpkg-divert --local --rename --add /sbin/initctl
 
 RUN apt-get -y update; apt-get -y install gnupg2 wget ca-certificates rpl pwgen
+
 RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ stretch-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
 RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
 
@@ -15,7 +16,7 @@ RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-k
 # We add postgis as well to prevent build errors (that we dont see on local builds)
 # on docker hub e.g.
 # The following packages have unmet dependencies:
-RUN apt-get update; apt-get install -y postgresql-client-10 postgresql-common postgresql-10 postgresql-10-postgis-2.4 postgresql-10-pgrouting netcat
+RUN apt-get update; apt-get install -y postgresql-client-11 postgresql-common postgresql-11 postgresql-11-postgis-2.5 postgresql-11-pgrouting netcat
 
 # Open port 5432 so linked containers can see them
 EXPOSE 5432

build.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 docker build -t kartoza/postgis:manual-build .
-docker build -t kartoza/postgis:10.0-2.4 .
+docker build -t kartoza/postgis:11 .

docker-entrypoint.sh

@@ -4,15 +4,36 @@
 set -e
 
 # Setup postgres CONF file
-source /setup-conf.sh
+if grep -rlq "#user-settings" /etc/postgresql/9.6/main/postgresql.conf
+then
+    echo "postgres conf already configured"
+else
+    source /setup-conf.sh
+fi
+
 
 # Setup ssl
-source /setup-ssl.sh
+
+# Setup ssl
+if grep -rlq "ssl-cert-snakeoil.pem" /etc/postgresql/9.6/main/postgresql.conf
+then
+    echo "ssl already configured"
+else
+    echo "SSL not configures so proceed to setup"
+    source /setup-ssl.sh
+
+fi
 
 # Setup pg_hba.conf
-source /setup-pg_hba.sh
+if grep -rlq "172.0.0.0/8" /etc/postgresql/9.6/main/pg_hba.conf
+then
+    echo "pg_hba  already configured"
+else
+    echo "we will setup pg_hba conf"
+    source /setup-pg_hba.sh
+fi
 
-if [ -z "$REPLICATE_FROM" ]; then
+if [[ -z "$REPLICATE_FROM" ]]; then
 	# This means this is a master instance. We check that database exists
 	echo "Setup master database"
 	source /setup-database.sh
@@ -37,7 +58,7 @@ for f in /docker-entrypoint-initdb.d/*; do
 done
 
 # If no arguments passed to entrypoint, then run postgres by default
-if [ $# -eq 0 ];
+if [[ $# -eq 0 ]];
 then
 	echo "Postgres initialisation process completed .... restarting in foreground"
 

env-data.sh

@@ -1,19 +1,19 @@
 #!/usr/bin/env bash
 
-DATADIR="/var/lib/postgresql/10/main"
+DATADIR="/var/lib/postgresql/11/main"
-ROOT_CONF="/etc/postgresql/10/main"
+ROOT_CONF="/etc/postgresql/11/main"
 CONF="$ROOT_CONF/postgresql.conf"
 WAL_ARCHIVE="/opt/archivedir"
 RECOVERY_CONF="$ROOT_CONF/recovery.conf"
-POSTGRES="/usr/lib/postgresql/10/bin/postgres"
+POSTGRES="/usr/lib/postgresql/11/bin/postgres"
-INITDB="/usr/lib/postgresql/10/bin/initdb"
+INITDB="/usr/lib/postgresql/11/bin/initdb"
-SQLDIR="/usr/share/postgresql/10/contrib/postgis-2.4/"
+SQLDIR="/usr/share/postgresql/11/contrib/postgis-2.5/"
 SETVARS="POSTGIS_ENABLE_OUTDB_RASTERS=1 POSTGIS_GDAL_ENABLED_DRIVERS=ENABLE_ALL"
 LOCALONLY="-c listen_addresses='127.0.0.1'"
 PG_BASEBACKUP="/usr/bin/pg_basebackup"
 PROMOTE_FILE="/tmp/pg_promote_master"
 PGSTAT_TMP="/var/run/postgresql/"
-PG_PID="/var/run/postgresql/10-main.pid"
+PG_PID="/var/run/postgresql/11-main.pid"
 
 # Make sure we have a user set up
 if [ -z "${POSTGRES_USER}" ]; then

setup-conf.sh

@@ -24,6 +24,7 @@ random_page_cost = 2.0
 xmloption = 'document'
 #archive_mode=on
 #archive_command = 'test ! -f ${WAL_ARCHIVE}/%f && cp -r %p ${WAL_ARCHIVE}/%f'
+#user-settings
 EOF
 
 # Optimise PostgreSQL shared memory for PostGIS

setup-database.sh

@@ -5,7 +5,7 @@ source /env-data.sh
 # This script will setup the necessary folder for database
 
 # test if DATADIR is existent
-if [ ! -d ${DATADIR} ]; then
+if [[ ! -d ${DATADIR} ]]; then
 	echo "Creating Postgres data at ${DATADIR}"
 	mkdir -p ${DATADIR}
 fi
@@ -17,7 +17,7 @@ chown -R postgres:postgres ${DATADIR}
 
 
 # test if DATADIR has content
-if [ ! "$(ls -A ${DATADIR})" ]; then
+if [[ ! "$(ls -A ${DATADIR})" ]]; then
 	# No content yet - first time pg is being run!
 	# No Replicate From settings. Assume that this is a master database.
 	# Initialise db
@@ -28,7 +28,6 @@ fi
 
 # test database existing
 trap "echo \"Sending SIGTERM to postgres\"; killall -s SIGTERM postgres" SIGTERM
-echo "Use modified postgresql.conf for greater speed (spatial and replication)"
 
 
 
@@ -102,10 +101,10 @@ fi
 su - postgres -c "psql -l"
 
 # Kill postgres
-PID=`cat $PG_PID`
+PID=`cat ${PG_PID}`
 kill -TERM ${PID}
 
 # Wait for background postgres main process to exit
-while [ "$(ls -A ${PG_PID} 2>/dev/null)" ]; do
+while [[ "$(ls -A ${PG_PID} 2>/dev/null)" ]]; do
   sleep 1
 done

setup-replication.sh

@@ -6,14 +6,12 @@ source /env-data.sh
 
 # Adapted from https://github.com/DanielDent/docker-postgres-replication
 # To set up replication
-if [[ "$DESTROY_DATABASE_ON_RESTART" =~ [Tt][Rr][Uu][Ee] ]]; then
-	echo "Destroy initial database, if any."
-	rm -rf $DATADIR
-fi
 
-mkdir -p $DATADIR
+
-chown -R postgres:postgres $DATADIR
+
-chmod -R 700 $DATADIR
+mkdir -p ${DATADIR}
+chown -R postgres:postgres ${DATADIR}
+chmod -R 700 ${DATADIR}
 
 # No content yet - but this is a slave database
 until ping -c 1 -W 1 ${REPLICATE_FROM}
@@ -22,22 +20,52 @@ do
 	sleep 1s
 done
 
-if [[ "$DESTROY_DATABASE_ON_RESTART" =~ [Tt][Rr][Uu][Ee] ]]; then
+function configure_replication_permissions {
-	echo "Get initial database from master"
 
+    echo "Setup data permissions"
+    echo "----------------------"
     chown -R postgres:postgres $(getent passwd postgres  | cut -d: -f6)
         su - postgres -c "echo \"${REPLICATE_FROM}:${REPLICATE_PORT}:*:${POSTGRES_USER}:${POSTGRES_PASS}\" > ~/.pgpass"
         su - postgres -c "chmod 0600 ~/.pgpass"
+}
 
-	until su - postgres -c "${PG_BASEBACKUP} -X stream -h ${REPLICATE_FROM} -p ${REPLICATE_PORT} -D ${DATADIR} -U ${POSTGRES_USER} -vP -w"
+function streaming_replication {
+until su - postgres -c "${PG_BASEBACKUP} -X stream -h ${REPLICATE_FROM} -p ${REPLICATE_PORT} -D ${DATADIR} -U ${POSTGRES_USER} -vP -w"
 	do
 		echo "Waiting for master to connect..."
 		sleep 1s
-		if [ "$(ls -A $DATADIR)" ]; then
+		if [[ "$(ls -A ${DATADIR})" ]]; then
 			echo "Need empty folder. Cleaning directory..."
-			rm -rf $DATADIR/*
+			rm -rf ${DATADIR}/*
 		fi
 	done
+
+}
+
+var=`du -sh /var/lib/postgresql/11/main/pg_wal | awk '{print $1}'`
+var_size=${var:0:2}
+
+if [[ "$DESTROY_DATABASE_ON_RESTART" =~ [Tt][Rr][Uu][Ee] ]]; then
+	echo "Get initial database from master"
+
+	configure_replication_permissions
+
+	streaming_replication
+else
+    echo "Destroy database has been set to false: Check Backup directory if it already exists"
+    configure_replication_permissions
+    # We need a clever way to identify if base backup exists
+
+
+    if [[ "${var_size} -gt  40" ]]; then
+			echo "Base directory exist - Please startup the database"
+	else
+        echo "Base directory does not exists- Create a new one"
+	   streaming_replication
+	fi
+
+
+
 fi
 
 # Setup recovery.conf, a configuration file for slave
@@ -45,12 +73,13 @@ cat > ${DATADIR}/recovery.conf <<EOF
 standby_mode = on
 primary_conninfo = 'host=${REPLICATE_FROM} port=${REPLICATE_PORT} user=${POSTGRES_USER} password=${POSTGRES_PASS} sslmode=${PGSSLMODE}'
 trigger_file = '${PROMOTE_FILE}'
+#restore_command = 'cp /opt/archive/%f "%p"' Use if you are syncing the wal segments from master
 EOF
 # Setup permissions. Postgres won't start without this.
 chown postgres ${DATADIR}/recovery.conf
 chmod 600 ${DATADIR}/recovery.conf
 
 # Promote to master if desired
-if [ ! -z "$PROMOTE_MASTER" ]; then
+if [[ ! -z "${PROMOTE_MASTER}" ]]; then
-	touch $PROMOTE_FILE
+	touch ${PROMOTE_FILE}
 fi

setup.sh

@@ -14,15 +14,6 @@ echo "host    all             all             192.168.0.0/16               md5"
 echo "listen_addresses = '*'" >> $CONF
 echo "port = 5432" >> $CONF
 
-# Enable ssl
-
-echo "ssl = true" >> $CONF
-#echo "ssl_ciphers = 'DEFAULT:!LOW:!EXP:!MD5:@STRENGTH' " >> $CONF
-#echo "ssl_renegotiation_limit = 512MB "  >> $CONF
-echo "ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem'" >> $CONF
-echo "ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key'" >> $CONF
-#echo "ssl_ca_file = ''                       # (change requires restart)" >> $CONF
-#echo "ssl_crl_file = ''" >> $CONF
 
 # Create backup template for conf
 cat $CONF > $CONF.template