Port changes to 11 branch (#115)

* Fix typo in helper script

* Port work in develop to 10 branch (#90)

* Port 9.6 to develop (#89)

* Part one of porting work from 9.6 to 10

* Backported more scripts from 9.6 branch

* Added missing apt update in dockerfile

* Updates to entrypoint to reference image and update docker-compose to reference 10 pg

* Added sample and docs from 9.6 branch

* Removed my diagram as Rizky had already added one

* Fix env paths for pg 10

* Fixes for backporting work from 9.6 to 10 - dbb now spins up and accepts connections properly

* Update README.md

* Backport from branch: 9.6-2.4

Fix default datadir

- Change into default datadir
- Add small unittest

* Optimise PostgreSQL performance and align with the changes done in 9.6
version

* Fix version numbers

* Minor change to correct env file

* Merged 10 branch into develop

* Remove reduntant conf file from dockerfile

* Remove reduntant conf file directive from setup script

* Remove reduntant conf file directive from  script

* Migrate settings to be compatible with PostgreSQL version 11

* fix ssl setup

* Align changes to postgresql version 11

* Set destroy database to false

* Commented out code for destroy database

* Fix version number for build

build.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 docker build -t kartoza/postgis:manual-build .
-docker build -t kartoza/postgis:11 .
+docker build -t kartoza/postgis:11.0-2.5 .

docker-entrypoint.sh

@@ -3,35 +3,17 @@
 # This script will run as the postgres user due to the Dockerfile USER directive
 set -e
 
+#TODO Prepare lock files that prevent running the setup-conf,setup-pg_hba,setup-ssl.sh on each restart
 # Setup postgres CONF file
-if grep -rlq "#user-settings" /etc/postgresql/9.6/main/postgresql.conf
-then
-    echo "postgres conf already configured"
-else
-    source /setup-conf.sh
-fi
 
+source /setup-conf.sh
 
 # Setup ssl
-
-# Setup ssl
-if grep -rlq "ssl-cert-snakeoil.pem" /etc/postgresql/9.6/main/postgresql.conf
-then
-    echo "ssl already configured"
-else
-    echo "SSL not configures so proceed to setup"
-    source /setup-ssl.sh
-
-fi
+source /setup-ssl.sh
 
 # Setup pg_hba.conf
-if grep -rlq "172.0.0.0/8" /etc/postgresql/9.6/main/pg_hba.conf
-then
-    echo "pg_hba  already configured"
-else
-    echo "we will setup pg_hba conf"
-    source /setup-pg_hba.sh
-fi
+
+source /setup-pg_hba.sh
 
 if [[ -z "$REPLICATE_FROM" ]]; then
 	# This means this is a master instance. We check that database exists

sample/replication/docker-compose.yml

@@ -1,5 +1,10 @@
 
-version: '2'
+version: '2.1'
+
+volumes:
+  pg-master-data-dir:
+  pg-slave-data-dir:
+
 
 services:
   pg-master:
@@ -8,8 +13,8 @@ services:
     # You can optionally mount to volume, to play with the persistence and
     # observe how the slave will behave after restarts.
     volumes:
-      - './pg-master:/var/lib/postgresql'
-      - './tests:/tests'
+      - pg-master-data-dir:/var/lib/postgresql
+      - ./tests:/tests
     environment:
       # ALLOW_IP_RANGE option is used to specify additionals allowed domains
       # in pg_hba.
@@ -17,11 +22,13 @@ services:
       ALLOW_IP_RANGE: '0.0.0.0/0'
 
       # We can specify optional credentials
-      POSTGRES_USER: 'superadmin'
-      POSTGRES_PASS: 'superstrongpassword'
+      POSTGRES_USER: 'docker'
+      POSTGRES_PASS: 'docker'
     # You can expose the port to observe it in your local machine
     ports:
       - "7777:5432"
+    healthcheck:
+      test: "exit 0"
 
   pg-slave:
     image: 'kartoza/postgis:manual-build'
@@ -31,8 +38,9 @@ services:
     # The slave will always destroy its database and copy from master at
     # runtime
     volumes:
-      - './pg-slave:/var/lib/postgresql'
-      - './tests:/tests'
+      - pg-slave-data-dir:/var/lib/postgresql
+      - ./tests:/tests
+
     environment:
       # ALLOW_IP_RANGE option is used to specify additionals allowed domains
       # in pg_hba.
@@ -42,13 +50,13 @@ services:
       # connect to this slave
       ALLOW_IP_RANGE: '0.0.0.0/0'
 
-      # REPLICATE_FROM options accepts domain-name or IP adress
+      # REPLICATE_FROM options accepts domain-name or IP address
       # with this in mind, you can also put docker service name, because it
       # will be resolved as host name.
       REPLICATE_FROM: 'pg-master'
 
       # REPLICATE_PORT will default to 5432 if not specified.
-      REPLICATE_PORT: '5432'
+      # REPLICATE_PORT: '5432'
       # In the case where you need to replicate from outside service,
       # you can put the server address and port here, as long as the target
       # where configured as master, and replicable.
@@ -58,7 +66,7 @@ services:
       # DESTROY_DATABASE_ON_RESTART will default to True if not specified.
       # If specified other than True, it will prevent slave from destroying
       # database on restart
-#      DESTROY_DATABASE_ON_RESTART: 'False'
+      DESTROY_DATABASE_ON_RESTART: 'True'
 
       # PROMOTE_MASTER Default empty.
       # If specified with any value, then it will convert current slave into
@@ -69,10 +77,11 @@ services:
       # For now we don't support different credentials for replication
       # so we use the same credentials as master's superuser, or anything that
       # have replication role.
-      POSTGRES_USER: 'superadmin'
-      POSTGRES_PASS: 'superstrongpassword'
-    links:
-      - 'pg-master'
+      POSTGRES_USER: 'docker'
+      POSTGRES_PASS: 'docker'
+    depends_on:
+        pg-master:
+          condition: service_healthy
     # You can expose the port to observe it in your local machine
     # For this sample, it was disabled by default to allow scaling test
     ports:

setup-pg_hba.sh

@@ -5,19 +5,19 @@ source /env-data.sh
 # This script will setup pg_hba.conf
 
 # Reconfigure pg_hba if environment settings changed
-cat $ROOT_CONF/pg_hba.conf.template > $ROOT_CONF/pg_hba.conf
+cat ${ROOT_CONF}/pg_hba.conf.template > ${ROOT_CONF}/pg_hba.conf
 
 # Custom IP range via docker run -e (https://docs.docker.com/engine/reference/run/#env-environment-variables)
 # Usage is: docker run [...] -e ALLOW_IP_RANGE='192.168.0.0/16'
-if [ "$ALLOW_IP_RANGE" ]
+if [[ "$ALLOW_IP_RANGE" ]]
 then
 	echo "Add rule to pg_hba: $ALLOW_IP_RANGE"
- 	echo "host    all             all             $ALLOW_IP_RANGE              md5" >> $ROOT_CONF/pg_hba.conf
+ 	echo "host    all             all             $ALLOW_IP_RANGE              md5" >> ${ROOT_CONF}/pg_hba.conf
 fi
 
 # check password first so we can output the warning before postgres
 # messes it up
-if [ "$POSTGRES_PASS" ]; then
+if [[ "$POSTGRES_PASS" ]]; then
 	pass="PASSWORD '$POSTGRES_PASS'"
 	authMethod=md5
 else
@@ -40,9 +40,9 @@ else
 	authMethod=trust
 fi
 
-if [ -z "$REPLICATE_FROM" ]; then
+if [[ -z "$REPLICATE_FROM" ]]; then
 	# if env not set, then assume this is master instance
 	# add rules to pg_hba.conf to allow replication from all
 	echo "Add rule to pg_hba: replication user"
-	echo "host replication all 0.0.0.0/0 $authMethod" >> $ROOT_CONF/pg_hba.conf
+	echo "host replication all 0.0.0.0/0 $authMethod" >> ${ROOT_CONF}/pg_hba.conf
 fi

setup-replication.sh

@@ -42,8 +42,7 @@ until su - postgres -c "${PG_BASEBACKUP} -X stream -h ${REPLICATE_FROM} -p ${REP
 
 }
 
-var=`du -sh /var/lib/postgresql/11/main/pg_wal | awk '{print $1}'`
-var_size=${var:0:2}
+
 
 if [[ "$DESTROY_DATABASE_ON_RESTART" =~ [Tt][Rr][Uu][Ee] ]]; then
 	echo "Get initial database from master"
@@ -51,23 +50,25 @@ if [[ "$DESTROY_DATABASE_ON_RESTART" =~ [Tt][Rr][Uu][Ee] ]]; then
 	configure_replication_permissions
 
 	streaming_replication
-else
-    echo "Destroy database has been set to false: Check Backup directory if it already exists"
-    configure_replication_permissions
-    # We need a clever way to identify if base backup exists
-
-
-    if [[ "${var_size} -gt  40" ]]; then
-			echo "Base directory exist - Please startup the database"
-	else
-        echo "Base directory does not exists- Create a new one"
-	   streaming_replication
-	fi
-
-
 
 fi
 
+#TODO We need a clever way to identify if base backup exists - Incoperate it as an else statement in destroy logic
+
+
+#configure_replication_permissions
+#var=`du -sh /var/lib/postgresql/11/main/pg_wal | awk '{print $1}'`
+#var_size=${var:0:2}
+
+#if [[ "${var_size} -gt  33 " ]]; then
+        #echo ${var_size}
+        #echo "Base directory exist - Please startup the database"
+#else
+    #echo "Base directory does not exists- Create a new one"
+   #streaming_replication
+#fi
+
+
 # Setup recovery.conf, a configuration file for slave
 cat > ${DATADIR}/recovery.conf <<EOF
 standby_mode = on

setup-ssl.sh

@@ -15,3 +15,12 @@ mv /tmp/ssl-copy /etc/ssl
 # Needed under debian, wasnt needed under ubuntu
 mkdir -p ${PGSTAT_TMP}
 chmod 0777 ${PGSTAT_TMP}
+
+# moved from setup.sh
+echo "ssl = true" >> $CONF
+#echo "ssl_ciphers = 'DEFAULT:!LOW:!EXP:!MD5:@STRENGTH' " >> $CONF
+#echo "ssl_renegotiation_limit = 512MB "  >> $CONF
+echo "ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem'" >> $CONF
+echo "ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key'" >> $CONF
+#echo "ssl_ca_file = ''                       # (change requires restart)" >> $CONF
+#echo "ssl_crl_file = ''" >> $CONF