From 15392391f3d508e6b4451a11a7cc67c2930da077 Mon Sep 17 00:00:00 2001 From: mazano Date: Wed, 30 Jan 2019 21:06:28 +0200 Subject: [PATCH] Port changes for Postgres version 11 (#114) * Fix typo in helper script * Port work in develop to 10 branch (#90) * Port 9.6 to develop (#89) * Part one of porting work from 9.6 to 10 * Backported more scripts from 9.6 branch * Added missing apt update in dockerfile * Updates to entrypoint to reference image and update docker-compose to reference 10 pg * Added sample and docs from 9.6 branch * Removed my diagram as Rizky had already added one * Fix env paths for pg 10 * Fixes for backporting work from 9.6 to 10 - dbb now spins up and accepts connections properly * Update README.md * Backport from branch: 9.6-2.4 Fix default datadir - Change into default datadir - Add small unittest * Optimise PostgreSQL performance and align with the changes done in 9.6 version * Fix version numbers * Minor change to correct env file * Merged 10 branch into develop * Remove reduntant conf file from dockerfile * Remove reduntant conf file directive from setup script * Remove reduntant conf file directive from script * Migrate settings to be compatible with PostgreSQL version 11 * fix ssl setup --- Dockerfile | 3 ++- build.sh | 2 +- docker-entrypoint.sh | 31 ++++++++++++++++++---- env-data.sh | 12 ++++----- setup-conf.sh | 1 + setup-database.sh | 9 +++---- setup-replication.sh | 63 ++++++++++++++++++++++++++++++++------------ setup.sh | 9 ------- 8 files changed, 86 insertions(+), 44 deletions(-) diff --git a/Dockerfile b/Dockerfile index c44021e..6eb024b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ ENV DEBIAN_FRONTEND noninteractive RUN dpkg-divert --local --rename --add /sbin/initctl RUN apt-get -y update; apt-get -y install gnupg2 wget ca-certificates rpl pwgen + RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ stretch-pgdg main" > /etc/apt/sources.list.d/pgdg.list' RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - @@ -15,7 +16,7 @@ RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-k # We add postgis as well to prevent build errors (that we dont see on local builds) # on docker hub e.g. # The following packages have unmet dependencies: -RUN apt-get update; apt-get install -y postgresql-client-10 postgresql-common postgresql-10 postgresql-10-postgis-2.4 postgresql-10-pgrouting netcat +RUN apt-get update; apt-get install -y postgresql-client-11 postgresql-common postgresql-11 postgresql-11-postgis-2.5 postgresql-11-pgrouting netcat # Open port 5432 so linked containers can see them EXPOSE 5432 diff --git a/build.sh b/build.sh index 94e418f..16caf98 100755 --- a/build.sh +++ b/build.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash docker build -t kartoza/postgis:manual-build . -docker build -t kartoza/postgis:10.0-2.4 . +docker build -t kartoza/postgis:11 . diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index fc908c8..9b3a34a 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -4,15 +4,36 @@ set -e # Setup postgres CONF file -source /setup-conf.sh +if grep -rlq "#user-settings" /etc/postgresql/9.6/main/postgresql.conf +then + echo "postgres conf already configured" +else + source /setup-conf.sh +fi + # Setup ssl -source /setup-ssl.sh + +# Setup ssl +if grep -rlq "ssl-cert-snakeoil.pem" /etc/postgresql/9.6/main/postgresql.conf +then + echo "ssl already configured" +else + echo "SSL not configures so proceed to setup" + source /setup-ssl.sh + +fi # Setup pg_hba.conf -source /setup-pg_hba.sh +if grep -rlq "172.0.0.0/8" /etc/postgresql/9.6/main/pg_hba.conf +then + echo "pg_hba already configured" +else + echo "we will setup pg_hba conf" + source /setup-pg_hba.sh +fi -if [ -z "$REPLICATE_FROM" ]; then +if [[ -z "$REPLICATE_FROM" ]]; then # This means this is a master instance. We check that database exists echo "Setup master database" source /setup-database.sh @@ -37,7 +58,7 @@ for f in /docker-entrypoint-initdb.d/*; do done # If no arguments passed to entrypoint, then run postgres by default -if [ $# -eq 0 ]; +if [[ $# -eq 0 ]]; then echo "Postgres initialisation process completed .... restarting in foreground" diff --git a/env-data.sh b/env-data.sh index 56aac91..be1731b 100644 --- a/env-data.sh +++ b/env-data.sh @@ -1,19 +1,19 @@ #!/usr/bin/env bash -DATADIR="/var/lib/postgresql/10/main" -ROOT_CONF="/etc/postgresql/10/main" +DATADIR="/var/lib/postgresql/11/main" +ROOT_CONF="/etc/postgresql/11/main" CONF="$ROOT_CONF/postgresql.conf" WAL_ARCHIVE="/opt/archivedir" RECOVERY_CONF="$ROOT_CONF/recovery.conf" -POSTGRES="/usr/lib/postgresql/10/bin/postgres" -INITDB="/usr/lib/postgresql/10/bin/initdb" -SQLDIR="/usr/share/postgresql/10/contrib/postgis-2.4/" +POSTGRES="/usr/lib/postgresql/11/bin/postgres" +INITDB="/usr/lib/postgresql/11/bin/initdb" +SQLDIR="/usr/share/postgresql/11/contrib/postgis-2.5/" SETVARS="POSTGIS_ENABLE_OUTDB_RASTERS=1 POSTGIS_GDAL_ENABLED_DRIVERS=ENABLE_ALL" LOCALONLY="-c listen_addresses='127.0.0.1'" PG_BASEBACKUP="/usr/bin/pg_basebackup" PROMOTE_FILE="/tmp/pg_promote_master" PGSTAT_TMP="/var/run/postgresql/" -PG_PID="/var/run/postgresql/10-main.pid" +PG_PID="/var/run/postgresql/11-main.pid" # Make sure we have a user set up if [ -z "${POSTGRES_USER}" ]; then diff --git a/setup-conf.sh b/setup-conf.sh index 6335787..b5cf4b3 100644 --- a/setup-conf.sh +++ b/setup-conf.sh @@ -24,6 +24,7 @@ random_page_cost = 2.0 xmloption = 'document' #archive_mode=on #archive_command = 'test ! -f ${WAL_ARCHIVE}/%f && cp -r %p ${WAL_ARCHIVE}/%f' +#user-settings EOF # Optimise PostgreSQL shared memory for PostGIS diff --git a/setup-database.sh b/setup-database.sh index 32e683e..13abc1d 100644 --- a/setup-database.sh +++ b/setup-database.sh @@ -5,7 +5,7 @@ source /env-data.sh # This script will setup the necessary folder for database # test if DATADIR is existent -if [ ! -d ${DATADIR} ]; then +if [[ ! -d ${DATADIR} ]]; then echo "Creating Postgres data at ${DATADIR}" mkdir -p ${DATADIR} fi @@ -17,7 +17,7 @@ chown -R postgres:postgres ${DATADIR} # test if DATADIR has content -if [ ! "$(ls -A ${DATADIR})" ]; then +if [[ ! "$(ls -A ${DATADIR})" ]]; then # No content yet - first time pg is being run! # No Replicate From settings. Assume that this is a master database. # Initialise db @@ -28,7 +28,6 @@ fi # test database existing trap "echo \"Sending SIGTERM to postgres\"; killall -s SIGTERM postgres" SIGTERM -echo "Use modified postgresql.conf for greater speed (spatial and replication)" @@ -102,10 +101,10 @@ fi su - postgres -c "psql -l" # Kill postgres -PID=`cat $PG_PID` +PID=`cat ${PG_PID}` kill -TERM ${PID} # Wait for background postgres main process to exit -while [ "$(ls -A ${PG_PID} 2>/dev/null)" ]; do +while [[ "$(ls -A ${PG_PID} 2>/dev/null)" ]]; do sleep 1 done diff --git a/setup-replication.sh b/setup-replication.sh index e45d6ed..9b30476 100755 --- a/setup-replication.sh +++ b/setup-replication.sh @@ -6,14 +6,12 @@ source /env-data.sh # Adapted from https://github.com/DanielDent/docker-postgres-replication # To set up replication -if [[ "$DESTROY_DATABASE_ON_RESTART" =~ [Tt][Rr][Uu][Ee] ]]; then - echo "Destroy initial database, if any." - rm -rf $DATADIR -fi -mkdir -p $DATADIR -chown -R postgres:postgres $DATADIR -chmod -R 700 $DATADIR + + +mkdir -p ${DATADIR} +chown -R postgres:postgres ${DATADIR} +chmod -R 700 ${DATADIR} # No content yet - but this is a slave database until ping -c 1 -W 1 ${REPLICATE_FROM} @@ -22,22 +20,52 @@ do sleep 1s done -if [[ "$DESTROY_DATABASE_ON_RESTART" =~ [Tt][Rr][Uu][Ee] ]]; then - echo "Get initial database from master" +function configure_replication_permissions { - chown -R postgres:postgres $(getent passwd postgres | cut -d: -f6) - su - postgres -c "echo \"${REPLICATE_FROM}:${REPLICATE_PORT}:*:${POSTGRES_USER}:${POSTGRES_PASS}\" > ~/.pgpass" - su - postgres -c "chmod 0600 ~/.pgpass" + echo "Setup data permissions" + echo "----------------------" + chown -R postgres:postgres $(getent passwd postgres | cut -d: -f6) + su - postgres -c "echo \"${REPLICATE_FROM}:${REPLICATE_PORT}:*:${POSTGRES_USER}:${POSTGRES_PASS}\" > ~/.pgpass" + su - postgres -c "chmod 0600 ~/.pgpass" +} - until su - postgres -c "${PG_BASEBACKUP} -X stream -h ${REPLICATE_FROM} -p ${REPLICATE_PORT} -D ${DATADIR} -U ${POSTGRES_USER} -vP -w" +function streaming_replication { +until su - postgres -c "${PG_BASEBACKUP} -X stream -h ${REPLICATE_FROM} -p ${REPLICATE_PORT} -D ${DATADIR} -U ${POSTGRES_USER} -vP -w" do echo "Waiting for master to connect..." sleep 1s - if [ "$(ls -A $DATADIR)" ]; then + if [[ "$(ls -A ${DATADIR})" ]]; then echo "Need empty folder. Cleaning directory..." - rm -rf $DATADIR/* + rm -rf ${DATADIR}/* fi done + +} + +var=`du -sh /var/lib/postgresql/11/main/pg_wal | awk '{print $1}'` +var_size=${var:0:2} + +if [[ "$DESTROY_DATABASE_ON_RESTART" =~ [Tt][Rr][Uu][Ee] ]]; then + echo "Get initial database from master" + + configure_replication_permissions + + streaming_replication +else + echo "Destroy database has been set to false: Check Backup directory if it already exists" + configure_replication_permissions + # We need a clever way to identify if base backup exists + + + if [[ "${var_size} -gt 40" ]]; then + echo "Base directory exist - Please startup the database" + else + echo "Base directory does not exists- Create a new one" + streaming_replication + fi + + + fi # Setup recovery.conf, a configuration file for slave @@ -45,12 +73,13 @@ cat > ${DATADIR}/recovery.conf <> $CONF echo "port = 5432" >> $CONF -# Enable ssl - -echo "ssl = true" >> $CONF -#echo "ssl_ciphers = 'DEFAULT:!LOW:!EXP:!MD5:@STRENGTH' " >> $CONF -#echo "ssl_renegotiation_limit = 512MB " >> $CONF -echo "ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem'" >> $CONF -echo "ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key'" >> $CONF -#echo "ssl_ca_file = '' # (change requires restart)" >> $CONF -#echo "ssl_crl_file = ''" >> $CONF # Create backup template for conf cat $CONF > $CONF.template