From ac9ec47e3bc0de204a60fafaf77ba2a8439b886c Mon Sep 17 00:00:00 2001 From: mazano Date: Fri, 23 Dec 2022 15:50:45 +0200 Subject: [PATCH] install tagged versions of extensions (#400) * fix #391 --- README.md | 9 +++ scenario_tests/extensions/docker-compose.yml | 2 +- .../extensions/tests/test_extensions.py | 13 ++++- scripts/docker-entrypoint.sh | 10 ++-- scripts/env-data.sh | 19 ++++++ scripts/setup-database.sh | 58 +++++++++---------- 6 files changed, 73 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 4f120dc..7aa6842 100644 --- a/README.md +++ b/README.md @@ -262,6 +262,15 @@ they will cause the container to exit. Users should also consult documentation relating to that specific extension i.e. [timescaledb](https://github.com/timescale/timescaledb), [pg_cron](https://github.com/citusdata/pg_cron), [pgrouting](https://pgrouting.org/) +You can also install tagged version of extensions i.e + +```bash +POSTGRES_MULTIPLE_EXTENSIONS=postgis,pgrouting:3.4.0 +``` + +where `pgrouting:3.4.0` The extension name is fixed with the version name with the delimiter being a +colon. + #### Shared preload libraries Some PostgreSQL extensions require shared_preload_libraries to be specified in the conf files. diff --git a/scenario_tests/extensions/docker-compose.yml b/scenario_tests/extensions/docker-compose.yml index 00eb3e3..76d5cb0 100644 --- a/scenario_tests/extensions/docker-compose.yml +++ b/scenario_tests/extensions/docker-compose.yml @@ -33,7 +33,7 @@ services: environment: ALLOW_IP_RANGE: '0.0.0.0/0' TEST_CLASS: test_extensions.TestExtensions - POSTGRES_MULTIPLE_EXTENSIONS: postgis,pgrouting + POSTGRES_MULTIPLE_EXTENSIONS: postgis,pgcrypto:1.3 POSTGRES_PASS: 'docker' ports: - "7776:5432" diff --git a/scenario_tests/extensions/tests/test_extensions.py b/scenario_tests/extensions/tests/test_extensions.py index ac87eb0..e043136 100644 --- a/scenario_tests/extensions/tests/test_extensions.py +++ b/scenario_tests/extensions/tests/test_extensions.py @@ -3,8 +3,19 @@ import os from utils.utils import DBConnection from pprint import pprint + class TestExtensionsBase(unittest.TestCase): - SPECIFIED_EXT = os.getenv('POSTGRES_MULTIPLE_EXTENSIONS').split(',') + if os.getenv('POSTGRES_MULTIPLE_EXTENSIONS').find(':') != -1: + SPECIFIED_EXT = [] + PG_EXT = os.getenv('POSTGRES_MULTIPLE_EXTENSIONS').split(',') + for versions in PG_EXT: + if versions.find(':') != -1: + ext_name = versions.split(":")[0] + SPECIFIED_EXT.append(ext_name) + else: + SPECIFIED_EXT.append(versions) + else: + SPECIFIED_EXT = os.getenv('POSTGRES_MULTIPLE_EXTENSIONS').split(',') DEFAULT_EXT = ['plpgsql', 'pg_cron'] # get installed in any case def setUp(self): diff --git a/scripts/docker-entrypoint.sh b/scripts/docker-entrypoint.sh index 948304d..ff07f60 100755 --- a/scripts/docker-entrypoint.sh +++ b/scripts/docker-entrypoint.sh @@ -21,15 +21,13 @@ figlet -t "Kartoza Docker PostGIS" if [[ -f /scripts/.pass_20.txt ]]; then USER_CREDENTIAL_PASS=$(cat /scripts/.pass_20.txt) cp /scripts/.pass_20.txt /tmp/PGPASSWORD.txt - echo -e "[Entrypoint] GENERATED Postgres PASSWORD: \e[1;31m $USER_CREDENTIAL_PASS" - echo -e "\033[0m PGPASSWORD Generated above: " + echo -e "[Entrypoint] GENERATED Postgres PASSWORD: \e[1;31m $USER_CREDENTIAL_PASS \033[0m" fi if [[ -f /scripts/.pass_22.txt ]]; then USER_CREDENTIAL_PASS=$(cat /scripts/.pass_22.txt) cp /scripts/.pass_22.txt /tmp/REPLPASSWORD.txt - echo -e "[Entrypoint] GENERATED Replication PASSWORD: \e[1;34m $USER_CREDENTIAL_PASS" - echo -e "\033[0m Replication password Generated above: " + echo -e "[Entrypoint] GENERATED Replication PASSWORD: \e[1;34m $USER_CREDENTIAL_PASS \033[0m" fi @@ -46,11 +44,11 @@ else fi + # If no arguments passed to entrypoint, then run postgres by default if [[ $# -eq 0 ]]; then - echo "Postgres initialisation process completed .... restarting in foreground" - + echo -e "[Entrypoint] \e[1;31m Postgres initialisation process completed .... restarting in foreground \033[0m" su - postgres -c "$SETVARS $POSTGRES -D $DATADIR -c config_file=$CONF" fi diff --git a/scripts/env-data.sh b/scripts/env-data.sh index 2fe3046..2cd60da 100644 --- a/scripts/env-data.sh +++ b/scripts/env-data.sh @@ -494,3 +494,22 @@ function over_write_conf() { } +function extension_install() { + DATABASE=$1 + IFS=':' + read -a strarr <<< "$ext" + EXTENSION_NAME=${strarr[0]} + EXTENSION_VERSION=${strarr[1]} + if [[ -z ${EXTENSION_VERSION} ]];then + if [[ ${EXTENSION_NAME} != 'pg_cron' ]]; then + echo -e "\e[32m [Entrypoint] Enabling extension \e[1;31m ${EXTENSION_NAME} \e[32m in the database : \e[1;31m ${DATABASE} \033[0m" + psql ${DATABASE} -U ${POSTGRES_USER} -p 5432 -h localhost -c "CREATE EXTENSION IF NOT EXISTS \"${EXTENSION_NAME}\" cascade;" + fi + else + echo -e "\e[32m [Entrypoint] Installing extension \e[1;31m ${EXTENSION_NAME} \e[32m with version \e[1;31m ${EXTENSION_VERSION} \e[32m in the database : \e[1;31m ${DATABASE} \033[0m" + if [[ ${EXTENSION_NAME} != 'pg_cron' ]]; then + psql ${DATABASE} -U ${POSTGRES_USER} -p 5432 -h localhost -c "CREATE EXTENSION IF NOT EXISTS \"${EXTENSION_NAME}\" WITH VERSION '${EXTENSION_VERSION}' cascade;" + fi + fi + +} \ No newline at end of file diff --git a/scripts/setup-database.sh b/scripts/setup-database.sh index 01aed39..d467d7b 100644 --- a/scripts/setup-database.sh +++ b/scripts/setup-database.sh @@ -36,13 +36,12 @@ if [[ -z "$(ls -A ${DATADIR} 2> /dev/null)" || "${RECREATE_DATADIR}" =~ [Tt][Rr] # Only attempt reinitializations if ${RECREATE_DATADIR} is true # No Replicate From settings. Assume that this is a master database. # Initialise db - echo "Initializing Postgres Database at ${DATADIR}" + echo -e "\e[32m [Entrypoint] Initializing Postgres Database at \e[1;31m ${DATADIR} \033[0m" create_dir "${DATADIR}" rm -rf ${DATADIR}/* chown -R postgres:postgres "${DATADIR}" - echo "Initializing with command:" command="$INITDB -U postgres --pwfile=<(echo "$POSTGRES_PASS") -E ${DEFAULT_ENCODING} --lc-collate=${DEFAULT_COLLATION} --lc-ctype=${DEFAULT_CTYPE} --wal-segsize=${WAL_SEGSIZE} --auth=${PASSWORD_AUTHENTICATION} -D ${DATADIR} ${INITDB_WALDIR_FLAG} ${INITDB_EXTRA_ARGS}" - echo "$command" + echo -e "\e[32m [Entrypoint] Initializing Cluster with the following commands Postgres Database at \e[1;31m $command \033[0m" su - postgres -c "$command" else # If using existing datadir: @@ -91,57 +90,56 @@ echo "postgres ready" # Setup user source /scripts/setup-user.sh -# enable extensions in template1 if env variable set to true -if [[ "$(boolean ${POSTGRES_TEMPLATE_EXTENSIONS})" =~ [Tt][Rr][Uu][Ee] ]] ; then - for ext in $(echo ${POSTGRES_MULTIPLE_EXTENSIONS} | tr ',' ' '); do - echo "Enabling \"${ext}\" in the database template1" - su - postgres -c "psql -c 'CREATE EXTENSION IF NOT EXISTS \"${ext}\" cascade;' template1" - done -fi + +export PGPASSWORD=${POSTGRES_PASS} # Create a default db called 'gis' or $POSTGRES_DBNAME that you can use to get up and running quickly # It will be owned by the docker db user # Since we now pass a comma separated list in database creation we need to search for all databases as a test - - for db in $(echo ${POSTGRES_DBNAME} | tr ',' ' '); do RESULT=`su - postgres -c "psql -t -c \"SELECT count(1) from pg_database where datname='${db}';\""` - if [[ ${RESULT} -eq 0 ]]; then - echo "Create db ${db}" - su - postgres -c "createdb -O ${POSTGRES_USER} ${db}" - su - postgres -c "psql -c 'CREATE EXTENSION IF NOT EXISTS pg_cron cascade;' ${SINGLE_DB}" + echo -e "\e[32m [Entrypoint] Create database \e[1;31m ${db} \033[0m" + DB_CREATE=$(createdb -h localhost -p 5432 -U ${POSTGRES_USER} ${db}) + eval ${DB_CREATE} + psql ${SINGLE_DB} -U ${POSTGRES_USER} -p 5432 -h localhost -c 'CREATE EXTENSION IF NOT EXISTS pg_cron cascade;' for ext in $(echo ${POSTGRES_MULTIPLE_EXTENSIONS} | tr ',' ' '); do - echo "Enabling \"${ext}\" in the database ${db}" - if [[ ${ext} != 'pg_cron' ]]; then - su - postgres -c "psql -c 'CREATE EXTENSION IF NOT EXISTS \"${ext}\" cascade;' $db" + extension_install ${db} + # enable extensions in template1 if env variable set to true + if [[ "$(boolean ${POSTGRES_TEMPLATE_EXTENSIONS})" =~ [Tt][Rr][Uu][Ee] ]] ; then + extension_install template1 fi done - echo "Loading legacy sql" - su - postgres -c "psql ${db} -f ${SQLDIR}/legacy_minimal.sql" || true - su - postgres -c "psql ${db} -f ${SQLDIR}/legacy_gist.sql" || true - if [[ "$WAL_LEVEL" == 'logical' ]];then - PGPASSWORD=${POSTGRES_PASS} psql ${db} -U ${POSTGRES_USER} -p 5432 -h localhost -c "CREATE PUBLICATION logical_replication;" + echo -e "\e[32m [Entrypoint] loading legacy sql in database \e[1;31m ${db} \033[0m" + psql ${db} -U ${POSTGRES_USER} -p 5432 -h localhost -f ${SQLDIR}/legacy_minimal.sql || true + psql ${db} -U ${POSTGRES_USER} -p 5432 -h localhost -f ${SQLDIR}/legacy_gist.sql || true + if [[ "$WAL_LEVEL" =~ [Ll][Oo][Gg][Ii][Cc][Aa][Ll] ]];then + psql ${db} -U ${POSTGRES_USER} -p 5432 -h localhost -c "CREATE PUBLICATION logical_replication;" fi else - echo "${db} db already exists" + echo -e "\e[32m [Entrypoint] Database \e[1;31m ${db} \e[32m already exists \033[0m" + fi done + + # Create schemas in the DB for db in $(echo ${POSTGRES_DBNAME} | tr ',' ' '); do for schema in $(echo ${SCHEMA_NAME} | tr ',' ' '); do - SCHEMA_RESULT=`PGPASSWORD=${POSTGRES_PASS} psql -t ${db} -U ${POSTGRES_USER} -p 5432 -h localhost -c "select count(1) from information_schema.schemata where schema_name = '${schemas}' and catalog_name = '${db}';"` + SCHEMA_RESULT=$(psql -t ${db} -U ${POSTGRES_USER} -p 5432 -h localhost -c "select count(1) from information_schema.schemata where schema_name = '${schemas}' and catalog_name = '${db}';") if [[ ${SCHEMA_RESULT} -eq 0 ]] && [[ "${ALL_DATABASES}" =~ [Ff][Aa][Ll][Ss][Ee] ]]; then - echo "Creating schema ${schema} in database ${SINGLE_DB}" - PGPASSWORD=${POSTGRES_PASS} psql ${SINGLE_DB} -U ${POSTGRES_USER} -p 5432 -h localhost -c " CREATE SCHEMA IF NOT EXISTS ${schema};" + echo -e "\e[32m [Entrypoint] Creating schema \e[1;31m ${schema} \e[32m in database \e[1;31m ${SINGLE_DB} \033[0m" + psql ${SINGLE_DB} -U ${POSTGRES_USER} -p 5432 -h localhost -c " CREATE SCHEMA IF NOT EXISTS ${schema};" elif [[ ${SCHEMA_RESULT} -eq 0 ]] && [[ "${ALL_DATABASES}" =~ [Tt][Rr][Uu][Ee] ]]; then - echo "Creating schema ${schema} in database ${db}" - PGPASSWORD=${POSTGRES_PASS} psql ${db} -U ${POSTGRES_USER} -p 5432 -h localhost -c " CREATE SCHEMA IF NOT EXISTS ${schema};" + echo -e "\e[32m [Entrypoint] Creating schema \e[1;31m ${schema} \e[32m in database \e[1;31m ${db} \033[0m" + psql ${db} -U ${POSTGRES_USER} -p 5432 -h localhost -c " CREATE SCHEMA IF NOT EXISTS ${schema};" fi done done + + # This should show up in docker logs afterwards su - postgres -c "psql -l 2>&1"