diff --git a/.gitignore b/.gitignore index 193c1ae..05e944f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ */replication/pg-* */replication/docker-compose.override.yml .DS_Store +.python-version +venv +__pycache__ diff --git a/.travis.yml b/.travis.yml index d8ad827..243de6a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,22 +6,18 @@ services: - docker python: - - '2.7' - - '3.5' - - '3.6.7' - - '3.7.1' + - '3.7' + +env: + - SCENARIO=datadir_init + - SCENARIO=replications + - SCENARIO=collations + +before_script: + - ./build-test.sh script: - - ./build.sh - - pushd sample/replication - - make up - - make status - # Check for database status - - until make check-master-running; do echo "Retrying"; sleep 5; done - - until make check-slave-running; do echo "Retrying"; sleep 5; done - - # Check replications - - until make check-master-replication; do echo "Retrying"; make master-log-tail; sleep 5; done - - sleep 60 # Wait replication finished - - until make check-slave-replication; do echo "Retrying"; make slave-log-tail; sleep 5; done + - pushd scenario_tests/${SCENARIO} + - ./test.sh - popd + diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 0000000..c897705 --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1,16 @@ +#--------- Generic stuff all our Dockerfiles should start with so we get caching ------------ +FROM kartoza/postgis:manual-build + +# For testing + +COPY scenario_tests/utils/requirements.txt /lib/utils/requirements.txt + +RUN set -eux \ + && export DEBIAN_FRONTEND=noninteractive \ + && apt-get update \ + && apt-get -y --no-install-recommends install python3-pip \ + && apt-get -y --purge autoremove \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN pip3 install -r /lib/utils/requirements.txt diff --git a/README.md b/README.md index f8adf13..3b314b9 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,50 @@ docker run --name "postgis" -p 25432:5432 -d -t kartoza/postgis ## Environment variables +#### Cluster Initializations + +With minimum setup, our image will use initial cluster located in the +`DATADIR` environment variable. If you want to use persistence, mount these +location into your volume/host. By default, `DATADIR` will point to `/var/lib/postgresql/{major-version}`. +You can instead mount the parent location like this: + +* `-v data-volume:/var/lib/postgresql` + +This default cluster will be initialized with default locale settings `C.UTF-8`. +If, for instance, you want to create a new cluster with your own settings (not using the default cluster). +You need to specify different empty directory, like this + +```shell script +-v data-volume:/opt/postgres/data \ +-e DATADIR:/opt/postgres/data \ +-e DEFAULT_ENCODING="UTF8" \ +-e DEFAULT_COLLATION="id_ID.utf8" \ +-e DEFAULT_CTYPE="id_ID.utf8" \ +-e INITDB_EXTRA_ARGS="" +``` + +The containers will use above parameters to initialize a new db cluster in the +specified directory. If the directory is not empty, then initialization parameter will be ignored. + +These are some initialization parameter that will only be used to initialize new cluster. +If the container uses existing cluster, it will be ignored (for example, when the container restarts). + +* `DEFAULT_ENCODING`: cluster encoding +* `DEFAULT_COLLATION`: cluster collation +* `DEFAULT_CTYPE`: cluster ctype +* `WAL_SEGSIZE`: WAL segsize option +* `INITDB_EXTRA_ARGS`: extra parameter that will be passed down to `initdb` command + +In addition to that, we have another parameter: `RECREATE_DATADIR` that can be used to force database reinitializations. +If this parameter is specified as `TRUE` it will act as explicit consent to delete `DATADIR` and create +new db cluster. + +* `RECREATE_DATADIR`: Force database reinitializations in the location `DATADIR` + +If you used `RECREATE_DATADIR` and successfully created new cluster. Remember +that you should remove this parameter afterwards. Because, if it was not omitted, +it will always recreate new db cluster after every container restarts. + #### Basic configuration You can use the following environment variables to pass a @@ -152,10 +196,11 @@ You can also define any other configuration to add to `postgres.conf`, separated * `-e EXTRA_CONF="log_destination = 'stderr'\nlogging_collector = on"` -If you plan on migrating the image and continue using the data directory you need to pass - -* EXISTING_DATA_DIR=true +If you want to reinitialize the data directory from scratch, you need to do: +1. Do backup, move data, etc. Any preparations before deleting your data directory. +2. Set environment variables `RECREATE_DATADIR=TRUE`. Restart the service +3. The service will delete your `DATADIR` directory and start reinitializing your data directory from scratch. ## Docker secrets @@ -414,13 +459,25 @@ The database cluster is initialised with the following encoding settings -E "UTF8" --lc-collate="en_US.UTF-8" --lc-ctype="en_US.UTF-8" ` +or + +` +-E "UTF8" --lc-collate="C.UTF-8" --lc-ctype="C.UTF-8" +` + +If you use default `DATADIR` location. + If you need to setup a database cluster with other encoding parameters you need -to pass the environment variables +to pass the environment variables when you initialize the cluster. * -e DEFAULT_ENCODING="UTF8" * -e DEFAULT_COLLATION="en_US.UTF-8" * -e DEFAULT_CTYPE="en_US.UTF-8" +Initializing a new cluster can be done by using different `DATADIR` location and +mounting an empty volume. Or use parameter `RECREATE_DATADIR` to forcefully +delete the current cluster and create a new one. Make sure to remove parameter +`RECREATE_DATADIR` after creating the cluster. See [the postgres documentation about encoding](https://www.postgresql.org/docs/11/multibyte.html) for more information. @@ -429,6 +486,6 @@ See [the postgres documentation about encoding](https://www.postgresql.org/docs/ Tim Sutton (tim@kartoza.com) Gavin Fleming (gavin@kartoza.com) -Risky Maulana (rizky@kartoza.com) +Rizky Maulana (rizky@kartoza.com) Admire Nyakudya (admire@kartoza.com) December 2018 diff --git a/build-test.sh b/build-test.sh new file mode 100755 index 0000000..f01d44c --- /dev/null +++ b/build-test.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +./build.sh + +docker build -t kartoza/postgis:manual-build -f Dockerfile.test . diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 5e92d0a..aa27599 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash -# This script will run as the postgres user due to the Dockerfile USER directive set -e +source /env-data.sh + # Setup postgres CONF file source /setup-conf.sh @@ -14,46 +15,6 @@ source /setup-ssl.sh source /setup-pg_hba.sh - - -# Running extended script or sql if provided. -# Useful for people who extends the image. -function entry_point_script { -SETUP_LOCKFILE="/docker-entrypoint-initdb.d/.entry_point.lock" -if [[ -f "${SETUP_LOCKFILE}" ]]; then - return 0 -else - if find "/docker-entrypoint-initdb.d" -mindepth 1 -print -quit 2>/dev/null | grep -q .; then - for f in /docker-entrypoint-initdb.d/*; do - export PGPASSWORD=${POSTGRES_PASS} - case "$f" in - *.sql) echo "$0: running $f"; psql ${SINGLE_DB} -U ${POSTGRES_USER} -p 5432 -h localhost -f ${f} || true ;; - *.sql.gz) echo "$0: running $f"; gunzip < "$f" | psql ${SINGLE_DB} -U ${POSTGRES_USER} -p 5432 -h localhost || true ;; - *.sh) echo "$0: running $f"; . $f || true;; - *) echo "$0: ignoring $f" ;; - esac - echo - done - # Put lock file to make sure entry point scripts were run - touch ${SETUP_LOCKFILE} - else - return 0 - fi - -fi - -} - -function kill_postgres { -PID=`cat ${PG_PID}` -kill -TERM ${PID} - -# Wait for background postgres main process to exit -while [[ "$(ls -A ${PG_PID} 2>/dev/null)" ]]; do - sleep 1 -done -} - if [[ -z "$REPLICATE_FROM" ]]; then # This means this is a master instance. We check that database exists echo "Setup master database" diff --git a/env-data.sh b/env-data.sh index 82ef14c..867b4ae 100644 --- a/env-data.sh +++ b/env-data.sh @@ -41,6 +41,17 @@ function file_env { unset "$fileVar" } +function boolean() { + case $1 in + [Tt][Rr][Uu][Ee] | [Yy][Ee][Ss]) + echo 'TRUE' + ;; + *) + echo 'FALSE' + ;; + esac +} + file_env 'POSTGRES_PASS' file_env 'POSTGRES_USER' file_env 'POSTGRES_DBNAME' @@ -55,6 +66,18 @@ fi if [ -z "${POSTGRES_DBNAME}" ]; then POSTGRES_DBNAME=gis fi +# If datadir is defined use this +if [ -n "${DATADIR}" ]; then + DATADIR=${DATADIR} +fi +# RECREATE_DATADIR flag default value +# Always assume that we don't want to recreate datadir if not explicitly defined +# For issue: https://github.com/kartoza/docker-postgis/issues/226 +if [ -z "${RECREATE_DATADIR}" ]; then + RECREATE_DATADIR=FALSE +else + RECREATE_DATADIR=$(boolean ${RECREATE_DATADIR}) +fi # SSL mode if [ -z "${PGSSLMODE}" ]; then PGSSLMODE=require @@ -192,37 +215,79 @@ fi # Compatibility with official postgres variable # Official postgres variable gets priority -if [ ! -z "${POSTGRES_PASSWORD}" ]; then +if [ -n "${POSTGRES_PASSWORD}" ]; then POSTGRES_PASS=${POSTGRES_PASSWORD} fi -if [ ! -z "${PGDATA}" ]; then +if [ -n "${PGDATA}" ]; then DATADIR=${PGDATA} fi -if [ ! -z "$POSTGRES_DB" ]; then +if [ -n "${POSTGRES_DB}" ]; then POSTGRES_DBNAME=${POSTGRES_DB} fi +if [ -n "${POSTGRES_INITDB_ARGS}" ]; then + INITDB_EXTRA_ARGS=${POSTGRES_INITDB_ARGS} +fi + list=(`echo ${POSTGRES_DBNAME} | tr ',' ' '`) arr=(${list}) SINGLE_DB=${arr[0]} + + # usable function definitions -function restart_postgres { -PID=`cat ${PG_PID}` -kill -TERM ${PID} +function kill_postgres { + PID=`cat ${PG_PID}` + kill -TERM ${PID} -# Wait for background postgres main process to exit -while [[ "$(ls -A ${PG_PID} 2>/dev/null)" ]]; do - sleep 1 -done + # Wait for background postgres main process to exit + # wait until PID file gets deleted + while ls -A ${PG_PID} 2> /dev/null; do + sleep 1 + done -# Brought postgres back up again -source /env-data.sh -su - postgres -c "${POSTGRES} -D ${DATADIR} -c config_file=${CONF} ${LOCALONLY} &" - -# wait for postgres to come up -until su - postgres -c "psql -l"; do - sleep 1 -done -echo "postgres ready" + return 0 +} + +function restart_postgres { + + kill_postgres + + # Brought postgres back up again + source /env-data.sh + su - postgres -c "$SETVARS $POSTGRES -D $DATADIR -c config_file=$CONF &" + + # wait for postgres to come up + until su - postgres -c "pg_isready"; do + sleep 1 + done + echo "postgres ready" + return 0 +} + + + +# Running extended script or sql if provided. +# Useful for people who extends the image. +function entry_point_script { + SETUP_LOCKFILE="/docker-entrypoint-initdb.d/.entry_point.lock" + # If lockfile doesn't exists, proceed. + if [[ ! -f "${SETUP_LOCKFILE}" ]]; then + if find "/docker-entrypoint-initdb.d" -mindepth 1 -print -quit 2>/dev/null | grep -q .; then + for f in /docker-entrypoint-initdb.d/*; do + export PGPASSWORD=${POSTGRES_PASS} + case "$f" in + *.sql) echo "$0: running $f"; psql ${SINGLE_DB} -U ${POSTGRES_USER} -p 5432 -h localhost -f ${f} || true ;; + *.sql.gz) echo "$0: running $f"; gunzip < "$f" | psql ${SINGLE_DB} -U ${POSTGRES_USER} -p 5432 -h localhost || true ;; + *.sh) echo "$0: running $f"; . $f || true;; + *) echo "$0: ignoring $f" ;; + esac + echo + done + # Put lock file to make sure entry point scripts were run + touch ${SETUP_LOCKFILE} + fi + fi + + return 0 } diff --git a/scenario_tests/.gitignore b/scenario_tests/.gitignore new file mode 100644 index 0000000..7376571 --- /dev/null +++ b/scenario_tests/.gitignore @@ -0,0 +1 @@ +docker-compose.override.yml diff --git a/scenario_tests/README.md b/scenario_tests/README.md new file mode 100644 index 0000000..0af9888 --- /dev/null +++ b/scenario_tests/README.md @@ -0,0 +1,75 @@ +# TESTING GUIDE + + +## TL;DR; How to run the test + +Go into root repo and run + +``` +./build-test.sh +``` + +It will create a tagged image `kartoza/postgis:manual-build` + +Each scenario tests in this directory use this image. + +To run each scenario test, go into the scenario directory and run test script: + +``` +# Testing collations scenario +cd collations +./test.sh +``` + +## Testing architecture + +We make two stage of testing in travis. +First build stage, we build the image, push it into docker hub. +Then the testing stage will +use this image to run each scenario tests (run in parallel). + +Any testing framework can be used, but we use python `unittest` module for simplicity and readability. + +Due to the shared testing image used for other scenarios, for simplicity, put +relevant python dependencies on `utils/requirement.txt`. + +For OS level dependencies, include your setup in the root folder's `Dockerfile.test` + +## Making new tests + +Create new directory in this folder (`scenario_tests`). +Directory should contains: + +- Host level test script called `test.sh` +- `docker-compose.yml` file for the service setup +- `.env` file if needed for `docker-compose.yml` settings +- `tests` directory which contains your test scripts. The testing architecture will + execute `test.sh` script in this directory (service level), if you use generic host level `test.sh` script. + + +Explanations: + +Host level test script is used to setup the docker service, then run the unit test when +the service is ready. You can copy paste from existing `collations` for generic script. + +`docker-compose.yml` file should mount your `tests` directory and provides settings +needed by the service that are going to be tested. + +`tests` directory contains the actual test script that will be run from *inside* +the service. For example, in `collations` scenario `test.sh` (service level scripts) +will start python unittest script with necessary variables. + +Add your scenario to travis config: + +In `jobs.include[]` list there will be `stage: test` entry. +Add your environment variable needed to run your test in `stage.env` list. +For example, if you have new scenario folder `my_test`, then the env key +will look like this: + +``` +- stage: test + env: + - SCENARIO=replications + - SCENARIO=collations + - SCENARIO=my_test EXTRA_SETTING_1=value1 EXTRA_SETTING_2=value2 EXTRA_SETTING_3=value3 +``` diff --git a/scenario_tests/collations/docker-compose.yml b/scenario_tests/collations/docker-compose.yml new file mode 100644 index 0000000..f3239ed --- /dev/null +++ b/scenario_tests/collations/docker-compose.yml @@ -0,0 +1,54 @@ + +version: '2.1' + +volumes: + pg-data-dir: + new-pg-data-dir: + +services: + pg: + image: 'kartoza/postgis:${TAG:-manual-build}' + restart: 'always' + # You can optionally mount to volume, to play with the persistence and + # observe how the node will behave after restarts. + volumes: + - pg-data-dir:/var/lib/postgresql + - ./tests:/tests + - ../utils:/lib/utils + environment: + DEFAULT_COLLATION: ${DEFAULT_COLLATION:-id_ID.utf8} + DEFAULT_CTYPE: ${DEFAULT_COLLATION:-id_ID.utf8} + ALLOW_IP_RANGE: '0.0.0.0/0' + TEST_CLASS: test_collation.TestCollationDefault + ports: + - "7777:5432" + healthcheck: + interval: 60s + timeout: 30s + retries: 3 + test: "pg_isready" + + pg-new: + image: 'kartoza/postgis:${TAG:-manual-build}' + restart: 'always' + # You can optionally mount to volume, to play with the persistence and + # observe how the node will behave after restarts. + volumes: + - new-pg-data-dir:/opt/data/postgis + - ./tests:/tests + - ../utils:/lib/utils + hostname: 'pg-new' + environment: + RECREATE_DATADIR: "True" + PGDATA: /opt/data/postgis + DEFAULT_COLLATION: ${DEFAULT_COLLATION:-id_ID.utf8} + DEFAULT_CTYPE: ${DEFAULT_COLLATION:-id_ID.utf8} + ALLOW_IP_RANGE: '0.0.0.0/0' + TEST_CLASS: test_collation.TestCollationInitialization + ports: + - "7776:5432" + healthcheck: + interval: 60s + timeout: 30s + retries: 3 + test: "pg_isready" diff --git a/scenario_tests/collations/test.sh b/scenario_tests/collations/test.sh new file mode 100755 index 0000000..1b36eef --- /dev/null +++ b/scenario_tests/collations/test.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# exit immediately if test fails +set -e + +source ../test-env.sh + +# Run service +docker-compose up -d + +sleep 5 + +services=("pg" "pg-new") + +for service in "${services[@]}"; do + + # Execute tests + until docker-compose exec $service pg_isready; do + sleep 1 + done; + docker-compose exec $service /bin/bash /tests/test.sh + +done + +docker-compose down -v diff --git a/scenario_tests/collations/tests/__init__.py b/scenario_tests/collations/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scenario_tests/collations/tests/test.sh b/scenario_tests/collations/tests/test.sh new file mode 100644 index 0000000..f1ad178 --- /dev/null +++ b/scenario_tests/collations/tests/test.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -e + +source /env-data.sh + +# execute tests +pushd /tests + +cat << EOF +Settings used: + +DEFAULT_COLLATION: ${DEFAULT_COLLATION} +DEFAULT_CTYPE: ${DEFAULT_CTYPE} +EOF + +PGHOST=localhost \ +PGDATABASE=gis \ +PYTHONPATH=/lib \ + python3 -m unittest -v ${TEST_CLASS} diff --git a/scenario_tests/collations/tests/test_collation.py b/scenario_tests/collations/tests/test_collation.py new file mode 100644 index 0000000..8b96094 --- /dev/null +++ b/scenario_tests/collations/tests/test_collation.py @@ -0,0 +1,95 @@ +import unittest +import os +from utils.utils import DBConnection + + +class TestCollationBase(unittest.TestCase): + + def setUp(self): + self.db = DBConnection() + + def fetch_collation(self, cursor, dbname): + cursor.execute( + """ + select datcollate, datctype from pg_database where datname = '{}'; + """.format(dbname) + ) + + row = cursor.fetchone() + return row + + +class TestCollationDefault(TestCollationBase): + + def test_check_collation(self): + # create new table + self.db.conn.autocommit = True + with self.db.cursor() as c: + + # Specified database created by entrypoint script should have + # the correct collation from database clusters + # DEFAULT_COLLATION and DEFAULT_CTYPE will be ignored + dbcollate, dbctype = self.fetch_collation(c, 'gis') + self.assertEqual(dbcollate, 'C.UTF-8') + self.assertEqual(dbctype, 'C.UTF-8') + + c.execute( + """ + drop database if exists sample_db; + """ + ) + c.execute( + """ + create database sample_db; + """ + ) + + # Database created manually will have default settings + dbcollate, dbctype = self.fetch_collation(c, 'sample_db') + self.assertEqual(dbcollate, 'C.UTF-8') + self.assertEqual(dbctype, 'C.UTF-8') + + # Default database created by entrypoint script have + # default collation + dbcollate, dbctype = self.fetch_collation(c, 'postgres') + self.assertEqual(dbcollate, 'C.UTF-8') + self.assertEqual(dbctype, 'C.UTF-8') + + +class TestCollationInitialization(TestCollationBase): + + def test_check_collation_in_new_datadir(self): + # create new table + default_collation = os.environ.get('DEFAULT_COLLATION') + default_ctype = os.environ.get('DEFAULT_CTYPE') + self.db.conn.autocommit = True + with self.db.cursor() as c: + + # Specified database created by entrypoint script should have + # the correct collation from database clusters + # DEFAULT_COLLATION and DEFAULT_CTYPE will be ignored + dbcollate, dbctype = self.fetch_collation(c, 'gis') + self.assertEqual(dbcollate, default_collation) + self.assertEqual(dbctype, default_ctype) + + c.execute( + """ + drop database if exists sample_db; + """ + ) + c.execute( + """ + create database sample_db; + """ + ) + + # Database created manually will have default settings + dbcollate, dbctype = self.fetch_collation(c, 'sample_db') + self.assertEqual(dbcollate, default_collation) + self.assertEqual(dbctype, default_ctype) + + # Default database created by entrypoint script have + # default collation + dbcollate, dbctype = self.fetch_collation(c, 'postgres') + self.assertEqual(dbcollate, default_collation) + self.assertEqual(dbctype, default_ctype) diff --git a/scenario_tests/datadir_init/docker-compose.yml b/scenario_tests/datadir_init/docker-compose.yml new file mode 100644 index 0000000..6755a3d --- /dev/null +++ b/scenario_tests/datadir_init/docker-compose.yml @@ -0,0 +1,57 @@ +version: '2.1' +volumes: + default-pg-data-dir: + new-pg-data-dir: + recreate-pg-data-dir: +services: + pg-default: + image: 'kartoza/postgis:${TAG:-manual-build}' + volumes: + # By default persisted volumes should be in /var/lib/postgresql + - default-pg-data-dir:/var/lib/postgresql + - ./tests:/tests + - ../utils:/lib/utils + environment: + # Default usage, no datadir location defined + TEST_CLASS: TestDefault + healthcheck: + interval: 60s + timeout: 30s + retries: 3 + test: "pg_isready" + + pg-new: + image: 'kartoza/postgis:${TAG:-manual-build}' + volumes: + # Mount to new locations where there are no initial data + - new-pg-data-dir:/opt/mypostgis/data + - ./tests:/tests + - ../utils:/lib/utils + environment: + # Tell the new location + TEST_CLASS: TestNew + DATADIR: /opt/mypostgis/data + healthcheck: + interval: 60s + timeout: 30s + retries: 3 + test: "pg_isready" + + pg-recreate: + image: 'kartoza/postgis:${TAG:-manual-build}' + volumes: + - recreate-pg-data-dir:/var/lib/postgresql + - ./tests:/tests + - ../utils:/lib/utils + environment: + # Tell that you are going to perform cluster reinitialization + TEST_CLASS: TestRecreate + RECREATE_DATADIR: "True" + DEFAULT_ENCODING: ${DEFAULT_ENCODING:-UTF-8} + DEFAULT_COLLATION: ${DEFAULT_COLLATION:-id_ID.utf8} + DEFAULT_CTYPE: ${DEFAULT_COLLATION:-id_ID.utf8} + healthcheck: + interval: 60s + timeout: 30s + retries: 3 + test: "pg_isready" diff --git a/scenario_tests/datadir_init/test.sh b/scenario_tests/datadir_init/test.sh new file mode 100755 index 0000000..d46f2c6 --- /dev/null +++ b/scenario_tests/datadir_init/test.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# exit immediately if test fails +set -e + +source ../test-env.sh + +# Run service +docker-compose up -d + +sleep 5 + +services=("pg-default" "pg-new" "pg-recreate") + +for service in "${services[@]}"; do + + # Execute tests + until docker-compose exec $service pg_isready; do + sleep 1 + done; + docker-compose exec $service /bin/bash /tests/test.sh + +done + +docker-compose down -v diff --git a/scenario_tests/datadir_init/tests/__init__.py b/scenario_tests/datadir_init/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scenario_tests/datadir_init/tests/test.sh b/scenario_tests/datadir_init/tests/test.sh new file mode 100644 index 0000000..b30adf9 --- /dev/null +++ b/scenario_tests/datadir_init/tests/test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -e + +source /env-data.sh + +# execute tests +pushd /tests + +cat << EOF +Settings used: + +RECREATE_DATADIR: ${RECREATE_DATADIR} +DATADIR: ${DATADIR} +PGDATA: ${PGDATA} +INITDB_EXTRA_ARGS: ${INITDB_EXTRA_ARGS} +EOF + +PGHOST=localhost \ +PGDATABASE=gis \ +PYTHONPATH=/lib \ + python3 -m unittest -v test_datadir.${TEST_CLASS} diff --git a/scenario_tests/datadir_init/tests/test_datadir.py b/scenario_tests/datadir_init/tests/test_datadir.py new file mode 100644 index 0000000..99897b0 --- /dev/null +++ b/scenario_tests/datadir_init/tests/test_datadir.py @@ -0,0 +1,80 @@ +import unittest +import os +from utils.utils import DBConnection + + +class TestCollationBase(unittest.TestCase): + + def setUp(self): + self.db = DBConnection() + + def fetch_collation(self, cursor, dbname): + cursor.execute( + """ + select datcollate, datctype from pg_database where datname = '{}'; + """.format(dbname) + ) + + row = cursor.fetchone() + return row + + def fetch_datadir_location(self, cursor): + cursor.execute( + """ + show data_directory; + """ + ) + row = cursor.fetchone() + return row[0] + + +class TestDefault(TestCollationBase): + + def test_check_collation(self): + # create new table + self.db.conn.autocommit = True + with self.db.cursor() as c: + + # Check datadir locations + self.assertTrue( + self.fetch_datadir_location(c).startswith( + '/var/lib/postgresql' + ) + ) + + +class TestNew(TestCollationBase): + + def test_check_collation_in_new_datadir(self): + # create new table + self.db.conn.autocommit = True + with self.db.cursor() as c: + + # Check datadir locations + self.assertTrue( + self.fetch_datadir_location(c).startswith( + os.environ.get('DATADIR') + ) + ) + + +class TestRecreate(TestCollationBase): + + def test_check_collation_in_new_datadir(self): + # create new table + self.db.conn.autocommit = True + with self.db.cursor() as c: + + # Check datadir locations + self.assertTrue( + self.fetch_datadir_location(c).startswith( + '/var/lib/postgresql' + ) + ) + + # Check that the new cluster is not the default cluster + # from it's collations + dbcollate, dbctype = self.fetch_collation(c, 'gis') + self.assertEqual(dbcollate, os.environ.get('DEFAULT_COLLATION')) + self.assertEqual(dbctype, os.environ.get('DEFAULT_CTYPE')) + diff --git a/scenario_tests/replications/docker-compose.yml b/scenario_tests/replications/docker-compose.yml new file mode 100644 index 0000000..e934b02 --- /dev/null +++ b/scenario_tests/replications/docker-compose.yml @@ -0,0 +1,100 @@ + +version: '2.1' + +volumes: + pg-master-data-dir: + pg-node-data-dir: + +services: + pg-master: + image: 'kartoza/postgis:${TAG:-manual-build}' + restart: 'always' + # You can optionally mount to volume, to play with the persistence and + # observe how the node will behave after restarts. + volumes: + - pg-master-data-dir:/var/lib/postgresql + - ./tests:/tests + - ../utils:/lib/utils + environment: + # ALLOW_IP_RANGE option is used to specify additionals allowed domains + # in pg_hba. + # This range should allow nodes to connect to master + ALLOW_IP_RANGE: '0.0.0.0/0' + + # We can specify optional credentials + REPLICATION_USER: 'replicator' + REPLICATION_PASS: 'replicator' + # Setup master replication variables + #PG_MAX_WAL_SENDERS: 8 + #PG_WAL_KEEP_SEGMENTS: 100 + # You can expose the port to observe it in your local machine + ports: + - "7777:5432" + healthcheck: + interval: 60s + timeout: 30s + retries: 3 + test: "pg_isready" + + pg-node: + image: 'kartoza/postgis:${TAG:-manual-build}' + restart: 'always' + # You can optionally mount to volume, but we're not able to scale it + # in that case. + # The node will always destroy its database and copy from master at + # runtime + volumes: + - pg-node-data-dir:/var/lib/postgresql + - ./tests:/tests + - ../utils:/lib/utils + + environment: + # ALLOW_IP_RANGE option is used to specify additionals allowed domains + # in pg_hba. + # Not really needed in nodes for the replication, but optionally can + # be put when nodes are needed to be a failover server when master + # is down. The IP Range are generally needed if other services wants to + # connect to this node + ALLOW_IP_RANGE: '0.0.0.0/0' + + # REPLICATE_FROM options accepts domain-name or IP address + # with this in mind, you can also put docker service name, because it + # will be resolved as host name. + REPLICATE_FROM: 'pg-master' + + # REPLICATE_PORT will default to 5432 if not specified. + # REPLICATE_PORT: '5432' + # In the case where you need to replicate from outside service, + # you can put the server address and port here, as long as the target + # where configured as master, and replicable. +# REPLICATE_FROM: '192.168.1.8' +# REPLICATE_PORT: '7777' + + # DESTROY_DATABASE_ON_RESTART will default to True if not specified. + # If specified other than True, it will prevent node from destroying + # database on restart + DESTROY_DATABASE_ON_RESTART: 'True' + + # PROMOTE_MASTER Default empty. + # If specified with any value, then it will convert current node into + # a writable state. Useful if master is down and the current node needs + # to be promoted until manual recovery. +# PROMOTE_MASTER: 'True' + + # For now we don't support different credentials for replication + # so we use the same credentials as master's superuser, or anything that + # have replication role. + REPLICATION_USER: 'replicator' + REPLICATION_PASS: 'replicator' + depends_on: + pg-master: + condition: service_healthy + # You can expose the port to observe it in your local machine + # For this sample, it was disabled by default to allow scaling test + ports: + - "7776:5432" + healthcheck: + interval: 60s + timeout: 30s + retries: 3 + test: "pg_isready" diff --git a/scenario_tests/replications/test.sh b/scenario_tests/replications/test.sh new file mode 100755 index 0000000..cd7b07c --- /dev/null +++ b/scenario_tests/replications/test.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# exit immediately if test fails +set -e + +source ../test-env.sh + +# Run service +docker-compose up -d + +sleep 5 + +# Preparing master cluster +until docker-compose exec pg-master pg_isready; do + sleep 1 +done; + +# Execute tests +docker-compose exec pg-master /bin/bash /tests/test_master.sh + +# Preparing node cluster +until docker-compose exec pg-node pg_isready; do + sleep 1 +done; + +# Execute tests +docker-compose exec pg-node /bin/bash /tests/test_node.sh + +docker-compose down -v diff --git a/scenario_tests/replications/tests/__init__.py b/scenario_tests/replications/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scenario_tests/replications/tests/test_master.sh b/scenario_tests/replications/tests/test_master.sh new file mode 100644 index 0000000..e9f0840 --- /dev/null +++ b/scenario_tests/replications/tests/test_master.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -e + +source /env-data.sh + +# execute tests +pushd /tests + +PGHOST=localhost \ +PGDATABASE=gis \ +PYTHONPATH=/lib \ + python3 -m unittest -v test_replication.TestReplicationMaster diff --git a/scenario_tests/replications/tests/test_node.sh b/scenario_tests/replications/tests/test_node.sh new file mode 100644 index 0000000..ca2068e --- /dev/null +++ b/scenario_tests/replications/tests/test_node.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -e + +source /env-data.sh + +# execute tests +pushd /tests + +PGHOST=localhost \ +PGDATABASE=gis \ +PYTHONPATH=/lib \ + python3 -m unittest -v test_replication.TestReplicationNode diff --git a/scenario_tests/replications/tests/test_replication.py b/scenario_tests/replications/tests/test_replication.py new file mode 100644 index 0000000..06bbf62 --- /dev/null +++ b/scenario_tests/replications/tests/test_replication.py @@ -0,0 +1,59 @@ +import unittest +from utils.utils import DBConnection + + +class TestReplicationMaster(unittest.TestCase): + + def setUp(self): + self.db = DBConnection() + + def test_create_new_data(self): + # create new table + self.db.conn.autocommit = True + with self.db.cursor() as c: + c.execute( + """ + CREATE TABLE IF NOT EXISTS test_replication_table ( + id integer not null + constraint pkey primary key, + geom geometry(Point, 4326), + name varchar(30), + alias varchar(30), + description varchar(255) + ); + """ + ) + + c.execute( + """ + INSERT INTO test_replication_table (id, geom, name, alias, description) + VALUES + ( + 1, + st_setsrid(st_point(107.6097, 6.9120), 4326), + 'Bandung', + 'Paris van Java', + 'Asia-Africa conference was held here' + ) ON CONFLICT DO NOTHING; + """ + ) + + +class TestReplicationNode(unittest.TestCase): + + def setUp(self): + self.db = DBConnection() + + def test_read_data(self): + # create new table + self.db.conn.autocommit = True + with self.db.cursor() as c: + c.execute( + """ + SELECT * FROM test_replication_table; + """ + ) + + rows = c.fetchall() + self.assertEqual(len(rows), 1) + diff --git a/scenario_tests/test-env.sh b/scenario_tests/test-env.sh new file mode 100644 index 0000000..e368c89 --- /dev/null +++ b/scenario_tests/test-env.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +# Display test environment variable + +cat << EOF +Test environment: + +Compose Project : ${COMPOSE_PROJECT_NAME} +Compose File : ${COMPOSE_PROJECT_FILE} +Image tag : ${TAG} + +EOF diff --git a/scenario_tests/utils/__init__.py b/scenario_tests/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scenario_tests/utils/requirements.txt b/scenario_tests/utils/requirements.txt new file mode 100644 index 0000000..37ec460 --- /dev/null +++ b/scenario_tests/utils/requirements.txt @@ -0,0 +1 @@ +psycopg2-binary diff --git a/scenario_tests/utils/utils.py b/scenario_tests/utils/utils.py new file mode 100644 index 0000000..1bae017 --- /dev/null +++ b/scenario_tests/utils/utils.py @@ -0,0 +1,43 @@ +import os + +import psycopg2 + + +class DBConnection: + + def __init__(self): + self.conn = DBConnection.create_conn() + + def table_exists(self, table_name, table_schema='public'): + cur = self.conn.cursor() + query = ( + 'select ' + 'exists(' + 'select 1 ' + 'from information_schema.tables ' + 'where table_name = %s and table_schema = %s)') + cur.execute(query, (table_name, table_schema)) + try: + row = cur.fetchone() + return row[0] + except: + return False + + @staticmethod + def create_conn(): + """ + :return: psycopg2.connection + """ + return psycopg2.connect( + host=os.environ.get('POSTGRES_HOST'), + database=os.environ.get('POSTGRES_DB'), + user=os.environ.get('POSTGRES_USER'), + password=os.environ.get('POSTGRES_PASS'), + port=os.environ.get('POSTGRES_PORT') + ) + + def cursor(self): + """ + :return: psycopg2.cursor + """ + return self.conn.cursor() diff --git a/setup-conf.sh b/setup-conf.sh index 44325d6..e0fe92d 100644 --- a/setup-conf.sh +++ b/setup-conf.sh @@ -15,6 +15,11 @@ SINGLE_DB=${arr[0]} # Refresh configuration in case environment settings changed. cat $CONF.template > $CONF +# Reflect DATADIR loaction +# Delete any data_dir declarations +sed -i '/data_directory/d' $CONF +echo "data_directory = '${DATADIR}'" >> $CONF + # This script will setup necessary configuration to optimise for PostGIS and to enable replications cat >> $CONF < /dev/null)" || "${RECREATE_DATADIR}" == 'TRUE' ]]; then + # Only attempt reinitializations if ${RECREATE_DATADIR} is true + # No Replicate From settings. Assume that this is a master database. + # Initialise db + echo "Initializing Postgres Database at ${DATADIR}" + mkdir -p ${DATADIR} + rm -rf ${DATADIR}/* + chown -R postgres:postgres ${DATADIR} + echo "Initializing with command:" + command="$INITDB -U postgres -E ${DEFAULT_ENCODING} --lc-collate=${DEFAULT_COLLATION} --lc-ctype=${DEFAULT_CTYPE} --wal-segsize=${WAL_SEGSIZE} -D ${DATADIR} ${INITDB_EXTRA_ARGS}" + su - postgres -c "$command" fi; + # Set proper permissions # needs to be done as root: chown -R postgres:postgres ${DATADIR} +chmod -R 750 ${DATADIR} # test database existing trap "echo \"Sending SIGTERM to postgres\"; killall -s SIGTERM postgres" SIGTERM - +# Run as local only for config setup phase to avoid outside access su - postgres -c "${POSTGRES} -D ${DATADIR} -c config_file=${CONF} ${LOCALONLY} &" # wait for postgres to come up -until su - postgres -c "psql -l"; do +until su - postgres -c "pg_isready"; do sleep 1 done echo "postgres ready" @@ -42,7 +39,7 @@ echo "postgres ready" source /setup-user.sh # enable extensions in template1 if env variable set to true -if [ "$POSTGRES_TEMPLATE_EXTENSIONS" = true ] ; then +if [[ "$(boolean ${POSTGRES_TEMPLATE_EXTENSIONS})" == TRUE ]] ; then for ext in $(echo ${POSTGRES_MULTIPLE_EXTENSIONS} | tr ',' ' '); do echo "Enabling ${ext} in the database template1" su - postgres -c "psql -c 'CREATE EXTENSION IF NOT EXISTS ${ext} cascade;' template1" @@ -60,7 +57,7 @@ for db in $(echo ${POSTGRES_DBNAME} | tr ',' ' '); do RESULT=`su - postgres -c "psql -t -c \"SELECT count(1) from pg_database where datname='${db}';\""` if [[ ${RESULT} -eq 0 ]]; then echo "Create db ${db}" - su - postgres -c "createdb -O ${POSTGRES_USER} ${db}" + su - postgres -c "createdb -O ${POSTGRES_USER} ${db}" for ext in $(echo ${POSTGRES_MULTIPLE_EXTENSIONS} | tr ',' ' '); do echo "Enabling ${ext} in the database ${db}" if [[ ${ext} = 'pg_cron' ]]; then @@ -72,8 +69,7 @@ for db in $(echo ${POSTGRES_DBNAME} | tr ',' ' '); do echo "Loading legacy sql" su - postgres -c "psql ${db} -f ${SQLDIR}/legacy_minimal.sql" || true su - postgres -c "psql ${db} -f ${SQLDIR}/legacy_gist.sql" || true - export PGPASSWORD=${POSTGRES_PASS} - psql ${db} -U ${POSTGRES_USER} -p 5432 -h localhost -f custom.sql + PGPASSWORD=${POSTGRES_PASS} psql ${db} -U ${POSTGRES_USER} -p 5432 -h localhost -f custom.sql else echo "${db} db already exists" @@ -87,4 +83,4 @@ fi rm custom.sql # This should show up in docker logs afterwards -su - postgres -c "psql -l" +su - postgres -c "psql -l 2>&1"