diff --git a/.github/workflows/build-latest.yaml b/.github/workflows/build-latest.yaml index 0bd2a77..8d24c4c 100644 --- a/.github/workflows/build-latest.yaml +++ b/.github/workflows/build-latest.yaml @@ -135,6 +135,7 @@ jobs: - collations - extensions - logical_replication + - logical_replications include: - distro: debian imageVersion: bullseye @@ -177,6 +178,7 @@ jobs: - name: Run scenario test env: COMPOSE_INTERACTIVE_NO_CLI: 1 + PRINT_TEST_LOGS: 1 run: | pushd scenario_tests/${{ matrix.scenario }} ./test.sh diff --git a/scenario_tests/collations/test.sh b/scenario_tests/collations/test.sh index 28a3953..703fb2e 100755 --- a/scenario_tests/collations/test.sh +++ b/scenario_tests/collations/test.sh @@ -8,6 +8,10 @@ source ../test-env.sh # Run service docker-compose up -d +if [[ -n "${PRINT_TEST_LOGS}" ]]; then + docker-compose logs -f & +fi + sleep 30 services=("pg" "pg-new") @@ -15,10 +19,12 @@ services=("pg" "pg-new") for service in "${services[@]}"; do # Execute tests - until docker-compose exec $service pg_isready; do + until docker-compose exec -T $service pg_isready; do sleep 30 + echo "Wait service to be ready" done; - docker-compose exec $service /bin/bash /tests/test.sh + echo "Execute test for $service" + docker-compose exec -T $service /bin/bash /tests/test.sh done diff --git a/scenario_tests/datadir_init/test.sh b/scenario_tests/datadir_init/test.sh index 4c041c5..78b212c 100755 --- a/scenario_tests/datadir_init/test.sh +++ b/scenario_tests/datadir_init/test.sh @@ -8,6 +8,10 @@ source ../test-env.sh # Run service docker-compose up -d +if [[ -n "${PRINT_TEST_LOGS}" ]]; then + docker-compose logs -f & +fi + sleep 60 services=("pg-default" "pg-new" "pg-recreate") @@ -15,10 +19,12 @@ services=("pg-default" "pg-new" "pg-recreate") for service in "${services[@]}"; do # Execute tests - until docker-compose exec $service pg_isready; do - sleep 1 + until docker-compose exec -T $service pg_isready; do + sleep 5 + echo "Wait service to be ready" done; - docker-compose exec $service /bin/bash /tests/test.sh + echo "Execute test for $service" + docker-compose exec -T $service /bin/bash /tests/test.sh done diff --git a/scenario_tests/extensions/test.sh b/scenario_tests/extensions/test.sh index d1e4dd7..06131df 100755 --- a/scenario_tests/extensions/test.sh +++ b/scenario_tests/extensions/test.sh @@ -8,6 +8,10 @@ source ../test-env.sh # Run service docker-compose up -d +if [[ -n "${PRINT_TEST_LOGS}" ]]; then + docker-compose logs -f & +fi + sleep 30 services=("pg" "pg-two-extensions") @@ -15,10 +19,12 @@ services=("pg" "pg-two-extensions") for service in "${services[@]}"; do # Execute tests - until docker-compose exec $service pg_isready; do + until docker-compose exec -T $service pg_isready; do sleep 30 + echo "Wait service to be ready" done; - docker-compose exec $service /bin/bash /tests/test.sh + echo "Execute test for $service" + docker-compose exec -T $service /bin/bash /tests/test.sh done diff --git a/scenario_tests/logical_replications/docker-compose.yml b/scenario_tests/logical_replications/docker-compose.yml new file mode 100644 index 0000000..7681b66 --- /dev/null +++ b/scenario_tests/logical_replications/docker-compose.yml @@ -0,0 +1,104 @@ + +version: '2.1' + +volumes: + pg-master-data-dir: + pg-node-data-dir: + +services: + pg-publisher: + image: 'kartoza/postgis:${TAG:-manual-build}' + restart: 'always' + # You can optionally mount to volume, to play with the persistence and + # observe how the node will behave after restarts. + volumes: + - pg-master-data-dir:/var/lib/postgresql + - ./scripts/setup-publisher.sql:/docker-entrypoint-initdb.d/setup-publisher.sql + - ./tests:/tests + - ../utils:/lib/utils + environment: + # ALLOW_IP_RANGE option is used to specify additionals allowed domains + # in pg_hba. + # This range should allow nodes to connect to master + ALLOW_IP_RANGE: '0.0.0.0/0' + + # We can specify optional credentials + REPLICATION_USER: 'replicator' + REPLICATION_PASS: 'replicator' + WAL_LEVEL: 'logical' + # Setup master replication variables + #PG_MAX_WAL_SENDERS: 8 + #PG_WAL_KEEP_SEGMENTS: 100 + # You can expose the port to observe it in your local machine + ports: + - "7777:5432" + healthcheck: + interval: 60s + timeout: 30s + retries: 3 + test: "pg_isready" + + pg-subscriber: + image: 'kartoza/postgis:${TAG:-manual-build}' + restart: 'always' + # You can optionally mount to volume, but we're not able to scale it + # in that case. + # The node will always destroy its database and copy from master at + # runtime + volumes: + - pg-node-data-dir:/var/lib/postgresql + - ./scripts/setup-subscriber.sql:/docker-entrypoint-initdb.d/setup-subscriber.sql + - ./tests:/tests + - ../utils:/lib/utils + + environment: + # ALLOW_IP_RANGE option is used to specify additionals allowed domains + # in pg_hba. + # Not really needed in nodes for the replication, but optionally can + # be put when nodes are needed to be a failover server when master + # is down. The IP Range are generally needed if other services wants to + # connect to this node + ALLOW_IP_RANGE: '0.0.0.0/0' + + # REPLICATE_FROM options accepts domain-name or IP address + # with this in mind, you can also put docker service name, because it + # will be resolved as host name. + #REPLICATE_FROM: 'pg-publisher' + + # REPLICATE_PORT will default to 5432 if not specified. + # REPLICATE_PORT: '5432' + # In the case where you need to replicate from outside service, + # you can put the server address and port here, as long as the target + # where configured as master, and replicable. +# REPLICATE_FROM: '192.168.1.8' +# REPLICATE_PORT: '7777' + + # DESTROY_DATABASE_ON_RESTART will default to True if not specified. + # If specified other than True, it will prevent node from destroying + # database on restart + DESTROY_DATABASE_ON_RESTART: 'True' + + # PROMOTE_MASTER Default empty. + # If specified with any value, then it will convert current node into + # a writable state. Useful if master is down and the current node needs + # to be promoted until manual recovery. +# PROMOTE_MASTER: 'True' + + # For now we don't support different credentials for replication + # so we use the same credentials as master's superuser, or anything that + # have replication role. + REPLICATION_USER: 'replicator' + REPLICATION_PASS: 'replicator' + WAL_LEVEL: 'logical' + depends_on: + pg-publisher: + condition: service_healthy + # You can expose the port to observe it in your local machine + # For this sample, it was disabled by default to allow scaling test + ports: + - "7776:5432" +# healthcheck: +# interval: 60s +# timeout: 30s +# retries: 3 +# test: "pg_isready" diff --git a/scenario_tests/logical_replications/scripts/setup-publisher.sql b/scenario_tests/logical_replications/scripts/setup-publisher.sql new file mode 100644 index 0000000..ffac7ba --- /dev/null +++ b/scenario_tests/logical_replications/scripts/setup-publisher.sql @@ -0,0 +1,22 @@ +-- Create a table +CREATE TABLE sweets + ( + id SERIAL, + name TEXT, + price DECIMAL, + CONSTRAINT sweets_pkey PRIMARY KEY (id) + ); + +CREATE TABLE public.block ( + id serial NOT NULL, + geom public.geometry(Polygon,4326), + fid bigint, + tile_name character varying, + location character varying +); + +-- Add table to publication called logical_replication which is created by the scripts +ALTER PUBLICATION logical_replication ADD TABLE sweets; +ALTER PUBLICATION logical_replication ADD TABLE block; +-- Inserts records into the table +INSERT INTO sweets (name, price) VALUES ('strawberry', 4.50), ('Coffee', 6.20), ('lollipop', 3.80); diff --git a/scenario_tests/logical_replications/scripts/setup-subscriber.sql b/scenario_tests/logical_replications/scripts/setup-subscriber.sql new file mode 100644 index 0000000..0d0e456 --- /dev/null +++ b/scenario_tests/logical_replications/scripts/setup-subscriber.sql @@ -0,0 +1,20 @@ +-- Create a table +CREATE TABLE sweets + ( + id SERIAL, + name TEXT, + price DECIMAL, + CONSTRAINT sweets_pkey PRIMARY KEY (id) + ); + +CREATE TABLE public.block ( + id serial NOT NULL, + geom public.geometry(Polygon,4326), + fid bigint, + tile_name character varying, + location character varying +); +-- Create a publication +CREATE SUBSCRIPTION logical_subscription + CONNECTION 'host=pg-publisher port=5432 password=docker user=docker dbname=gis' + PUBLICATION logical_replication; diff --git a/scenario_tests/logical_replications/test.sh b/scenario_tests/logical_replications/test.sh new file mode 100755 index 0000000..4d4105a --- /dev/null +++ b/scenario_tests/logical_replications/test.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# exit immediately if test fails +set -e + +source ../test-env.sh + +# Run service +docker-compose up -d + +if [[ -n "${PRINT_TEST_LOGS}" ]]; then + docker-compose logs -f & +fi + +sleep 30 + +# Preparing master cluster +until docker-compose exec -T pg-publisher pg_isready; do + sleep 1 +done; + +# Execute tests +docker-compose exec -T pg-publisher /bin/bash /tests/test_master.sh + +# Preparing node cluster +until docker-compose exec -T pg-subscriber pg_isready; do + sleep 1 +done; + +# Execute tests +docker-compose exec -T pg-node /bin/bash /tests/test_node.sh + +docker-compose down -v diff --git a/scenario_tests/logical_replications/tests/__init__.py b/scenario_tests/logical_replications/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scenario_tests/logical_replications/tests/test_master.sh b/scenario_tests/logical_replications/tests/test_master.sh new file mode 100644 index 0000000..fdc4042 --- /dev/null +++ b/scenario_tests/logical_replications/tests/test_master.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -e + +source /scripts/env-data.sh + +# execute tests +pushd /tests + +PGHOST=localhost \ +PGDATABASE=gis \ +PYTHONPATH=/lib \ + python3 -m unittest -v test_replication.TestReplicationMaster diff --git a/scenario_tests/logical_replications/tests/test_node.sh b/scenario_tests/logical_replications/tests/test_node.sh new file mode 100644 index 0000000..b21e3d5 --- /dev/null +++ b/scenario_tests/logical_replications/tests/test_node.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -e + +source /scripts/env-data.sh + +# execute tests +pushd /tests + +PGHOST=localhost \ +PGDATABASE=gis \ +PYTHONPATH=/lib \ + python3 -m unittest -v test_replication.TestReplicationNode diff --git a/scenario_tests/logical_replications/tests/test_replication.py b/scenario_tests/logical_replications/tests/test_replication.py new file mode 100644 index 0000000..b0e46aa --- /dev/null +++ b/scenario_tests/logical_replications/tests/test_replication.py @@ -0,0 +1,75 @@ +import unittest +from utils.utils import DBConnection + + +class TestReplicationMaster(unittest.TestCase): + + def setUp(self): + self.db = DBConnection() + + def test_create_new_data(self): + # create new table + self.db.conn.autocommit = True + with self.db.cursor() as c: + c.execute( + """ + CREATE TABLE IF NOT EXISTS test_replication_table ( + id integer not null + constraint pkey primary key, + geom geometry(Point, 4326), + name varchar(30), + alias varchar(30), + description varchar(255) + ); + """ + ) + + c.execute( + """ + ALTER PUBLICATION logical_replication + ADD TABLE test_replication_table; + """ + ) + + c.execute( + """ + INSERT INTO test_replication_table (id, geom, name, alias, description) + VALUES + ( + 1, + st_setsrid(st_point(107.6097, 6.9120), 4326), + 'Bandung', + 'Paris van Java', + 'Asia-Africa conference was held here' + ) ON CONFLICT DO NOTHING; + """ + ) + + +class TestReplicationNode(unittest.TestCase): + + def setUp(self): + self.db = DBConnection() + + def test_read_data(self): + # create new table + self.db.conn.autocommit = True + with self.db.cursor() as c: + c.execute( + """ + CREATE SUBSCRIPTION logical_subscription + CONNECTION 'host=${PG_PUBLISHER} port=5432 + password=${PG_PASSWORD} user=${PG_USER} dbname=gis' + PUBLICATION logical_replication; + """ + ) + + c.execute( + """ + SELECT * FROM test_replication_table; + """ + ) + + rows = c.fetchall() + self.assertEqual(len(rows), 1) + diff --git a/scenario_tests/streaming_replication/test.sh b/scenario_tests/streaming_replication/test.sh index 9395fc9..5ddb0e8 100755 --- a/scenario_tests/streaming_replication/test.sh +++ b/scenario_tests/streaming_replication/test.sh @@ -8,22 +8,26 @@ source ../test-env.sh # Run service docker-compose up -d +if [[ -n "${PRINT_TEST_LOGS}" ]]; then + docker-compose logs -f & +fi + sleep 30 # Preparing master cluster -until docker-compose exec pg-master pg_isready; do +until docker-compose exec -T pg-master pg_isready; do sleep 30 done; # Execute tests -docker-compose exec pg-master /bin/bash /tests/test_master.sh +docker-compose exec -T pg-master /bin/bash /tests/test_master.sh # Preparing node cluster -until docker-compose exec pg-node pg_isready; do +until docker-compose exec -T pg-node pg_isready; do sleep 30 done; # Execute tests -docker-compose exec pg-node /bin/bash /tests/test_node.sh +docker-compose exec -T pg-node /bin/bash /tests/test_node.sh docker-compose down -v