Improve scenario tests logging

.github/workflows/build-latest.yaml

@@ -135,6 +135,7 @@ jobs:
           - collations
           - extensions
           - logical_replication
+          - logical_replications
         include:
           - distro: debian
             imageVersion: bullseye
@@ -177,6 +178,7 @@ jobs:
       - name: Run scenario test
         env:
           COMPOSE_INTERACTIVE_NO_CLI: 1
+          PRINT_TEST_LOGS: 1
         run: |
           pushd scenario_tests/${{ matrix.scenario }}
           ./test.sh

scenario_tests/collations/test.sh

@@ -8,6 +8,10 @@ source ../test-env.sh
 # Run service
 docker-compose up -d
 
+if [[ -n "${PRINT_TEST_LOGS}" ]]; then
+  docker-compose logs -f &
+fi
+
 sleep 30
 
 services=("pg" "pg-new")
@@ -15,10 +19,12 @@ services=("pg" "pg-new")
 for service in "${services[@]}"; do
 
   # Execute tests
-  until docker-compose exec $service pg_isready; do
+  until docker-compose exec -T $service pg_isready; do
     sleep 30
+    echo "Wait service to be ready"
   done;
-  docker-compose exec $service /bin/bash /tests/test.sh
+  echo "Execute test for $service"
+  docker-compose exec -T $service /bin/bash /tests/test.sh
 
 done
 

scenario_tests/datadir_init/test.sh

@@ -8,6 +8,10 @@ source ../test-env.sh
 # Run service
 docker-compose up -d
 
+if [[ -n "${PRINT_TEST_LOGS}" ]]; then
+  docker-compose logs -f &
+fi
+
 sleep 60
 
 services=("pg-default" "pg-new" "pg-recreate")
@@ -15,10 +19,12 @@ services=("pg-default" "pg-new" "pg-recreate")
 for service in "${services[@]}"; do
 
   # Execute tests
-  until docker-compose exec $service pg_isready; do
+  until docker-compose exec -T $service pg_isready; do
-    sleep 1
+    sleep 5
+    echo "Wait service to be ready"
   done;
-  docker-compose exec $service /bin/bash /tests/test.sh
+  echo "Execute test for $service"
+  docker-compose exec -T $service /bin/bash /tests/test.sh
 
 done
 

scenario_tests/extensions/test.sh

@@ -8,6 +8,10 @@ source ../test-env.sh
 # Run service
 docker-compose up -d
 
+if [[ -n "${PRINT_TEST_LOGS}" ]]; then
+  docker-compose logs -f &
+fi
+
 sleep 30
 
 services=("pg" "pg-two-extensions")
@@ -15,10 +19,12 @@ services=("pg" "pg-two-extensions")
 for service in "${services[@]}"; do
 
   # Execute tests
-  until docker-compose exec $service pg_isready; do
+  until docker-compose exec -T $service pg_isready; do
     sleep 30
+    echo "Wait service to be ready"
   done;
-  docker-compose exec $service /bin/bash /tests/test.sh
+  echo "Execute test for $service"
+  docker-compose exec -T $service /bin/bash /tests/test.sh
 
 done
 

scenario_tests/logical_replications/docker-compose.yml

@@ -0,0 +1,104 @@
+
+version: '2.1'
+
+volumes:
+  pg-master-data-dir:
+  pg-node-data-dir:
+
+services:
+  pg-publisher:
+    image: 'kartoza/postgis:${TAG:-manual-build}'
+    restart: 'always'
+    # You can optionally mount to volume, to play with the persistence and
+    # observe how the node will behave after restarts.
+    volumes:
+      - pg-master-data-dir:/var/lib/postgresql
+      - ./scripts/setup-publisher.sql:/docker-entrypoint-initdb.d/setup-publisher.sql
+      - ./tests:/tests
+      - ../utils:/lib/utils
+    environment:
+      # ALLOW_IP_RANGE option is used to specify additionals allowed domains
+      # in pg_hba.
+      # This range should allow nodes to connect to master
+      ALLOW_IP_RANGE: '0.0.0.0/0'
+
+      # We can specify optional credentials
+      REPLICATION_USER: 'replicator'
+      REPLICATION_PASS: 'replicator'
+      WAL_LEVEL: 'logical'
+      # Setup master replication variables
+      #PG_MAX_WAL_SENDERS: 8
+      #PG_WAL_KEEP_SEGMENTS: 100
+    # You can expose the port to observe it in your local machine
+    ports:
+      - "7777:5432"
+    healthcheck:
+      interval: 60s
+      timeout: 30s
+      retries: 3
+      test: "pg_isready"
+
+  pg-subscriber:
+    image: 'kartoza/postgis:${TAG:-manual-build}'
+    restart: 'always'
+    # You can optionally mount to volume, but we're not able to scale it
+    # in that case.
+    # The node will always destroy its database and copy from master at
+    # runtime
+    volumes:
+      - pg-node-data-dir:/var/lib/postgresql
+      - ./scripts/setup-subscriber.sql:/docker-entrypoint-initdb.d/setup-subscriber.sql
+      - ./tests:/tests
+      - ../utils:/lib/utils
+
+    environment:
+      # ALLOW_IP_RANGE option is used to specify additionals allowed domains
+      # in pg_hba.
+      # Not really needed in nodes for the replication, but optionally can
+      # be put when nodes are needed to be a failover server when master
+      # is down. The IP Range are generally needed if other services wants to
+      # connect to this node
+      ALLOW_IP_RANGE: '0.0.0.0/0'
+
+      # REPLICATE_FROM options accepts domain-name or IP address
+      # with this in mind, you can also put docker service name, because it
+      # will be resolved as host name.
+      #REPLICATE_FROM: 'pg-publisher'
+
+      # REPLICATE_PORT will default to 5432 if not specified.
+      # REPLICATE_PORT: '5432'
+      # In the case where you need to replicate from outside service,
+      # you can put the server address and port here, as long as the target
+      # where configured as master, and replicable.
+#      REPLICATE_FROM: '192.168.1.8'
+#      REPLICATE_PORT: '7777'
+
+      # DESTROY_DATABASE_ON_RESTART will default to True if not specified.
+      # If specified other than True, it will prevent node from destroying
+      # database on restart
+      DESTROY_DATABASE_ON_RESTART: 'True'
+
+      # PROMOTE_MASTER Default empty.
+      # If specified with any value, then it will convert current node into
+      # a writable state. Useful if master is down and the current node needs
+      # to be promoted until manual recovery.
+#      PROMOTE_MASTER: 'True'
+
+      # For now we don't support different credentials for replication
+      # so we use the same credentials as master's superuser, or anything that
+      # have replication role.
+      REPLICATION_USER: 'replicator'
+      REPLICATION_PASS: 'replicator'
+      WAL_LEVEL: 'logical'
+    depends_on:
+        pg-publisher:
+          condition: service_healthy
+    # You can expose the port to observe it in your local machine
+    # For this sample, it was disabled by default to allow scaling test
+    ports:
+      - "7776:5432"
+#    healthcheck:
+#      interval: 60s
+#      timeout: 30s
+#      retries: 3
+#      test: "pg_isready"

scenario_tests/logical_replications/scripts/setup-publisher.sql

@@ -0,0 +1,22 @@
+-- Create a table
+CREATE TABLE sweets
+    (
+        id SERIAL,
+        name TEXT,
+        price DECIMAL,
+        CONSTRAINT sweets_pkey PRIMARY KEY (id)
+    );
+
+CREATE TABLE public.block (
+    id serial NOT NULL,
+    geom public.geometry(Polygon,4326),
+    fid bigint,
+    tile_name character varying,
+    location character varying
+);
+
+-- Add table to publication called logical_replication which is created by the scripts
+ALTER PUBLICATION logical_replication ADD TABLE sweets;
+ALTER PUBLICATION logical_replication ADD TABLE block;
+-- Inserts records into the table
+INSERT INTO sweets (name, price) VALUES ('strawberry', 4.50), ('Coffee', 6.20), ('lollipop', 3.80);

scenario_tests/logical_replications/scripts/setup-subscriber.sql

@@ -0,0 +1,20 @@
+-- Create a table
+CREATE TABLE sweets
+    (
+        id SERIAL,
+        name TEXT,
+        price DECIMAL,
+        CONSTRAINT sweets_pkey PRIMARY KEY (id)
+    );
+
+CREATE TABLE public.block (
+    id serial NOT NULL,
+    geom public.geometry(Polygon,4326),
+    fid bigint,
+    tile_name character varying,
+    location character varying
+);
+-- Create a publication
+CREATE SUBSCRIPTION logical_subscription
+    CONNECTION 'host=pg-publisher port=5432 password=docker user=docker dbname=gis'
+    PUBLICATION logical_replication;

scenario_tests/logical_replications/test.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# exit immediately if test fails
+set -e
+
+source ../test-env.sh
+
+# Run service
+docker-compose up -d
+
+if [[ -n "${PRINT_TEST_LOGS}" ]]; then
+  docker-compose logs -f &
+fi
+
+sleep 30
+
+# Preparing master cluster
+until docker-compose exec -T pg-publisher pg_isready; do
+  sleep 1
+done;
+
+# Execute tests
+docker-compose exec -T pg-publisher /bin/bash /tests/test_master.sh
+
+# Preparing node cluster
+until docker-compose exec -T pg-subscriber pg_isready; do
+  sleep 1
+done;
+
+# Execute tests
+docker-compose exec -T pg-node /bin/bash /tests/test_node.sh
+
+docker-compose down -v

scenario_tests/logical_replications/tests/test_master.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -e
+
+source /scripts/env-data.sh
+
+# execute tests
+pushd /tests
+
+PGHOST=localhost \
+PGDATABASE=gis \
+PYTHONPATH=/lib \
+  python3 -m unittest -v test_replication.TestReplicationMaster

scenario_tests/logical_replications/tests/test_node.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -e
+
+source /scripts/env-data.sh
+
+# execute tests
+pushd /tests
+
+PGHOST=localhost \
+PGDATABASE=gis \
+PYTHONPATH=/lib \
+  python3 -m unittest -v test_replication.TestReplicationNode

scenario_tests/logical_replications/tests/test_replication.py

@@ -0,0 +1,75 @@
+import unittest
+from utils.utils import DBConnection
+
+
+class TestReplicationMaster(unittest.TestCase):
+
+    def setUp(self):
+        self.db = DBConnection()
+
+    def test_create_new_data(self):
+        # create new table
+        self.db.conn.autocommit = True
+        with self.db.cursor() as c:
+            c.execute(
+                """
+                CREATE TABLE IF NOT EXISTS test_replication_table (
+                    id integer not null
+                        constraint pkey primary key,
+                    geom geometry(Point, 4326),
+                    name varchar(30),
+                    alias varchar(30),
+                    description varchar(255)
+                );
+                """
+            )
+
+            c.execute(
+                """
+                ALTER PUBLICATION logical_replication 
+                    ADD TABLE test_replication_table; 
+                """
+            )
+
+            c.execute(
+                """
+                INSERT INTO test_replication_table (id, geom, name, alias, description)
+                VALUES 
+                (
+                    1,
+                    st_setsrid(st_point(107.6097, 6.9120), 4326),
+                    'Bandung',
+                    'Paris van Java',
+                    'Asia-Africa conference was held here'                     
+                ) ON CONFLICT DO NOTHING;
+                """
+            )
+
+
+class TestReplicationNode(unittest.TestCase):
+
+    def setUp(self):
+        self.db = DBConnection()
+
+    def test_read_data(self):
+        # create new table
+        self.db.conn.autocommit = True
+        with self.db.cursor() as c:
+            c.execute(
+                """
+                CREATE SUBSCRIPTION logical_subscription
+                    CONNECTION 'host=${PG_PUBLISHER} port=5432 
+                    password=${PG_PASSWORD} user=${PG_USER} dbname=gis'
+                    PUBLICATION logical_replication;
+                """
+            )
+
+            c.execute(
+                """
+                SELECT * FROM test_replication_table;
+                """
+            )
+
+            rows = c.fetchall()
+            self.assertEqual(len(rows), 1)
+

scenario_tests/streaming_replication/test.sh

@@ -8,22 +8,26 @@ source ../test-env.sh
 # Run service
 docker-compose up -d
 
+if [[ -n "${PRINT_TEST_LOGS}" ]]; then
+  docker-compose logs -f &
+fi
+
 sleep 30
 
 # Preparing master cluster
-until docker-compose exec pg-master pg_isready; do
+until docker-compose exec -T pg-master pg_isready; do
   sleep 30
 done;
 
 # Execute tests
-docker-compose exec pg-master /bin/bash /tests/test_master.sh
+docker-compose exec -T pg-master /bin/bash /tests/test_master.sh
 
 # Preparing node cluster
-until docker-compose exec pg-node pg_isready; do
+until docker-compose exec -T pg-node pg_isready; do
   sleep 30
 done;
 
 # Execute tests
-docker-compose exec pg-node /bin/bash /tests/test_node.sh
+docker-compose exec -T pg-node /bin/bash /tests/test_node.sh
 
 docker-compose down -v