From 9961432fc581567a11a294a3c98618da86696850 Mon Sep 17 00:00:00 2001 From: admire Date: Sat, 29 May 2021 14:40:34 +0200 Subject: [PATCH] Fix SSL authentification when PostgreSQL requires ssl connections for clients --- .env | 9 ++++ docker-compose.yml | 3 ++ docker-imposm3/importer.py | 87 ++++++++++++++++++++++++++++---------- docker-osmenrich/enrich.py | 34 +++++++++++++-- 4 files changed, 107 insertions(+), 26 deletions(-) diff --git a/.env b/.env index d08c5be..53a6083 100644 --- a/.env +++ b/.env @@ -51,3 +51,12 @@ COMPRESSION_LEVEL=1 BASE_URL=http://planet.openstreetmap.org/replication/ PGADMIN_DEFAULT_EMAIL=docker@gmail.com PGADMIN_DEFAULT_PASSWORD=docker +# https://github.com/kartoza/docker-postgis#postgres-ssl-setup +FORCE_SSL=false +# Force client connection to require ssl mode in connecting +SSL_MODE=disable + +# Activate these in the osmenrich and imposm if the SSL_MODE is set to verify-full or verify-ca +#SSL_CERT=/etc/certs/cert.pem +#SSL_ROOT_CERT=/etc/certs/root.crt +#SSL_KEY=/etc/certs/key.pem \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 17819d4..8de89a3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,7 @@ services: - POSTGRES_PASS=${POSTGRES_PASS} - POSTGRES_DBNAME=${POSTGRES_DBNAME} - ALLOW_IP_RANGE=${ALLOW_IP_RANGE} + - FORCE_SSL=${FORCE_SSL} volumes: - osm-postgis-data:/var/lib/postgresql ports: @@ -55,6 +56,7 @@ services: - DBSCHEMA_BACKUP=${DBSCHEMA_BACKUP} - QGIS_STYLE=${QGIS_STYLE} - CLIP=${CLIP} + - SSL_MODE=${SSL_MODE} command: bash -c "while [ ! -f /home/settings/country.pbf ] ; do sleep 1; done && python3 -u /home/importer.py" osmupdate: @@ -104,3 +106,4 @@ services: - IMPORT_DONE=${IMPORT_DONE} - TIME=${TIME} - DBSCHEMA_PRODUCTION=${DBSCHEMA_PRODUCTION} + - SSL_MODE=${SSL_MODE} diff --git a/docker-imposm3/importer.py b/docker-imposm3/importer.py index 16624d3..4a9e010 100644 --- a/docker-imposm3/importer.py +++ b/docker-imposm3/importer.py @@ -18,10 +18,9 @@ * * ***************************************************************************/ """ - +import sys from os import environ, listdir from os.path import join, exists, abspath, isabs -from pathlib import Path from shutil import move from subprocess import call from sys import exit, stderr @@ -51,7 +50,11 @@ class Importer(object): 'DBSCHEMA_IMPORT': 'import', 'DBSCHEMA_BACKUP': 'backup', 'CLIP': 'no', - 'QGIS_STYLE': 'yes' + 'QGIS_STYLE': 'yes', + 'SSL_MODE': 'disable', + 'SSL_CERT': None, + 'SSL_ROOT_CERT': None, + 'SSL_KEY': None } self.osm_file = None self.mapping_file = None @@ -191,24 +194,70 @@ class Importer(object): def check_postgis(self): """Test connection to PostGIS and create the URI.""" + if self.default['SSL_MODE'] == 'verify-ca' or self.default['SSL_MODE'] == 'verify-full': + if self.default['SSL_CERT'] is None and self.default['SSL_KEY'] is None and self.default['SSL_ROOT_CERT'] \ + is None: + sys.exit() + else: + + conn_parameters = "dbname='%s' user='%s' host='%s' port='%s' password='%s'" \ + " sslmode='%s' sslcert='%s' sslkey='%s' sslrootcert='%s' " % ( + self.default['POSTGRES_DBNAME'], + self.default['POSTGRES_USER'], + self.default['POSTGRES_HOST'], + self.default['POSTGRES_PORT'], + self.default['POSTGRES_PASS'], + self.default['SSL_MODE'], + self.default['SSL_CERT'], + self.default['SSL_KEY'], + self.default['SSL_ROOT_CERT']) + else: + conn_parameters = "dbname='%s' user='%s' host='%s' port='%s' password='%s' sslmode='%s' " % ( + self.default['POSTGRES_DBNAME'], + self.default['POSTGRES_USER'], + self.default['POSTGRES_HOST'], + self.default['POSTGRES_PORT'], + self.default['POSTGRES_PASS'], + self.default['SSL_MODE']) + try: - connection = connect( - "dbname='%s' user='%s' host='%s' port='%s' password='%s'" % ( - self.default['POSTGRES_DBNAME'], - self.default['POSTGRES_USER'], - self.default['POSTGRES_HOST'], - self.default['POSTGRES_PORT'], - self.default['POSTGRES_PASS'])) + connection = connect(conn_parameters) self.cursor = connection.cursor() except OperationalError as e: self.error(e) - self.postgis_uri = 'postgis://%s:%s@%s:%s/%s' % ( - self.default['POSTGRES_USER'], - self.default['POSTGRES_PASS'], - self.default['POSTGRES_HOST'], - self.default['POSTGRES_PORT'], - self.default['POSTGRES_DBNAME']) + if self.default['SSL_MODE'] == 'verify-ca' or self.default['SSL_MODE'] == 'verify-full': + if self.default['SSL_CERT'] is None and self.default['SSL_KEY'] is None and self.default['SSL_ROOT_CERT'] \ + is None: + sys.exit() + else: + self.postgis_uri = \ + 'postgis://%s:%s@%s:%s/%s?sslmode=%s&sslcert=%s&sslkey=%s&sslrootcert=%s' % ( + self.default['POSTGRES_USER'], + self.default['POSTGRES_PASS'], + self.default['POSTGRES_HOST'], + self.default['POSTGRES_PORT'], + self.default['POSTGRES_DBNAME'], + self.default['SSL_MODE'], + self.default['SSL_CERT'], + self.default['SSL_KEY'], + self.default['SSL_ROOT_CERT']) + elif self.default['SSL_MODE'] == 'require' or self.default['SSL_MODE'] == 'prefer': + self.postgis_uri = 'postgis://%s:%s@%s:%s/%s?sslmode=%s' \ + % ( + self.default['POSTGRES_USER'], + self.default['POSTGRES_PASS'], + self.default['POSTGRES_HOST'], + self.default['POSTGRES_PORT'], + self.default['POSTGRES_DBNAME'], + self.default['SSL_MODE']) + else: + self.postgis_uri = 'postgis://%s:%s@%s:%s/%s' % ( + self.default['POSTGRES_USER'], + self.default['POSTGRES_PASS'], + self.default['POSTGRES_HOST'], + self.default['POSTGRES_PORT'], + self.default['POSTGRES_DBNAME']) def import_custom_sql(self): """Import the custom SQL file into the database.""" @@ -240,11 +289,6 @@ class Importer(object): # noinspection PyUnboundLocalVariable return self.cursor.fetchone()[0] - def lockfile(self): - setup_lockfile = join(self.default['SETTINGS'], 'importer.lock') - if not exists(setup_lockfile): - Path(setup_lockfile).touch() - def run(self): """First checker.""" @@ -292,7 +336,6 @@ class Importer(object): self.error(msg) else: self.info('Import PBF successful : %s' % self.osm_file) - self.lockfile() if self.post_import_file or self.qgis_style: # Set the password for psql diff --git a/docker-osmenrich/enrich.py b/docker-osmenrich/enrich.py index b5fa070..a4abfdd 100644 --- a/docker-osmenrich/enrich.py +++ b/docker-osmenrich/enrich.py @@ -19,6 +19,7 @@ ***************************************************************************/ """ +import sys import gzip from os import environ, listdir, mkdir from os.path import join, exists, getsize @@ -64,7 +65,11 @@ class Enrich(object): 'CACHE': 'cache', 'MAX_DIFF_FILE_SIZE': 100000000, 'DBSCHEMA_PRODUCTION': 'public', - 'CACHE_MODIFY_CHECK': '' + 'CACHE_MODIFY_CHECK': '', + 'SSL_MODE': 'disable', + 'SSL_CERT': None, + 'SSL_ROOT_CERT': None, + 'SSL_KEY': None } self.mapping_file = None self.mapping_database_schema = {} @@ -207,12 +212,33 @@ class Enrich(object): ) def create_connection(self): - return connect( - "dbname='%s' user='%s' host='%s' password='%s'" % ( + if self.default['SSL_MODE'] == 'verify-ca' or self.default['SSL_MODE'] == 'verify-full': + if self.default['SSL_CERT'] is None and self.default['SSL_KEY'] is None and self.default['SSL_ROOT_CERT'] \ + is None: + sys.exit() + else: + + conn_parameters = "dbname='%s' user='%s' host='%s' port='%s' password='%s'" \ + " sslmode='%s' sslcert='%s' sslkey='%s' sslrootcert='%s' " % ( + self.default['POSTGRES_DBNAME'], + self.default['POSTGRES_USER'], + self.default['POSTGRES_HOST'], + self.default['POSTGRES_PORT'], + self.default['POSTGRES_PASS'], + self.default['SSL_MODE'], + self.default['SSL_CERT'], + self.default['SSL_KEY'], + self.default['SSL_ROOT_CERT']) + else: + conn_parameters = "dbname='%s' user='%s' host='%s' port='%s' password='%s' sslmode='%s' " % ( self.default['POSTGRES_DBNAME'], self.default['POSTGRES_USER'], self.default['POSTGRES_HOST'], - self.default['POSTGRES_PASS'])) + self.default['POSTGRES_PORT'], + self.default['POSTGRES_PASS'], + self.default['SSL_MODE']) + + return connect(conn_parameters) def check_database(self): """Test connection to PostGIS and create the URI."""