gitops/patroni
1
0
Fork 0
mirror of https://github.com/outbackdingo/patroni.git synced 2026-08-25 14:53:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Enable SSL in REST API and Postgres if possible when running behave (#2498)

Browse Source 
If openssl binary is available use it to generate a self-signed certificate. Use it to protect Patroni REST API
(`verify_client: required`).

In case if Postgres is compiled with SSL support enable it in the configuration and configure pg_hba.conf to check client certificates (`verify-ca`) in addition to passwords. Also configure superuser/replication/rewind users to use client certificates and verify server certificate (`verify-ca`)
This commit is contained in:
Alexander Kukushkin
2022-12-21 10:20:30 +01:00
committed by GitHub
parent 4d77b444dc
commit 49f1ccf874
2 changed files with 74 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
features/environment.py
+70 -10
View File
@@ -2,6 +2,7 @@ import abc
import datetime
import os
import json
import re
import shutil
import signal
import six
@@ -202,7 +203,32 @@ class PatroniController(AbstractController):
'logging_collector': 'on', 'log_destination': 'csvlog',
'log_directory': self._output_dir.replace('\\', '/'),
'log_filename': name + '.log', 'log_statement': 'all', 'log_min_messages': 'debug1',
'unix_socket_directories': tempfile.gettempdir().replace('\\', '/')})
'shared_buffers': '1MB', 'unix_socket_directories': tempfile.gettempdir().replace('\\', '/')})
config['postgresql']['pg_hba'] = [
'local all all trust',
'local replication all trust',
'host replication replicator all md5',
'host all all all md5'
]
if self._context.postgres_supports_ssl and self._context.certfile:
config['postgresql']['parameters'].update({
'ssl': 'on',
'ssl_ca_file': self._context.certfile.replace('\\', '/'),
'ssl_cert_file': self._context.certfile.replace('\\', '/'),
'ssl_key_file': self._context.keyfile.replace('\\', '/')
})
for user in config['postgresql'].get('authentication').keys():
config['postgresql'].get('authentication', {}).get(user, {}).update({
'sslmode': 'verify-ca',
'sslrootcert': self._context.certfile,
'sslcert': self._context.certfile,
'sslkey': self._context.keyfile
})
for i, line in enumerate(list(config['postgresql']['pg_hba'])):
if line.endswith('md5'):
# we want to verify client cert first and than password
config['postgresql']['pg_hba'][i] = 'hostssl' + line[4:] + ' clientcert=verify-ca'
if 'bootstrap' in config:
config['bootstrap']['post_bootstrap'] = 'psql -w -c "SELECT 1"'
@@ -220,13 +246,15 @@ class PatroniController(AbstractController):
with open(patroni_config_path, 'w') as f:
yaml.safe_dump(config, f, default_flow_style=False)
user = config['postgresql'].get('authentication', config['postgresql']).get('superuser', {})
self._connkwargs = {k: user[n] for n, k in [('username', 'user'), ('password', 'password')] if n in user}
self._connkwargs.update({'host': host, 'port': self.__PORT, 'dbname': 'postgres'})
self._connkwargs = config['postgresql'].get('authentication', config['postgresql']).get('superuser', {})
self._connkwargs.update({'host': host, 'port': self.__PORT, 'dbname': 'postgres',
'user': self._connkwargs.pop('username', None)})
self._replication = config['postgresql'].get('authentication', config['postgresql']).get('replication', {})
self._replication.update({'host': host, 'port': self.__PORT, 'dbname': 'postgres'})
self._replication.update({'host': host, 'port': self.__PORT, 'user': self._replication.pop('username', None)})
self._restapi_url = 'http://{0}'.format(config['restapi']['connect_address'])
if self._context.certfile:
self._restapi_url = self._restapi_url.replace('http://', 'https://')
return patroni_config_path
@@ -285,7 +313,10 @@ class PatroniController(AbstractController):
@property
def backup_source(self):
return 'postgres://{username}:{password}@{host}:{port}/{dbname}'.format(**self._replication)
def escape(value):
return re.sub(r'([\'\\ ])', r'\\\1', str(value))
return ' '.join('{0}={1}'.format(k, escape(v)) for k, v in self._replication.items())
def backup(self, dest=os.path.join('data', 'basebackup')):
subprocess.call(PatroniPoolController.BACKUP_SCRIPT + ['--walmethod=none',
@@ -661,8 +692,17 @@ class PatroniPoolController(object):
self._patroni_path = None
self._processes = {}
self.create_and_set_output_directory('')
self._check_postgres_ssl()
self.known_dcs = {subclass.name(): subclass for subclass in AbstractDcsController.get_subclasses()}
def _check_postgres_ssl(self):
try:
subprocess.check_output(['postgres', '-D', os.devnull, '-c', 'ssl=on'], stderr=subprocess.STDOUT)
raise Exception # this one should never happen because the previous line will always raise and exception
except Exception as e:
self._context.postgres_supports_ssl = isinstance(e, subprocess.CalledProcessError)\
and 'SSL is not supported by this build' not in e.output.decode()
@property
def patroni_path(self):
if self._patroni_path is None:
@@ -713,7 +753,8 @@ class PatroniPoolController(object):
'bootstrap': {
'method': 'pg_basebackup',
'pg_basebackup': {
'command': " ".join(self.BACKUP_SCRIPT) + ' --walmethod=stream --dbname=' + f.backup_source
'command': " ".join(self.BACKUP_SCRIPT +
['--walmethod=stream', '--dbname="{0}"'.format(f.backup_source)])
},
'dcs': {
'postgresql': {
@@ -890,13 +931,32 @@ class WatchdogMonitor(object):
# actions to execute on start/stop of the tests and before running individual features
def before_all(context):
os.environ.update({'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password'})
context.request_executor = PatroniRequest({'ctl': {'auth': os.environ['PATRONI_RESTAPI_USERNAME'] +
':' + os.environ['PATRONI_RESTAPI_PASSWORD']}})
context.ci = os.name == 'nt' or\
any(a in os.environ for a in ('TRAVIS_BUILD_NUMBER', 'BUILD_NUMBER', 'GITHUB_ACTIONS'))
context.timeout_multiplier = 5 if context.ci else 1 # MacOS sometimes is VERY slow
context.pctl = PatroniPoolController(context)
context.keyfile = os.path.join(context.pctl.output_dir, 'patroni.key')
context.certfile = os.path.join(context.pctl.output_dir, 'patroni.crt')
try:
with open(os.devnull, 'w') as null:
ret = subprocess.call(['openssl', 'req', '-nodes', '-new', '-x509', '-subj', '/CN=batman.patroni',
'-keyout', context.keyfile, '-out', context.certfile], stdout=null, stderr=null)
if ret != 0:
raise Exception
except Exception:
context.keyfile = context.certfile = None
os.environ.update({'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password'})
ctl = {'auth': os.environ['PATRONI_RESTAPI_USERNAME'] + ':' + os.environ['PATRONI_RESTAPI_PASSWORD']}
if context.certfile:
os.environ.update({'PATRONI_RESTAPI_CAFILE': context.certfile,
'PATRONI_RESTAPI_CERTFILE': context.certfile,
'PATRONI_RESTAPI_KEYFILE': context.keyfile,
'PATRONI_RESTAPI_VERIFY_CLIENT': 'required',
'PATRONI_CTL_INSECURE': 'on'})
ctl.update({'cacert': context.certfile, 'certfile': context.certfile, 'keyfile': context.keyfile})
context.request_executor = PatroniRequest({'ctl': ctl}, True)
context.dcs_ctl = context.pctl.known_dcs[context.pctl.dcs](context)
context.dcs_ctl.start()
try:
features/steps/patroni_api.py
+4
View File
@@ -70,6 +70,8 @@ def do_post_empty(context, url):
@step('I issue a {request_method:w} request to {url:url} with {data}')
def do_request(context, request_method, url, data):
if context.certfile:
url = url.replace('http://', 'https://')
data = data and json.loads(data)
try:
r = context.request_executor.request(request_method, url, data)
@@ -131,6 +133,8 @@ def add_tag_to_config(context, tag, value, pg_name):
@then('Response on GET {url} contains {value} after {timeout:d} seconds')
def check_http_response(context, url, value, timeout, negate=False):
if context.certfile:
url = url.replace('http://', 'https://')
timeout *= context.timeout_multiplier
for _ in range(int(timeout)):
r = context.request_executor.request('GET', url)