gitops/patroni
1
0
Fork 0
mirror of https://github.com/outbackdingo/patroni.git synced 2026-08-25 14:53:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve GUCs validation (#3130)

Browse Source 
Due to postgres --describe-config not showing GUCs defined as GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE, Patroni was always ignoring some GUCs that a user might want to have configured with non-default values.

- remove postgres --describe-config validation.
- define minor versions for availability bounds of some back-patched GUCs
This commit is contained in:
Polina Bungina
2024-08-23 14:20:16 +02:00
committed by GitHub
parent 31cf951b69
commit 8c5ab4c07d
12 changed files with 223 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/__init__.py
+1 -10
View File
@@ -3,7 +3,7 @@ import os
import shutil
import unittest
from unittest.mock import Mock, patch, PropertyMock
from unittest.mock import Mock, patch
import urllib3
@@ -20,15 +20,6 @@ class SleepException(Exception):
pass
mock_available_gucs = PropertyMock(return_value={
'cluster_name', 'constraint_exclusion', 'force_parallel_mode', 'hot_standby', 'listen_addresses', 'max_connections',
'max_locks_per_transaction', 'max_prepared_transactions', 'max_replication_slots', 'max_stack_depth',
'max_wal_senders', 'max_worker_processes', 'port', 'search_path', 'shared_preload_libraries',
'stats_temp_directory', 'synchronous_standby_names', 'track_commit_timestamp', 'unix_socket_directories',
'vacuum_cost_delay', 'vacuum_cost_limit', 'wal_keep_size', 'wal_level', 'wal_log_hints', 'zero_damaged_pages',
'autovacuum', 'wal_segment_size', 'wal_block_size', 'shared_buffers', 'wal_buffers',
})
GET_PG_SETTINGS_RESULT = [
('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
('wal_block_size', '8192', None, 'integer', 'internal'),
tests/test_bootstrap.py
+2 -2
View File
@@ -10,13 +10,13 @@ from patroni.postgresql.bootstrap import Bootstrap
from patroni.postgresql.cancellable import CancellableSubprocess
from patroni.postgresql.config import ConfigHandler, get_param_diff
from . import BaseTestPostgresql, mock_available_gucs, psycopg_connect
from . import BaseTestPostgresql, psycopg_connect
@patch('subprocess.call', Mock(return_value=0))
@patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 12.1"))
@patch('patroni.psycopg.connect', psycopg_connect)
@patch('os.rename', Mock())
@patch.object(Postgresql, 'available_gucs', mock_available_gucs)
class TestBootstrap(BaseTestPostgresql):
@patch('patroni.postgresql.CallbackExecutor', Mock())
tests/test_patroni.py
-2
View File
@@ -78,7 +78,6 @@ class TestPatroni(unittest.TestCase):
@patch.object(etcd.Client, 'read', etcd_read)
@patch.object(Thread, 'start', Mock())
@patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379']))
@patch.object(Postgresql, '_get_gucs', Mock(return_value={'foo': True, 'bar': True}))
def setUp(self):
self._handlers = logging.getLogger().handlers[:]
RestApiServer._BaseServer__is_shut_down = Mock()
@@ -102,7 +101,6 @@ class TestPatroni(unittest.TestCase):
@patch.object(etcd.Client, 'delete', Mock())
@patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379']))
@patch.object(Thread, 'join', Mock())
@patch.object(Postgresql, '_get_gucs', Mock(return_value={'foo': True, 'bar': True}))
def test_patroni_patroni_main(self):
with patch('subprocess.call', Mock(return_value=1)):
with patch.object(Patroni, 'run', Mock(side_effect=SleepException)):
tests/test_postgresql.py
+41 -15
View File
@@ -15,7 +15,7 @@ import patroni.psycopg as psycopg
from patroni import global_config
from patroni.async_executor import CriticalTask
from patroni.collections import CaseInsensitiveDict, CaseInsensitiveSet
from patroni.collections import CaseInsensitiveDict
from patroni.dcs import RemoteMember
from patroni.exceptions import PatroniException, PostgresConnectionException
from patroni.postgresql import Postgresql, STATE_NO_RESPONSE, STATE_REJECT
@@ -24,12 +24,12 @@ from patroni.postgresql.callback_executor import CallbackAction
from patroni.postgresql.config import _false_validator, get_param_diff
from patroni.postgresql.postmaster import PostmasterProcess
from patroni.postgresql.validator import _get_postgres_guc_validators, _load_postgres_gucs_validators, \
_read_postgres_gucs_validators_file, Bool, Enum, EnumBool, Integer, InvalidGucValidatorsFile, Real, String, \
ValidatorFactory, ValidatorFactoryInvalidSpec, ValidatorFactoryInvalidType, ValidatorFactoryNoType
_read_postgres_gucs_validators_file, Bool, Enum, EnumBool, Integer, InvalidGucValidatorsFile, \
Real, String, transform_postgresql_parameter_value, ValidatorFactory, ValidatorFactoryInvalidSpec, \
ValidatorFactoryInvalidType, ValidatorFactoryNoType
from patroni.utils import RetryFailedError
from . import BaseTestPostgresql, GET_PG_SETTINGS_RESULT, \
mock_available_gucs, MockCursor, MockPostmaster, psycopg_connect
from . import BaseTestPostgresql, GET_PG_SETTINGS_RESULT, MockCursor, MockPostmaster, psycopg_connect
mtime_ret = {}
@@ -98,8 +98,8 @@ Data page checksum version: 0
@patch('subprocess.call', Mock(return_value=0))
@patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 12.1"))
@patch('patroni.psycopg.connect', psycopg_connect)
@patch.object(Postgresql, 'available_gucs', mock_available_gucs)
class TestPostgresql(BaseTestPostgresql):
@patch('subprocess.call', Mock(return_value=0))
@@ -107,7 +107,6 @@ class TestPostgresql(BaseTestPostgresql):
@patch('patroni.postgresql.CallbackExecutor', Mock())
@patch.object(Postgresql, 'get_major_version', Mock(return_value=140000))
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
@patch.object(Postgresql, 'available_gucs', mock_available_gucs)
def setUp(self):
super(TestPostgresql, self).setUp()
self.p.config.write_postgresql_conf()
@@ -530,9 +529,16 @@ class TestPostgresql(BaseTestPostgresql):
self.assertEqual(self.p.controldata(), {})
@patch('patroni.postgresql.Postgresql._version_file_exists', Mock(return_value=True))
@patch('subprocess.check_output', MagicMock(return_value=0, side_effect=pg_controldata_string))
def test_sysid(self):
self.assertEqual(self.p.sysid, "6200971513092291716")
with patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string)):
self.assertEqual(self.p.sysid, "6200971513092291716")
def test_pg_version(self):
self.assertEqual(self.p.config.pg_version, 99999) # server_version
with patch.object(Postgresql, 'server_version', PropertyMock(side_effect=AttributeError)):
self.assertEqual(self.p.config.pg_version, 140000) # PG_VERSION==14, postgres --version == 12.1
with patch('subprocess.check_output', Mock(return_value=b"postgres (PostgreSQL) 14.1")):
self.assertEqual(self.p.config.pg_version, 140001)
@patch('os.path.isfile', Mock(return_value=True))
@patch('shutil.copy', Mock(side_effect=IOError))
@@ -896,6 +902,32 @@ class TestPostgresql(BaseTestPostgresql):
def test_handle_parameter_change(self):
self.p.handle_parameter_change()
@patch('patroni.postgresql.validator.logger.warning')
def test_transform_postgresql_parameter_value(self, mock_warning):
not_none_values = (
('foo.bar', 'foo', 160003), # name, value, version
("allow_in_place_tablespaces", 'true', 130008),
("restrict_nonsystem_relation_kind", 'view', 160005)
)
for i in not_none_values:
self.assertIsNotNone(
transform_postgresql_parameter_value(i[2], i[0], i[1])
)
none_values = (
("archive_cleanup_command", 'foo', 160003, False), # name, value, version, unexpected param
("allow_in_place_tablespaces", 'true', 130005, True),
("restrict_nonsystem_relation_kind", 'view', 160001, True),
)
for i in none_values:
self.assertIsNone(
transform_postgresql_parameter_value(i[2], i[0], i[1])
)
if i[3]:
mock_warning.assert_called_once_with(
'Removing unexpected parameter=%s value=%s from the config', i[0], i[1])
mock_warning.reset_mock()
def test_validator_factory(self):
# validator with no type
validator = {
@@ -1115,12 +1147,6 @@ class TestPostgresql2(BaseTestPostgresql):
def setUp(self):
super(TestPostgresql2, self).setUp()
@patch('subprocess.check_output', Mock(return_value='\n'.join(mock_available_gucs.return_value).encode('utf-8')))
def test_available_gucs(self):
gucs = self.p.available_gucs
self.assertIsInstance(gucs, CaseInsensitiveSet)
self.assertEqual(gucs, mock_available_gucs.return_value)
def test_cluster_info_query(self):
self.assertIn('diff(pg_catalog.pg_current_wal_flush_lsn(', self.p.cluster_info_query)
self.p._major_version = 90600
tests/test_sync.py
+1 -3
View File
@@ -7,12 +7,11 @@ from patroni.collections import CaseInsensitiveSet
from patroni.dcs import Cluster, ClusterConfig, Status, SyncState
from patroni.postgresql import Postgresql
from . import BaseTestPostgresql, mock_available_gucs, psycopg_connect
from . import BaseTestPostgresql, psycopg_connect
@patch('subprocess.call', Mock(return_value=0))
@patch('patroni.psycopg.connect', psycopg_connect)
@patch.object(Postgresql, 'available_gucs', mock_available_gucs)
class TestSync(BaseTestPostgresql):
@patch('subprocess.call', Mock(return_value=0))
@@ -20,7 +19,6 @@ class TestSync(BaseTestPostgresql):
@patch('patroni.postgresql.CallbackExecutor', Mock())
@patch.object(Postgresql, 'get_major_version', Mock(return_value=140000))
@patch.object(Postgresql, 'is_running', Mock(return_value=True))
@patch.object(Postgresql, 'available_gucs', mock_available_gucs)
def setUp(self):
super(TestSync, self).setUp()
self.p.config.write_postgresql_conf()
tests/test_utils.py
+43 -1
View File
@@ -3,7 +3,8 @@ import unittest
from unittest.mock import Mock, patch
from patroni.exceptions import PatroniException
from patroni.utils import enable_keepalive, polling_loop, Retry, RetryFailedError, unquote, validate_directory
from patroni.utils import enable_keepalive, get_major_version, get_postgres_version, \
polling_loop, Retry, RetryFailedError, unquote, validate_directory
class TestUtils(unittest.TestCase):
@@ -67,6 +68,47 @@ class TestUtils(unittest.TestCase):
'\'value with a \'"\'"\' single quote\''),
'value with a \' single quote')
def test_get_postgres_version(self):
with patch('subprocess.check_output', Mock(return_value=b'postgres (PostgreSQL) 9.6.24\n')):
self.assertEqual(get_postgres_version(), '9.6.24')
with patch('subprocess.check_output',
Mock(return_value=b'postgres (PostgreSQL) 10.23 (Ubuntu 10.23-4.pgdg22.04+1)\n')):
self.assertEqual(get_postgres_version(), '10.23')
with patch('subprocess.check_output',
Mock(return_value=b'postgres (PostgreSQL) 17beta3 (Ubuntu 17~beta3-1.pgdg22.04+1)\n')):
self.assertEqual(get_postgres_version(), '17.0')
with patch('subprocess.check_output',
Mock(return_value=b'postgres (PostgreSQL) 9.6beta3\n')):
self.assertEqual(get_postgres_version(), '9.6.0')
with patch('subprocess.check_output', Mock(return_value=b'postgres (PostgreSQL) 9.6rc2\n')):
self.assertEqual(get_postgres_version(), '9.6.0')
# because why not
with patch('subprocess.check_output', Mock(return_value=b'postgres (PostgreSQL) 10\n')):
self.assertEqual(get_postgres_version(), '10.0')
with patch('subprocess.check_output', Mock(return_value=b'postgres (PostgreSQL) 10wow, something new\n')):
self.assertEqual(get_postgres_version(), '10.0')
with patch('subprocess.check_output', Mock(side_effect=OSError)):
self.assertRaises(PatroniException, get_postgres_version, 'postgres')
def test_get_major_version(self):
with patch('subprocess.check_output', Mock(return_value=b'postgres (PostgreSQL) 9.6.24\n')):
self.assertEqual(get_major_version(), '9.6')
with patch('subprocess.check_output',
Mock(return_value=b'postgres (PostgreSQL) 10.23 (Ubuntu 10.23-4.pgdg22.04+1)\n')):
self.assertEqual(get_major_version(), '10')
with patch('subprocess.check_output',
Mock(return_value=b'postgres (PostgreSQL) 17beta3 (Ubuntu 17~beta3-1.pgdg22.04+1)\n')):
self.assertEqual(get_major_version(), '17')
with patch('subprocess.check_output',
Mock(return_value=b'postgres (PostgreSQL) 9.6beta3\n')):
self.assertEqual(get_major_version(), '9.6')
with patch('subprocess.check_output', Mock(return_value=b'postgres (PostgreSQL) 9.6rc2\n')):
self.assertEqual(get_major_version(), '9.6')
with patch('subprocess.check_output', Mock(return_value=b'postgres (PostgreSQL) 10\n')):
self.assertEqual(get_major_version(), '10')
with patch('subprocess.check_output', Mock(side_effect=OSError)):
self.assertRaises(PatroniException, get_major_version, 'postgres')
@patch('time.sleep', Mock())
class TestRetrySleeper(unittest.TestCase):