gitops/patroni
1
0
Fork 0
mirror of https://github.com/outbackdingo/patroni.git synced 2026-08-25 14:53:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Factor out global configuration into a dedicated class (#2628)

Browse Source 
It will help to avoid code duplications.
This commit is contained in:
Alexander Kukushkin
2023-04-03 08:09:29 +02:00
committed by GitHub
parent c549ea7d5c
commit 6f357a4e17
14 changed files with 329 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_api.py
+31 -33
View File
@@ -11,6 +11,7 @@ from mock import Mock, PropertyMock, patch
from socketserver import ThreadingMixIn
from patroni.api import RestApiHandler, RestApiServer
from patroni.config import GlobalConfig
from patroni.dcs import ClusterConfig, Member
from patroni.ha import _MemberStatus
from patroni.utils import tzutc
@@ -116,10 +117,6 @@ class MockHa(object):
def is_paused():
return True
@staticmethod
def is_standby_cluster():
return False
class MockLogger(object):
@@ -128,10 +125,16 @@ class MockLogger(object):
records_lost = 1
class MockConfig(object):
def get_global_config(self, _):
return GlobalConfig({})
class MockPatroni(object):
ha = MockHa()
config = Mock()
config = MockConfig()
postgresql = ha.state_handler
dcs = Mock()
logger = MockLogger()
@@ -185,7 +188,8 @@ class TestRestApiHandler(unittest.TestCase):
def test_do_GET(self):
MockPatroni.dcs.cluster.last_lsn = 20
MockPatroni.dcs.cluster.sync.members = [MockPostgresql.name]
MockRestApiServer(RestApiHandler, 'GET /replica')
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=True)):
MockRestApiServer(RestApiHandler, 'GET /replica')
MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M')
MockRestApiServer(RestApiHandler, 'GET /replica?lag=10MB')
MockRestApiServer(RestApiHandler, 'GET /replica?lag=10485760')
@@ -207,7 +211,7 @@ class TestRestApiHandler(unittest.TestCase):
with patch.object(MockHa, 'is_leader', Mock(return_value=True)):
MockRestApiServer(RestApiHandler, 'GET /replica')
MockRestApiServer(RestApiHandler, 'GET /read-only-sync')
with patch.object(MockHa, 'is_standby_cluster', Mock(return_value=True)):
with patch.object(GlobalConfig, 'is_standby_cluster', Mock(return_value=True)):
MockRestApiServer(RestApiHandler, 'GET /standby_leader')
MockPatroni.dcs.cluster = None
with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'primary'})):
@@ -217,7 +221,8 @@ class TestRestApiHandler(unittest.TestCase):
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /primary'))
with patch.object(RestApiServer, 'query', Mock(return_value=[('', 1, '', '', '', '', False, '')])):
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
with patch.object(MockHa, 'is_standby_cluster', Mock(return_value=True)):
with patch.object(GlobalConfig, 'is_standby_cluster', Mock(return_value=True)),\
patch.object(GlobalConfig, 'is_paused', Mock(return_value=True)):
MockRestApiServer(RestApiHandler, 'GET /standby_leader')
# test tags
@@ -405,9 +410,7 @@ class TestRestApiHandler(unittest.TestCase):
def test_do_POST_sigterm(self):
self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /sigterm HTTP/1.0' + self._authorization))
@patch.object(MockPatroni, 'dcs')
def test_do_POST_restart(self, mock_dcs):
mock_dcs.get_cluster.return_value.is_paused.return_value = False
def test_do_POST_restart(self):
request = 'POST /restart HTTP/1.0' + self._authorization
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
@@ -449,12 +452,12 @@ class TestRestApiHandler(unittest.TestCase):
request = make_request(role='primary', postgres_version='9.5.2')
MockRestApiServer(RestApiHandler, request)
mock_dcs.get_cluster.return_value.is_paused.return_value = True
MockRestApiServer(RestApiHandler, make_request(schedule='2016-08-42 12:45TZ+1', role='primary'))
# Valid timeout
MockRestApiServer(RestApiHandler, make_request(timeout='60s'))
# Invalid timeout
MockRestApiServer(RestApiHandler, make_request(timeout='42towels'))
with patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
MockRestApiServer(RestApiHandler, make_request(schedule='2016-08-42 12:45TZ+1', role='primary'))
# Valid timeout
MockRestApiServer(RestApiHandler, make_request(timeout='60s'))
# Invalid timeout
MockRestApiServer(RestApiHandler, make_request(timeout='42towels'))
def test_do_DELETE_restart(self):
for retval in (True, False):
@@ -471,10 +474,7 @@ class TestRestApiHandler(unittest.TestCase):
mock_dcs.get_cluster.return_value.failover = None
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
@patch.object(MockPatroni, 'dcs')
def test_do_POST_reinitialize(self, mock_dcs):
cluster = mock_dcs.get_cluster.return_value
cluster.is_paused.return_value = False
def test_do_POST_reinitialize(self):
request = 'POST /reinitialize HTTP/1.0' + self._authorization + '\nContent-Length: 15\n\n{"force": true}'
MockRestApiServer(RestApiHandler, request)
with patch.object(MockHa, 'reinitialize', Mock(return_value=None)):
@@ -492,8 +492,6 @@ class TestRestApiHandler(unittest.TestCase):
def test_do_POST_switchover(self, dcs):
dcs.loop_wait = 10
cluster = dcs.get_cluster.return_value
cluster.is_synchronous_mode.return_value = False
cluster.is_paused.return_value = False
post = 'POST /switchover HTTP/1.0' + self._authorization + '\nContent-Length: '
@@ -507,21 +505,22 @@ class TestRestApiHandler(unittest.TestCase):
request = post + '25\n\n{"leader": "postgresql1"}'
cluster.is_paused.return_value = True
MockRestApiServer(RestApiHandler, request)
cluster.is_paused.return_value = False
for cluster.is_synchronous_mode.return_value in (True, False):
with patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
MockRestApiServer(RestApiHandler, request)
for is_synchronous_mode in (True, False):
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)):
MockRestApiServer(RestApiHandler, request)
cluster.leader.name = 'postgresql2'
request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
MockRestApiServer(RestApiHandler, request)
cluster.leader.name = 'postgresql1'
cluster.sync.matches.return_value = False
for cluster.is_synchronous_mode.return_value in (True, False):
MockRestApiServer(RestApiHandler, request)
for is_synchronous_mode in (True, False):
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)):
MockRestApiServer(RestApiHandler, request)
cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
Member(0, 'postgresql2', 30, {'api_url': 'http'})]
@@ -555,7 +554,8 @@ class TestRestApiHandler(unittest.TestCase):
request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2",' +\
' "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}'
MockRestApiServer(RestApiHandler, request)
with patch.object(MockPatroni, 'dcs') as d:
with patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)),\
patch.object(MockPatroni, 'dcs') as d:
d.manual_failover.return_value = False
MockRestApiServer(RestApiHandler, request)
@@ -571,13 +571,11 @@ class TestRestApiHandler(unittest.TestCase):
# Invalid date
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}'))
@patch.object(MockPatroni, 'dcs', Mock())
def test_do_POST_failover(self):
post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}')
MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}')
@patch.object(MockPatroni, 'dcs', Mock())
@patch.object(MockHa, 'is_leader', Mock(return_value=True))
def test_do_POST_citus(self):
post = 'POST /citus HTTP/1.0' + self._authorization + '\nContent-Length: '
tests/test_config.py
+1 -3
View File
@@ -20,9 +20,7 @@ class TestConfig(unittest.TestCase):
def test_set_dynamic_configuration(self):
with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
self.assertIsNone(self.config.set_dynamic_configuration({'foo': 'bar'}))
self.assertTrue(self.config.set_dynamic_configuration({'synchronous_mode': True,
'standby_cluster': {}, 'master_start_timeout': 1}))
self.assertEqual(self.config.get('primary_start_timeout'), 1)
self.assertTrue(self.config.set_dynamic_configuration({'standby_cluster': {}}))
def test_reload_local_configuration(self):
os.environ.update({
tests/test_ctl.py
+6 -6
View File
@@ -5,7 +5,7 @@ import unittest
from click.testing import CliRunner
from datetime import datetime, timedelta
from mock import patch, Mock
from mock import patch, Mock, PropertyMock
from patroni.ctl import ctl, load_config, output_members, get_dcs, parse_dcs, \
get_all_members, get_any_member, get_cursor, query_member, PatroniCtlException, apply_config_changes, \
format_config_for_editing, show_diff, invoke_editor, format_pg_version, CONFIG_FILE_PATH, PatronictlPrettyTable
@@ -98,7 +98,7 @@ class TestCtl(unittest.TestCase):
input='leader\nother\n2300-01-01T12:23:00\ny')
assert result.exit_code == 0
with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
'--force', '--scheduled', '2015-01-01T12:00:00'])
assert result.exit_code == 1
@@ -309,7 +309,7 @@ class TestCtl(unittest.TestCase):
result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
assert 'Failed: flush scheduled restart' in result.output
with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
result = self.runner.invoke(ctl,
['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
assert result.exit_code == 1
@@ -491,7 +491,7 @@ class TestCtl(unittest.TestCase):
assert 'Failed' in result.output
mock_post.return_value.status = 200
with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
result = self.runner.invoke(ctl, ['pause', 'dummy'])
assert 'Cluster is already paused' in result.output
@@ -512,11 +512,11 @@ class TestCtl(unittest.TestCase):
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
mock_post.return_value.status = 200
with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=False)):
with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=False)):
result = self.runner.invoke(ctl, ['resume', 'dummy'])
assert 'Cluster is not paused' in result.output
with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
result = self.runner.invoke(ctl, ['resume', 'dummy'])
assert 'Success' in result.output
tests/test_etcd.py
-1
View File
@@ -256,7 +256,6 @@ class TestEtcd(unittest.TestCase):
def test_get_cluster(self):
cluster = self.etcd.get_cluster()
self.assertIsInstance(cluster, Cluster)
self.assertFalse(cluster.is_synchronous_mode())
self.etcd._base_path = '/service/legacy'
self.assertIsInstance(self.etcd.get_cluster(), Cluster)
self.etcd._base_path = '/service/broken'
tests/test_ha.py
+18 -9
View File
@@ -136,7 +136,6 @@ zookeeper:
sys.argv = sys.argv[:1]
self.config = Config(None)
self.config.set_dynamic_configuration({'maximum_lag_on_failover': 5})
self.version = '1.5.7'
self.postgresql = p
self.dcs = d
@@ -300,7 +299,8 @@ class TestHa(PostgresInit):
self.ha._async_executor.schedule('doing crash recovery in a single user mode')
self.ha.state_handler.cancellable._process = Mock()
self.ha._crash_recovery_started -= 600
self.ha.patroni.config.set_dynamic_configuration({'maximum_lag_on_failover': 10})
self.ha.cluster.config.data.update({'maximum_lag_on_failover': 10})
self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
self.assertEqual(self.ha.run_cycle(), 'terminated crash recovery because of startup timeout')
@patch.object(Rewind, 'ensure_clean_shutdown', Mock())
@@ -488,6 +488,7 @@ class TestHa(PostgresInit):
def test_check_failsafe_topology(self):
self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
self.ha.cluster = get_cluster_initialized_with_leader_and_failsafe()
self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
self.ha.dcs._last_failsafe = self.ha.cluster.failsafe
self.assertEqual(self.ha.run_cycle(), 'demoting self because DCS is not accessible and I was a leader')
self.ha.state_handler.name = self.ha.cluster.leader.name
@@ -507,6 +508,7 @@ class TestHa(PostgresInit):
def test_no_dcs_connection_primary_failsafe(self):
self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
self.ha.cluster = get_cluster_initialized_with_leader_and_failsafe()
self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
self.ha.dcs._last_failsafe = self.ha.cluster.failsafe
self.ha.state_handler.name = self.ha.cluster.leader.name
self.assertEqual(self.ha.run_cycle(),
@@ -523,6 +525,7 @@ class TestHa(PostgresInit):
def test_no_dcs_connection_replica_failsafe(self):
self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
self.ha.cluster = get_cluster_initialized_with_leader_and_failsafe()
self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
self.ha.update_failsafe({'name': 'leader', 'api_url': 'http://127.0.0.1:8008/patroni',
'conn_url': 'postgres://127.0.0.1:5432/postgres', 'slots': {'foo': 1000}})
self.p.is_leader = false
@@ -685,6 +688,8 @@ class TestHa(PostgresInit):
self.ha.fetch_node_status = get_node_status(timeline=1)
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
self.ha.fetch_node_status = get_node_status(wal_position=1)
self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5})
self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
# manual failover from the previous leader to us won't happen if we hold the nofailover flag
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, None))
@@ -841,6 +846,7 @@ class TestHa(PostgresInit):
def test__is_healthiest_node(self):
self.ha.cluster = get_cluster_initialized_without_leader(sync=('postgresql1', self.p.name))
self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
self.p.is_leader = false
self.ha.fetch_node_status = get_node_status() # accessible, in_recovery
@@ -852,6 +858,8 @@ class TestHa(PostgresInit):
# in synchronous_mode consider itself healthy if the former leader is accessible in read-only and ahead of us
with patch.object(Ha, 'is_synchronous_mode', Mock(return_value=True)):
self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5})
self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
with patch('patroni.postgresql.Postgresql.last_operation', return_value=1):
self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
with patch('patroni.postgresql.Postgresql.replica_cached_timeline', return_value=1):
@@ -1067,8 +1075,8 @@ class TestHa(PostgresInit):
def test_failover_immediately_on_zero_primary_start_timeout(self, demote):
self.p.is_running = false
self.ha.cluster = get_cluster_initialized_with_leader(sync=(self.p.name, 'other'))
self.ha.cluster.config.data['synchronous_mode'] = True
self.ha.patroni.config.set_dynamic_configuration({'primary_start_timeout': 0})
self.ha.cluster.config.data.update({'synchronous_mode': True, 'primary_start_timeout': 0})
self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
self.ha.has_lock = true
self.ha.update_lock = true
self.ha.fetch_node_status = get_node_status() # accessible, in_recovery
@@ -1077,13 +1085,14 @@ class TestHa(PostgresInit):
def test_primary_stop_timeout(self):
self.assertEqual(self.ha.primary_stop_timeout(), None)
self.ha.patroni.config.set_dynamic_configuration({'primary_stop_timeout': 30})
self.ha.cluster.config.data.update({'primary_stop_timeout': 30})
self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
with patch.object(Ha, 'is_synchronous_mode', Mock(return_value=True)):
self.assertEqual(self.ha.primary_stop_timeout(), 30)
self.ha.patroni.config.set_dynamic_configuration({'primary_stop_timeout': 30})
with patch.object(Ha, 'is_synchronous_mode', Mock(return_value=False)):
self.assertEqual(self.ha.primary_stop_timeout(), None)
self.ha.patroni.config.set_dynamic_configuration({'primary_stop_timeout': None})
self.ha.cluster.config.data['primary_stop_timeout'] = None
self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
self.assertEqual(self.ha.primary_stop_timeout(), None)
@patch('patroni.postgresql.Postgresql.follow')
@@ -1171,9 +1180,9 @@ class TestHa(PostgresInit):
# Test sync set to '*' when synchronous_mode_strict is enabled
mock_set_sync.reset_mock()
self.ha.is_synchronous_mode_strict = true
self.p.sync_handler.current_state = Mock(return_value=([], []))
self.ha.run_cycle()
with patch('patroni.config.GlobalConfig.is_synchronous_mode_strict', PropertyMock(return_value=True)):
self.ha.run_cycle()
mock_set_sync.assert_called_once_with(['*'])
def test_sync_replication_become_primary(self):
tests/test_postgresql.py
+4 -2
View File
@@ -10,6 +10,7 @@ from mock import Mock, MagicMock, PropertyMock, patch, mock_open
import patroni.psycopg as psycopg
from patroni.async_executor import CriticalTask
from patroni.config import GlobalConfig
from patroni.dcs import RemoteMember
from patroni.exceptions import PostgresConnectionException, PatroniException
from patroni.postgresql import Postgresql, STATE_REJECT, STATE_NO_RESPONSE
@@ -644,9 +645,10 @@ class TestPostgresql(BaseTestPostgresql):
self.assertIsNone(self.p.wait_for_startup())
def test_get_server_parameters(self):
config = {'synchronous_mode': True, 'parameters': {'wal_level': 'hot_standby'}, 'listen': '0'}
config = {'parameters': {'wal_level': 'hot_standby'}, 'listen': '0'}
self.p._global_config = GlobalConfig({'synchronous_mode': True})
self.p.config.get_server_parameters(config)
config['synchronous_mode_strict'] = True
self.p._global_config = GlobalConfig({'synchronous_mode': True, 'synchronous_mode_strict': True})
self.p.config.get_server_parameters(config)
self.p.config.set_synchronous_standby_names('foo')
self.assertTrue(str(self.p.config.get_server_parameters(config)).startswith('{'))