mirror of
https://github.com/outbackdingo/patroni.git
synced 2026-08-25 14:53:37 +00:00
Ensure strict failover/switchover definition difference (#2784)
- Don't set leader in failover key from patronictl failover - Show warning and execute switchover if leader option is provided for patronictl failover command - Be more precise in the log messages - Allow to failover to an async candidate in sync mode - Check if candidate is the same as the leader specified in api - Fix and extend some tests - Add documentation
This commit is contained in:
Polina Bungina
+120
-45
@@ -516,86 +516,161 @@ class TestRestApiHandler(unittest.TestCase):
|
||||
|
||||
post = 'POST /switchover HTTP/1.0' + self._authorization + '\nContent-Length: '
|
||||
|
||||
MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}')
|
||||
# Invalid content
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}')
|
||||
response_mock.assert_called_with(400, 'Switchover could be performed only from a specific leader')
|
||||
|
||||
# Empty content
|
||||
request = post + '0\n\n'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
cluster.leader.name = 'postgresql1'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
# [Switchover without a candidate]
|
||||
|
||||
request = post + '25\n\n{"leader": "postgresql1"}'
|
||||
|
||||
with patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
|
||||
# Cluster with only a leader
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
cluster.leader.name = 'postgresql1'
|
||||
request = post + '25\n\n{"leader": "postgresql1"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(
|
||||
412, 'switchover is not possible: cluster does not have members except leader')
|
||||
|
||||
for is_synchronous_mode in (True, False):
|
||||
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)):
|
||||
# Switchover in pause mode
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock, \
|
||||
patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(
|
||||
400, 'Switchover is possible only to a specific candidate in a paused state')
|
||||
|
||||
# No healthy nodes to promote in both sync and async mode
|
||||
for is_synchronous_mode, response in (
|
||||
(True, 'switchover is not possible: can not find sync_standby'),
|
||||
(False, 'switchover is not possible: cluster does not have members except leader')):
|
||||
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \
|
||||
patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(412, response)
|
||||
|
||||
cluster.leader.name = 'postgresql2'
|
||||
request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
# [Switchover to the candidate specified]
|
||||
|
||||
# Candidate to promote is the same as the leader specified
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
request = post + '53\n\n{"leader": "postgresql2", "candidate": "postgresql2"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(400, 'Switchover target and source are the same')
|
||||
|
||||
# Current leader is different from the one specified
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
cluster.leader.name = 'postgresql2'
|
||||
request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(412, 'leader name does not match')
|
||||
|
||||
# Candidate to promote is not a member of the cluster
|
||||
cluster.leader.name = 'postgresql1'
|
||||
cluster.sync.matches.return_value = False
|
||||
for is_synchronous_mode in (True, False):
|
||||
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)):
|
||||
for is_synchronous_mode, response in (
|
||||
(True, 'candidate name does not match with sync_standby'), (False, 'candidate does not exists')):
|
||||
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \
|
||||
patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(412, response)
|
||||
|
||||
cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
|
||||
Member(0, 'postgresql2', 30, {'api_url': 'http'})]
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
|
||||
cluster.failover = None
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
# Failover key is empty in DCS
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
cluster.failover = None
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(503, 'Switchover failed')
|
||||
|
||||
dcs.get_cluster.side_effect = [cluster]
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
# Result polling failed
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
dcs.get_cluster.side_effect = [cluster]
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(503, 'Switchover status unknown')
|
||||
|
||||
cluster2 = cluster.copy()
|
||||
cluster2.leader.name = 'postgresql0'
|
||||
cluster2.is_unlocked.return_value = False
|
||||
dcs.get_cluster.side_effect = [cluster, cluster2]
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
# Switchover to a node different from the candidate specified
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
cluster2 = cluster.copy()
|
||||
cluster2.leader.name = 'postgresql0'
|
||||
cluster2.is_unlocked.return_value = False
|
||||
dcs.get_cluster.side_effect = [cluster, cluster2]
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(200, 'Switched over to "postgresql0" instead of "postgresql2"')
|
||||
|
||||
cluster2.leader.name = 'postgresql2'
|
||||
dcs.get_cluster.side_effect = [cluster, cluster2]
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
# Successful switchover to the candidate
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
cluster2.leader.name = 'postgresql2'
|
||||
dcs.get_cluster.side_effect = [cluster, cluster2]
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(200, 'Successfully switched over to "postgresql2"')
|
||||
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
dcs.manual_failover.return_value = False
|
||||
dcs.get_cluster.side_effect = None
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(503, 'failed to write failover key into DCS')
|
||||
|
||||
dcs.get_cluster.side_effect = None
|
||||
dcs.manual_failover.return_value = False
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
dcs.manual_failover.return_value = True
|
||||
|
||||
with patch.object(MockHa, 'fetch_nodes_statuses', Mock(return_value=[])):
|
||||
# Candidate is not healthy to be promoted
|
||||
with patch.object(MockHa, 'fetch_nodes_statuses', Mock(return_value=[])), \
|
||||
patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(412, 'switchover is not possible: no good candidates have been found')
|
||||
|
||||
# [Scheduled switchover]
|
||||
|
||||
# Valid future date
|
||||
request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2",' +\
|
||||
' "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
with patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)), \
|
||||
patch.object(MockPatroni, 'dcs') as d:
|
||||
d.manual_failover.return_value = False
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2",' + \
|
||||
' "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(202, 'Switchover scheduled')
|
||||
|
||||
# Exception: No timezone specified
|
||||
request = post + '97\n\n{"leader": "postgresql1", "member": "postgresql2",' +\
|
||||
' "scheduled_at": "6016-02-15T18:13:30.568224"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
# Schedule in paused mode
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock, \
|
||||
patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
|
||||
dcs.manual_failover.return_value = False
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(400, "Can't schedule switchover in the paused state")
|
||||
|
||||
# No timezone specified
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
request = post + '97\n\n{"leader": "postgresql1", "member": "postgresql2",' + \
|
||||
' "scheduled_at": "6016-02-15T18:13:30.568224"}'
|
||||
MockRestApiServer(RestApiHandler, request)
|
||||
response_mock.assert_called_with(400, 'Timezone information is mandatory for the scheduled switchover')
|
||||
|
||||
# Exception: Scheduled in the past
|
||||
request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "'
|
||||
MockRestApiServer(RestApiHandler, request + '1016-02-15T18:13:30.568224+01:00"}')
|
||||
|
||||
# Scheduled in the past
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
MockRestApiServer(RestApiHandler, request + '1016-02-15T18:13:30.568224+01:00"}')
|
||||
response_mock.assert_called_with(422, 'Cannot schedule switchover in the past')
|
||||
|
||||
# Invalid date
|
||||
self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}'))
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}')
|
||||
response_mock.assert_called_with(
|
||||
422, 'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601')
|
||||
|
||||
def test_do_POST_failover(self):
|
||||
post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
|
||||
MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}')
|
||||
MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}')
|
||||
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}')
|
||||
response_mock.assert_called_once_with(400, 'Failover could be performed only to a specific candidate')
|
||||
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}')
|
||||
response_mock.assert_called_once_with(400, "Failover can't be scheduled")
|
||||
|
||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||
MockRestApiServer(RestApiHandler, post + '30\n\n{"leader":"1","candidate":"2"}')
|
||||
response_mock.assert_called_once_with(412, 'leader name does not match')
|
||||
|
||||
@patch.object(MockHa, 'is_leader', Mock(return_value=True))
|
||||
def test_do_POST_citus(self):
|
||||
|
||||
+92
-45
@@ -21,10 +21,17 @@ from .test_ha import get_cluster_initialized_without_leader, get_cluster_initial
|
||||
get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader, get_cluster, Member
|
||||
|
||||
|
||||
@patch('patroni.ctl.load_config', Mock(return_value={
|
||||
'scope': 'alpha', 'restapi': {'listen': '::', 'certfile': 'a'}, 'ctl': {'certfile': 'a'},
|
||||
'etcd': {'host': 'localhost:2379'}, 'citus': {'database': 'citus', 'group': 0},
|
||||
'postgresql': {'data_dir': '.', 'pgpass': './pgpass', 'parameters': {}, 'retry_timeout': 5}}))
|
||||
DEFAULT_CONFIG = {
|
||||
'scope': 'alpha',
|
||||
'restapi': {'listen': '::', 'certfile': 'a'},
|
||||
'ctl': {'certfile': 'a'},
|
||||
'etcd': {'host': 'localhost:2379'},
|
||||
'citus': {'database': 'citus', 'group': 0},
|
||||
'postgresql': {'data_dir': '.', 'pgpass': './pgpass', 'parameters': {}, 'retry_timeout': 5}
|
||||
}
|
||||
|
||||
|
||||
@patch('patroni.ctl.load_config', Mock(return_value=DEFAULT_CONFIG))
|
||||
class TestCtl(unittest.TestCase):
|
||||
TEST_ROLES = ('master', 'primary', 'leader')
|
||||
|
||||
@@ -96,91 +103,131 @@ class TestCtl(unittest.TestCase):
|
||||
mock_get_dcs.return_value = self.e
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
|
||||
mock_get_dcs.return_value.set_failover_value = Mock()
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
|
||||
assert 'leader' in result.output
|
||||
|
||||
# Confirm
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
|
||||
self.assertEqual(result.exit_code, 0)
|
||||
|
||||
# Abort
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\nN')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
|
||||
# Without a candidate with --force option
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force'])
|
||||
self.assertEqual(result.exit_code, 0)
|
||||
|
||||
# Scheduled (confirm)
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'],
|
||||
input='leader\nother\n2300-01-01T12:23:00\ny')
|
||||
assert result.exit_code == 0
|
||||
self.assertEqual(result.exit_code, 0)
|
||||
|
||||
# Scheduled (abort)
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||
'--scheduled', '2015-01-01T12:00:00+01:00'], input='leader\nother\n\nN')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
|
||||
# Scheduled with --force option
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||
'--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
|
||||
self.assertEqual(result.exit_code, 0)
|
||||
|
||||
# Scheduled in pause mode
|
||||
with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||
'--force', '--scheduled', '2015-01-01T12:00:00'])
|
||||
assert result.exit_code == 1
|
||||
|
||||
# Aborting switchover, as we answer NO to the confirmation
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\nN')
|
||||
assert result.exit_code == 1
|
||||
|
||||
# Aborting scheduled switchover, as we answer NO to the confirmation
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||
'--scheduled', '2015-01-01T12:00:00+01:00'], input='leader\nother\n\nN')
|
||||
assert result.exit_code == 1
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn("Can't schedule switchover in the paused state", result.output)
|
||||
|
||||
# Target and source are equal
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nleader\n\ny')
|
||||
assert result.exit_code == 1
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn('Switchover target and source are the same', result.output)
|
||||
|
||||
# Reality is not part of this cluster
|
||||
# Candidate is not a member of the cluster
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nReality\n\ny')
|
||||
assert result.exit_code == 1
|
||||
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force'])
|
||||
assert 'Member' in result.output
|
||||
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||
'--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
|
||||
assert result.exit_code == 0
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn('Member Reality does not exist in cluster dummy or is tagged as nofailover', result.output)
|
||||
|
||||
# Invalid timestamp
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force', '--scheduled', 'invalid'])
|
||||
assert result.exit_code != 0
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn('Unable to parse scheduled timestamp', result.output)
|
||||
|
||||
# Invalid timestamp
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||
'--force', '--scheduled', '2115-02-30T12:00:00+01:00'])
|
||||
assert result.exit_code != 0
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn('Unable to parse scheduled timestamp', result.output)
|
||||
|
||||
# Specifying wrong leader
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='dummy')
|
||||
assert result.exit_code == 1
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn('Member dummy is not the leader of cluster dummy', result.output)
|
||||
|
||||
# Errors while sending Patroni REST API request
|
||||
with patch.object(PoolManager, 'request', Mock(side_effect=Exception)):
|
||||
# Non-responding patroni
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'],
|
||||
input='leader\nother\n2300-01-01T12:23:00\ny')
|
||||
assert 'falling back to DCS' in result.output
|
||||
self.assertIn('falling back to DCS', result.output)
|
||||
|
||||
with patch.object(PoolManager, 'request') as mocked:
|
||||
mocked.return_value.status = 500
|
||||
with patch.object(PoolManager, 'request') as mock_api_request:
|
||||
mock_api_request.return_value.status = 500
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
|
||||
assert 'Switchover failed' in result.output
|
||||
self.assertIn('Switchover failed', result.output)
|
||||
|
||||
mocked.return_value.status = 501
|
||||
mocked.return_value.data = b'Server does not support this operation'
|
||||
mock_api_request.return_value.status = 501
|
||||
mock_api_request.return_value.data = b'Server does not support this operation'
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
|
||||
assert 'Switchover failed' in result.output
|
||||
self.assertIn('Switchover failed', result.output)
|
||||
|
||||
# No members available
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
|
||||
assert result.exit_code == 1
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn('No candidates found to switchover to', result.output)
|
||||
|
||||
# No leader available
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
|
||||
assert result.exit_code == 1
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn('This cluster has no leader', result.output)
|
||||
|
||||
# Citus cluster, no group number specified
|
||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force'], input='\n')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
self.assertIn('For Citus clusters the --group must me specified', result.output)
|
||||
|
||||
@patch('patroni.ctl.get_dcs')
|
||||
@patch.object(PoolManager, 'request', Mock(return_value=MockResponse()))
|
||||
@patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
|
||||
def test_failover(self, mock_get_dcs):
|
||||
mock_get_dcs.return_value = self.e
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
|
||||
mock_get_dcs.return_value.set_failover_value = Mock()
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--force'], input='\n')
|
||||
assert 'For Citus clusters the --group must me specified' in result.output
|
||||
|
||||
# No candidate specified
|
||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='0\n')
|
||||
assert 'Failover could be performed only to a specific candidate' in result.output
|
||||
self.assertIn('Failover could be performed only to a specific candidate', result.output)
|
||||
|
||||
cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
|
||||
|
||||
# Temp test to check a fallback to switchover if leader is specified
|
||||
with patch('patroni.ctl._do_failover_or_switchover') as failover_func_mock:
|
||||
result = self.runner.invoke(ctl, ['failover', '--leader', 'leader', 'dummy'], input='0\n')
|
||||
self.assertIn('Supplying a leader name using this command is deprecated', result.output)
|
||||
failover_func_mock.assert_called_once_with(
|
||||
DEFAULT_CONFIG, 'switchover', 'dummy', None, 'leader', None, False)
|
||||
|
||||
# Failover to an async member in sync mode (confirm)
|
||||
cluster.members.append(Member(0, 'async', 28, {'api_url': 'http://127.0.0.1:8012/patroni'}))
|
||||
cluster.config.data['synchronous_mode'] = True
|
||||
mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster)
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--group', '0', '--candidate', 'async'], input='y\ny')
|
||||
self.assertIn('Are you sure you want to failover to the asynchronous node async', result.output)
|
||||
|
||||
# Failover to an async member in sync mode (abort)
|
||||
mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster)
|
||||
result = self.runner.invoke(ctl, ['failover', 'dummy', '--group', '0', '--candidate', 'async'], input='N')
|
||||
self.assertEqual(result.exit_code, 1)
|
||||
|
||||
@patch('patroni.dcs.dcs_modules', Mock(return_value=['patroni.dcs.dummy', 'patroni.dcs.etcd']))
|
||||
def test_get_dcs(self):
|
||||
|
||||
+265
-98
@@ -435,6 +435,7 @@ class TestHa(PostgresInit):
|
||||
|
||||
def test_promote_without_watchdog(self):
|
||||
self.ha.has_lock = true
|
||||
self.p.is_primary = true
|
||||
with patch.object(Watchdog, 'activate', Mock(return_value=False)):
|
||||
self.assertEqual(self.ha.run_cycle(), 'Demoting self because watchdog could not be activated')
|
||||
self.p.is_primary = false
|
||||
@@ -614,6 +615,7 @@ class TestHa(PostgresInit):
|
||||
self.ha.cluster = get_cluster_not_initialized_without_leader()
|
||||
self.e.initialize = true
|
||||
self.ha.bootstrap()
|
||||
self.p.is_primary = true
|
||||
with patch.object(Watchdog, 'activate', Mock(return_value=False)), \
|
||||
patch('patroni.ha.logger.error') as mock_logger:
|
||||
self.assertEqual(self.ha.post_bootstrap(), 'running post_bootstrap')
|
||||
@@ -687,110 +689,289 @@ class TestHa(PostgresInit):
|
||||
|
||||
@patch('patroni.postgresql.citus.CitusHandler.is_coordinator', Mock(return_value=False))
|
||||
def test_manual_failover_from_leader(self):
|
||||
self.ha.has_lock = true # I am the leader
|
||||
|
||||
# to me
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', self.p.name, None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
mock_warning.assert_called_with('%s: I am already the leader, no need to %s', 'manual failover', 'failover')
|
||||
|
||||
# to a non-existent candidate
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'blabla', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
mock_warning.assert_called_with(
|
||||
'%s: no healthy members found, %s is not possible', 'manual failover', 'failover')
|
||||
|
||||
# to an existent candidate
|
||||
self.ha.fetch_node_status = get_node_status()
|
||||
self.ha.has_lock = true
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', self.p.name, None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'blabla', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
f = Failover(0, self.p.name, '', None)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(f)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'b', None))
|
||||
self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
|
||||
self.assertEqual(self.ha.run_cycle(), 'manual failover: demoting myself')
|
||||
|
||||
# to a candidate on an older timeline
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.fetch_node_status = get_node_status(timeline=1)
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.assertEqual(mock_info.call_args_list[0][0],
|
||||
('Timeline %s of member %s is behind the cluster timeline %s', 1, 'b', 2))
|
||||
|
||||
# to a lagging candidate
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.fetch_node_status = get_node_status(wal_position=1)
|
||||
self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5})
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.assertEqual(mock_info.call_args_list[0][0],
|
||||
('Member %s exceeds maximum replication lag', 'b'))
|
||||
self.ha.cluster.members.pop()
|
||||
|
||||
@patch('patroni.postgresql.citus.CitusHandler.is_coordinator', Mock(return_value=False))
|
||||
def test_manual_switchover_from_leader(self):
|
||||
self.ha.has_lock = true # I am the leader
|
||||
|
||||
self.ha.fetch_node_status = get_node_status()
|
||||
|
||||
# different leader specified in failover key, no candidate
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
mock_warning.assert_called_with(
|
||||
'%s: leader name does not match: %s != %s', 'switchover', 'blabla', 'postgresql0')
|
||||
|
||||
# no candidate
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, '', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'switchover: demoting myself')
|
||||
|
||||
self.ha._rewind.rewind_or_reinitialize_needed_and_possible = true
|
||||
self.assertEqual(self.ha.run_cycle(), 'manual failover: demoting myself')
|
||||
self.ha.fetch_node_status = get_node_status(nofailover=True)
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.ha.fetch_node_status = get_node_status(watchdog_failed=True)
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.ha.fetch_node_status = get_node_status(timeline=1)
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.ha.fetch_node_status = get_node_status(wal_position=1)
|
||||
self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5})
|
||||
self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
# manual failover from the previous leader to us won't happen if we hold the nofailover flag
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.assertEqual(self.ha.run_cycle(), 'switchover: demoting myself')
|
||||
|
||||
# Failover scheduled time must include timezone
|
||||
scheduled = datetime.datetime.now()
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.ha.run_cycle()
|
||||
# other members with failover_limitation_s
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.fetch_node_status = get_node_status(nofailover=True)
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.assertEqual(mock_info.call_args_list[0][0], ('Member %s is %s', 'leader', 'not allowed to promote'))
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.fetch_node_status = get_node_status(watchdog_failed=True)
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.assertEqual(mock_info.call_args_list[0][0], ('Member %s is %s', 'leader', 'not watchdog capable'))
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.fetch_node_status = get_node_status(timeline=1)
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.assertEqual(mock_info.call_args_list[0][0],
|
||||
('Timeline %s of member %s is behind the cluster timeline %s', 1, 'leader', 2))
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.fetch_node_status = get_node_status(wal_position=1)
|
||||
self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5})
|
||||
self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.assertEqual(mock_info.call_args_list[0][0], ('Member %s exceeds maximum replication lag', 'leader'))
|
||||
|
||||
@patch('patroni.postgresql.citus.CitusHandler.is_coordinator', Mock(return_value=False))
|
||||
def test_scheduled_switchover_from_leader(self):
|
||||
self.ha.has_lock = true # I am the leader
|
||||
|
||||
self.ha.fetch_node_status = get_node_status()
|
||||
|
||||
# switchover scheduled time must include timezone
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
scheduled = datetime.datetime.now()
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'blabla', scheduled))
|
||||
self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
|
||||
self.assertIn('Incorrect value of scheduled_at: %s', mock_warning.call_args_list[0][0])
|
||||
|
||||
# scheduled now
|
||||
scheduled = datetime.datetime.utcnow().replace(tzinfo=tzutc)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'b', scheduled))
|
||||
self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
|
||||
self.assertEqual('switchover: demoting myself', self.ha.run_cycle())
|
||||
|
||||
scheduled = scheduled + datetime.timedelta(seconds=30)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
# scheduled in the future
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
scheduled = scheduled + datetime.timedelta(seconds=30)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'blabla', scheduled))
|
||||
self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
self.assertIn('Awaiting %s at %s (in %.0f seconds)', mock_info.call_args_list[0][0])
|
||||
|
||||
scheduled = scheduled + datetime.timedelta(seconds=-600)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
# stale value
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
scheduled = scheduled + datetime.timedelta(seconds=-600)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'b', scheduled))
|
||||
self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
|
||||
self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
self.assertIn('Found a stale %s value, cleaning up: %s', mock_warning.call_args_list[0][0])
|
||||
|
||||
scheduled = None
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
def test_manual_switchover_from_leader_in_pause(self):
|
||||
self.ha.has_lock = true # I am the leader
|
||||
self.ha.is_paused = true
|
||||
|
||||
# no candidate
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, '', None))
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
self.assertEqual('PAUSE: no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
mock_warning.assert_called_with(
|
||||
'%s is possible only to a specific candidate in a paused state', 'Switchover')
|
||||
|
||||
def test_manual_failover_from_leader_in_pause(self):
|
||||
self.ha.has_lock = true
|
||||
self.ha.fetch_node_status = get_node_status()
|
||||
self.ha.is_paused = true
|
||||
scheduled = datetime.datetime.now()
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
|
||||
self.assertEqual('PAUSE: no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, '', None))
|
||||
self.assertEqual('PAUSE: no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
|
||||
# failover from me, candidate is healthy
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, None, 'b', None))
|
||||
self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
|
||||
self.assertEqual('PAUSE: manual failover: demoting myself', self.ha.run_cycle())
|
||||
self.ha.cluster.members.pop()
|
||||
|
||||
def test_manual_failover_from_leader_in_synchronous_mode(self):
|
||||
self.ha.has_lock = true
|
||||
self.ha.is_synchronous_mode = true
|
||||
self.ha.process_sync_replication = Mock()
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'a', None), (self.p.name, None))
|
||||
self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'a', None), (self.p.name, 'a'))
|
||||
self.ha.is_failover_possible = true
|
||||
self.ha.fetch_node_status = get_node_status()
|
||||
|
||||
# I am the leader
|
||||
self.p.is_primary = true
|
||||
self.ha.has_lock = true
|
||||
|
||||
# the candidate is not in sync members but we allow failover to an async candidate
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, None, 'b', None), sync=(self.p.name, 'a'))
|
||||
self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
|
||||
self.assertEqual('manual failover: demoting myself', self.ha.run_cycle())
|
||||
self.ha.cluster.members.pop()
|
||||
|
||||
def test_manual_switchover_from_leader_in_synchronous_mode(self):
|
||||
self.ha.is_synchronous_mode = true
|
||||
self.ha.process_sync_replication = Mock()
|
||||
|
||||
# I am the leader
|
||||
self.p.is_primary = true
|
||||
self.ha.has_lock = true
|
||||
|
||||
# candidate specified is not in sync members
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'a', None),
|
||||
sync=(self.p.name, 'blabla'))
|
||||
self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
self.assertEqual(mock_warning.call_args_list[0][0],
|
||||
('%s candidate=%s does not match with sync_standbys=%s', 'Switchover', 'a', 'blabla'))
|
||||
|
||||
# the candidate is in sync members and is healthy
|
||||
self.ha.fetch_node_status = get_node_status(wal_position=305419896)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'a', None),
|
||||
sync=(self.p.name, 'a'))
|
||||
self.ha.cluster.members.append(Member(0, 'a', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
|
||||
self.assertEqual('switchover: demoting myself', self.ha.run_cycle())
|
||||
|
||||
# the candidate is in sync members but is not healthy
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.fetch_node_status = get_node_status(nofailover=true)
|
||||
self.assertEqual('no action. I am (postgresql0), the leader with the lock', self.ha.run_cycle())
|
||||
self.assertEqual(mock_info.call_args_list[0][0], ('Member %s is %s', 'a', 'not allowed to promote'))
|
||||
|
||||
def test_manual_failover_process_no_leader(self):
|
||||
self.p.is_primary = false
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', self.p.name, None))
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader', None))
|
||||
self.p.set_role('replica')
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
self.ha.fetch_node_status = get_node_status() # accessible, in_recovery
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, self.p.name, '', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
self.ha.fetch_node_status = get_node_status(reachable=False) # inaccessible, in_recovery
|
||||
|
||||
# failover to another member, fetch_node_status for candidate fails
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
self.assertEqual(mock_warning.call_args_list[1][0],
|
||||
('%s: member %s is %s', 'manual failover', 'leader', 'not reachable'))
|
||||
|
||||
# failover to another member, candidate is accessible, in_recovery
|
||||
self.p.set_role('replica')
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
# set failover flag to True for all members of the cluster
|
||||
self.ha.fetch_node_status = get_node_status()
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
|
||||
# set nofailover flag to True for all members of the cluster
|
||||
# this should elect the current member, as we are not going to call the API for it.
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
|
||||
self.ha.fetch_node_status = get_node_status(nofailover=True) # accessible, in_recovery
|
||||
self.p.set_role('replica')
|
||||
self.ha.fetch_node_status = get_node_status(nofailover=True)
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
# same as previous, but set the current member to nofailover. In no case it should be elected as a leader
|
||||
|
||||
# failover to me but I am set to nofailover. In no case I should be elected as a leader
|
||||
self.p.set_role('replica')
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'postgresql0', None))
|
||||
self.ha.patroni.nofailover = True
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because I am not allowed to promote')
|
||||
|
||||
self.ha.patroni.nofailover = False
|
||||
|
||||
# failover to another member that is on an older timeline (only failover_limitation() is checked)
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'b', None))
|
||||
self.ha.cluster.members.append(Member(0, 'b', 28, {'api_url': 'http://127.0.0.1:8011/patroni'}))
|
||||
self.ha.fetch_node_status = get_node_status(timeline=1)
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
mock_info.assert_called_with('%s: to %s, i am %s', 'manual failover', 'b', 'postgresql0')
|
||||
|
||||
# failover to another member lagging behind the cluster_lsn (only failover_limitation() is checked)
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5})
|
||||
self.ha.fetch_node_status = get_node_status(wal_position=1)
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
mock_info.assert_called_with('%s: to %s, i am %s', 'manual failover', 'b', 'postgresql0')
|
||||
|
||||
def test_manual_switchover_process_no_leader(self):
|
||||
self.p.is_primary = false
|
||||
self.p.set_role('replica')
|
||||
|
||||
# I was the leader, other members are healthy
|
||||
self.ha.fetch_node_status = get_node_status()
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, self.p.name, '', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
|
||||
# I was the leader, I am the only healthy member
|
||||
with patch('patroni.ha.logger.info') as mock_info:
|
||||
self.ha.fetch_node_status = get_node_status(reachable=False) # inaccessible, in_recovery
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
self.assertEqual(mock_info.call_args_list[0][0], ('Member %s is %s', 'leader', 'not reachable'))
|
||||
self.assertEqual(mock_info.call_args_list[1][0], ('Member %s is %s', 'other', 'not reachable'))
|
||||
|
||||
def test_manual_failover_process_no_leader_in_synchronous_mode(self):
|
||||
self.ha.is_synchronous_mode = true
|
||||
self.p.is_primary = false
|
||||
self.ha.fetch_node_status = get_node_status(nofailover=True) # other nodes are not healthy
|
||||
|
||||
# switchover to a specific node, which name doesn't match our name (postgresql0)
|
||||
# manual failover when our name (postgresql0) isn't in the /sync key and the candidate node is not available
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None),
|
||||
sync=('leader1', 'blabla'))
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
|
||||
# manual failover when the candidate node isn't available but our name is in the /sync key
|
||||
# while other sync node is nofailover
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None),
|
||||
sync=('leader1', 'postgresql0'))
|
||||
self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(), CaseInsensitiveSet()))
|
||||
self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
self.assertEqual(mock_warning.call_args_list[0][0],
|
||||
('%s: member %s is %s', 'manual failover', 'other', 'not allowed to promote'))
|
||||
|
||||
# manual failover to our node (postgresql0),
|
||||
# which name is not in sync nodes list (some sync nodes are available)
|
||||
self.p.set_role('replica')
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'postgresql0', None),
|
||||
sync=('leader1', 'other'))
|
||||
self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(['leader1']),
|
||||
CaseInsensitiveSet(['leader1'])))
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
|
||||
def test_manual_switchover_process_no_leader_in_synchronous_mode(self):
|
||||
self.ha.is_synchronous_mode = true
|
||||
self.p.is_primary = false
|
||||
|
||||
# to a specific node, which name doesn't match our name (postgresql0)
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'other', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
|
||||
# switchover to our node (postgresql0), which name is not in sync nodes list
|
||||
# to our node (postgresql0), which name is not in sync nodes list
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'postgresql0', None),
|
||||
sync=('leader1', 'blabla'))
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
|
||||
# switchover from a specific leader, but our name (postgresql0) is not in the sync nodes list
|
||||
# without candidate, our name (postgresql0) is not in the sync nodes list
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', '', None),
|
||||
sync=('leader', 'blabla'))
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
@@ -800,45 +981,31 @@ class TestHa(PostgresInit):
|
||||
sync=('postgresql0'))
|
||||
self.ha.patroni.nofailover = True
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because I am not allowed to promote')
|
||||
self.ha.patroni.nofailover = False
|
||||
|
||||
# manual failover when our name (postgresql0) isn't in the /sync key and the `other` node is not available
|
||||
self.ha.fetch_node_status = get_node_status(nofailover=True) # accessible, in_recovery
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None),
|
||||
sync=('leader1', 'blabla'))
|
||||
self.assertEqual(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
|
||||
|
||||
# manual failover when the `other` node isn't available but our name is in the /sync key
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None),
|
||||
sync=('leader1', 'postgresql0'))
|
||||
self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(), CaseInsensitiveSet()))
|
||||
self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
|
||||
# manual failover to our node (postgresql0),
|
||||
# which name is not in sync nodes list (the leader and all sync nodes are not available)
|
||||
self.p.set_role('replica')
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'postgresql0', None),
|
||||
sync=('leader1', 'other'))
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
|
||||
# manual failover to our node (postgresql0),
|
||||
# which name is not in sync nodes list (some sync nodes are available)
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'postgresql0', None),
|
||||
sync=('leader1', 'other'))
|
||||
self.p.set_role('replica')
|
||||
self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(['leader1']),
|
||||
CaseInsensitiveSet(['leader1'])))
|
||||
self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
|
||||
|
||||
def test_manual_failover_process_no_leader_in_pause(self):
|
||||
self.ha.is_paused = true
|
||||
|
||||
# I am running as primary, cluster is unlocked, the candidate is allowed to promote
|
||||
# but we are in pause
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'PAUSE: continue to run as primary without lock')
|
||||
|
||||
def test_manual_switchover_process_no_leader_in_pause(self):
|
||||
self.ha.is_paused = true
|
||||
|
||||
# I am running as primary, cluster is unlocked, no candidate specified
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', '', None))
|
||||
self.assertEqual(self.ha.run_cycle(), 'PAUSE: continue to run as primary without lock')
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'blabla', None))
|
||||
self.assertEqual('PAUSE: acquired session lock as a leader', self.ha.run_cycle())
|
||||
|
||||
# the candidate is not running
|
||||
with patch('patroni.ha.logger.warning') as mock_warning:
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'blabla', None))
|
||||
self.assertEqual('PAUSE: acquired session lock as a leader', self.ha.run_cycle())
|
||||
self.assertEqual(
|
||||
mock_warning.call_args_list[0][0],
|
||||
('%s: removing failover key because failover candidate is not running', 'switchover'))
|
||||
|
||||
# switchover to me, I am not leader
|
||||
self.p.is_primary = false
|
||||
self.p.set_role('replica')
|
||||
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', self.p.name, None))
|
||||
@@ -846,7 +1013,7 @@ class TestHa(PostgresInit):
|
||||
|
||||
def test_is_healthiest_node(self):
|
||||
self.ha.is_failsafe_mode = true
|
||||
self.p.is_primary = false
|
||||
self.ha.state_handler.is_primary = false
|
||||
self.ha.patroni.nofailover = False
|
||||
self.ha.fetch_node_status = get_node_status()
|
||||
self.ha.dcs._last_failsafe = {'foo': ''}
|
||||
@@ -1088,7 +1255,7 @@ class TestHa(PostgresInit):
|
||||
f = Failover(0, self.p.name, '', None)
|
||||
self.ha.cluster = get_cluster_initialized_with_leader(f)
|
||||
self.ha.fetch_node_status = get_node_status() # accessible, in_recovery
|
||||
self.assertEqual(self.ha.run_cycle(), 'manual failover: demoting myself')
|
||||
self.assertEqual(self.ha.run_cycle(), 'switchover: demoting myself')
|
||||
|
||||
@patch('patroni.ha.Ha.demote')
|
||||
def test_failover_immediately_on_zero_primary_start_timeout(self, demote):
|
||||
|
||||
Reference in New Issue
Block a user