From 7024d0a9873937b2806ba6a207134fd491127fe1 Mon Sep 17 00:00:00 2001 From: Polina Bungina Date: Sun, 27 Aug 2023 17:48:43 +0200 Subject: [PATCH] Yet another refactoring --- patroni/api.py | 4 - patroni/ctl.py | 29 ++++--- patroni/utils.py | 2 +- tests/test_api.py | 94 +++++++++++++-------- tests/test_ctl.py | 205 +++++++++++++++++++++++++++++----------------- 5 files changed, 205 insertions(+), 129 deletions(-) diff --git a/patroni/api.py b/patroni/api.py index 2d36fd3a..b4a050b9 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -1015,10 +1015,6 @@ class RestApiHandler(BaseHTTPRequestHandler): logger.info("received %s request with leader=%s candidate=%s scheduled_at=%s", action, leader, candidate, scheduled_at) - if action == 'failover' and leader: - logger.warning('received failover request with leader specifed - performing switchover') - action = 'switchover' - manual_failover = ManualFailover(action, cluster, leader, candidate, scheduled_at, global_config.is_paused, global_config.is_synchronous_mode, self.server.patroni) diff --git a/patroni/ctl.py b/patroni/ctl.py index f617959d..f444d9a6 100644 --- a/patroni/ctl.py +++ b/patroni/ctl.py @@ -1030,6 +1030,7 @@ def restart(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M') scheduled = click.prompt('When should the restart take place (e.g. ' + next_hour + ') ', type=str, default='now') + scheduled = scheduled if scheduled != 'now' else None parse_result, scheduled_at = parse_schedule(scheduled) if parse_result: @@ -1190,24 +1191,21 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s global_config = get_global_config(cluster) # Leader is required for switchover only - if action == 'switchover': + if action == 'switchover' and leader is None: if cluster.leader is None or not cluster.leader.name: raise PatroniCtlException('This cluster has no leader') + if force: + leader = cluster.leader.name + else: + prompt = 'Standby Leader' if global_config.is_standby_cluster else 'Primary' + leader = click.prompt(prompt, type=str, default=(cluster.leader and cluster.leader.name)) - if leader is None: - if force: - leader = cluster.leader.name - else: - prompt = 'Standby Leader' if global_config.is_standby_cluster else 'Primary' - leader = click.prompt(prompt, type=str, default=(cluster.leader and cluster.leader.name)) - - # Check if there are any candidates available at all - candidate_names = [str(m.name) for m in cluster.members if m.name != leader and not m.nofailover] - if not candidate_names: - raise PatroniCtlException('No candidates found to {0} to'.format(action)) - - candidate_names.sort() # we sort the names for consistent output to the client if candidate is None and not force: + # Check if there are any candidates available at all + candidate_names = [str(m.name) for m in cluster.members if m.name != leader and not m.nofailover] + if not candidate_names: + raise PatroniCtlException('No candidates found to {0} to'.format(action)) + candidate_names.sort() # we sort the names for consistent output to the client candidate = click.prompt('Candidate ' + str(candidate_names), type=str, default='') # We allow manual failover to an aync node in the sync mode, so we better ask for the confirmation @@ -1219,8 +1217,9 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s if action == 'switchover' and scheduled is None and not force: next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M') - scheduled = click.prompt('When should the switchover take place (e.g. ' + next_hour + ' ) ', + scheduled = click.prompt('When should the switchover take place (e.g. ' + next_hour + ') ', type=str, default='now') + scheduled = scheduled if scheduled != 'now' else None # Now, when we collected all the possible info, run checks manual_failover = ManualFailover(action, cluster, leader, candidate, scheduled, diff --git a/patroni/utils.py b/patroni/utils.py index 3b95e599..881a2245 100644 --- a/patroni/utils.py +++ b/patroni/utils.py @@ -1074,7 +1074,7 @@ class ParseScheduleErrors(Enum): def parse_schedule(schedule: Optional[str]) -> Tuple[Optional[ParseScheduleErrors], Optional[datetime.datetime]]: scheduled_at = None - if schedule is not None and (schedule or 'now') != 'now': + if schedule is not None: try: scheduled_at = dateutil.parser.parse(schedule) if scheduled_at.tzinfo is None: diff --git a/tests/test_api.py b/tests/test_api.py index fe7270bc..882fe898 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -12,9 +12,10 @@ from patroni.api import RestApiHandler, RestApiServer from patroni.config import GlobalConfig from patroni.dcs import ClusterConfig, Member from patroni.ha import _MemberStatus -from patroni.utils import RetryFailedError, tzutc +from patroni.manual_failover import ManualFailoverPrecheckStatus +from patroni.utils import ParseScheduleErrors, RetryFailedError, tzutc -from .test_ha import get_cluster_initialized_without_leader +from .test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5) @@ -504,7 +505,7 @@ class TestRestApiHandler(unittest.TestCase): # Invalid content with patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}') - response_mock.assert_called_with(400, 'Switchover could be performed only from a specific leader') + response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SWITCHOVER_NO_LEADER.value[::-1]) # Empty content request = post + '0\n\n' @@ -512,25 +513,19 @@ class TestRestApiHandler(unittest.TestCase): # [Switchover without a candidate] - # Cluster with only a leader - with patch.object(RestApiHandler, 'write_response') as response_mock: - cluster.leader.name = 'postgresql1' - request = post + '25\n\n{"leader": "postgresql1"}' - MockRestApiServer(RestApiHandler, request) - response_mock.assert_called_with( - 412, 'switchover is not possible: cluster does not have members except leader') + cluster.leader.name = 'postgresql1' + request = post + '25\n\n{"leader": "postgresql1"}' - # Switchover in pause mode + # No candidate in pause mode with patch.object(RestApiHandler, 'write_response') as response_mock, \ patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)): MockRestApiServer(RestApiHandler, request) - response_mock.assert_called_with( - 400, 'Switchover is possible only to a specific candidate in a paused state') + response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SWITCHOVER_PAUSE_NO_CANDIDATE.value[::-1]) # No healthy nodes to promote in both sync and async mode for is_synchronous_mode, response in ( - (True, 'switchover is not possible: can not find sync_standby'), - (False, 'switchover is not possible: cluster does not have members except leader')): + (True, ManualFailoverPrecheckStatus.NO_SYNC_CANDIDATE.value[0].format(action='switchover')), + (False, ManualFailoverPrecheckStatus.ONLY_LEADER.value[0].format(action='switchover'))): with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \ patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, request) @@ -542,21 +537,25 @@ class TestRestApiHandler(unittest.TestCase): with patch.object(RestApiHandler, 'write_response') as response_mock: request = post + '53\n\n{"leader": "postgresql2", "candidate": "postgresql2"}' MockRestApiServer(RestApiHandler, request) - response_mock.assert_called_with(400, 'Switchover target and source are the same') + response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SWITCHOVER_TO_LEADER.value[::-1]) # Current leader is different from the one specified with patch.object(RestApiHandler, 'write_response') as response_mock: cluster.leader.name = 'postgresql2' request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}' MockRestApiServer(RestApiHandler, request) - response_mock.assert_called_with(412, 'Member postgresql1 is not the leader of cluster dummy') + response_mock.assert_called_with( + ManualFailoverPrecheckStatus.LEADER_NOT_MEMBER.value[1], + ManualFailoverPrecheckStatus.LEADER_NOT_MEMBER.value[0].format(leader='postgresql1', + cluster_name='dummy')) - # Candidate to promote is not a member of the cluster + # Candidate to promote is not a sync standby/a member of the cluster cluster.leader.name = 'postgresql1' cluster.sync.matches.return_value = False for is_synchronous_mode, response in ( - (True, 'candidate name does not match with sync_standby'), - (False, 'Member postgresql2 does not exist in cluster dummy or is tagged as nofailover')): + (True, ManualFailoverPrecheckStatus.CANDIDATE_NOT_SYNC_STANDBY.value[0]), + (False, ManualFailoverPrecheckStatus.CANDIDATE_NOT_MEMEBER.value[0].format(candidate="postgresql2", + cluster_name='dummy'))): with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \ patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, request) @@ -565,9 +564,21 @@ class TestRestApiHandler(unittest.TestCase): cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}), Member(0, 'postgresql2', 30, {'api_url': 'http'})] + # Cluster has no leader + cluster.leader.name = None + with patch.object(RestApiHandler, 'write_response') as response_mock: + request = post + '53\n\n{"leader": "postgresql1"}' + MockRestApiServer(RestApiHandler, request) + response_mock.assert_called_with( + ManualFailoverPrecheckStatus.CLUSTER_NO_LEADER.value[1], + ManualFailoverPrecheckStatus.CLUSTER_NO_LEADER.value[0].format(leader='leader', cluster_name='dummy')) + + cluster.leader.name = 'postgresql1' + # Failover key is empty in DCS with patch.object(RestApiHandler, 'write_response') as response_mock: cluster.failover = None + request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}' MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with(503, 'Switchover failed') @@ -605,7 +616,9 @@ class TestRestApiHandler(unittest.TestCase): with patch.object(MockHa, 'has_members_eligible_to_promote', Mock(return_value=False)), \ patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, request) - response_mock.assert_called_with(412, 'switchover is not possible: no good candidates have been found') + response_mock.assert_called_with( + ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[1], + ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[0].format(action='switchover')) # [Scheduled switchover] @@ -616,47 +629,58 @@ class TestRestApiHandler(unittest.TestCase): MockRestApiServer(RestApiHandler, request) response_mock.assert_called_with(202, 'Switchover scheduled') - # Schedule in paused mode + # Scheduled in pause mode with patch.object(RestApiHandler, 'write_response') as response_mock, \ patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)): dcs.manual_failover.return_value = False MockRestApiServer(RestApiHandler, request) - response_mock.assert_called_with(400, "Can't schedule switchover in the paused state") + response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SCHEDULED_SWITCHOVER_PAUSE.value[::-1]) # No timezone specified with patch.object(RestApiHandler, 'write_response') as response_mock: request = post + '97\n\n{"leader": "postgresql1", "member": "postgresql2",' + \ ' "scheduled_at": "6016-02-15T18:13:30.568224"}' MockRestApiServer(RestApiHandler, request) - response_mock.assert_called_with(400, 'Timezone information is mandatory for the scheduled switchover') + response_mock.assert_called_with( + ParseScheduleErrors.NO_TIMEZONE.value[1], + ParseScheduleErrors.NO_TIMEZONE.value[0].format(action='switchover')) request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "' # Scheduled in the past with patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, request + '1016-02-15T18:13:30.568224+01:00"}') - response_mock.assert_called_with(422, 'Cannot schedule switchover in the past') + response_mock.assert_called_with( + ParseScheduleErrors.SCHEDULED_IN_PAST.value[1], + ParseScheduleErrors.SCHEDULED_IN_PAST.value[0].format(action='switchover')) # Invalid date with patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}') - response_mock.assert_called_with( - 422, 'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601') + response_mock.assert_called_with(*ParseScheduleErrors.PARSING_ERROR.value[::-1]) - def test_do_POST_failover(self): + @patch.object(MockPatroni, 'dcs') + def test_do_POST_failover(self, mock_dcs): post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: ' + cluster = mock_dcs.get_cluster.return_value - # with patch.object(RestApiHandler, 'write_response') as response_mock: - # MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}') - # response_mock.assert_called_once_with(400, 'Failover could be performed only to a specific candidate') + with patch.object(RestApiHandler, 'write_response') as response_mock: + MockRestApiServer(RestApiHandler, post + '19\n\n{"leader":"leader"}') + response_mock.assert_called_once_with(*ManualFailoverPrecheckStatus.FAILOVER_NO_CANDIDATE.value[::-1]) with patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}') - response_mock.assert_called_once_with(400, "Failover can't be scheduled") + response_mock.assert_called_once_with(*ManualFailoverPrecheckStatus.SCHEDULED_FAILOVER.value[::-1]) - with patch.object(RestApiHandler, 'write_response') as response_mock: - MockRestApiServer(RestApiHandler, post + '30\n\n{"leader":"1","candidate":"2"}') - response_mock.assert_called_once_with(412, 'Member 1 is not the leader of cluster dummy') + # Candidate is not healthy to be promoted + cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}), + Member(0, 'postgresql2', 30, {'api_url': 'http'})] + with patch.object(MockHa, 'has_members_eligible_to_promote', Mock(return_value=False)), \ + patch.object(RestApiHandler, 'write_response') as response_mock: + MockRestApiServer(RestApiHandler, post + '27\n\n{"candidate":"postgresql2"}') + response_mock.assert_called_with( + ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[1], + ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[0].format(action='failover')) @patch.object(MockHa, 'is_leader', Mock(return_value=True)) def test_do_POST_citus(self): diff --git a/tests/test_ctl.py b/tests/test_ctl.py index 05394533..466188c3 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -6,12 +6,14 @@ import unittest from click.testing import CliRunner from datetime import datetime, timedelta from mock import patch, Mock, PropertyMock +from patroni.config import GlobalConfig from patroni.ctl import ctl, load_config, output_members, get_dcs, parse_dcs, \ get_all_members, get_any_member, get_cursor, query_member, PatroniCtlException, apply_config_changes, \ format_config_for_editing, show_diff, invoke_editor, format_pg_version, CONFIG_FILE_PATH, PatronictlPrettyTable from patroni.dcs.etcd import AbstractEtcdClientWithFailover, Cluster, Failover +from patroni.manual_failover import ManualFailoverPrecheckStatus from patroni.psycopg import OperationalError -from patroni.utils import tzutc +from patroni.utils import ParseScheduleErrors, tzutc from prettytable import PrettyTable, ALL from urllib3 import PoolManager @@ -113,77 +115,6 @@ class TestCtl(unittest.TestCase): result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force']) self.assertEqual(result.exit_code, 0) - # Scheduled (confirm) - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], - input=f'leader\nother\n{self.SCHEDULED_TS}\ny') - self.assertEqual(result.exit_code, 0) - - # Scheduled (abort) - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', - '--scheduled', self.SCHEDULED_TS], input='leader\nother\n\nN') - self.assertEqual(result.exit_code, 1) - - # Scheduled with --force option - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', - '--force', '--scheduled', self.SCHEDULED_TS]) - self.assertEqual(result.exit_code, 0) - - # Scheduled in pause mode - with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)): - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', - '--force', '--scheduled', self.SCHEDULED_TS]) - self.assertEqual(result.exit_code, 1) - self.assertIn("Can't schedule switchover in the paused state", result.output) - - # Target and source are equal - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nleader\n\ny') - self.assertEqual(result.exit_code, 1) - self.assertIn('Switchover target and source are the same', result.output) - - # Candidate is not a member of the cluster - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nReality\n\ny') - self.assertEqual(result.exit_code, 1) - self.assertIn('Member Reality does not exist in cluster dummy or is tagged as nofailover', result.output) - - # Invalid timestamp with force - result = self.runner.invoke(ctl,['switchover', 'dummy', '--group', '0', '--force', '--scheduled', - self.SCHEDULED_TS_INVALID]) - self.assertEqual(result.exit_code, 1) - self.assertIn('Unable to parse scheduled timestamp', result.output) - - # Invalid timestamp - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', - '--force', '--scheduled', self.SCHEDULED_TS_INVALID]) - self.assertEqual(result.exit_code, 1) - self.assertIn('Unable to parse scheduled timestamp', result.output) - - # Invalid timestamp - no timezone - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', - '--force', '--scheduled', self.SCHEDULED_TS_NO_TZ]) - self.assertEqual(result.exit_code, 1) - self.assertIn('Timezone information is mandatory for the scheduled switchover', result.output) - - # Specifying wrong leader - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='dummy') - self.assertEqual(result.exit_code, 1) - self.assertIn('Member dummy is not the leader of cluster dummy', result.output) - - # Errors while sending Patroni REST API request - with patch.object(PoolManager, 'request', Mock(side_effect=Exception)): - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], - input=f'leader\nother\n{self.SCHEDULED_TS}\ny') - self.assertIn('falling back to DCS', result.output) - - with patch.object(PoolManager, 'request') as mock_api_request: - mock_api_request.return_value.status = 500 - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') - self.assertIn('Switchover failed', result.output) - - mock_api_request.return_value.status = 501 - mock_api_request.return_value.data = b'Server does not support this operation' - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') - self.assertIn('Switchover failed', result.output) - # No members available mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') @@ -201,6 +132,132 @@ class TestCtl(unittest.TestCase): self.assertEqual(result.exit_code, 1) self.assertIn('For Citus clusters the --group must me specified', result.output) + # [Scheduled] + + mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader + + # Scheduled (confirm) + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], + input=f'leader\nother\n{self.SCHEDULED_TS}\ny') + self.assertEqual(result.exit_code, 0) + self.assertIn(f'Are you sure you want to schedule switchover of cluster dummy ' + f'at {self.SCHEDULED_TS}, demoting current leader', result.output) + + # Scheduled (abort) + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', + '--scheduled', self.SCHEDULED_TS], input='leader\nother\n\nN') + self.assertEqual(result.exit_code, 1) + + # Scheduled with --force option + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', + '--force', '--scheduled', self.SCHEDULED_TS]) + self.assertEqual(result.exit_code, 0) + + # Scheduled in pause mode + with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)): + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', + '--force', '--scheduled', self.SCHEDULED_TS]) + self.assertEqual(result.exit_code, 1) + self.assertIn(ManualFailoverPrecheckStatus.SCHEDULED_SWITCHOVER_PAUSE.value[0], result.output) + + # Invalid timestamp with force + result = self.runner.invoke(ctl,['switchover', 'dummy', '--group', '0', '--force', '--scheduled', + self.SCHEDULED_TS_INVALID]) + self.assertEqual(result.exit_code, 1) + self.assertIn('Unable to parse scheduled timestamp', result.output) + + # Invalid timestamp + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', + '--force', '--scheduled', self.SCHEDULED_TS_INVALID]) + self.assertEqual(result.exit_code, 1) + self.assertIn('Unable to parse scheduled timestamp', result.output) + + # Invalid timestamp - no timezone + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', + '--force', '--scheduled', self.SCHEDULED_TS_NO_TZ]) + self.assertEqual(result.exit_code, 1) + self.assertIn(ParseScheduleErrors.NO_TIMEZONE.value[0].format(action='switchover'), result.output) + + # [Other erroneous combinations] + + # No candidate in pause mode + with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)): + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\n\n\ny') + self.assertEqual(result.exit_code, 1) + self.assertIn(ManualFailoverPrecheckStatus.SWITCHOVER_PAUSE_NO_CANDIDATE.value[0], result.output) + + # Target and source are equal + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nleader\n\ny') + self.assertEqual(result.exit_code, 1) + self.assertIn(ManualFailoverPrecheckStatus.SWITCHOVER_TO_LEADER.value[0], result.output) + + # Candidate is not a member of the cluster + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nReality\n\ny') + self.assertEqual(result.exit_code, 1) + self.assertIn(ManualFailoverPrecheckStatus.CANDIDATE_NOT_MEMEBER.value[0].format(candidate='Reality', + cluster_name='dummy'), + result.output) + + # Specifying wrong leader + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='dummy') + self.assertEqual(result.exit_code, 1) + self.assertIn( + ManualFailoverPrecheckStatus.LEADER_NOT_MEMBER.value[0].format(leader='dummy', + cluster_name='dummy'), + result.output) + + mock_get_dcs.return_value.get_cluster = Mock( + return_value=get_cluster_initialized_with_leader(sync=('leader', 'other'))) + + # Candidate is not a sync standby + with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=True)): + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\notherMember\n\ny') + self.assertEqual(result.exit_code, 1) + self.assertIn(ManualFailoverPrecheckStatus.CANDIDATE_NOT_SYNC_STANDBY.value[0], result.output) + + # No healthy nodes to promote in sync mode + mock_get_dcs.return_value.get_cluster = Mock(return_value=get_cluster_initialized_with_leader(sync=('leader'))) + with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=True)): + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force']) + self.assertEqual(result.exit_code, 1) + self.assertIn(ManualFailoverPrecheckStatus.NO_SYNC_CANDIDATE.value[0].format(action='switchover'), + result.output) + + # No healthy nodes to promote in async mode + mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader + with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=False)): + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force']) + self.assertEqual(result.exit_code, 1) + self.assertIn(ManualFailoverPrecheckStatus.ONLY_LEADER.value[0].format(action='switchover'), + result.output) + + # Cluster has no leader + mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--leader', 'leader', '--force']) + self.assertEqual(result.exit_code, 1) + self.assertIn( + ManualFailoverPrecheckStatus.CLUSTER_NO_LEADER.value[0].format(leader='leader', cluster_name='dummy'), + result.output) + + # [Errors while sending Patroni REST API request] + + mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader + + with patch.object(PoolManager, 'request', Mock(side_effect=Exception)): + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], + input=f'leader\nother\n{self.SCHEDULED_TS}\ny') + self.assertIn('falling back to DCS', result.output) + + with patch.object(PoolManager, 'request') as mock_api_request: + mock_api_request.return_value.status = 500 + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') + self.assertIn('Switchover failed', result.output) + + mock_api_request.return_value.status = 501 + mock_api_request.return_value.data = b'Server does not support this operation' + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') + self.assertIn('Switchover failed', result.output) + @patch('patroni.ctl.get_dcs') @patch.object(PoolManager, 'request', Mock(return_value=MockResponse())) @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse())) @@ -210,7 +267,7 @@ class TestCtl(unittest.TestCase): # No candidate specified mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader result = self.runner.invoke(ctl, ['failover', 'dummy'], input='0\n') - self.assertIn('Failover could be performed only to a specific candidate', result.output) + self.assertIn(ManualFailoverPrecheckStatus.FAILOVER_NO_CANDIDATE.value[0], result.output) # Failover to an async member in sync mode (confirm) cluster = get_cluster_initialized_with_leader(sync=('leader', 'other')) @@ -374,7 +431,7 @@ class TestCtl(unittest.TestCase): result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force', '--scheduled', self.SCHEDULED_TS_NO_TZ]) self.assertEqual(result.exit_code, 1) - self.assertIn('Timezone information is mandatory for the scheduled restart', result.output) + self.assertIn(ParseScheduleErrors.NO_TIMEZONE.value[0].format(action='restart'), result.output) # Error parsing scheduled flag value (invalid date) result = self.runner.invoke(ctl,