Yet another refactoring

This commit is contained in:
Polina Bungina
2023-08-27 21:57:18 +02:00
parent cc076c40aa
commit 7024d0a987
5 changed files with 205 additions and 129 deletions
-4
View File
@@ -1015,10 +1015,6 @@ class RestApiHandler(BaseHTTPRequestHandler):
logger.info("received %s request with leader=%s candidate=%s scheduled_at=%s", logger.info("received %s request with leader=%s candidate=%s scheduled_at=%s",
action, leader, candidate, scheduled_at) action, leader, candidate, scheduled_at)
if action == 'failover' and leader:
logger.warning('received failover request with leader specifed - performing switchover')
action = 'switchover'
manual_failover = ManualFailover(action, cluster, leader, candidate, scheduled_at, manual_failover = ManualFailover(action, cluster, leader, candidate, scheduled_at,
global_config.is_paused, global_config.is_synchronous_mode, global_config.is_paused, global_config.is_synchronous_mode,
self.server.patroni) self.server.patroni)
+14 -15
View File
@@ -1030,6 +1030,7 @@ def restart(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member
next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M') next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M')
scheduled = click.prompt('When should the restart take place (e.g. ' + next_hour + ') ', scheduled = click.prompt('When should the restart take place (e.g. ' + next_hour + ') ',
type=str, default='now') type=str, default='now')
scheduled = scheduled if scheduled != 'now' else None
parse_result, scheduled_at = parse_schedule(scheduled) parse_result, scheduled_at = parse_schedule(scheduled)
if parse_result: if parse_result:
@@ -1190,24 +1191,21 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s
global_config = get_global_config(cluster) global_config = get_global_config(cluster)
# Leader is required for switchover only # Leader is required for switchover only
if action == 'switchover': if action == 'switchover' and leader is None:
if cluster.leader is None or not cluster.leader.name: if cluster.leader is None or not cluster.leader.name:
raise PatroniCtlException('This cluster has no leader') raise PatroniCtlException('This cluster has no leader')
if force:
leader = cluster.leader.name
else:
prompt = 'Standby Leader' if global_config.is_standby_cluster else 'Primary'
leader = click.prompt(prompt, type=str, default=(cluster.leader and cluster.leader.name))
if leader is None:
if force:
leader = cluster.leader.name
else:
prompt = 'Standby Leader' if global_config.is_standby_cluster else 'Primary'
leader = click.prompt(prompt, type=str, default=(cluster.leader and cluster.leader.name))
# Check if there are any candidates available at all
candidate_names = [str(m.name) for m in cluster.members if m.name != leader and not m.nofailover]
if not candidate_names:
raise PatroniCtlException('No candidates found to {0} to'.format(action))
candidate_names.sort() # we sort the names for consistent output to the client
if candidate is None and not force: if candidate is None and not force:
# Check if there are any candidates available at all
candidate_names = [str(m.name) for m in cluster.members if m.name != leader and not m.nofailover]
if not candidate_names:
raise PatroniCtlException('No candidates found to {0} to'.format(action))
candidate_names.sort() # we sort the names for consistent output to the client
candidate = click.prompt('Candidate ' + str(candidate_names), type=str, default='') candidate = click.prompt('Candidate ' + str(candidate_names), type=str, default='')
# We allow manual failover to an aync node in the sync mode, so we better ask for the confirmation # We allow manual failover to an aync node in the sync mode, so we better ask for the confirmation
@@ -1219,8 +1217,9 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s
if action == 'switchover' and scheduled is None and not force: if action == 'switchover' and scheduled is None and not force:
next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M') next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M')
scheduled = click.prompt('When should the switchover take place (e.g. ' + next_hour + ' ) ', scheduled = click.prompt('When should the switchover take place (e.g. ' + next_hour + ') ',
type=str, default='now') type=str, default='now')
scheduled = scheduled if scheduled != 'now' else None
# Now, when we collected all the possible info, run checks # Now, when we collected all the possible info, run checks
manual_failover = ManualFailover(action, cluster, leader, candidate, scheduled, manual_failover = ManualFailover(action, cluster, leader, candidate, scheduled,
+1 -1
View File
@@ -1074,7 +1074,7 @@ class ParseScheduleErrors(Enum):
def parse_schedule(schedule: Optional[str]) -> Tuple[Optional[ParseScheduleErrors], Optional[datetime.datetime]]: def parse_schedule(schedule: Optional[str]) -> Tuple[Optional[ParseScheduleErrors], Optional[datetime.datetime]]:
scheduled_at = None scheduled_at = None
if schedule is not None and (schedule or 'now') != 'now': if schedule is not None:
try: try:
scheduled_at = dateutil.parser.parse(schedule) scheduled_at = dateutil.parser.parse(schedule)
if scheduled_at.tzinfo is None: if scheduled_at.tzinfo is None:
+59 -35
View File
@@ -12,9 +12,10 @@ from patroni.api import RestApiHandler, RestApiServer
from patroni.config import GlobalConfig from patroni.config import GlobalConfig
from patroni.dcs import ClusterConfig, Member from patroni.dcs import ClusterConfig, Member
from patroni.ha import _MemberStatus from patroni.ha import _MemberStatus
from patroni.utils import RetryFailedError, tzutc from patroni.manual_failover import ManualFailoverPrecheckStatus
from patroni.utils import ParseScheduleErrors, RetryFailedError, tzutc
from .test_ha import get_cluster_initialized_without_leader from .test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader
future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5) future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5)
@@ -504,7 +505,7 @@ class TestRestApiHandler(unittest.TestCase):
# Invalid content # Invalid content
with patch.object(RestApiHandler, 'write_response') as response_mock: with patch.object(RestApiHandler, 'write_response') as response_mock:
MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}') MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}')
response_mock.assert_called_with(400, 'Switchover could be performed only from a specific leader') response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SWITCHOVER_NO_LEADER.value[::-1])
# Empty content # Empty content
request = post + '0\n\n' request = post + '0\n\n'
@@ -512,25 +513,19 @@ class TestRestApiHandler(unittest.TestCase):
# [Switchover without a candidate] # [Switchover without a candidate]
# Cluster with only a leader cluster.leader.name = 'postgresql1'
with patch.object(RestApiHandler, 'write_response') as response_mock: request = post + '25\n\n{"leader": "postgresql1"}'
cluster.leader.name = 'postgresql1'
request = post + '25\n\n{"leader": "postgresql1"}'
MockRestApiServer(RestApiHandler, request)
response_mock.assert_called_with(
412, 'switchover is not possible: cluster does not have members except leader')
# Switchover in pause mode # No candidate in pause mode
with patch.object(RestApiHandler, 'write_response') as response_mock, \ with patch.object(RestApiHandler, 'write_response') as response_mock, \
patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)): patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
MockRestApiServer(RestApiHandler, request) MockRestApiServer(RestApiHandler, request)
response_mock.assert_called_with( response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SWITCHOVER_PAUSE_NO_CANDIDATE.value[::-1])
400, 'Switchover is possible only to a specific candidate in a paused state')
# No healthy nodes to promote in both sync and async mode # No healthy nodes to promote in both sync and async mode
for is_synchronous_mode, response in ( for is_synchronous_mode, response in (
(True, 'switchover is not possible: can not find sync_standby'), (True, ManualFailoverPrecheckStatus.NO_SYNC_CANDIDATE.value[0].format(action='switchover')),
(False, 'switchover is not possible: cluster does not have members except leader')): (False, ManualFailoverPrecheckStatus.ONLY_LEADER.value[0].format(action='switchover'))):
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \ with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \
patch.object(RestApiHandler, 'write_response') as response_mock: patch.object(RestApiHandler, 'write_response') as response_mock:
MockRestApiServer(RestApiHandler, request) MockRestApiServer(RestApiHandler, request)
@@ -542,21 +537,25 @@ class TestRestApiHandler(unittest.TestCase):
with patch.object(RestApiHandler, 'write_response') as response_mock: with patch.object(RestApiHandler, 'write_response') as response_mock:
request = post + '53\n\n{"leader": "postgresql2", "candidate": "postgresql2"}' request = post + '53\n\n{"leader": "postgresql2", "candidate": "postgresql2"}'
MockRestApiServer(RestApiHandler, request) MockRestApiServer(RestApiHandler, request)
response_mock.assert_called_with(400, 'Switchover target and source are the same') response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SWITCHOVER_TO_LEADER.value[::-1])
# Current leader is different from the one specified # Current leader is different from the one specified
with patch.object(RestApiHandler, 'write_response') as response_mock: with patch.object(RestApiHandler, 'write_response') as response_mock:
cluster.leader.name = 'postgresql2' cluster.leader.name = 'postgresql2'
request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}' request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
MockRestApiServer(RestApiHandler, request) MockRestApiServer(RestApiHandler, request)
response_mock.assert_called_with(412, 'Member postgresql1 is not the leader of cluster dummy') response_mock.assert_called_with(
ManualFailoverPrecheckStatus.LEADER_NOT_MEMBER.value[1],
ManualFailoverPrecheckStatus.LEADER_NOT_MEMBER.value[0].format(leader='postgresql1',
cluster_name='dummy'))
# Candidate to promote is not a member of the cluster # Candidate to promote is not a sync standby/a member of the cluster
cluster.leader.name = 'postgresql1' cluster.leader.name = 'postgresql1'
cluster.sync.matches.return_value = False cluster.sync.matches.return_value = False
for is_synchronous_mode, response in ( for is_synchronous_mode, response in (
(True, 'candidate name does not match with sync_standby'), (True, ManualFailoverPrecheckStatus.CANDIDATE_NOT_SYNC_STANDBY.value[0]),
(False, 'Member postgresql2 does not exist in cluster dummy or is tagged as nofailover')): (False, ManualFailoverPrecheckStatus.CANDIDATE_NOT_MEMEBER.value[0].format(candidate="postgresql2",
cluster_name='dummy'))):
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \ with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \
patch.object(RestApiHandler, 'write_response') as response_mock: patch.object(RestApiHandler, 'write_response') as response_mock:
MockRestApiServer(RestApiHandler, request) MockRestApiServer(RestApiHandler, request)
@@ -565,9 +564,21 @@ class TestRestApiHandler(unittest.TestCase):
cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}), cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
Member(0, 'postgresql2', 30, {'api_url': 'http'})] Member(0, 'postgresql2', 30, {'api_url': 'http'})]
# Cluster has no leader
cluster.leader.name = None
with patch.object(RestApiHandler, 'write_response') as response_mock:
request = post + '53\n\n{"leader": "postgresql1"}'
MockRestApiServer(RestApiHandler, request)
response_mock.assert_called_with(
ManualFailoverPrecheckStatus.CLUSTER_NO_LEADER.value[1],
ManualFailoverPrecheckStatus.CLUSTER_NO_LEADER.value[0].format(leader='leader', cluster_name='dummy'))
cluster.leader.name = 'postgresql1'
# Failover key is empty in DCS # Failover key is empty in DCS
with patch.object(RestApiHandler, 'write_response') as response_mock: with patch.object(RestApiHandler, 'write_response') as response_mock:
cluster.failover = None cluster.failover = None
request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
MockRestApiServer(RestApiHandler, request) MockRestApiServer(RestApiHandler, request)
response_mock.assert_called_with(503, 'Switchover failed') response_mock.assert_called_with(503, 'Switchover failed')
@@ -605,7 +616,9 @@ class TestRestApiHandler(unittest.TestCase):
with patch.object(MockHa, 'has_members_eligible_to_promote', Mock(return_value=False)), \ with patch.object(MockHa, 'has_members_eligible_to_promote', Mock(return_value=False)), \
patch.object(RestApiHandler, 'write_response') as response_mock: patch.object(RestApiHandler, 'write_response') as response_mock:
MockRestApiServer(RestApiHandler, request) MockRestApiServer(RestApiHandler, request)
response_mock.assert_called_with(412, 'switchover is not possible: no good candidates have been found') response_mock.assert_called_with(
ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[1],
ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[0].format(action='switchover'))
# [Scheduled switchover] # [Scheduled switchover]
@@ -616,47 +629,58 @@ class TestRestApiHandler(unittest.TestCase):
MockRestApiServer(RestApiHandler, request) MockRestApiServer(RestApiHandler, request)
response_mock.assert_called_with(202, 'Switchover scheduled') response_mock.assert_called_with(202, 'Switchover scheduled')
# Schedule in paused mode # Scheduled in pause mode
with patch.object(RestApiHandler, 'write_response') as response_mock, \ with patch.object(RestApiHandler, 'write_response') as response_mock, \
patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)): patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
dcs.manual_failover.return_value = False dcs.manual_failover.return_value = False
MockRestApiServer(RestApiHandler, request) MockRestApiServer(RestApiHandler, request)
response_mock.assert_called_with(400, "Can't schedule switchover in the paused state") response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SCHEDULED_SWITCHOVER_PAUSE.value[::-1])
# No timezone specified # No timezone specified
with patch.object(RestApiHandler, 'write_response') as response_mock: with patch.object(RestApiHandler, 'write_response') as response_mock:
request = post + '97\n\n{"leader": "postgresql1", "member": "postgresql2",' + \ request = post + '97\n\n{"leader": "postgresql1", "member": "postgresql2",' + \
' "scheduled_at": "6016-02-15T18:13:30.568224"}' ' "scheduled_at": "6016-02-15T18:13:30.568224"}'
MockRestApiServer(RestApiHandler, request) MockRestApiServer(RestApiHandler, request)
response_mock.assert_called_with(400, 'Timezone information is mandatory for the scheduled switchover') response_mock.assert_called_with(
ParseScheduleErrors.NO_TIMEZONE.value[1],
ParseScheduleErrors.NO_TIMEZONE.value[0].format(action='switchover'))
request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "' request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "'
# Scheduled in the past # Scheduled in the past
with patch.object(RestApiHandler, 'write_response') as response_mock: with patch.object(RestApiHandler, 'write_response') as response_mock:
MockRestApiServer(RestApiHandler, request + '1016-02-15T18:13:30.568224+01:00"}') MockRestApiServer(RestApiHandler, request + '1016-02-15T18:13:30.568224+01:00"}')
response_mock.assert_called_with(422, 'Cannot schedule switchover in the past') response_mock.assert_called_with(
ParseScheduleErrors.SCHEDULED_IN_PAST.value[1],
ParseScheduleErrors.SCHEDULED_IN_PAST.value[0].format(action='switchover'))
# Invalid date # Invalid date
with patch.object(RestApiHandler, 'write_response') as response_mock: with patch.object(RestApiHandler, 'write_response') as response_mock:
MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}') MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}')
response_mock.assert_called_with( response_mock.assert_called_with(*ParseScheduleErrors.PARSING_ERROR.value[::-1])
422, 'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601')
def test_do_POST_failover(self): @patch.object(MockPatroni, 'dcs')
def test_do_POST_failover(self, mock_dcs):
post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: ' post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
cluster = mock_dcs.get_cluster.return_value
# with patch.object(RestApiHandler, 'write_response') as response_mock: with patch.object(RestApiHandler, 'write_response') as response_mock:
# MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}') MockRestApiServer(RestApiHandler, post + '19\n\n{"leader":"leader"}')
# response_mock.assert_called_once_with(400, 'Failover could be performed only to a specific candidate') response_mock.assert_called_once_with(*ManualFailoverPrecheckStatus.FAILOVER_NO_CANDIDATE.value[::-1])
with patch.object(RestApiHandler, 'write_response') as response_mock: with patch.object(RestApiHandler, 'write_response') as response_mock:
MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}') MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}')
response_mock.assert_called_once_with(400, "Failover can't be scheduled") response_mock.assert_called_once_with(*ManualFailoverPrecheckStatus.SCHEDULED_FAILOVER.value[::-1])
with patch.object(RestApiHandler, 'write_response') as response_mock: # Candidate is not healthy to be promoted
MockRestApiServer(RestApiHandler, post + '30\n\n{"leader":"1","candidate":"2"}') cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
response_mock.assert_called_once_with(412, 'Member 1 is not the leader of cluster dummy') Member(0, 'postgresql2', 30, {'api_url': 'http'})]
with patch.object(MockHa, 'has_members_eligible_to_promote', Mock(return_value=False)), \
patch.object(RestApiHandler, 'write_response') as response_mock:
MockRestApiServer(RestApiHandler, post + '27\n\n{"candidate":"postgresql2"}')
response_mock.assert_called_with(
ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[1],
ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[0].format(action='failover'))
@patch.object(MockHa, 'is_leader', Mock(return_value=True)) @patch.object(MockHa, 'is_leader', Mock(return_value=True))
def test_do_POST_citus(self): def test_do_POST_citus(self):
+131 -74
View File
@@ -6,12 +6,14 @@ import unittest
from click.testing import CliRunner from click.testing import CliRunner
from datetime import datetime, timedelta from datetime import datetime, timedelta
from mock import patch, Mock, PropertyMock from mock import patch, Mock, PropertyMock
from patroni.config import GlobalConfig
from patroni.ctl import ctl, load_config, output_members, get_dcs, parse_dcs, \ from patroni.ctl import ctl, load_config, output_members, get_dcs, parse_dcs, \
get_all_members, get_any_member, get_cursor, query_member, PatroniCtlException, apply_config_changes, \ get_all_members, get_any_member, get_cursor, query_member, PatroniCtlException, apply_config_changes, \
format_config_for_editing, show_diff, invoke_editor, format_pg_version, CONFIG_FILE_PATH, PatronictlPrettyTable format_config_for_editing, show_diff, invoke_editor, format_pg_version, CONFIG_FILE_PATH, PatronictlPrettyTable
from patroni.dcs.etcd import AbstractEtcdClientWithFailover, Cluster, Failover from patroni.dcs.etcd import AbstractEtcdClientWithFailover, Cluster, Failover
from patroni.manual_failover import ManualFailoverPrecheckStatus
from patroni.psycopg import OperationalError from patroni.psycopg import OperationalError
from patroni.utils import tzutc from patroni.utils import ParseScheduleErrors, tzutc
from prettytable import PrettyTable, ALL from prettytable import PrettyTable, ALL
from urllib3 import PoolManager from urllib3 import PoolManager
@@ -113,77 +115,6 @@ class TestCtl(unittest.TestCase):
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force']) result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force'])
self.assertEqual(result.exit_code, 0) self.assertEqual(result.exit_code, 0)
# Scheduled (confirm)
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'],
input=f'leader\nother\n{self.SCHEDULED_TS}\ny')
self.assertEqual(result.exit_code, 0)
# Scheduled (abort)
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
'--scheduled', self.SCHEDULED_TS], input='leader\nother\n\nN')
self.assertEqual(result.exit_code, 1)
# Scheduled with --force option
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
'--force', '--scheduled', self.SCHEDULED_TS])
self.assertEqual(result.exit_code, 0)
# Scheduled in pause mode
with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
'--force', '--scheduled', self.SCHEDULED_TS])
self.assertEqual(result.exit_code, 1)
self.assertIn("Can't schedule switchover in the paused state", result.output)
# Target and source are equal
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nleader\n\ny')
self.assertEqual(result.exit_code, 1)
self.assertIn('Switchover target and source are the same', result.output)
# Candidate is not a member of the cluster
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nReality\n\ny')
self.assertEqual(result.exit_code, 1)
self.assertIn('Member Reality does not exist in cluster dummy or is tagged as nofailover', result.output)
# Invalid timestamp with force
result = self.runner.invoke(ctl,['switchover', 'dummy', '--group', '0', '--force', '--scheduled',
self.SCHEDULED_TS_INVALID])
self.assertEqual(result.exit_code, 1)
self.assertIn('Unable to parse scheduled timestamp', result.output)
# Invalid timestamp
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
'--force', '--scheduled', self.SCHEDULED_TS_INVALID])
self.assertEqual(result.exit_code, 1)
self.assertIn('Unable to parse scheduled timestamp', result.output)
# Invalid timestamp - no timezone
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
'--force', '--scheduled', self.SCHEDULED_TS_NO_TZ])
self.assertEqual(result.exit_code, 1)
self.assertIn('Timezone information is mandatory for the scheduled switchover', result.output)
# Specifying wrong leader
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='dummy')
self.assertEqual(result.exit_code, 1)
self.assertIn('Member dummy is not the leader of cluster dummy', result.output)
# Errors while sending Patroni REST API request
with patch.object(PoolManager, 'request', Mock(side_effect=Exception)):
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'],
input=f'leader\nother\n{self.SCHEDULED_TS}\ny')
self.assertIn('falling back to DCS', result.output)
with patch.object(PoolManager, 'request') as mock_api_request:
mock_api_request.return_value.status = 500
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
self.assertIn('Switchover failed', result.output)
mock_api_request.return_value.status = 501
mock_api_request.return_value.data = b'Server does not support this operation'
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
self.assertIn('Switchover failed', result.output)
# No members available # No members available
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
@@ -201,6 +132,132 @@ class TestCtl(unittest.TestCase):
self.assertEqual(result.exit_code, 1) self.assertEqual(result.exit_code, 1)
self.assertIn('For Citus clusters the --group must me specified', result.output) self.assertIn('For Citus clusters the --group must me specified', result.output)
# [Scheduled]
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
# Scheduled (confirm)
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'],
input=f'leader\nother\n{self.SCHEDULED_TS}\ny')
self.assertEqual(result.exit_code, 0)
self.assertIn(f'Are you sure you want to schedule switchover of cluster dummy '
f'at {self.SCHEDULED_TS}, demoting current leader', result.output)
# Scheduled (abort)
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
'--scheduled', self.SCHEDULED_TS], input='leader\nother\n\nN')
self.assertEqual(result.exit_code, 1)
# Scheduled with --force option
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
'--force', '--scheduled', self.SCHEDULED_TS])
self.assertEqual(result.exit_code, 0)
# Scheduled in pause mode
with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
'--force', '--scheduled', self.SCHEDULED_TS])
self.assertEqual(result.exit_code, 1)
self.assertIn(ManualFailoverPrecheckStatus.SCHEDULED_SWITCHOVER_PAUSE.value[0], result.output)
# Invalid timestamp with force
result = self.runner.invoke(ctl,['switchover', 'dummy', '--group', '0', '--force', '--scheduled',
self.SCHEDULED_TS_INVALID])
self.assertEqual(result.exit_code, 1)
self.assertIn('Unable to parse scheduled timestamp', result.output)
# Invalid timestamp
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
'--force', '--scheduled', self.SCHEDULED_TS_INVALID])
self.assertEqual(result.exit_code, 1)
self.assertIn('Unable to parse scheduled timestamp', result.output)
# Invalid timestamp - no timezone
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
'--force', '--scheduled', self.SCHEDULED_TS_NO_TZ])
self.assertEqual(result.exit_code, 1)
self.assertIn(ParseScheduleErrors.NO_TIMEZONE.value[0].format(action='switchover'), result.output)
# [Other erroneous combinations]
# No candidate in pause mode
with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\n\n\ny')
self.assertEqual(result.exit_code, 1)
self.assertIn(ManualFailoverPrecheckStatus.SWITCHOVER_PAUSE_NO_CANDIDATE.value[0], result.output)
# Target and source are equal
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nleader\n\ny')
self.assertEqual(result.exit_code, 1)
self.assertIn(ManualFailoverPrecheckStatus.SWITCHOVER_TO_LEADER.value[0], result.output)
# Candidate is not a member of the cluster
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nReality\n\ny')
self.assertEqual(result.exit_code, 1)
self.assertIn(ManualFailoverPrecheckStatus.CANDIDATE_NOT_MEMEBER.value[0].format(candidate='Reality',
cluster_name='dummy'),
result.output)
# Specifying wrong leader
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='dummy')
self.assertEqual(result.exit_code, 1)
self.assertIn(
ManualFailoverPrecheckStatus.LEADER_NOT_MEMBER.value[0].format(leader='dummy',
cluster_name='dummy'),
result.output)
mock_get_dcs.return_value.get_cluster = Mock(
return_value=get_cluster_initialized_with_leader(sync=('leader', 'other')))
# Candidate is not a sync standby
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=True)):
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\notherMember\n\ny')
self.assertEqual(result.exit_code, 1)
self.assertIn(ManualFailoverPrecheckStatus.CANDIDATE_NOT_SYNC_STANDBY.value[0], result.output)
# No healthy nodes to promote in sync mode
mock_get_dcs.return_value.get_cluster = Mock(return_value=get_cluster_initialized_with_leader(sync=('leader')))
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=True)):
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force'])
self.assertEqual(result.exit_code, 1)
self.assertIn(ManualFailoverPrecheckStatus.NO_SYNC_CANDIDATE.value[0].format(action='switchover'),
result.output)
# No healthy nodes to promote in async mode
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=False)):
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force'])
self.assertEqual(result.exit_code, 1)
self.assertIn(ManualFailoverPrecheckStatus.ONLY_LEADER.value[0].format(action='switchover'),
result.output)
# Cluster has no leader
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--leader', 'leader', '--force'])
self.assertEqual(result.exit_code, 1)
self.assertIn(
ManualFailoverPrecheckStatus.CLUSTER_NO_LEADER.value[0].format(leader='leader', cluster_name='dummy'),
result.output)
# [Errors while sending Patroni REST API request]
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
with patch.object(PoolManager, 'request', Mock(side_effect=Exception)):
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'],
input=f'leader\nother\n{self.SCHEDULED_TS}\ny')
self.assertIn('falling back to DCS', result.output)
with patch.object(PoolManager, 'request') as mock_api_request:
mock_api_request.return_value.status = 500
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
self.assertIn('Switchover failed', result.output)
mock_api_request.return_value.status = 501
mock_api_request.return_value.data = b'Server does not support this operation'
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
self.assertIn('Switchover failed', result.output)
@patch('patroni.ctl.get_dcs') @patch('patroni.ctl.get_dcs')
@patch.object(PoolManager, 'request', Mock(return_value=MockResponse())) @patch.object(PoolManager, 'request', Mock(return_value=MockResponse()))
@patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse())) @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
@@ -210,7 +267,7 @@ class TestCtl(unittest.TestCase):
# No candidate specified # No candidate specified
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='0\n') result = self.runner.invoke(ctl, ['failover', 'dummy'], input='0\n')
self.assertIn('Failover could be performed only to a specific candidate', result.output) self.assertIn(ManualFailoverPrecheckStatus.FAILOVER_NO_CANDIDATE.value[0], result.output)
# Failover to an async member in sync mode (confirm) # Failover to an async member in sync mode (confirm)
cluster = get_cluster_initialized_with_leader(sync=('leader', 'other')) cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
@@ -374,7 +431,7 @@ class TestCtl(unittest.TestCase):
result = self.runner.invoke(ctl, result = self.runner.invoke(ctl,
['restart', 'alpha', 'other', '--force', '--scheduled', self.SCHEDULED_TS_NO_TZ]) ['restart', 'alpha', 'other', '--force', '--scheduled', self.SCHEDULED_TS_NO_TZ])
self.assertEqual(result.exit_code, 1) self.assertEqual(result.exit_code, 1)
self.assertIn('Timezone information is mandatory for the scheduled restart', result.output) self.assertIn(ParseScheduleErrors.NO_TIMEZONE.value[0].format(action='restart'), result.output)
# Error parsing scheduled flag value (invalid date) # Error parsing scheduled flag value (invalid date)
result = self.runner.invoke(ctl, result = self.runner.invoke(ctl,