mirror of
https://github.com/outbackdingo/patroni.git
synced 2026-08-25 14:53:37 +00:00
Yet another refactoring
This commit is contained in:
@@ -1015,10 +1015,6 @@ class RestApiHandler(BaseHTTPRequestHandler):
|
|||||||
logger.info("received %s request with leader=%s candidate=%s scheduled_at=%s",
|
logger.info("received %s request with leader=%s candidate=%s scheduled_at=%s",
|
||||||
action, leader, candidate, scheduled_at)
|
action, leader, candidate, scheduled_at)
|
||||||
|
|
||||||
if action == 'failover' and leader:
|
|
||||||
logger.warning('received failover request with leader specifed - performing switchover')
|
|
||||||
action = 'switchover'
|
|
||||||
|
|
||||||
manual_failover = ManualFailover(action, cluster, leader, candidate, scheduled_at,
|
manual_failover = ManualFailover(action, cluster, leader, candidate, scheduled_at,
|
||||||
global_config.is_paused, global_config.is_synchronous_mode,
|
global_config.is_paused, global_config.is_synchronous_mode,
|
||||||
self.server.patroni)
|
self.server.patroni)
|
||||||
|
|||||||
+14
-15
@@ -1030,6 +1030,7 @@ def restart(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member
|
|||||||
next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M')
|
next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M')
|
||||||
scheduled = click.prompt('When should the restart take place (e.g. ' + next_hour + ') ',
|
scheduled = click.prompt('When should the restart take place (e.g. ' + next_hour + ') ',
|
||||||
type=str, default='now')
|
type=str, default='now')
|
||||||
|
scheduled = scheduled if scheduled != 'now' else None
|
||||||
|
|
||||||
parse_result, scheduled_at = parse_schedule(scheduled)
|
parse_result, scheduled_at = parse_schedule(scheduled)
|
||||||
if parse_result:
|
if parse_result:
|
||||||
@@ -1190,24 +1191,21 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s
|
|||||||
global_config = get_global_config(cluster)
|
global_config = get_global_config(cluster)
|
||||||
|
|
||||||
# Leader is required for switchover only
|
# Leader is required for switchover only
|
||||||
if action == 'switchover':
|
if action == 'switchover' and leader is None:
|
||||||
if cluster.leader is None or not cluster.leader.name:
|
if cluster.leader is None or not cluster.leader.name:
|
||||||
raise PatroniCtlException('This cluster has no leader')
|
raise PatroniCtlException('This cluster has no leader')
|
||||||
|
if force:
|
||||||
|
leader = cluster.leader.name
|
||||||
|
else:
|
||||||
|
prompt = 'Standby Leader' if global_config.is_standby_cluster else 'Primary'
|
||||||
|
leader = click.prompt(prompt, type=str, default=(cluster.leader and cluster.leader.name))
|
||||||
|
|
||||||
if leader is None:
|
|
||||||
if force:
|
|
||||||
leader = cluster.leader.name
|
|
||||||
else:
|
|
||||||
prompt = 'Standby Leader' if global_config.is_standby_cluster else 'Primary'
|
|
||||||
leader = click.prompt(prompt, type=str, default=(cluster.leader and cluster.leader.name))
|
|
||||||
|
|
||||||
# Check if there are any candidates available at all
|
|
||||||
candidate_names = [str(m.name) for m in cluster.members if m.name != leader and not m.nofailover]
|
|
||||||
if not candidate_names:
|
|
||||||
raise PatroniCtlException('No candidates found to {0} to'.format(action))
|
|
||||||
|
|
||||||
candidate_names.sort() # we sort the names for consistent output to the client
|
|
||||||
if candidate is None and not force:
|
if candidate is None and not force:
|
||||||
|
# Check if there are any candidates available at all
|
||||||
|
candidate_names = [str(m.name) for m in cluster.members if m.name != leader and not m.nofailover]
|
||||||
|
if not candidate_names:
|
||||||
|
raise PatroniCtlException('No candidates found to {0} to'.format(action))
|
||||||
|
candidate_names.sort() # we sort the names for consistent output to the client
|
||||||
candidate = click.prompt('Candidate ' + str(candidate_names), type=str, default='')
|
candidate = click.prompt('Candidate ' + str(candidate_names), type=str, default='')
|
||||||
|
|
||||||
# We allow manual failover to an aync node in the sync mode, so we better ask for the confirmation
|
# We allow manual failover to an aync node in the sync mode, so we better ask for the confirmation
|
||||||
@@ -1219,8 +1217,9 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s
|
|||||||
|
|
||||||
if action == 'switchover' and scheduled is None and not force:
|
if action == 'switchover' and scheduled is None and not force:
|
||||||
next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M')
|
next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M')
|
||||||
scheduled = click.prompt('When should the switchover take place (e.g. ' + next_hour + ' ) ',
|
scheduled = click.prompt('When should the switchover take place (e.g. ' + next_hour + ') ',
|
||||||
type=str, default='now')
|
type=str, default='now')
|
||||||
|
scheduled = scheduled if scheduled != 'now' else None
|
||||||
|
|
||||||
# Now, when we collected all the possible info, run checks
|
# Now, when we collected all the possible info, run checks
|
||||||
manual_failover = ManualFailover(action, cluster, leader, candidate, scheduled,
|
manual_failover = ManualFailover(action, cluster, leader, candidate, scheduled,
|
||||||
|
|||||||
+1
-1
@@ -1074,7 +1074,7 @@ class ParseScheduleErrors(Enum):
|
|||||||
|
|
||||||
def parse_schedule(schedule: Optional[str]) -> Tuple[Optional[ParseScheduleErrors], Optional[datetime.datetime]]:
|
def parse_schedule(schedule: Optional[str]) -> Tuple[Optional[ParseScheduleErrors], Optional[datetime.datetime]]:
|
||||||
scheduled_at = None
|
scheduled_at = None
|
||||||
if schedule is not None and (schedule or 'now') != 'now':
|
if schedule is not None:
|
||||||
try:
|
try:
|
||||||
scheduled_at = dateutil.parser.parse(schedule)
|
scheduled_at = dateutil.parser.parse(schedule)
|
||||||
if scheduled_at.tzinfo is None:
|
if scheduled_at.tzinfo is None:
|
||||||
|
|||||||
+59
-35
@@ -12,9 +12,10 @@ from patroni.api import RestApiHandler, RestApiServer
|
|||||||
from patroni.config import GlobalConfig
|
from patroni.config import GlobalConfig
|
||||||
from patroni.dcs import ClusterConfig, Member
|
from patroni.dcs import ClusterConfig, Member
|
||||||
from patroni.ha import _MemberStatus
|
from patroni.ha import _MemberStatus
|
||||||
from patroni.utils import RetryFailedError, tzutc
|
from patroni.manual_failover import ManualFailoverPrecheckStatus
|
||||||
|
from patroni.utils import ParseScheduleErrors, RetryFailedError, tzutc
|
||||||
|
|
||||||
from .test_ha import get_cluster_initialized_without_leader
|
from .test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader
|
||||||
|
|
||||||
|
|
||||||
future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5)
|
future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5)
|
||||||
@@ -504,7 +505,7 @@ class TestRestApiHandler(unittest.TestCase):
|
|||||||
# Invalid content
|
# Invalid content
|
||||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||||
MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}')
|
MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}')
|
||||||
response_mock.assert_called_with(400, 'Switchover could be performed only from a specific leader')
|
response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SWITCHOVER_NO_LEADER.value[::-1])
|
||||||
|
|
||||||
# Empty content
|
# Empty content
|
||||||
request = post + '0\n\n'
|
request = post + '0\n\n'
|
||||||
@@ -512,25 +513,19 @@ class TestRestApiHandler(unittest.TestCase):
|
|||||||
|
|
||||||
# [Switchover without a candidate]
|
# [Switchover without a candidate]
|
||||||
|
|
||||||
# Cluster with only a leader
|
cluster.leader.name = 'postgresql1'
|
||||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
request = post + '25\n\n{"leader": "postgresql1"}'
|
||||||
cluster.leader.name = 'postgresql1'
|
|
||||||
request = post + '25\n\n{"leader": "postgresql1"}'
|
|
||||||
MockRestApiServer(RestApiHandler, request)
|
|
||||||
response_mock.assert_called_with(
|
|
||||||
412, 'switchover is not possible: cluster does not have members except leader')
|
|
||||||
|
|
||||||
# Switchover in pause mode
|
# No candidate in pause mode
|
||||||
with patch.object(RestApiHandler, 'write_response') as response_mock, \
|
with patch.object(RestApiHandler, 'write_response') as response_mock, \
|
||||||
patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
|
patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
|
||||||
MockRestApiServer(RestApiHandler, request)
|
MockRestApiServer(RestApiHandler, request)
|
||||||
response_mock.assert_called_with(
|
response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SWITCHOVER_PAUSE_NO_CANDIDATE.value[::-1])
|
||||||
400, 'Switchover is possible only to a specific candidate in a paused state')
|
|
||||||
|
|
||||||
# No healthy nodes to promote in both sync and async mode
|
# No healthy nodes to promote in both sync and async mode
|
||||||
for is_synchronous_mode, response in (
|
for is_synchronous_mode, response in (
|
||||||
(True, 'switchover is not possible: can not find sync_standby'),
|
(True, ManualFailoverPrecheckStatus.NO_SYNC_CANDIDATE.value[0].format(action='switchover')),
|
||||||
(False, 'switchover is not possible: cluster does not have members except leader')):
|
(False, ManualFailoverPrecheckStatus.ONLY_LEADER.value[0].format(action='switchover'))):
|
||||||
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \
|
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \
|
||||||
patch.object(RestApiHandler, 'write_response') as response_mock:
|
patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||||
MockRestApiServer(RestApiHandler, request)
|
MockRestApiServer(RestApiHandler, request)
|
||||||
@@ -542,21 +537,25 @@ class TestRestApiHandler(unittest.TestCase):
|
|||||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||||
request = post + '53\n\n{"leader": "postgresql2", "candidate": "postgresql2"}'
|
request = post + '53\n\n{"leader": "postgresql2", "candidate": "postgresql2"}'
|
||||||
MockRestApiServer(RestApiHandler, request)
|
MockRestApiServer(RestApiHandler, request)
|
||||||
response_mock.assert_called_with(400, 'Switchover target and source are the same')
|
response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SWITCHOVER_TO_LEADER.value[::-1])
|
||||||
|
|
||||||
# Current leader is different from the one specified
|
# Current leader is different from the one specified
|
||||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||||
cluster.leader.name = 'postgresql2'
|
cluster.leader.name = 'postgresql2'
|
||||||
request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
|
request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
|
||||||
MockRestApiServer(RestApiHandler, request)
|
MockRestApiServer(RestApiHandler, request)
|
||||||
response_mock.assert_called_with(412, 'Member postgresql1 is not the leader of cluster dummy')
|
response_mock.assert_called_with(
|
||||||
|
ManualFailoverPrecheckStatus.LEADER_NOT_MEMBER.value[1],
|
||||||
|
ManualFailoverPrecheckStatus.LEADER_NOT_MEMBER.value[0].format(leader='postgresql1',
|
||||||
|
cluster_name='dummy'))
|
||||||
|
|
||||||
# Candidate to promote is not a member of the cluster
|
# Candidate to promote is not a sync standby/a member of the cluster
|
||||||
cluster.leader.name = 'postgresql1'
|
cluster.leader.name = 'postgresql1'
|
||||||
cluster.sync.matches.return_value = False
|
cluster.sync.matches.return_value = False
|
||||||
for is_synchronous_mode, response in (
|
for is_synchronous_mode, response in (
|
||||||
(True, 'candidate name does not match with sync_standby'),
|
(True, ManualFailoverPrecheckStatus.CANDIDATE_NOT_SYNC_STANDBY.value[0]),
|
||||||
(False, 'Member postgresql2 does not exist in cluster dummy or is tagged as nofailover')):
|
(False, ManualFailoverPrecheckStatus.CANDIDATE_NOT_MEMEBER.value[0].format(candidate="postgresql2",
|
||||||
|
cluster_name='dummy'))):
|
||||||
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \
|
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \
|
||||||
patch.object(RestApiHandler, 'write_response') as response_mock:
|
patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||||
MockRestApiServer(RestApiHandler, request)
|
MockRestApiServer(RestApiHandler, request)
|
||||||
@@ -565,9 +564,21 @@ class TestRestApiHandler(unittest.TestCase):
|
|||||||
cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
|
cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
|
||||||
Member(0, 'postgresql2', 30, {'api_url': 'http'})]
|
Member(0, 'postgresql2', 30, {'api_url': 'http'})]
|
||||||
|
|
||||||
|
# Cluster has no leader
|
||||||
|
cluster.leader.name = None
|
||||||
|
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||||
|
request = post + '53\n\n{"leader": "postgresql1"}'
|
||||||
|
MockRestApiServer(RestApiHandler, request)
|
||||||
|
response_mock.assert_called_with(
|
||||||
|
ManualFailoverPrecheckStatus.CLUSTER_NO_LEADER.value[1],
|
||||||
|
ManualFailoverPrecheckStatus.CLUSTER_NO_LEADER.value[0].format(leader='leader', cluster_name='dummy'))
|
||||||
|
|
||||||
|
cluster.leader.name = 'postgresql1'
|
||||||
|
|
||||||
# Failover key is empty in DCS
|
# Failover key is empty in DCS
|
||||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||||
cluster.failover = None
|
cluster.failover = None
|
||||||
|
request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
|
||||||
MockRestApiServer(RestApiHandler, request)
|
MockRestApiServer(RestApiHandler, request)
|
||||||
response_mock.assert_called_with(503, 'Switchover failed')
|
response_mock.assert_called_with(503, 'Switchover failed')
|
||||||
|
|
||||||
@@ -605,7 +616,9 @@ class TestRestApiHandler(unittest.TestCase):
|
|||||||
with patch.object(MockHa, 'has_members_eligible_to_promote', Mock(return_value=False)), \
|
with patch.object(MockHa, 'has_members_eligible_to_promote', Mock(return_value=False)), \
|
||||||
patch.object(RestApiHandler, 'write_response') as response_mock:
|
patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||||
MockRestApiServer(RestApiHandler, request)
|
MockRestApiServer(RestApiHandler, request)
|
||||||
response_mock.assert_called_with(412, 'switchover is not possible: no good candidates have been found')
|
response_mock.assert_called_with(
|
||||||
|
ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[1],
|
||||||
|
ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[0].format(action='switchover'))
|
||||||
|
|
||||||
# [Scheduled switchover]
|
# [Scheduled switchover]
|
||||||
|
|
||||||
@@ -616,47 +629,58 @@ class TestRestApiHandler(unittest.TestCase):
|
|||||||
MockRestApiServer(RestApiHandler, request)
|
MockRestApiServer(RestApiHandler, request)
|
||||||
response_mock.assert_called_with(202, 'Switchover scheduled')
|
response_mock.assert_called_with(202, 'Switchover scheduled')
|
||||||
|
|
||||||
# Schedule in paused mode
|
# Scheduled in pause mode
|
||||||
with patch.object(RestApiHandler, 'write_response') as response_mock, \
|
with patch.object(RestApiHandler, 'write_response') as response_mock, \
|
||||||
patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
|
patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
|
||||||
dcs.manual_failover.return_value = False
|
dcs.manual_failover.return_value = False
|
||||||
MockRestApiServer(RestApiHandler, request)
|
MockRestApiServer(RestApiHandler, request)
|
||||||
response_mock.assert_called_with(400, "Can't schedule switchover in the paused state")
|
response_mock.assert_called_with(*ManualFailoverPrecheckStatus.SCHEDULED_SWITCHOVER_PAUSE.value[::-1])
|
||||||
|
|
||||||
# No timezone specified
|
# No timezone specified
|
||||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||||
request = post + '97\n\n{"leader": "postgresql1", "member": "postgresql2",' + \
|
request = post + '97\n\n{"leader": "postgresql1", "member": "postgresql2",' + \
|
||||||
' "scheduled_at": "6016-02-15T18:13:30.568224"}'
|
' "scheduled_at": "6016-02-15T18:13:30.568224"}'
|
||||||
MockRestApiServer(RestApiHandler, request)
|
MockRestApiServer(RestApiHandler, request)
|
||||||
response_mock.assert_called_with(400, 'Timezone information is mandatory for the scheduled switchover')
|
response_mock.assert_called_with(
|
||||||
|
ParseScheduleErrors.NO_TIMEZONE.value[1],
|
||||||
|
ParseScheduleErrors.NO_TIMEZONE.value[0].format(action='switchover'))
|
||||||
|
|
||||||
request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "'
|
request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "'
|
||||||
|
|
||||||
# Scheduled in the past
|
# Scheduled in the past
|
||||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||||
MockRestApiServer(RestApiHandler, request + '1016-02-15T18:13:30.568224+01:00"}')
|
MockRestApiServer(RestApiHandler, request + '1016-02-15T18:13:30.568224+01:00"}')
|
||||||
response_mock.assert_called_with(422, 'Cannot schedule switchover in the past')
|
response_mock.assert_called_with(
|
||||||
|
ParseScheduleErrors.SCHEDULED_IN_PAST.value[1],
|
||||||
|
ParseScheduleErrors.SCHEDULED_IN_PAST.value[0].format(action='switchover'))
|
||||||
|
|
||||||
# Invalid date
|
# Invalid date
|
||||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||||
MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}')
|
MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}')
|
||||||
response_mock.assert_called_with(
|
response_mock.assert_called_with(*ParseScheduleErrors.PARSING_ERROR.value[::-1])
|
||||||
422, 'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601')
|
|
||||||
|
|
||||||
def test_do_POST_failover(self):
|
@patch.object(MockPatroni, 'dcs')
|
||||||
|
def test_do_POST_failover(self, mock_dcs):
|
||||||
post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
|
post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
|
||||||
|
cluster = mock_dcs.get_cluster.return_value
|
||||||
|
|
||||||
# with patch.object(RestApiHandler, 'write_response') as response_mock:
|
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||||
# MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}')
|
MockRestApiServer(RestApiHandler, post + '19\n\n{"leader":"leader"}')
|
||||||
# response_mock.assert_called_once_with(400, 'Failover could be performed only to a specific candidate')
|
response_mock.assert_called_once_with(*ManualFailoverPrecheckStatus.FAILOVER_NO_CANDIDATE.value[::-1])
|
||||||
|
|
||||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||||
MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}')
|
MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}')
|
||||||
response_mock.assert_called_once_with(400, "Failover can't be scheduled")
|
response_mock.assert_called_once_with(*ManualFailoverPrecheckStatus.SCHEDULED_FAILOVER.value[::-1])
|
||||||
|
|
||||||
with patch.object(RestApiHandler, 'write_response') as response_mock:
|
# Candidate is not healthy to be promoted
|
||||||
MockRestApiServer(RestApiHandler, post + '30\n\n{"leader":"1","candidate":"2"}')
|
cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
|
||||||
response_mock.assert_called_once_with(412, 'Member 1 is not the leader of cluster dummy')
|
Member(0, 'postgresql2', 30, {'api_url': 'http'})]
|
||||||
|
with patch.object(MockHa, 'has_members_eligible_to_promote', Mock(return_value=False)), \
|
||||||
|
patch.object(RestApiHandler, 'write_response') as response_mock:
|
||||||
|
MockRestApiServer(RestApiHandler, post + '27\n\n{"candidate":"postgresql2"}')
|
||||||
|
response_mock.assert_called_with(
|
||||||
|
ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[1],
|
||||||
|
ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES.value[0].format(action='failover'))
|
||||||
|
|
||||||
@patch.object(MockHa, 'is_leader', Mock(return_value=True))
|
@patch.object(MockHa, 'is_leader', Mock(return_value=True))
|
||||||
def test_do_POST_citus(self):
|
def test_do_POST_citus(self):
|
||||||
|
|||||||
+131
-74
@@ -6,12 +6,14 @@ import unittest
|
|||||||
from click.testing import CliRunner
|
from click.testing import CliRunner
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from mock import patch, Mock, PropertyMock
|
from mock import patch, Mock, PropertyMock
|
||||||
|
from patroni.config import GlobalConfig
|
||||||
from patroni.ctl import ctl, load_config, output_members, get_dcs, parse_dcs, \
|
from patroni.ctl import ctl, load_config, output_members, get_dcs, parse_dcs, \
|
||||||
get_all_members, get_any_member, get_cursor, query_member, PatroniCtlException, apply_config_changes, \
|
get_all_members, get_any_member, get_cursor, query_member, PatroniCtlException, apply_config_changes, \
|
||||||
format_config_for_editing, show_diff, invoke_editor, format_pg_version, CONFIG_FILE_PATH, PatronictlPrettyTable
|
format_config_for_editing, show_diff, invoke_editor, format_pg_version, CONFIG_FILE_PATH, PatronictlPrettyTable
|
||||||
from patroni.dcs.etcd import AbstractEtcdClientWithFailover, Cluster, Failover
|
from patroni.dcs.etcd import AbstractEtcdClientWithFailover, Cluster, Failover
|
||||||
|
from patroni.manual_failover import ManualFailoverPrecheckStatus
|
||||||
from patroni.psycopg import OperationalError
|
from patroni.psycopg import OperationalError
|
||||||
from patroni.utils import tzutc
|
from patroni.utils import ParseScheduleErrors, tzutc
|
||||||
from prettytable import PrettyTable, ALL
|
from prettytable import PrettyTable, ALL
|
||||||
from urllib3 import PoolManager
|
from urllib3 import PoolManager
|
||||||
|
|
||||||
@@ -113,77 +115,6 @@ class TestCtl(unittest.TestCase):
|
|||||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force'])
|
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force'])
|
||||||
self.assertEqual(result.exit_code, 0)
|
self.assertEqual(result.exit_code, 0)
|
||||||
|
|
||||||
# Scheduled (confirm)
|
|
||||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'],
|
|
||||||
input=f'leader\nother\n{self.SCHEDULED_TS}\ny')
|
|
||||||
self.assertEqual(result.exit_code, 0)
|
|
||||||
|
|
||||||
# Scheduled (abort)
|
|
||||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
|
||||||
'--scheduled', self.SCHEDULED_TS], input='leader\nother\n\nN')
|
|
||||||
self.assertEqual(result.exit_code, 1)
|
|
||||||
|
|
||||||
# Scheduled with --force option
|
|
||||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
|
||||||
'--force', '--scheduled', self.SCHEDULED_TS])
|
|
||||||
self.assertEqual(result.exit_code, 0)
|
|
||||||
|
|
||||||
# Scheduled in pause mode
|
|
||||||
with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
|
|
||||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
|
||||||
'--force', '--scheduled', self.SCHEDULED_TS])
|
|
||||||
self.assertEqual(result.exit_code, 1)
|
|
||||||
self.assertIn("Can't schedule switchover in the paused state", result.output)
|
|
||||||
|
|
||||||
# Target and source are equal
|
|
||||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nleader\n\ny')
|
|
||||||
self.assertEqual(result.exit_code, 1)
|
|
||||||
self.assertIn('Switchover target and source are the same', result.output)
|
|
||||||
|
|
||||||
# Candidate is not a member of the cluster
|
|
||||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nReality\n\ny')
|
|
||||||
self.assertEqual(result.exit_code, 1)
|
|
||||||
self.assertIn('Member Reality does not exist in cluster dummy or is tagged as nofailover', result.output)
|
|
||||||
|
|
||||||
# Invalid timestamp with force
|
|
||||||
result = self.runner.invoke(ctl,['switchover', 'dummy', '--group', '0', '--force', '--scheduled',
|
|
||||||
self.SCHEDULED_TS_INVALID])
|
|
||||||
self.assertEqual(result.exit_code, 1)
|
|
||||||
self.assertIn('Unable to parse scheduled timestamp', result.output)
|
|
||||||
|
|
||||||
# Invalid timestamp
|
|
||||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
|
||||||
'--force', '--scheduled', self.SCHEDULED_TS_INVALID])
|
|
||||||
self.assertEqual(result.exit_code, 1)
|
|
||||||
self.assertIn('Unable to parse scheduled timestamp', result.output)
|
|
||||||
|
|
||||||
# Invalid timestamp - no timezone
|
|
||||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
|
||||||
'--force', '--scheduled', self.SCHEDULED_TS_NO_TZ])
|
|
||||||
self.assertEqual(result.exit_code, 1)
|
|
||||||
self.assertIn('Timezone information is mandatory for the scheduled switchover', result.output)
|
|
||||||
|
|
||||||
# Specifying wrong leader
|
|
||||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='dummy')
|
|
||||||
self.assertEqual(result.exit_code, 1)
|
|
||||||
self.assertIn('Member dummy is not the leader of cluster dummy', result.output)
|
|
||||||
|
|
||||||
# Errors while sending Patroni REST API request
|
|
||||||
with patch.object(PoolManager, 'request', Mock(side_effect=Exception)):
|
|
||||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'],
|
|
||||||
input=f'leader\nother\n{self.SCHEDULED_TS}\ny')
|
|
||||||
self.assertIn('falling back to DCS', result.output)
|
|
||||||
|
|
||||||
with patch.object(PoolManager, 'request') as mock_api_request:
|
|
||||||
mock_api_request.return_value.status = 500
|
|
||||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
|
|
||||||
self.assertIn('Switchover failed', result.output)
|
|
||||||
|
|
||||||
mock_api_request.return_value.status = 501
|
|
||||||
mock_api_request.return_value.data = b'Server does not support this operation'
|
|
||||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
|
|
||||||
self.assertIn('Switchover failed', result.output)
|
|
||||||
|
|
||||||
# No members available
|
# No members available
|
||||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
|
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
|
||||||
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
|
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
|
||||||
@@ -201,6 +132,132 @@ class TestCtl(unittest.TestCase):
|
|||||||
self.assertEqual(result.exit_code, 1)
|
self.assertEqual(result.exit_code, 1)
|
||||||
self.assertIn('For Citus clusters the --group must me specified', result.output)
|
self.assertIn('For Citus clusters the --group must me specified', result.output)
|
||||||
|
|
||||||
|
# [Scheduled]
|
||||||
|
|
||||||
|
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
|
||||||
|
|
||||||
|
# Scheduled (confirm)
|
||||||
|
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'],
|
||||||
|
input=f'leader\nother\n{self.SCHEDULED_TS}\ny')
|
||||||
|
self.assertEqual(result.exit_code, 0)
|
||||||
|
self.assertIn(f'Are you sure you want to schedule switchover of cluster dummy '
|
||||||
|
f'at {self.SCHEDULED_TS}, demoting current leader', result.output)
|
||||||
|
|
||||||
|
# Scheduled (abort)
|
||||||
|
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||||
|
'--scheduled', self.SCHEDULED_TS], input='leader\nother\n\nN')
|
||||||
|
self.assertEqual(result.exit_code, 1)
|
||||||
|
|
||||||
|
# Scheduled with --force option
|
||||||
|
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||||
|
'--force', '--scheduled', self.SCHEDULED_TS])
|
||||||
|
self.assertEqual(result.exit_code, 0)
|
||||||
|
|
||||||
|
# Scheduled in pause mode
|
||||||
|
with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
|
||||||
|
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||||
|
'--force', '--scheduled', self.SCHEDULED_TS])
|
||||||
|
self.assertEqual(result.exit_code, 1)
|
||||||
|
self.assertIn(ManualFailoverPrecheckStatus.SCHEDULED_SWITCHOVER_PAUSE.value[0], result.output)
|
||||||
|
|
||||||
|
# Invalid timestamp with force
|
||||||
|
result = self.runner.invoke(ctl,['switchover', 'dummy', '--group', '0', '--force', '--scheduled',
|
||||||
|
self.SCHEDULED_TS_INVALID])
|
||||||
|
self.assertEqual(result.exit_code, 1)
|
||||||
|
self.assertIn('Unable to parse scheduled timestamp', result.output)
|
||||||
|
|
||||||
|
# Invalid timestamp
|
||||||
|
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||||
|
'--force', '--scheduled', self.SCHEDULED_TS_INVALID])
|
||||||
|
self.assertEqual(result.exit_code, 1)
|
||||||
|
self.assertIn('Unable to parse scheduled timestamp', result.output)
|
||||||
|
|
||||||
|
# Invalid timestamp - no timezone
|
||||||
|
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
|
||||||
|
'--force', '--scheduled', self.SCHEDULED_TS_NO_TZ])
|
||||||
|
self.assertEqual(result.exit_code, 1)
|
||||||
|
self.assertIn(ParseScheduleErrors.NO_TIMEZONE.value[0].format(action='switchover'), result.output)
|
||||||
|
|
||||||
|
# [Other erroneous combinations]
|
||||||
|
|
||||||
|
# No candidate in pause mode
|
||||||
|
with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
|
||||||
|
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\n\n\ny')
|
||||||
|
self.assertEqual(result.exit_code, 1)
|
||||||
|
self.assertIn(ManualFailoverPrecheckStatus.SWITCHOVER_PAUSE_NO_CANDIDATE.value[0], result.output)
|
||||||
|
|
||||||
|
# Target and source are equal
|
||||||
|
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nleader\n\ny')
|
||||||
|
self.assertEqual(result.exit_code, 1)
|
||||||
|
self.assertIn(ManualFailoverPrecheckStatus.SWITCHOVER_TO_LEADER.value[0], result.output)
|
||||||
|
|
||||||
|
# Candidate is not a member of the cluster
|
||||||
|
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nReality\n\ny')
|
||||||
|
self.assertEqual(result.exit_code, 1)
|
||||||
|
self.assertIn(ManualFailoverPrecheckStatus.CANDIDATE_NOT_MEMEBER.value[0].format(candidate='Reality',
|
||||||
|
cluster_name='dummy'),
|
||||||
|
result.output)
|
||||||
|
|
||||||
|
# Specifying wrong leader
|
||||||
|
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='dummy')
|
||||||
|
self.assertEqual(result.exit_code, 1)
|
||||||
|
self.assertIn(
|
||||||
|
ManualFailoverPrecheckStatus.LEADER_NOT_MEMBER.value[0].format(leader='dummy',
|
||||||
|
cluster_name='dummy'),
|
||||||
|
result.output)
|
||||||
|
|
||||||
|
mock_get_dcs.return_value.get_cluster = Mock(
|
||||||
|
return_value=get_cluster_initialized_with_leader(sync=('leader', 'other')))
|
||||||
|
|
||||||
|
# Candidate is not a sync standby
|
||||||
|
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=True)):
|
||||||
|
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\notherMember\n\ny')
|
||||||
|
self.assertEqual(result.exit_code, 1)
|
||||||
|
self.assertIn(ManualFailoverPrecheckStatus.CANDIDATE_NOT_SYNC_STANDBY.value[0], result.output)
|
||||||
|
|
||||||
|
# No healthy nodes to promote in sync mode
|
||||||
|
mock_get_dcs.return_value.get_cluster = Mock(return_value=get_cluster_initialized_with_leader(sync=('leader')))
|
||||||
|
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=True)):
|
||||||
|
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force'])
|
||||||
|
self.assertEqual(result.exit_code, 1)
|
||||||
|
self.assertIn(ManualFailoverPrecheckStatus.NO_SYNC_CANDIDATE.value[0].format(action='switchover'),
|
||||||
|
result.output)
|
||||||
|
|
||||||
|
# No healthy nodes to promote in async mode
|
||||||
|
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
|
||||||
|
with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=False)):
|
||||||
|
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force'])
|
||||||
|
self.assertEqual(result.exit_code, 1)
|
||||||
|
self.assertIn(ManualFailoverPrecheckStatus.ONLY_LEADER.value[0].format(action='switchover'),
|
||||||
|
result.output)
|
||||||
|
|
||||||
|
# Cluster has no leader
|
||||||
|
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader
|
||||||
|
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--leader', 'leader', '--force'])
|
||||||
|
self.assertEqual(result.exit_code, 1)
|
||||||
|
self.assertIn(
|
||||||
|
ManualFailoverPrecheckStatus.CLUSTER_NO_LEADER.value[0].format(leader='leader', cluster_name='dummy'),
|
||||||
|
result.output)
|
||||||
|
|
||||||
|
# [Errors while sending Patroni REST API request]
|
||||||
|
|
||||||
|
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
|
||||||
|
|
||||||
|
with patch.object(PoolManager, 'request', Mock(side_effect=Exception)):
|
||||||
|
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'],
|
||||||
|
input=f'leader\nother\n{self.SCHEDULED_TS}\ny')
|
||||||
|
self.assertIn('falling back to DCS', result.output)
|
||||||
|
|
||||||
|
with patch.object(PoolManager, 'request') as mock_api_request:
|
||||||
|
mock_api_request.return_value.status = 500
|
||||||
|
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
|
||||||
|
self.assertIn('Switchover failed', result.output)
|
||||||
|
|
||||||
|
mock_api_request.return_value.status = 501
|
||||||
|
mock_api_request.return_value.data = b'Server does not support this operation'
|
||||||
|
result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
|
||||||
|
self.assertIn('Switchover failed', result.output)
|
||||||
|
|
||||||
@patch('patroni.ctl.get_dcs')
|
@patch('patroni.ctl.get_dcs')
|
||||||
@patch.object(PoolManager, 'request', Mock(return_value=MockResponse()))
|
@patch.object(PoolManager, 'request', Mock(return_value=MockResponse()))
|
||||||
@patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
|
@patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
|
||||||
@@ -210,7 +267,7 @@ class TestCtl(unittest.TestCase):
|
|||||||
# No candidate specified
|
# No candidate specified
|
||||||
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
|
mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
|
||||||
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='0\n')
|
result = self.runner.invoke(ctl, ['failover', 'dummy'], input='0\n')
|
||||||
self.assertIn('Failover could be performed only to a specific candidate', result.output)
|
self.assertIn(ManualFailoverPrecheckStatus.FAILOVER_NO_CANDIDATE.value[0], result.output)
|
||||||
|
|
||||||
# Failover to an async member in sync mode (confirm)
|
# Failover to an async member in sync mode (confirm)
|
||||||
cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
|
cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
|
||||||
@@ -374,7 +431,7 @@ class TestCtl(unittest.TestCase):
|
|||||||
result = self.runner.invoke(ctl,
|
result = self.runner.invoke(ctl,
|
||||||
['restart', 'alpha', 'other', '--force', '--scheduled', self.SCHEDULED_TS_NO_TZ])
|
['restart', 'alpha', 'other', '--force', '--scheduled', self.SCHEDULED_TS_NO_TZ])
|
||||||
self.assertEqual(result.exit_code, 1)
|
self.assertEqual(result.exit_code, 1)
|
||||||
self.assertIn('Timezone information is mandatory for the scheduled restart', result.output)
|
self.assertIn(ParseScheduleErrors.NO_TIMEZONE.value[0].format(action='restart'), result.output)
|
||||||
|
|
||||||
# Error parsing scheduled flag value (invalid date)
|
# Error parsing scheduled flag value (invalid date)
|
||||||
result = self.runner.invoke(ctl,
|
result = self.runner.invoke(ctl,
|
||||||
|
|||||||
Reference in New Issue
Block a user