diff --git a/patroni/api.py b/patroni/api.py index 360247a1..2d36fd3a 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -12,7 +12,6 @@ import json import logging import time import traceback -import dateutil.parser import datetime import os import socket @@ -29,9 +28,10 @@ from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, TYPE_CH from . import psycopg from .__main__ import Patroni from .exceptions import PostgresConnectionException, PostgresException +from .manual_failover import ManualFailover from .postgresql.misc import postgres_version_to_int from .utils import deep_compare, enable_keepalive, parse_bool, patch_config, Retry, \ - RetryFailedError, parse_int, split_host_port, tzutc, uri, cluster_as_json, parse_schedule, manual_failover_precheck + RetryFailedError, parse_int, parse_schedule, split_host_port, tzutc, uri, cluster_as_json logger = logging.getLogger(__name__) @@ -1019,12 +1019,13 @@ class RestApiHandler(BaseHTTPRequestHandler): logger.warning('received failover request with leader specifed - performing switchover') action = 'switchover' - data, status_code = manual_failover_precheck(action, cluster, leader, candidate, bool(scheduled_at), - global_config.is_paused, global_config.is_synchronous_mode, - self.server.patroni).value + manual_failover = ManualFailover(action, cluster, leader, candidate, scheduled_at, + global_config.is_paused, global_config.is_synchronous_mode, + self.server.patroni) + data, status_code = manual_failover.run_precheck().value if not data and scheduled_at: - parse_result, scheduled_at = parse_schedule(scheduled_at) + parse_result, scheduled_at = manual_failover.parse_scheduled() if parse_result: data, status_code = parse_result.value[0], parse_result.value[1] diff --git a/patroni/ctl.py b/patroni/ctl.py index 0962b17e..67a94fc4 100644 --- a/patroni/ctl.py +++ b/patroni/ctl.py @@ -16,8 +16,6 @@ import click import codecs import copy import datetime -import dateutil.parser -import dateutil.tz import difflib import io import json @@ -49,8 +47,9 @@ except ImportError: # pragma: no cover from .config import Config, get_global_config from .dcs import get_dcs as _get_dcs, AbstractDCS, Cluster, Member from .exceptions import PatroniException +from .manual_failover import ManualFailover from .postgresql.misc import postgres_version_to_int -from .utils import cluster_as_json, manual_failover_precheck, parse_schedule, patch_config, polling_loop +from .utils import cluster_as_json, parse_schedule, patch_config, polling_loop from .request import PatroniRequest from .version import __version__ @@ -1030,7 +1029,7 @@ def restart(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member scheduled = click.prompt('When should the restart take place (e.g. ' + next_hour + ') ', type=str, default='now') - parse_result, scheduled_at = parse_schedule(scheduled if scheduled != 'now' else None) + parse_result, scheduled_at = parse_schedule(scheduled) if parse_result: raise PatroniCtlException(parse_result.value[0].format(action='restart')) confirm_members_action(members, force, 'restart', scheduled_at) @@ -1222,9 +1221,10 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s type=str, default='now') # Now, when we collected all the possible info, run checks + manual_failover = ManualFailover(action, cluster, leader, candidate, scheduled, + global_config.is_paused, global_config.is_synchronous_mode) - result_text, _ = manual_failover_precheck(action, cluster, leader, candidate, bool(scheduled), - global_config.is_paused, global_config.is_synchronous_mode).value + result_text, _ = manual_failover.run_precheck().value if result_text: raise PatroniCtlException(result_text.format(action=action, leader=leader, candidate=candidate, cluster_name=cluster_name)) @@ -1232,7 +1232,7 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s scheduled_at_str = None scheduled_at = None if action == 'switchover': - parse_result, scheduled_at = parse_schedule(scheduled if scheduled != 'now' else None) + parse_result, scheduled_at = manual_failover.parse_scheduled() if parse_result: raise PatroniCtlException(parse_result.value[0].format(action=action)) if scheduled_at: diff --git a/patroni/manual_failover.py b/patroni/manual_failover.py new file mode 100644 index 00000000..96af819a --- /dev/null +++ b/patroni/manual_failover.py @@ -0,0 +1,97 @@ +from enum import Enum +from typing import Optional, Tuple, TYPE_CHECKING + +if TYPE_CHECKING: # pragma: no cover + import datetime + + from .dcs import Cluster + from .ha import Patroni + from .utils import ParseScheduleErrors + +from .utils import parse_schedule + + +class ManualFailoverPrecheckStatus(Enum): + FAILOVER_NO_CANDIDATE = ('Failover could be performed only to a specific candidate', 400) + SWITCHOVER_NO_LEADER = ('Switchover could be performed only from a specific leader', 400) + SCHEDULED_FAILOVER = ("Failover can't be scheduled", 400) + SCHEDULED_SWITCHOVER_PAUSE = ("Can't schedule switchover in the paused state", 400) + SWITCHOVER_PAUSE_NO_CANDIDATE = ('Switchover is possible only to a specific candidate in a paused state', 400) + SWITCHOVER_TO_LEADER = ('Switchover target and source are the same', 400) + + CLUSTER_NO_LEADER = ('Cluster {cluster_name} has no leader', 412) + LEADER_NOT_MEMBER = ('Member {leader} is not the leader of cluster {cluster_name}', 412) + CANDIDATE_NOT_SYNC_STANDBY = ('candidate name does not match with sync_standby', 412) + NO_SYNC_CANDIDATE = ('{action} is not possible: can not find sync_standby', 412) + ONLY_LEADER = ('{action} is not possible: cluster does not have members except leader', 412) + CANDIDATE_NOT_MEMEBER = ('Member {candidate} does not exist in cluster {cluster_name} or is tagged as nofailover', + 412) + NO_GOOD_CANDIDATES = ('{action} is not possible: no good candidates have been found', 412) + + CHECK_PASSED = ('', None) + + +class ManualFailover(object): + + def __init__(self, action: str, cluster: 'Cluster', + leader: Optional[str], candidate: Optional[str], scheduled: Optional[str], + paused: bool = False, sync_mode: bool = False, patroni_obj: Optional['Patroni'] = None) -> None: + self.action = action + self.cluster = cluster + self.leader = leader + self.candidate = candidate + self.scheduled = scheduled + self.paused = paused + self.sync_mode = sync_mode + self.patroni = patroni_obj + + def parse_scheduled(self) -> Tuple[Optional['ParseScheduleErrors'], Optional['datetime.datetime']]: + return parse_schedule(self.scheduled) + + def run_precheck(self) -> ManualFailoverPrecheckStatus: + if self.action == 'failover' and not self.candidate: + return ManualFailoverPrecheckStatus.FAILOVER_NO_CANDIDATE + elif self.action == 'switchover' and not self.leader: + return ManualFailoverPrecheckStatus.SWITCHOVER_NO_LEADER + + if self.scheduled: + if self.action == 'failover': + return ManualFailoverPrecheckStatus.SCHEDULED_FAILOVER + elif self.paused: + return ManualFailoverPrecheckStatus.SCHEDULED_SWITCHOVER_PAUSE + + if self.paused and not self.candidate: + return ManualFailoverPrecheckStatus.SWITCHOVER_PAUSE_NO_CANDIDATE + + if self.leader == self.candidate: + return ManualFailoverPrecheckStatus.SWITCHOVER_TO_LEADER + + if self.action == 'switchover': + if self.cluster.leader is None or not self.cluster.leader.name: + return ManualFailoverPrecheckStatus.CLUSTER_NO_LEADER + if self.cluster.leader.name != self.leader: + return ManualFailoverPrecheckStatus.LEADER_NOT_MEMBER + + if self.candidate: + if self.action == 'switchover' and self.sync_mode and not self.cluster.sync.matches(self.candidate): + return ManualFailoverPrecheckStatus.CANDIDATE_NOT_SYNC_STANDBY + members = [m for m in self.cluster.members if m.name == self.candidate] + if not members: + return ManualFailoverPrecheckStatus.CANDIDATE_NOT_MEMEBER + elif self.sync_mode: + members = [m for m in self.cluster.members if self.cluster.sync.matches(m.name)] + if not members: + return ManualFailoverPrecheckStatus.NO_SYNC_CANDIDATE + else: + members = [m for m in self.cluster.members if not self.cluster.leader or m.name != self.cluster.leader.name and m.api_url] + if not members: + return ManualFailoverPrecheckStatus.ONLY_LEADER + + if self.patroni: + for st in self.patroni.ha.fetch_nodes_statuses(members): + if st.failover_limitation() is None: + break + else: + return ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES + + return ManualFailoverPrecheckStatus.CHECK_PASSED diff --git a/patroni/utils.py b/patroni/utils.py index de7a02c6..3b95e599 100644 --- a/patroni/utils.py +++ b/patroni/utils.py @@ -35,9 +35,8 @@ from .exceptions import PatroniException from .version import __version__ if TYPE_CHECKING: # pragma: no cover - from .dcs import Cluster, Member + from .dcs import Cluster from .config import GlobalConfig - from .ha import Patroni tzutc = tz.tzutc() @@ -1073,29 +1072,9 @@ class ParseScheduleErrors(Enum): PARSING_ERROR = ('Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601', 422) -class ManualFailoverPrecheckStatus(Enum): - FAILOVER_NO_CANDIDATE = ('Failover could be performed only to a specific candidate', 400) - SWITCHOVER_NO_LEADER = ('Switchover could be performed only from a specific leader', 400) - SCHEDULED_FAILOVER = ("Failover can't be scheduled", 400) - SCHEDULED_SWITCHOVER_PAUSE = ("Can't schedule switchover in the paused state", 400) - SWITCHOVER_PAUSE_NO_CANDIDATE = ('Switchover is possible only to a specific candidate in a paused state', 400) - SWITCHOVER_TO_LEADER = ('Switchover target and source are the same', 400) - - CLUSTER_NO_LEADER = ('Cluster {cluster_name} has no leader', 412) - LEADER_NOT_MEMBER = ('Member {leader} is not the leader of cluster {cluster_name}', 412) - CANDIDATE_NOT_SYNC_STANDBY = ('candidate name does not match with sync_standby', 412) - NO_SYNC_CANDIDATE = ('{action} is not possible: can not find sync_standby', 412) - ONLY_LEADER = ('{action} is not possible: cluster does not have members except leader', 412) - CANDIDATE_NOT_MEMEBER = ('Member {candidate} does not exist in cluster {cluster_name} or is tagged as nofailover', - 412) - NO_GOOD_CANDIDATES = ('{action} is not possible: no good candidates have been found', 412) - - CHECK_PASSED = ('', None) - - def parse_schedule(schedule: Optional[str]) -> Tuple[Optional[ParseScheduleErrors], Optional[datetime.datetime]]: scheduled_at = None - if schedule is not None: + if schedule is not None and (schedule or 'now') != 'now': try: scheduled_at = dateutil.parser.parse(schedule) if scheduled_at.tzinfo is None: @@ -1105,55 +1084,3 @@ def parse_schedule(schedule: Optional[str]) -> Tuple[Optional[ParseScheduleError except (ValueError, TypeError): return ParseScheduleErrors.PARSING_ERROR, scheduled_at return None, scheduled_at - - -def manual_failover_precheck(action: str, cluster: 'Cluster', - leader: Optional[str], candidate: Optional[str], scheduled: bool = False, - paused: bool = False, sync_mode: bool = False, - patroni_obj: Optional['Patroni'] = None) -> ManualFailoverPrecheckStatus: - if action == 'failover' and not candidate: - return ManualFailoverPrecheckStatus.FAILOVER_NO_CANDIDATE - elif action == 'switchover' and not leader: - return ManualFailoverPrecheckStatus.SWITCHOVER_NO_LEADER - - if scheduled: - if action == 'failover': - return ManualFailoverPrecheckStatus.SCHEDULED_FAILOVER - elif paused: - return ManualFailoverPrecheckStatus.SCHEDULED_SWITCHOVER_PAUSE - - if paused and not candidate: - return ManualFailoverPrecheckStatus.SWITCHOVER_PAUSE_NO_CANDIDATE - - if leader == candidate: - return ManualFailoverPrecheckStatus.SWITCHOVER_TO_LEADER - - if action == 'switchover': - if cluster.leader is None or not cluster.leader.name: - return ManualFailoverPrecheckStatus.CLUSTER_NO_LEADER - if cluster.leader.name != leader: - return ManualFailoverPrecheckStatus.LEADER_NOT_MEMBER - - if candidate: - if action == 'switchover' and sync_mode and not cluster.sync.matches(candidate): - return ManualFailoverPrecheckStatus.CANDIDATE_NOT_SYNC_STANDBY - members = [m for m in cluster.members if m.name == candidate] - if not members: - return ManualFailoverPrecheckStatus.CANDIDATE_NOT_MEMEBER - elif sync_mode: - members = [m for m in cluster.members if cluster.sync.matches(m.name)] - if not members: - return ManualFailoverPrecheckStatus.NO_SYNC_CANDIDATE - else: - members = [m for m in cluster.members if not cluster.leader or m.name != cluster.leader.name and m.api_url] - if not members: - return ManualFailoverPrecheckStatus.ONLY_LEADER - - if patroni_obj: - for st in patroni_obj.ha.fetch_nodes_statuses(members): - if st.failover_limitation() is None: - break - else: - return ManualFailoverPrecheckStatus.NO_GOOD_CANDIDATES - - return ManualFailoverPrecheckStatus.CHECK_PASSED diff --git a/tests/test_ctl.py b/tests/test_ctl.py index c2c452e0..fc4cadfd 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -103,6 +103,7 @@ class TestCtl(unittest.TestCase): # Confirm result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') + print(result.output) self.assertEqual(result.exit_code, 0) # Abort