From 3e553df69d9639028a81ac8d0a62bf9770912f9e Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Mon, 24 Aug 2020 16:35:46 +0200 Subject: [PATCH] BUGFIX: pause on K8s (#1659) On K8s the `Cluster.leader` is a valid object even if the cluster has no leader because we need to know the `resourceVersion` for future CAS operation. Such a non-empty object broke HA loop and made other nodes to think that the leader is there. The right way to identify the missing leader which reliably works across all DCS is checking that the leader's name is empty. --- patroni/ctl.py | 4 ++-- patroni/ha.py | 2 +- tests/test_ha.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/patroni/ctl.py b/patroni/ctl.py index 94df4a5b..c666e797 100644 --- a/patroni/ctl.py +++ b/patroni/ctl.py @@ -225,7 +225,7 @@ def watching(w, watch, max_count=None, clear=True): def get_all_members(cluster, role='master'): if role == 'master': - if cluster.leader is not None: + if cluster.leader is not None and cluster.leader.name: yield cluster.leader return @@ -640,7 +640,7 @@ def _do_failover_or_switchover(obj, action, cluster_name, master, candidate, for dcs = get_dcs(obj, cluster_name) cluster = dcs.get_cluster() - if action == 'switchover' and cluster.leader is None: + if action == 'switchover' and (cluster.leader is None or not cluster.leader.name): raise PatroniCtlException('This cluster has no master') if master is None: diff --git a/patroni/ha.py b/patroni/ha.py index 3c9363e9..495ae341 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -366,7 +366,7 @@ class Ha(object): elif self.patroni.replicatefrom and self.patroni.replicatefrom != self.state_handler.name: node_to_follow = cluster.get_member(self.patroni.replicatefrom) else: - node_to_follow = cluster.leader + node_to_follow = cluster.leader if cluster.leader and cluster.leader.name else None node_to_follow = node_to_follow if node_to_follow and node_to_follow.name != self.state_handler.name else None diff --git a/tests/test_ha.py b/tests/test_ha.py index dfe13a81..30acd72b 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -48,7 +48,7 @@ def get_cluster_not_initialized_without_leader(cluster_config=None): def get_cluster_initialized_without_leader(leader=False, failover=None, sync=None, cluster_config=None): m1 = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:rep-pass@127.0.0.1:5435/postgres', 'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location': 4}) - leader = Leader(0, 0, m1) if leader else None + leader = Leader(0, 0, m1 if leader else Member(0, '', 28, {})) m2 = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:rep-pass@127.0.0.1:5436/postgres', 'api_url': 'http://127.0.0.1:8011/patroni', 'state': 'running',