From 0afdb816ba3652029908d90caac3987f8fa58184 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Tue, 30 Aug 2016 10:38:40 +0200 Subject: [PATCH] Unfinished promote may not break paused cluster. When a node to promote dies before finishing the promote and the cluster is in a standby mode, the failover key sticks indefinitely, preventing any master to take over the leader role. Prevent it by letting the node in a master role cleanup the failover key if the node to failover is not present among the members. The master check cannot be performed by the node role alone, since the node will not change its cached role on a manual promote. We need to check the DB state as well. --- patroni/ha.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/patroni/ha.py b/patroni/ha.py index 60734ee6..5d106512 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -237,6 +237,11 @@ class Ha(object): if failover.candidate == self.state_handler.name: # manual failover to me return True elif self.is_paused(): + # Remove failover key if the node to failover has terminated to avoid waiting for it indefinitely + # In order to avoid race conditions only the master is allowed to do so. + if (not self.cluster.get_member(failover.candidate, fallback_to_leader=False) and + (self.state_handler.is_leader() or self.state_handler.role == 'master')): + self.dcs.manual_failover('', '', index=self.cluster.failover.index) return False # find specific node and check that it is healthy