diff --git a/patroni/ha.py b/patroni/ha.py index 50250a15..c504a362 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -419,7 +419,10 @@ class Ha(object): time.sleep(2) picked, allow_promote = self.state_handler.pick_synchronous_standby(self.cluster) if allow_promote: - cluster = self.dcs.get_cluster() + try: + cluster = self.dcs.get_cluster() + except DCSError: + return logger.warning("Could not get cluster state from DCS during process_sync_replication()") if cluster.sync.leader and cluster.sync.leader != self.state_handler.name: logger.info("Synchronous replication key updated by someone else") return @@ -696,10 +699,14 @@ class Ha(object): return self._is_healthiest_node(members.values()) - def release_leader_key_voluntarily(self): + def _delete_leader(self): + self.set_is_leader(False) self.dcs.delete_leader() - self.touch_member() self.dcs.reset_cluster() + + def release_leader_key_voluntarily(self): + self._delete_leader() + self.touch_member() logger.info("Leader key released") def demote(self, mode): @@ -727,7 +734,8 @@ class Ha(object): self.set_is_leader(False) if mode_control['release']: - self.release_leader_key_voluntarily() + with self._async_executor: + self.release_leader_key_voluntarily() time.sleep(2) # Give a time to somebody to take the leader lock if mode_control['offline']: node_to_follow, leader = None, None @@ -875,8 +883,7 @@ class Ha(object): if self.cluster.failover and self.cluster.failover.candidate == self.state_handler.name: return 'waiting to become master after promote...' - self.dcs.delete_leader() - self.dcs.reset_cluster() + self._delete_leader() return 'removed leader lock because postgres is not running as master' if self.state_handler.is_leader() and self._leader_access_is_restricted: @@ -1089,8 +1096,7 @@ class Ha(object): self.watchdog.disable() if self.has_lock(): self.state_handler.set_role('demoted') - self.dcs.delete_leader() - self.dcs.reset_cluster() + self._delete_leader() return 'removed leader key after trying and failing to start postgres' return 'failed to start postgres' self._crash_recovery_executed = False @@ -1256,16 +1262,15 @@ class Ha(object): if not self.state_handler.is_healthy(): if self.is_paused(): if self.has_lock(): - self.dcs.delete_leader() - self.dcs.reset_cluster() + self._delete_leader() return 'removed leader lock because postgres is not running' # Normally we don't start Postgres in a paused state. We make an exception for the demoted primary # that needs to be started after it had been stopped by demote. When there is no need to call rewind # the demote code follows through to starting Postgres right away, however, in the rewind case # it returns from demote and reaches this point to start PostgreSQL again after rewind. In that # case it makes no sense to continue to recover() unless rewind has finished successfully. - elif self.state_handler.rewind_failed or not self.state_handler.need_rewind \ - or not self.state_handler.can_rewind_or_reinitialize_allowed: + elif self.state_handler.rewind_failed or not self.state_handler.rewind_executed and not \ + (self.state_handler.need_rewind and self.state_handler.can_rewind_or_reinitialize_allowed): return 'postgres is not running' # try to start dead postgres diff --git a/tests/test_ha.py b/tests/test_ha.py index d1ead91f..a03d78b8 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -864,11 +864,15 @@ class TestHa(unittest.TestCase): self.ha.run_cycle() self.assertEqual(self.ha.dcs.write_sync_state.call_count, 2) - # Test changing sync standby failed due to race + # Test updating sync standby key failed due to DCS being not accessible self.ha.dcs.write_sync_state = Mock(return_value=True) + self.ha.dcs.get_cluster = Mock(side_effect=DCSError('foo')) + self.ha.run_cycle() + + # Test changing sync standby failed due to race self.ha.dcs.get_cluster = Mock(return_value=get_cluster_initialized_with_leader(sync=('somebodyelse', None))) self.ha.run_cycle() - self.assertEqual(self.ha.dcs.write_sync_state.call_count, 1) + self.assertEqual(self.ha.dcs.write_sync_state.call_count, 2) # Test sync set to '*' when synchronous_mode_strict is enabled mock_set_sync.reset_mock()