From 89388c2e4bb3eb8f380ee18458dc74042ff29ab4 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 7 Oct 2021 16:08:10 +0200 Subject: [PATCH] Handle DCS exceptions when demoting (#2081) While doing demote due to failure to update leader lock it could happen that DCS goes completely down and the get_cluster() call raise the exception. Not being properly handled it results in postgres remaining stopped until DCS recovers. --- patroni/ha.py | 7 +++++-- tests/test_ha.py | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/patroni/ha.py b/patroni/ha.py index 3d6c6d1d..218e8621 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -870,8 +870,11 @@ class Ha(object): if mode_control['offline']: node_to_follow, leader = None, None else: - cluster = self.dcs.get_cluster() - node_to_follow, leader = self._get_node_to_follow(cluster), cluster.leader + try: + cluster = self.dcs.get_cluster() + node_to_follow, leader = self._get_node_to_follow(cluster), cluster.leader + except Exception: + node_to_follow, leader = None, None # FIXME: with mode offline called from DCS exception handler and handle_long_action_in_progress # there could be an async action already running, calling follow from here will lead diff --git a/tests/test_ha.py b/tests/test_ha.py index 945a083c..fdb348db 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -402,6 +402,8 @@ class TestHa(PostgresInit): self.ha.has_lock = true self.ha.update_lock = false self.assertEqual(self.ha.run_cycle(), 'demoted self because failed to update leader lock in DCS') + with patch.object(Ha, '_get_node_to_follow', Mock(side_effect=DCSError('foo'))): + self.assertEqual(self.ha.run_cycle(), 'demoted self because failed to update leader lock in DCS') self.p.is_leader = false self.assertEqual(self.ha.run_cycle(), 'not promoting because failed to update leader lock in DCS')