From 7ca55359de6b05d6173567c75c5e978df1c3a7e0 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Tue, 20 Sep 2016 15:45:21 +0200 Subject: [PATCH] Demote immediately if failed to update leader lock (#316) If the Etcd node partitioned from rest of the cluster it is still possible to read from it (though it returns some stale information), but it is not possible to write into it. Previously Patroni was trying to fetch the new cluster view from DCS in order to figure out is it still the leader or not and Etcd is always returning stale info where the node still owns the leader key, but with negative TTL. This weird bug clearly shows how dangerous premature optimization is. --- patroni/ha.py | 3 ++- tests/test_ha.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/patroni/ha.py b/patroni/ha.py index 2080e1ed..5f3a00cc 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -434,7 +434,8 @@ class Ha(object): else: # Either there is no connection to DCS or someone else acquired the lock logger.error('failed to update leader lock') - self.load_cluster_from_dcs() + self.demote(delete_leader=False) + return 'demoted self because failed to update leader lock in DCS' else: logger.info('does not have lock') return self.follow('demoting self because i do not have the lock and i was a leader', diff --git a/tests/test_ha.py b/tests/test_ha.py index 61d65718..f62a6f16 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -231,7 +231,7 @@ class TestHa(unittest.TestCase): self.ha.cluster.is_unlocked = false self.ha.has_lock = true self.ha.update_lock = false - self.assertEquals(self.ha.run_cycle(), 'demoting self because i do not have the lock and i was a leader') + self.assertEquals(self.ha.run_cycle(), 'demoted self because failed to update leader lock in DCS') def test_follow(self): self.ha.cluster.is_unlocked = false