From fef7d45208e9faac22a4f4a9bbd35bfc28f49419 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Tue, 17 Nov 2015 16:08:58 +0100 Subject: [PATCH] Handle unexpected exceptions in etcd. Previously, patroni would die after receiving an exception other than RetryFailedError, etcd.EtcdException from etcd. We have observed an AttributeError raised by etcd on some occasions. With this change, we demote ourselves, but not terminate on such exceptions. --- patroni/etcd.py | 4 ++++ tests/test_etcd.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/patroni/etcd.py b/patroni/etcd.py index 4a82f2d7..03e6dc08 100644 --- a/patroni/etcd.py +++ b/patroni/etcd.py @@ -143,6 +143,10 @@ def catch_etcd_errors(func): return not func(*args, **kwargs) is None except (RetryFailedError, etcd.EtcdException): return False + except: + logger.exception("") + raise EtcdError("unexpected error") + return wrapper diff --git a/tests/test_etcd.py b/tests/test_etcd.py index 915d5f9d..ea4f1b16 100644 --- a/tests/test_etcd.py +++ b/tests/test_etcd.py @@ -8,7 +8,7 @@ import unittest from dns.exception import DNSException from mock import Mock, patch from patroni.dcs import Cluster, DCSError, Leader -from patroni.etcd import Client, Etcd +from patroni.etcd import Client, Etcd, EtcdError class MockResponse: @@ -266,3 +266,7 @@ class TestEtcd(unittest.TestCase): self.etcd.watch(4.5) self.etcd.watch(9.5) self.etcd.watch(100) + + @patch('patroni.etcd.Etcd.retry', Mock(side_effect=AttributeError("foo"))) + def test_other_exceptions(self): + self.assertRaises(EtcdError, self.etcd.cancel_initialization)