Handle unexpected exceptions in etcd.

Previously, patroni would die after receiving an exception
other than RetryFailedError, etcd.EtcdException from etcd.
We have observed an AttributeError raised by etcd on some
occasions. With this change, we demote ourselves, but not
terminate on such exceptions.

patroni/etcd.py

@@ -143,6 +143,10 @@ def catch_etcd_errors(func):
             return not func(*args, **kwargs) is None
         except (RetryFailedError, etcd.EtcdException):
             return False
+        except:
+            logger.exception("")
+            raise EtcdError("unexpected error")
+
     return wrapper
 
 

tests/test_etcd.py

@@ -8,7 +8,7 @@ import unittest
 from dns.exception import DNSException
 from mock import Mock, patch
 from patroni.dcs import Cluster, DCSError, Leader
-from patroni.etcd import Client, Etcd
+from patroni.etcd import Client, Etcd, EtcdError
 
 
 class MockResponse:
@@ -266,3 +266,7 @@ class TestEtcd(unittest.TestCase):
         self.etcd.watch(4.5)
         self.etcd.watch(9.5)
         self.etcd.watch(100)
+
+    @patch('patroni.etcd.Etcd.retry', Mock(side_effect=AttributeError("foo")))
+    def test_other_exceptions(self):
+        self.assertRaises(EtcdError, self.etcd.cancel_initialization)