Demote master when etcd is not accessible

helpers/ha.py

@@ -89,6 +89,9 @@ class Ha:
                         return 'no action.  i am a secondary and i am following a leader'
         except EtcdError:
             logger.error('Error communicating with Etcd')
+            if self.state_handler.is_leader():
+                self.state_handler.demote(None)
+                return 'demoted self because etcd is not accessible and i was a leader'
         except OperationalError:
             logger.error('Error communicating with Postgresql.  Will try again')
         except HealthiestMemberError:

tests/test_ha.py

@@ -1,6 +1,7 @@
 import unittest
 import requests
 
+from helpers.errors import EtcdError
 from helpers.etcd import Cluster, Etcd
 from helpers.ha import Ha
 from test_etcd import requests_get, requests_put, requests_delete
@@ -54,6 +55,10 @@ def nop(*args, **kwargs):
     pass
 
 
+def dead_etcd():
+    raise EtcdError('Etcd is not responding properly')
+
+
 class TestHa(unittest.TestCase):
 
     def __init__(self, method_name='runTest'):
@@ -124,3 +129,7 @@ class TestHa(unittest.TestCase):
         self.ha.cluster.is_unlocked = false
         self.p.is_leader = false
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am a secondary and i am following a leader')
+
+    def test_no_etcd_connection_master_demote(self):
+        self.ha.load_cluster_from_etcd = dead_etcd
+        self.assertEquals(self.ha.run_cycle(), 'demoted self because etcd is not accessible and i was a leader')