From 34f9b666cfdd2acafa032e61938a03a23a3538b8 Mon Sep 17 00:00:00 2001
From: Alexander Kukushkin <alexander.kukushkin@zalando.de>
Date: Fri, 22 May 2015 12:26:19 +0200
Subject: [PATCH] Demote master when etcd is not accessible

---
 helpers/ha.py    | 3 +++
 tests/test_ha.py | 9 +++++++++
 2 files changed, 12 insertions(+)

diff --git a/helpers/ha.py b/helpers/ha.py
index b3e97d82..74c6a892 100644
--- a/helpers/ha.py
+++ b/helpers/ha.py
@@ -89,6 +89,9 @@ class Ha:
                         return 'no action.  i am a secondary and i am following a leader'
         except EtcdError:
             logger.error('Error communicating with Etcd')
+            if self.state_handler.is_leader():
+                self.state_handler.demote(None)
+                return 'demoted self because etcd is not accessible and i was a leader'
         except OperationalError:
             logger.error('Error communicating with Postgresql.  Will try again')
         except HealthiestMemberError:
diff --git a/tests/test_ha.py b/tests/test_ha.py
index c262c9ee..cabdf7d5 100644
--- a/tests/test_ha.py
+++ b/tests/test_ha.py
@@ -1,6 +1,7 @@
 import unittest
 import requests
 
+from helpers.errors import EtcdError
 from helpers.etcd import Cluster, Etcd
 from helpers.ha import Ha
 from test_etcd import requests_get, requests_put, requests_delete
@@ -54,6 +55,10 @@ def nop(*args, **kwargs):
     pass
 
 
+def dead_etcd():
+    raise EtcdError('Etcd is not responding properly')
+
+
 class TestHa(unittest.TestCase):
 
     def __init__(self, method_name='runTest'):
@@ -124,3 +129,7 @@ class TestHa(unittest.TestCase):
         self.ha.cluster.is_unlocked = false
         self.p.is_leader = false
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am a secondary and i am following a leader')
+
+    def test_no_etcd_connection_master_demote(self):
+        self.ha.load_cluster_from_etcd = dead_etcd
+        self.assertEquals(self.ha.run_cycle(), 'demoted self because etcd is not accessible and i was a leader')