From 2c6907465249c7c9af7af7f321646959a600ad9b Mon Sep 17 00:00:00 2001
From: Christopher Winslett <chris@compose.io>
Date: Tue, 12 May 2015 18:29:54 -0700
Subject: [PATCH] move stale protection from is_healthy to is_healthiest to
 allow stale secondaries to rejoin cluster

---
 helpers/ha.py         |  6 ++++--
 helpers/postgresql.py | 20 ++++++++++----------
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/helpers/ha.py b/helpers/ha.py
index d031e629..80736420 100644
--- a/helpers/ha.py
+++ b/helpers/ha.py
@@ -38,9 +38,9 @@ class Ha:
 
     def run_cycle(self):
         try:
-            if self.state_handler.is_healthy(self.etcd.last_leader_operation()):
+            if self.state_handler.is_healthy():
                 if self.is_unlocked():
-                    if self.state_handler.is_healthiest_node(self.etcd.members()):
+                    if self.state_handler.is_healthiest_node(self.etcd):
                         if self.acquire_lock():
                             if not self.state_handler.is_leader():
                                 self.state_handler.promote()
@@ -58,6 +58,8 @@ class Ha:
                         if self.state_handler.is_leader():
                             self.state_handler.demote(self.fetch_current_leader())
                             return "demoting self because i am not the healthiest node"
+                        elif self.fetch_current_leader() is None:
+                            return "waiting on leader to be elected because i am not the healthiest node"
                         else:
                             self.state_handler.follow_the_leader(self.fetch_current_leader())
                             return "following a different leader because i am not the healthiest node"
diff --git a/helpers/postgresql.py b/helpers/postgresql.py
index 06ed66e0..387ff19a 100644
--- a/helpers/postgresql.py
+++ b/helpers/postgresql.py
@@ -110,7 +110,7 @@ class Postgresql:
             options += " -c \"%s=%s\"" % (setting, value)
         return options
 
-    def is_healthy(self, last_leader_operation):
+    def is_healthy(self):
         if not self.is_running():
             logger.warning("Postgresql is not running.")
             return False
@@ -118,17 +118,17 @@ class Postgresql:
         if self.is_leader():
             return True
 
-        # this should only happen on initialization
-        if last_leader_operation is None:
-            return True
-
-        if (last_leader_operation - self.xlog_position()) > self.config["maximum_lag_on_failover"]:
-            return False
-
         return True
 
-    def is_healthiest_node(self, members):
-        for member in members:
+    def is_healthiest_node(self, state_store):
+        # this should only happen on initialization
+        if state_store.last_leader_operation() is None:
+            return True
+
+        if (state_store.last_leader_operation() - self.xlog_position()) > self.config["maximum_lag_on_failover"]:
+            return False
+
+        for member in state_store.members():
             if member["hostname"] == self.name:
                 continue
             try: