move stale protection from is_healthy to is_healthiest to allow stale secondaries to rejoin cluster

helpers/ha.py

@@ -38,9 +38,9 @@ class Ha:
 
     def run_cycle(self):
         try:
-            if self.state_handler.is_healthy(self.etcd.last_leader_operation()):
+            if self.state_handler.is_healthy():
                 if self.is_unlocked():
-                    if self.state_handler.is_healthiest_node(self.etcd.members()):
+                    if self.state_handler.is_healthiest_node(self.etcd):
                         if self.acquire_lock():
                             if not self.state_handler.is_leader():
                                 self.state_handler.promote()
@@ -58,6 +58,8 @@ class Ha:
                         if self.state_handler.is_leader():
                             self.state_handler.demote(self.fetch_current_leader())
                             return "demoting self because i am not the healthiest node"
+                        elif self.fetch_current_leader() is None:
+                            return "waiting on leader to be elected because i am not the healthiest node"
                         else:
                             self.state_handler.follow_the_leader(self.fetch_current_leader())
                             return "following a different leader because i am not the healthiest node"

helpers/postgresql.py

@@ -110,7 +110,7 @@ class Postgresql:
             options += " -c \"%s=%s\"" % (setting, value)
         return options
 
-    def is_healthy(self, last_leader_operation):
+    def is_healthy(self):
         if not self.is_running():
             logger.warning("Postgresql is not running.")
             return False
@@ -118,17 +118,17 @@ class Postgresql:
         if self.is_leader():
             return True
 
-        # this should only happen on initialization
-        if last_leader_operation is None:
-            return True
-
-        if (last_leader_operation - self.xlog_position()) > self.config["maximum_lag_on_failover"]:
-            return False
-
         return True
 
-    def is_healthiest_node(self, members):
-        for member in members:
+    def is_healthiest_node(self, state_store):
+        # this should only happen on initialization
+        if state_store.last_leader_operation() is None:
+            return True
+
+        if (state_store.last_leader_operation() - self.xlog_position()) > self.config["maximum_lag_on_failover"]:
+            return False
+
+        for member in state_store.members():
             if member["hostname"] == self.name:
                 continue
             try: