Consider sync node as a healthy even when the former leader is ahead (#1059)

Fixes https://github.com/zalando/patroni/issues/1054

patroni/ha.py

@@ -604,7 +604,11 @@ class Ha(object):
                         return False
                     if my_wal_position < st.wal_position:
                         logger.info('Wal position of %s is ahead of my wal position', st.member.name)
-                        return False
+                        # In synchronous mode the former leader might be still accessible and even be ahead of us.
+                        # We should not disqualify himself from the leader race in such a situation.
+                        if not self.is_synchronous_mode() or st.member.name != self.cluster.sync.leader:
+                            return False
+                        logger.info('Ignoring the former leader being ahead of us')
         return True
 
     def is_failover_possible(self, members):

tests/test_ha.py

@@ -588,6 +588,7 @@ class TestHa(unittest.TestCase):
 
     @patch('requests.get', requests_get)
     def test__is_healthiest_node(self):
+        self.ha.cluster = get_cluster_initialized_without_leader(sync=('postgresql1', self.p.name))
         self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.p.is_leader = false
         self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
@@ -596,6 +597,9 @@ class TestHa(unittest.TestCase):
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.fetch_node_status = get_node_status(wal_position=11)  # accessible, in_recovery, wal position ahead
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
+        # in synchronous_mode consider itself healthy if the former leader is accessible in read-only and ahead of us
+        with patch.object(Ha, 'is_synchronous_mode', Mock(return_value=True)):
+            self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         with patch('patroni.postgresql.Postgresql.timeline_wal_position', return_value=(1, 1)):
             self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         with patch('patroni.postgresql.Postgresql.replica_cached_timeline', return_value=1):