Reset is_leader status on demote (#777)

Make sure demoted cluster member stops responding with code 200 on the  /master API call.

Issue a new minor release.

Fixes https://github.com/zalando/patroni/issues/776

docs/releases.rst

@@ -3,6 +3,23 @@
 Release notes
 =============
 
+Version 1.4.6
+-------------
+
+**Bug fixes and stability improvements**
+
+This release fixes a critical issue with Patroni API /master endpoint returning 200 for the non-master node. This is a
+reporting issue, no actual split-brain, but under certain circumstances clients might be directed to the read-only node.
+
+- Reset is_leader status on demote (Alexander Kukushkin, Oleksii Kliukin)
+
+  Make sure demoted cluster member stops responding with code 200 on the /master API call.
+
+- Add new "cluster_unlocked" field to the API output (Dmitry Dolgov)
+
+  This field indicates whether the cluster has the master running. It can be used when it is not possible to query any
+  other node but one of the replicas.
+
 Version 1.4.5
 -------------
 

patroni/api.py

@@ -415,6 +415,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
     def get_postgresql_status(self, retry=False):
         try:
+            cluster = self.server.patroni.dcs.cluster
+
             if self.server.patroni.postgresql.state not in ('running', 'restarting', 'starting'):
                 raise RetryFailedError('')
             stmt = ("WITH replication_info AS ("
@@ -439,6 +441,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 'postmaster_start_time': row[0],
                 'role': 'replica' if row[1] == 0 else 'master',
                 'server_version': self.server.patroni.postgresql.server_version,
+                'cluster_unlocked': bool(not cluster or cluster.is_unlocked()),
                 'xlog': ({
                     'received_location': row[3],
                     'replayed_location': row[4],
@@ -451,7 +454,6 @@ class RestApiHandler(BaseHTTPRequestHandler):
             if row[1] > 0:
                 result['timeline'] = row[1]
             else:
-                cluster = self.server.patroni.dcs.cluster
                 leader_timeline = None if not cluster or cluster.is_unlocked() else cluster.leader.timeline
                 result['timeline'] = self.server.patroni.postgresql.replica_cached_timeline(leader_timeline)
 

patroni/ha.py

@@ -105,6 +105,9 @@ class Ha(object):
             self.old_cluster = cluster
         self.cluster = cluster
 
+        if self.cluster.is_unlocked() or self.cluster.leader.name != self.state_handler.name:
+            self.set_is_leader(False)
+
         self._leader_timeline = None if cluster.is_unlocked() else cluster.leader.timeline
 
     def acquire_lock(self):
@@ -620,6 +623,7 @@ class Ha(object):
         self.state_handler.stop(mode_control['stop'], checkpoint=mode_control['checkpoint'],
                                 on_safepoint=self.watchdog.disable if self.watchdog.is_running else None)
         self.state_handler.set_role('demoted')
+        self.set_is_leader(False)
 
         if mode_control['release']:
             self.release_leader_key_voluntarily()

patroni/version.py

@@ -1 +1 @@
-__version__ = '1.4.5'
+__version__ = '1.4.6'