Add dcs_last_seen field to API (#2051)

This field notes the last time (as unix epoch) a cluster member has successfully communicated with the DCS. This is useful to identify and/or analyze network partitions.

Also, expose dcs_last_seen in the MemberStatus class and its from_api_response() method.

patroni/api.py

@@ -288,6 +288,11 @@ class RestApiHandler(BaseHTTPRequestHandler):
         metrics.append("# TYPE patroni_postgres_timeline counter")
         metrics.append("patroni_postgres_timeline{0} {1}".format(scope_label, postgres.get('timeline', 0)))
 
+        metrics.append("# HELP patroni_dcs_last_seen Epoch timestamp when DCS was last contacted successfully"
+                       " by Patroni.")
+        metrics.append("# TYPE patroni_dcs_last_seen gauge")
+        metrics.append("patroni_dcs_last_seen{0} {1}".format(scope_label, postgres.get('dcs_last_seen', 0)))
+
         self._write_response(200, '\n'.join(metrics)+'\n', content_type='text/plain')
 
     def _read_json_content(self, body_is_optional=False):
@@ -600,6 +605,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 'role': 'replica' if row[1] == 0 else 'master',
                 'server_version': postgresql.server_version,
                 'cluster_unlocked': bool(not cluster or cluster.is_unlocked()),
+                'dcs_last_seen': self.server.patroni.dcs.last_seen,
                 'xlog': ({
                     'received_location': row[4] or row[3],
                     'replayed_location': row[3],

patroni/dcs/__init__.py

@@ -652,6 +652,7 @@ class AbstractDCS(object):
         self._cluster_valid_till = 0
         self._cluster_thread_lock = Lock()
         self._last_lsn = ''
+        self._last_seen = 0
         self._last_status = {}
         self.event = Event()
 
@@ -722,6 +723,10 @@ class AbstractDCS(object):
     def loop_wait(self):
         return self._loop_wait
 
+    @property
+    def last_seen(self):
+        return self._last_seen
+
     @abc.abstractmethod
     def _load_cluster(self):
         """Internally this method should build  `Cluster` object which
@@ -744,6 +749,8 @@ class AbstractDCS(object):
             self.reset_cluster()
             raise
 
+        self._last_seen = int(time.time())
+
         with self._cluster_thread_lock:
             self._cluster = cluster
             self._cluster_valid_till = time.time() + self.ttl

patroni/ha.py

@@ -21,13 +21,15 @@ from threading import RLock
 logger = logging.getLogger(__name__)
 
 
-class _MemberStatus(namedtuple('_MemberStatus', ['member', 'reachable', 'in_recovery', 'timeline',
-                                                 'wal_position', 'tags', 'watchdog_failed'])):
+class _MemberStatus(namedtuple('_MemberStatus', ['member', 'reachable', 'in_recovery',
+                                                 'dcs_last_seen', 'timeline', 'wal_position',
+                                                 'tags', 'watchdog_failed'])):
     """Node status distilled from API response:
 
         member - dcs.Member object of the node
         reachable - `!False` if the node is not reachable or is not responding with correct JSON
         in_recovery - `!True` if pg_is_in_recovery() == true
+        dcs_last_seen - timestamp from JSON of last succesful communication with DCS
         timeline - timeline value from JSON
         wal_position - maximum value of `replayed_location` or `received_location` from JSON
         tags - dictionary with values of different tags (i.e. nofailover)
@@ -37,12 +39,14 @@ class _MemberStatus(namedtuple('_MemberStatus', ['member', 'reachable', 'in_reco
     def from_api_response(cls, member, json):
         is_master = json['role'] == 'master'
         timeline = json.get('timeline', 0)
+        dcs_last_seen = json.get('dcs_last_seen', 0)
         wal = not is_master and max(json['xlog'].get('received_location', 0), json['xlog'].get('replayed_location', 0))
-        return cls(member, True, not is_master, timeline, wal, json.get('tags', {}), json.get('watchdog_failed', False))
+        return cls(member, True, not is_master, dcs_last_seen, timeline, wal,
+                   json.get('tags', {}), json.get('watchdog_failed', False))
 
     @classmethod
     def unknown(cls, member):
-        return cls(member, False, None, 0, 0, {}, False)
+        return cls(member, False, None, 0, 0, 0, {}, False)
 
     def failover_limitation(self):
         """Returns reason why this node can't promote or None if everything is ok."""

tests/test_api.py

@@ -54,6 +54,7 @@ class MockHa(object):
 
     state_handler = MockPostgresql()
     watchdog = MockWatchdog()
+    dcs_last_seen = 0
 
     @staticmethod
     def is_leader():
@@ -77,7 +78,7 @@ class MockHa(object):
 
     @staticmethod
     def fetch_nodes_statuses(members):
-        return [_MemberStatus(None, True, None, 0, None, {}, False)]
+        return [_MemberStatus(None, True, None, 0, 0, None, {}, False)]
 
     @staticmethod
     def schedule_future_restart(data):

tests/test_ha.py

@@ -80,13 +80,14 @@ def get_standby_cluster_initialized_with_only_leader(failover=None, sync=None):
     )
 
 
-def get_node_status(reachable=True, in_recovery=True, timeline=2,
-                    wal_position=10, nofailover=False, watchdog_failed=False):
+def get_node_status(reachable=True, in_recovery=True, dcs_last_seen=0,
+                    timeline=2, wal_position=10, nofailover=False,
+                    watchdog_failed=False):
     def fetch_node_status(e):
         tags = {}
         if nofailover:
             tags['nofailover'] = True
-        return _MemberStatus(e, reachable, in_recovery, timeline, wal_position, tags, watchdog_failed)
+        return _MemberStatus(e, reachable, in_recovery, dcs_last_seen, timeline, wal_position, tags, watchdog_failed)
     return fetch_node_status