Improve liveness probe (#2395)

it will start failing if the heartbeat loop isn't running longer than `ttl` on the primary or `2*ttl` on the replica.

Close https://github.com/zalando/patroni/issues/2388

docs/rest_api.rst

@@ -58,7 +58,7 @@ For all health check ``GET`` requests Patroni returns a JSON document with the s
 
 - ``GET /health``: returns HTTP status code **200** only when PostgreSQL is up and running.
 
-- ``GET /liveness``: always returns HTTP status code **200** what only indicates that Patroni is running. Could be used for ``livenessProbe``.
+- ``GET /liveness``: returns HTTP status code **200** if Patroni heartbeat loop is properly running and **503** if the last run was more than ``ttl`` seconds ago on the primary or ``2*ttl`` on the replica. Could be used for ``livenessProbe``.
 
 - ``GET /readiness``: returns HTTP status code **200** when the Patroni node is running as the leader or when PostgreSQL is up and running. The endpoint could be used for ``readinessProbe`` when it is not possible to use Kubernetes endpoints for leader elections (OpenShift).
 

patroni/api.py

@@ -190,7 +190,16 @@ class RestApiHandler(BaseHTTPRequestHandler):
         self.do_GET(write_status_code_only=True)
 
     def do_GET_liveness(self):
-        self._write_status_code_only(200)
+        patroni = self.server.patroni
+        is_primary = patroni.postgresql.role == 'master' and patroni.postgresql.is_running()
+        # We can tolerate Patroni problems longer on the replica.
+        # On the primary the liveness probe most likely will start failing only after the leader key expired.
+        # It should not be a big problem because replicas will see that the primary is still alive via REST API call.
+        liveness_threshold = patroni.dcs.ttl * (1 if is_primary else 2)
+
+        # In maintenance mode (pause) we are fine if heartbeat loop stuck.
+        status_code = 200 if patroni.ha.is_paused() or patroni.next_run + liveness_threshold > time.time() else 503
+        self._write_status_code_only(status_code)
 
     def do_GET_readiness(self):
         patroni = self.server.patroni

tests/test_api.py

@@ -46,6 +46,10 @@ class MockPostgresql(object):
     def replica_cached_timeline(_):
         return 2
 
+    @staticmethod
+    def is_running():
+        return True
+
 
 class MockWatchdog(object):
     is_healthy = False
@@ -289,7 +293,9 @@ class TestRestApiHandler(unittest.TestCase):
     def test_do_HEAD(self):
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'HEAD / HTTP/1.0'))
 
-    def test_do_GET_liveness(self):
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_GET_liveness(self, mock_dcs):
+        mock_dcs.ttl.return_value = PropertyMock(30)
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /liveness HTTP/1.0'))
 
     def test_do_GET_readiness(self):