From 88db6018ac65afa9231f36f0d3c135c24a16e350 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 1 Sep 2022 11:34:42 +0200 Subject: [PATCH] Improve liveness probe (#2395) it will start failing if the heartbeat loop isn't running longer than `ttl` on the primary or `2*ttl` on the replica. Close https://github.com/zalando/patroni/issues/2388 --- docs/rest_api.rst | 2 +- patroni/api.py | 11 ++++++++++- tests/test_api.py | 8 +++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/rest_api.rst b/docs/rest_api.rst index 5a2879d3..e6f9ecb3 100644 --- a/docs/rest_api.rst +++ b/docs/rest_api.rst @@ -58,7 +58,7 @@ For all health check ``GET`` requests Patroni returns a JSON document with the s - ``GET /health``: returns HTTP status code **200** only when PostgreSQL is up and running. -- ``GET /liveness``: always returns HTTP status code **200** what only indicates that Patroni is running. Could be used for ``livenessProbe``. +- ``GET /liveness``: returns HTTP status code **200** if Patroni heartbeat loop is properly running and **503** if the last run was more than ``ttl`` seconds ago on the primary or ``2*ttl`` on the replica. Could be used for ``livenessProbe``. - ``GET /readiness``: returns HTTP status code **200** when the Patroni node is running as the leader or when PostgreSQL is up and running. The endpoint could be used for ``readinessProbe`` when it is not possible to use Kubernetes endpoints for leader elections (OpenShift). diff --git a/patroni/api.py b/patroni/api.py index 963be433..d718a6e9 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -190,7 +190,16 @@ class RestApiHandler(BaseHTTPRequestHandler): self.do_GET(write_status_code_only=True) def do_GET_liveness(self): - self._write_status_code_only(200) + patroni = self.server.patroni + is_primary = patroni.postgresql.role == 'master' and patroni.postgresql.is_running() + # We can tolerate Patroni problems longer on the replica. + # On the primary the liveness probe most likely will start failing only after the leader key expired. + # It should not be a big problem because replicas will see that the primary is still alive via REST API call. + liveness_threshold = patroni.dcs.ttl * (1 if is_primary else 2) + + # In maintenance mode (pause) we are fine if heartbeat loop stuck. + status_code = 200 if patroni.ha.is_paused() or patroni.next_run + liveness_threshold > time.time() else 503 + self._write_status_code_only(status_code) def do_GET_readiness(self): patroni = self.server.patroni diff --git a/tests/test_api.py b/tests/test_api.py index 681814b0..73f81412 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -46,6 +46,10 @@ class MockPostgresql(object): def replica_cached_timeline(_): return 2 + @staticmethod + def is_running(): + return True + class MockWatchdog(object): is_healthy = False @@ -289,7 +293,9 @@ class TestRestApiHandler(unittest.TestCase): def test_do_HEAD(self): self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'HEAD / HTTP/1.0')) - def test_do_GET_liveness(self): + @patch.object(MockPatroni, 'dcs') + def test_do_GET_liveness(self, mock_dcs): + mock_dcs.ttl.return_value = PropertyMock(30) self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /liveness HTTP/1.0')) def test_do_GET_readiness(self):