From 250328b84b682021fc559593da2c461ce790a97a Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 7 Oct 2021 16:08:21 +0200 Subject: [PATCH] Use cached role as a fallback when postgres is slow (#2082) In some extreme cases Postgres could be so slow that the normal monitoring query doesn't finish in a few seconds. It results in the exception being raised from the `Postgresql._cluster_info_state_get()` method, which could lead to the situation that postgres isn't demoted on time. In order to make it reliable we will catch the exception and use the cached state of postgres (`is_running()` and `role`) to determine whether postgres is running as a primary. Close https://github.com/zalando/patroni/issues/2073 --- patroni/postgresql/__init__.py | 6 +++++- tests/test_postgresql.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/patroni/postgresql/__init__.py b/patroni/postgresql/__init__.py index 4949b456..38cd5f89 100644 --- a/patroni/postgresql/__init__.py +++ b/patroni/postgresql/__init__.py @@ -369,7 +369,11 @@ class Postgresql(object): return self._cluster_info_state_get('received_tli') def is_leader(self): - return bool(self._cluster_info_state_get('timeline')) + try: + return bool(self._cluster_info_state_get('timeline')) + except PostgresConnectionException: + logger.warning('Failed to determine PostgreSQL state from the connection, falling back to cached role') + return bool(self.is_running() and self.role == 'master') def replay_paused(self): return self._cluster_info_state_get('replay_paused') diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py index 884fbbc8..937db9c1 100644 --- a/tests/test_postgresql.py +++ b/tests/test_postgresql.py @@ -335,7 +335,7 @@ class TestPostgresql(BaseTestPostgresql): self.assertTrue(self.p.is_leader()) self.p.reset_cluster_info_state(None) with patch.object(Postgresql, '_query', Mock(side_effect=RetryFailedError(''))): - self.assertRaises(PostgresConnectionException, self.p.is_leader) + self.assertFalse(self.p.is_leader()) @patch.object(Postgresql, 'controldata', Mock(return_value={'Database cluster state': 'shut down', 'Latest checkpoint location': '0/1ADBC18',