From 09a7cf265dacbef62dcb66a87c8f9ad5cc956f10 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Mon, 11 Nov 2019 09:37:06 +0100 Subject: [PATCH] Fix 'start failed' issue (#1262) The start of postgres happens in two stages: 1. First Patroni is waiting for postgres port to be open 2. After that, it is waiting for postgres starts to accept connections There is a default timeout 60 seconds for both stages (in total). When the port isn't open, pg_isready exits with code=2. If postgres is rejecting connections due to recovery, exit code=1. In most cases postgres quickly opens the port and pg_isready starts returning 1, but in rare cases the whole timeout could spend in `1.` After that, the HA loop is still waiting for postgres to start, but executing only the check from `2.`. Since pg_isready exit code is still = 2, Patroni was falsely assuming that 'start failed' without taking into consideration the fact that the postmaster process is up and running. Fixes https://github.com/zalando/patroni/issues/1160 --- patroni/postgresql/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/patroni/postgresql/__init__.py b/patroni/postgresql/__init__.py index 08a2ce32..5dfa78c0 100644 --- a/patroni/postgresql/__init__.py +++ b/patroni/postgresql/__init__.py @@ -561,10 +561,12 @@ class Postgresql(object): if ready == STATE_REJECT: return False elif ready == STATE_NO_RESPONSE: - self.set_state('start failed') - self.slots_handler.schedule(False) # TODO: can remove this? - self.config.save_configuration_files(True) # TODO: maybe remove this? - return True + ret = not self.is_running() + if ret: + self.set_state('start failed') + self.slots_handler.schedule(False) # TODO: can remove this? + self.config.save_configuration_files(True) # TODO: maybe remove this? + return ret else: if ready != STATE_RUNNING: # Bad configuration or unexpected OS error. No idea of PostgreSQL status.