From 953ea749bf902e91ea6f957d6fd68ec8fd580dea Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Tue, 13 Oct 2015 15:00:16 +0200 Subject: [PATCH 1/2] Make sure patroni is not using stale connections. After the PostgreSQL crash (i.e. with kill -9), the backend patroni connects to may still exist. In this case, patroni will get stale postgres role from this backend, preventing a restarted node with a leader lock from being promoted. Easily reproducible and also observed in a staging environment after the postgres crash due to out of disk space. --- patroni/postgresql.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/patroni/postgresql.py b/patroni/postgresql.py index fc7956e9..a7f95a81 100644 --- a/patroni/postgresql.py +++ b/patroni/postgresql.py @@ -127,9 +127,15 @@ class Postgresql: def _cursor(self): if not self._cursor_holder or self._cursor_holder.closed or self._cursor_holder.connection.closed != 0: + logger.info("established a new patroni connection to the postgres cluster") self._cursor_holder = self.connection().cursor() return self._cursor_holder + def close_connection(self): + if self._cursor_holder and self._cursor_holder.connection and self._cursor_holder.connection.closed == 0: + self._cursor_holder.connection.close() + logger.info("closed patroni connection to the postgresql cluster") + def _query(self, sql, *params): cursor = None try: @@ -269,6 +275,12 @@ class Postgresql: logging.exception('Exception during CHECKPOINT') def stop(self, mode='fast', block_callbacks=False): + # make sure we close all connections established against + # the former node, otherwise, we might get a stalled one + # after kill -9, which would report incorrect data to + # patroni. + + self.close_connection() if not self.is_running(): if not block_callbacks: self.set_state('stopped') From 921e4fc32357d7b7fdecc57b76c5a7eb84aa6e5e Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 16 Oct 2015 10:28:15 +0200 Subject: [PATCH 2/2] psycopg2 should be not older than 2.6.1 --- requirements-py2.txt | 2 +- requirements-py3.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-py2.txt b/requirements-py2.txt index d57df720..fde9c79a 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -1,7 +1,7 @@ boto dnspython mock -psycopg2 +psycopg2>=2.6.1 PyYAML requests six >= 1.7 diff --git a/requirements-py3.txt b/requirements-py3.txt index 0fd9dfb3..cc00965b 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,7 +1,7 @@ boto mock dnspython3 -psycopg2 +psycopg2>=2.6.1 PyYAML requests six