From c5285bc293fd786e71fa8025a1bb64c0bc4b5550 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Wed, 9 Dec 2015 13:17:36 +0100 Subject: [PATCH 1/2] Call a checkpoint on master before pg_rewind. PostgreSQL does not run a checkpoint during promition. Since pg_rewind relies on the last checkpoint to get the timeline, there is a short race condition right after the promotion, when it can get the timeline wrong and fail. We work around this by calling the checkpoint manually. Make sure our test configuration does both archive and recovery. --- patroni/postgresql.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/patroni/postgresql.py b/patroni/postgresql.py index fba9a22e..0525fbcc 100644 --- a/patroni/postgresql.py +++ b/patroni/postgresql.py @@ -342,13 +342,13 @@ class Postgresql: ret and not block_callbacks and self.call_nowait(ACTION_ON_START) return ret - def checkpoint(self): + def checkpoint(self, connstring=None): try: - r = parseurl('postgres://{}/postgres'.format(self.local_address)) - r['options'] = '-c statement_timeout=0' - with psycopg2.connect(**r) as conn: + connstring = connstring or 'postgres://{}/postgres'.format(self.local_address) + with psycopg2.connect(connstring) as conn: conn.autocommit = True with conn.cursor() as cur: + cur.execute("SET statement_timeout = 0") cur.execute('CHECKPOINT') except: logging.exception('Exception during CHECKPOINT') @@ -454,6 +454,8 @@ recovery_target_timeline = 'latest' r['user'] = r['username'] env = self.write_pgpass(r) pc = "user={user} host={host} port={port} dbname=postgres sslmode=prefer sslcompression=1".format(**r) + # first run a checkpoint on a promoted master in order to make it store the new timeline (5540277D.8020309@iki.fi) + self.checkpoint(pc) logger.info("running pg_rewind from {}".format(pc)) pg_rewind = ['pg_rewind', '-D', self.data_dir, '--source-server', pc] try: From d0c84c87ba45f897ba2de42985d343f2b95f44b5 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Wed, 9 Dec 2015 13:56:32 +0100 Subject: [PATCH 2/2] Fix the formatting, add the missing changes to configuration files. --- patroni/postgresql.py | 3 ++- postgres0.yml | 4 +++- postgres1.yml | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/patroni/postgresql.py b/patroni/postgresql.py index 0525fbcc..5ea2b97c 100644 --- a/patroni/postgresql.py +++ b/patroni/postgresql.py @@ -454,7 +454,8 @@ recovery_target_timeline = 'latest' r['user'] = r['username'] env = self.write_pgpass(r) pc = "user={user} host={host} port={port} dbname=postgres sslmode=prefer sslcompression=1".format(**r) - # first run a checkpoint on a promoted master in order to make it store the new timeline (5540277D.8020309@iki.fi) + # first run a checkpoint on a promoted master in order + # to make it store the new timeline (5540277D.8020309@iki.fi) self.checkpoint(pc) logger.info("running pg_rewind from {}".format(pc)) pg_rewind = ['pg_rewind', '-D', self.data_dir, '--source-server', pc] diff --git a/postgres0.yml b/postgres0.yml index 84c74acd..36018ad5 100644 --- a/postgres0.yml +++ b/postgres0.yml @@ -82,10 +82,12 @@ postgresql: #use_iam: 1 #recovery_conf: #restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1 + recovery_conf: + restore_command: cp ../wal_archive/%f %p parameters: archive_mode: "on" wal_level: hot_standby - archive_command: mkdir -p ../wal_archive && cp %p ../wal_archive/%f + archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f max_wal_senders: 5 wal_keep_segments: 8 archive_timeout: 1800s diff --git a/postgres1.yml b/postgres1.yml index 32919843..e1b61b3b 100644 --- a/postgres1.yml +++ b/postgres1.yml @@ -82,10 +82,12 @@ postgresql: #use_iam: 1 #recovery_conf: #restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1 + recovery_conf: + restore_command: cp ../wal_archive/%f %p parameters: archive_mode: "on" wal_level: hot_standby - archive_command: mkdir -p ../wal_archive && cp %p ../wal_archive/%f + archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f max_wal_senders: 5 wal_keep_segments: 8 archive_timeout: 1800s