From fa7d36da9b1984f0cbe94299c13da58ba2911cba Mon Sep 17 00:00:00 2001 From: Josh Berkus Date: Thu, 22 Oct 2015 17:21:39 -0700 Subject: [PATCH] Merged basebackup into postgresql.py; changed things to provide alternative, configurable basebackup methods. --- README.rst | 8 ++++ patroni/postgresql.py | 90 ++++++++++++++++++++++++++++++++++++++----- postgres0.yml | 2 +- postgres1.yml | 2 +- 4 files changed, 91 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index 7c6bc3f8..b87438a3 100644 --- a/README.rst +++ b/README.rst @@ -108,6 +108,14 @@ settings: - *recovery\_conf*: additional configuration settings written to recovery.conf when configuring follower - *parameters*: list of configuration settings for Postgres. Many of these are required for replication to work. + + - *create_replica_methods*: an ordered list of the create methods for turning a patroni node into a new replica. + "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured + as its own config item. + + - *{replica_method}* for each create_replica_method other than basebackup, you would add a configuration section + of the same name. At a minimum, this should include "command" with a full path to the actual script to be + executed. Other configuration parameters will be passed along to the script in the form "parameter=value". Replication choices ------------------- diff --git a/patroni/postgresql.py b/patroni/postgresql.py index 81f5c02e..a68b8dfd 100644 --- a/patroni/postgresql.py +++ b/patroni/postgresql.py @@ -134,18 +134,64 @@ class Postgresql: @staticmethod def build_connstring(conn): - return "host={host} port={port} user={user}".format(**conn) + mconn = "" + for param, val in conn.iteritems(): + mconn = mconn + "{0}={1} ".format(param, val) + + return mconn def create_replica(self, master_connection, env): + # create the replica according to the replica_method + # defined by the user. this is a list, so we need to + # loop through all methods the user supplies connstring = self.build_connstring(master_connection) - cmd = self.config['restore'] - try: - ret = subprocess.call(shlex.split(cmd) + [self.scope, "replica", self.data_dir, connstring], env=env) - self.delete_trigger_file() - except: - logger.exception('Error when creating replica') - return 1 - return ret + env = os.environ.copy() + env['PGPASSFILE'] = 'pgpass' + # get list of replica methods from config + replica_list = self.config.get('create_replica_method', 'basebackup') + replica_methods = [rm.strip() for rm in replica_list.split(',')] + # go through them in priority order + for replica_method in replica_methods: + # if the method is basebackup, then use the built-in + if replica_method == "basebackup": + ret = self.basebackup(connstring, env) + if ret == 0: + # if basebackup succeeds, exit with success + return 0 + else: + # user-defined method; check for configuration + # not required, actually + if replica_method in self.config: + # look to see if the user has supplied a full command path + # if not, use the method name as the command + if "command" in self.config[replica_method]: + cmd = self.config[replica_method]["command"] + else: + cmd = replica_method + + # get the rest of the replica config + method_config = self.config[replica_method].copy() + # remove the command and turn it into a shlex set + del method_config["command"] + # add the default parameters + method_config.update({"scope": self.scope, + "role" : "replica", + "datadir" : self.data_dir, + "connstring" : self.connstring}) + params = ["--{0}={1}".format(arg, val) for arg, val in method_config.iteritems()] + + try: + # call script with the full set of parameters + ret = subprocess.call(shlex.split(cmd) + shlex.split(method_config), env=env) + # if we succeeded, stop + if ret == 0: + return ret + except Exception as e: + logger.exception('Error creating replica using method {0}: {1}'.format(replica_method, e.str)) + ret = 1 + + # out of methods, return 1 + return 1 def is_leader(self): return not self.query('SELECT pg_is_in_recovery()').fetchone()[0] @@ -427,3 +473,29 @@ recovery_target_timeline = 'latest' except: logger.exception('Could not remove data directory %s', self.data_dir) self.move_data_directory() + + def basebackup(self, master_connection, env): + # creates a replica data dir using pg_basebackup. + # this is the default, built-in create_replica_method + # tries twice, then returns failure (as 1) + # uses "stream" as the xlog-method to avoid sync issues + bbfailures = 0; + maxfailures = 2; + ret = 1 + while bbfailures < maxfailures: + try: + ret = subprocess.call(['pg_basebackup', '-R', '--pgdata=%s' % self.data_dir, + '--xlog-method=stream', "--dbname=%s" % master_connection], + env=env) + if ret == 0: + break + + except Exception as e: + logger.error('Error when fetching backup with pg_basebackup: {0}'.format(e)) + + bbfailures += 1 + if bbfailures < maxfailures: + logger.error('Trying again in 5 seconds') + time.sleep(5) + + return ret diff --git a/postgres0.yml b/postgres0.yml index f800183a..50f4065f 100644 --- a/postgres0.yml +++ b/postgres0.yml @@ -50,7 +50,7 @@ postgresql: env_dir: /home/postgres/etc/wal-e.d/env threshold_megabytes: 10240 threshold_backup_size_percentage: 30 - restore: patroni/scripts/restore.py + create_replica_method: basebackup #recovery_conf: #restore_command: cp ../wal_archive/%f %p parameters: diff --git a/postgres1.yml b/postgres1.yml index e1c3e663..dd2e2369 100644 --- a/postgres1.yml +++ b/postgres1.yml @@ -52,7 +52,7 @@ postgresql: env_dir: /home/postgres/etc/wal-e.d/env threshold_megabytes: 10240 threshold_backup_size_percentage: 30 - restore: patroni/scripts/restore.py + create_replica_method: basebackup parameters: archive_mode: "on" wal_level: hot_standby