From 9057ddeb7c80fb2fd54af83a09d62018beabc137 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Thu, 10 Mar 2016 16:06:31 +0100 Subject: [PATCH] First implementation of cloning from the replica. At the moment we just replace the master with the node at the 'clonefrom' tag if it's present. Master should be available anyway, otherwise, it will not even try to do cloning. Acceptance tests: https://github.com/zalando/patroni/pull/144/commits --- patroni/__init__.py | 4 ++++ patroni/dcs.py | 7 +++++++ patroni/ha.py | 4 +++- patroni/postgresql.py | 13 +++++++------ tests/test_ha.py | 1 + 5 files changed, 22 insertions(+), 7 deletions(-) diff --git a/patroni/__init__.py b/patroni/__init__.py index 36c8feac..85b6c5df 100644 --- a/patroni/__init__.py +++ b/patroni/__init__.py @@ -35,6 +35,10 @@ class Patroni(object): def replicatefrom(self): return self.tags.get('replicatefrom') + @property + def clonefrom(self): + return self.tags.get('clonefrom') + @staticmethod def get_dcs(name, config): if 'etcd' in config: diff --git a/patroni/dcs.py b/patroni/dcs.py index 4c2dddad..b705105d 100644 --- a/patroni/dcs.py +++ b/patroni/dcs.py @@ -72,6 +72,10 @@ class Member(namedtuple('Member', 'index,name,session,data')): def replicatefrom(self): return self.data.get('tags', {}).get('replicatefrom') + @property + def clonefrom(self): + return self.data.get('tags', {}).get('clonefrom') + class Leader(namedtuple('Leader', 'index,session,member')): @@ -147,6 +151,9 @@ class Cluster(namedtuple('Cluster', 'initialize,leader,last_leader_operation,mem def has_member(self, member_name): return any(m for m in self.members if m.name == member_name) + def get_member(self, member_name): + return ([m for m in self.members if m.name == member_name] or [None])[0] + class AbstractDCS(object): diff --git a/patroni/ha.py b/patroni/ha.py index 98ba4398..61e27814 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -76,7 +76,9 @@ class Ha(object): def bootstrap(self): if not self.cluster.is_unlocked(): # cluster already has leader self._async_executor.schedule('bootstrap from leader') - self._async_executor.run_async(self.clone, args=(self.cluster.leader, )) + clonefrom = self.patroni.clonefrom + source = self.cluster.get_member(clonefrom) if self.cluster.has_member(clonefrom) else self.cluster.leader + self._async_executor.run_async(self.clone, args=(source,)) return 'trying to bootstrap from leader' elif not self.cluster.initialize and not self.patroni.nofailover: # no initialize key if self.dcs.initialize(create_new=True): # race for initialization diff --git a/patroni/postgresql.py b/patroni/postgresql.py index a1cedb58..da561f8a 100644 --- a/patroni/postgresql.py +++ b/patroni/postgresql.py @@ -234,6 +234,7 @@ class Postgresql(object): return env def sync_replica(self, leader): + # add either the leader's or replica's credentials to pgpass env = self.write_pgpass(parseurl(leader.conn_url)) if leader else os.environ.copy() if self.create_replica(leader, env) == 0: self.delete_trigger_file() @@ -258,23 +259,23 @@ class Postgresql(object): replica_methods = self.config.get('create_replica_method', []) return any(self.replica_method_can_work_without_leader(replica_method) for replica_method in replica_methods) - def create_replica(self, leader, env): + def create_replica(self, source, env): # create the replica according to the replica_method # defined by the user. this is a list, so we need to # loop through all methods the user supplies - connstring = leader.conn_url if leader else "" + connstring = source.conn_url if source else "" # get list of replica methods from config. # If there is no configuration key, or no value is specified, use basebackup replica_methods = self.config.get('create_replica_method') or ['basebackup'] - # if we don't have any leader, leave only replica methods that work without it - replica_methods = [r for r in replica_methods if self.replica_method_can_work_without_leader(r)] if not leader \ + # if we don't have any source, leave only replica methods that work without it + replica_methods = [r for r in replica_methods if self.replica_method_can_work_without_leader(r)] if not source \ else replica_methods # go through them in priority order ret = 1 for replica_method in replica_methods: # if the method is basebackup, then use the built-in if replica_method == "basebackup": - ret = self.basebackup(leader, env) + ret = self.basebackup(source, env) if ret == 0: logger.info("replica has been created using basebackup") # if basebackup succeeds, exit with success @@ -702,7 +703,7 @@ $$""".format(name, options), name, password, password) """ Populate PostgreSQL data directory by doing one of the following: - create with initdb if there is no master. - - initialize the replica from an existing master + - initialize the replica from an existing master or replica - initialize the replica using the replica creation method that works without the master (i.e. restore from on-disk base backup) diff --git a/tests/test_ha.py b/tests/test_ha.py index 98b1ad1c..2b5e0ada 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -57,6 +57,7 @@ class MockPatroni(object): self.nap_time = 10 self.replicatefrom = None self.api.connection_string = 'http://127.0.0.1:8008' + self.clonefrom = None def run_async(func, args=()):