From 9dcbc75fd26d03b4a8d599bfba3e8b9892d87959 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Tue, 12 May 2015 10:19:45 +0200 Subject: [PATCH 1/2] Bugfix. It wasn't possible to start the old cluster if all its members were shutdown. Basically this is rollback to the original decision tree with the small exception: 1 - If leader is defined and it's not me - then slave would be started immidiately with the correct recovery conf. 2 - If leader is defined in and it's my host, then it will start instance in readonly (but without primary_conninfo in recovery.conf) 3 - And the third case - if the leader is not defined - it also will start instance in readonly, without primary_conninfo. After performing 2 or 3 it will perform usual decision tree. --- helpers/ha.py | 52 +++++++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/helpers/ha.py b/helpers/ha.py index 88b9bd6c..0399b0fe 100644 --- a/helpers/ha.py +++ b/helpers/ha.py @@ -45,65 +45,63 @@ class Ha: def run_cycle(self): try: self.load_cluster_from_etcd() + if not self.state_handler.is_healthy(): + has_lock = self.has_lock() + self.state_handler.write_recovery_conf(None if has_lock else self.cluster.leader) + self.state_handler.start() + if not has_lock: + return 'started as a secondary' + logging.info('started as readonly because i had the session lock') + self.load_cluster_from_etcd() + if self.is_unlocked(): - if not self.state_handler.is_healthy(): - return 'no action. not healthy enough to do anything.' - elif self.state_handler.is_healthiest_node(self.cluster.members): + if self.state_handler.is_healthiest_node(self.cluster.members): if self.acquire_lock(): if not self.state_handler.is_leader(): self.state_handler.promote() - return "promoted self to leader by acquiring session lock" - return "acquired session lock as a leader" + return 'promoted self to leader by acquiring session lock' + return 'acquired session lock as a leader' else: self.load_cluster_from_etcd() if self.state_handler.is_leader(): self.demote() - return "demoted self due after trying and failing to obtain lock" + return 'demoted self due after trying and failing to obtain lock' else: self.follow_the_leader() - return "following new leader after trying and failing to obtain lock" + return 'following new leader after trying and failing to obtain lock' else: self.load_cluster_from_etcd() if self.state_handler.is_leader(): self.demote() - return "demoting self because i am not the healthiest node" + return 'demoting self because i am not the healthiest node' else: self.follow_the_leader() - return "following a different leader because i am not the healthiest node" + return 'following a different leader because i am not the healthiest node' else: - if self.has_lock() and not self.state_handler.is_healthy(): - self.state_handler.write_recovery_conf(None) - self.state_handler.start() - self.load_cluster_from_etcd() - if self.has_lock() and self.update_lock(): try: if not self.state_handler.is_leader(): self.state_handler.promote() - return "promoted self to leader because i had the session lock" + return 'promoted self to leader because i had the session lock' else: - return "no action. i am the leader with the lock" + return 'no action. i am the leader with the lock' finally: # create replication slots self.state_handler.create_replication_slots([m.hostname for m in self.cluster.members]) else: - logger.info("does not have lock") - if not self.state_handler.is_healthy(): - self.state_handler.write_recovery_conf(self.cluster.leader) - self.state_handler.start() - return 'starting as a secondary' - elif self.state_handler.is_leader(): + logger.info('does not have lock') + if self.state_handler.is_leader(): self.demote() - return "demoting self because i do not have the lock and i was a leader" + return 'demoting self because i do not have the lock and i was a leader' else: self.follow_the_leader() - return "no action. i am a secondary and i am following a leader" + return 'no action. i am a secondary and i am following a leader' except EtcdError: - logger.error("Error communicating with Etcd") + logger.error('Error communicating with Etcd') except OperationalError: - logger.error("Error communicating with Postgresql. Will try again.") + logger.error('Error communicating with Postgresql. Will try again') except HealthiestMemberError: - logger.error("failed to determine healthiest member fromt etcd") + logger.error('failed to determine healthiest member fromt etcd') def run(self): while True: From 281b0b8455c83dfec5a97fdf5ed2d71575d45111 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Tue, 12 May 2015 11:07:52 +0200 Subject: [PATCH 2/2] Bugfix in has_lock: sometimes leader can be undefined --- helpers/ha.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/helpers/ha.py b/helpers/ha.py index 0399b0fe..3131a235 100644 --- a/helpers/ha.py +++ b/helpers/ha.py @@ -33,8 +33,9 @@ class Ha: return not (self.cluster.leader and self.cluster.leader.hostname) def has_lock(self): - logger.info('Lock owner: %s; I am %s', self.cluster.leader.hostname, self.state_handler.name) - return self.cluster.leader.hostname == self.state_handler.name + lock_owner = self.cluster.leader and self.cluster.leader.hostname + logger.info('Lock owner: %s; I am %s', lock_owner, self.state_handler.name) + return lock_owner == self.state_handler.name def demote(self): return self.state_handler.demote(self.cluster.leader)