diff --git a/helpers/ha.py b/helpers/ha.py index 88b9bd6c..3131a235 100644 --- a/helpers/ha.py +++ b/helpers/ha.py @@ -33,8 +33,9 @@ class Ha: return not (self.cluster.leader and self.cluster.leader.hostname) def has_lock(self): - logger.info('Lock owner: %s; I am %s', self.cluster.leader.hostname, self.state_handler.name) - return self.cluster.leader.hostname == self.state_handler.name + lock_owner = self.cluster.leader and self.cluster.leader.hostname + logger.info('Lock owner: %s; I am %s', lock_owner, self.state_handler.name) + return lock_owner == self.state_handler.name def demote(self): return self.state_handler.demote(self.cluster.leader) @@ -45,65 +46,63 @@ class Ha: def run_cycle(self): try: self.load_cluster_from_etcd() + if not self.state_handler.is_healthy(): + has_lock = self.has_lock() + self.state_handler.write_recovery_conf(None if has_lock else self.cluster.leader) + self.state_handler.start() + if not has_lock: + return 'started as a secondary' + logging.info('started as readonly because i had the session lock') + self.load_cluster_from_etcd() + if self.is_unlocked(): - if not self.state_handler.is_healthy(): - return 'no action. not healthy enough to do anything.' - elif self.state_handler.is_healthiest_node(self.cluster.members): + if self.state_handler.is_healthiest_node(self.cluster.members): if self.acquire_lock(): if not self.state_handler.is_leader(): self.state_handler.promote() - return "promoted self to leader by acquiring session lock" - return "acquired session lock as a leader" + return 'promoted self to leader by acquiring session lock' + return 'acquired session lock as a leader' else: self.load_cluster_from_etcd() if self.state_handler.is_leader(): self.demote() - return "demoted self due after trying and failing to obtain lock" + return 'demoted self due after trying and failing to obtain lock' else: self.follow_the_leader() - return "following new leader after trying and failing to obtain lock" + return 'following new leader after trying and failing to obtain lock' else: self.load_cluster_from_etcd() if self.state_handler.is_leader(): self.demote() - return "demoting self because i am not the healthiest node" + return 'demoting self because i am not the healthiest node' else: self.follow_the_leader() - return "following a different leader because i am not the healthiest node" + return 'following a different leader because i am not the healthiest node' else: - if self.has_lock() and not self.state_handler.is_healthy(): - self.state_handler.write_recovery_conf(None) - self.state_handler.start() - self.load_cluster_from_etcd() - if self.has_lock() and self.update_lock(): try: if not self.state_handler.is_leader(): self.state_handler.promote() - return "promoted self to leader because i had the session lock" + return 'promoted self to leader because i had the session lock' else: - return "no action. i am the leader with the lock" + return 'no action. i am the leader with the lock' finally: # create replication slots self.state_handler.create_replication_slots([m.hostname for m in self.cluster.members]) else: - logger.info("does not have lock") - if not self.state_handler.is_healthy(): - self.state_handler.write_recovery_conf(self.cluster.leader) - self.state_handler.start() - return 'starting as a secondary' - elif self.state_handler.is_leader(): + logger.info('does not have lock') + if self.state_handler.is_leader(): self.demote() - return "demoting self because i do not have the lock and i was a leader" + return 'demoting self because i do not have the lock and i was a leader' else: self.follow_the_leader() - return "no action. i am a secondary and i am following a leader" + return 'no action. i am a secondary and i am following a leader' except EtcdError: - logger.error("Error communicating with Etcd") + logger.error('Error communicating with Etcd') except OperationalError: - logger.error("Error communicating with Postgresql. Will try again.") + logger.error('Error communicating with Postgresql. Will try again') except HealthiestMemberError: - logger.error("failed to determine healthiest member fromt etcd") + logger.error('failed to determine healthiest member fromt etcd') def run(self): while True: