From e2faf641d53896e77503fe445b5887fe4ae8e738 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Mon, 11 May 2015 16:50:21 +0200 Subject: [PATCH] Start slave with the correct recovery.conf on governor start Also governor is able to pick up already running master and slave instances without restarting them. In case is you have a lock and postgres is not running behaviour remains the same: it will start master in read only mode and then promote if it still has the lock. --- governor.py | 22 ++++--------- helpers/etcd.py | 2 +- helpers/ha.py | 83 ++++++++++++++++++++++++++----------------------- 3 files changed, 51 insertions(+), 56 deletions(-) diff --git a/governor.py b/governor.py index bbca37a0..b3b2061f 100755 --- a/governor.py +++ b/governor.py @@ -1,11 +1,10 @@ #!/usr/bin/env python -import sys -import yaml -import time -import urllib2 import atexit import logging +import sys +import time +import yaml from helpers.etcd import Etcd from helpers.postgresql import Postgresql @@ -30,14 +29,9 @@ def stop_postgresql(): atexit.register(stop_postgresql) # wait for etcd to be available -etcd_ready = False -while not etcd_ready: - try: - etcd.touch_member(postgresql.name, postgresql.connection_string) - etcd_ready = True - except urllib2.URLError: - logging.info("waiting on etcd") - time.sleep(5) +while not etcd.touch_member(postgresql.name, postgresql.connection_string): + logging.info("waiting on etcd") + time.sleep(5) # is data directory empty? if postgresql.data_directory_empty(): @@ -60,11 +54,7 @@ if postgresql.data_directory_empty(): synced_from_leader = True else: time.sleep(5) -else: - postgresql.write_recovery_conf(None) - postgresql.start() while True: logging.info(ha.run_cycle()) - time.sleep(config["loop_wait"]) diff --git a/helpers/etcd.py b/helpers/etcd.py index 7e8182f8..112e8e4d 100644 --- a/helpers/etcd.py +++ b/helpers/etcd.py @@ -103,7 +103,7 @@ class Etcd: raise CurrentLeaderError("Etcd is not responding properly") def touch_member(self, member, connection_string): - self.put_client_path('/members/' + member, value=connection_string) + return self.put_client_path('/members/' + member, value=connection_string) def take_leader(self, value): return self.put_client_path('/leader', value=value, ttl=self.ttl) diff --git a/helpers/ha.py b/helpers/ha.py index da8e0d18..88b9bd6c 100644 --- a/helpers/ha.py +++ b/helpers/ha.py @@ -44,55 +44,60 @@ class Ha: def run_cycle(self): try: - if self.state_handler.is_healthy(): - self.load_cluster_from_etcd() - if self.is_unlocked(): - if self.state_handler.is_healthiest_node(self.cluster.members): - if self.acquire_lock(): - if not self.state_handler.is_leader(): - self.state_handler.promote() - return "promoted self to leader by acquiring session lock" - return "acquired session lock as a leader" - else: - self.load_cluster_from_etcd() - if self.state_handler.is_leader(): - self.demote() - return "demoted self due after trying and failing to obtain lock" - else: - self.follow_the_leader() - return "following new leader after trying and failing to obtain lock" + self.load_cluster_from_etcd() + if self.is_unlocked(): + if not self.state_handler.is_healthy(): + return 'no action. not healthy enough to do anything.' + elif self.state_handler.is_healthiest_node(self.cluster.members): + if self.acquire_lock(): + if not self.state_handler.is_leader(): + self.state_handler.promote() + return "promoted self to leader by acquiring session lock" + return "acquired session lock as a leader" else: self.load_cluster_from_etcd() if self.state_handler.is_leader(): self.demote() - return "demoting self because i am not the healthiest node" + return "demoted self due after trying and failing to obtain lock" else: self.follow_the_leader() - return "following a different leader because i am not the healthiest node" + return "following new leader after trying and failing to obtain lock" else: - if self.has_lock() and self.update_lock(): - try: - if not self.state_handler.is_leader(): - self.state_handler.promote() - return "promoted self to leader because i had the session lock" - else: - return "no action. i am the leader with the lock" - finally: - # create replication slots - self.state_handler.create_replication_slots([m.hostname for m in self.cluster.members]) + self.load_cluster_from_etcd() + if self.state_handler.is_leader(): + self.demote() + return "demoting self because i am not the healthiest node" else: - logger.info("does not have lock") - if self.state_handler.is_leader(): - self.demote() - return "demoting self because i do not have the lock and i was a leader" - else: - self.follow_the_leader() - return "no action. i am a secondary and i am following a leader" + self.follow_the_leader() + return "following a different leader because i am not the healthiest node" else: - if not self.state_handler.is_running(): # XXX is_running == is_healthy + if self.has_lock() and not self.state_handler.is_healthy(): + self.state_handler.write_recovery_conf(None) self.state_handler.start() - return "postgresql was stopped. starting again." - return "no action. not healthy enough to do anything." + self.load_cluster_from_etcd() + + if self.has_lock() and self.update_lock(): + try: + if not self.state_handler.is_leader(): + self.state_handler.promote() + return "promoted self to leader because i had the session lock" + else: + return "no action. i am the leader with the lock" + finally: + # create replication slots + self.state_handler.create_replication_slots([m.hostname for m in self.cluster.members]) + else: + logger.info("does not have lock") + if not self.state_handler.is_healthy(): + self.state_handler.write_recovery_conf(self.cluster.leader) + self.state_handler.start() + return 'starting as a secondary' + elif self.state_handler.is_leader(): + self.demote() + return "demoting self because i do not have the lock and i was a leader" + else: + self.follow_the_leader() + return "no action. i am a secondary and i am following a leader" except EtcdError: logger.error("Error communicating with Etcd") except OperationalError: