diff --git a/governor.py b/governor.py index 66967510..bf75efaa 100755 --- a/governor.py +++ b/governor.py @@ -96,7 +96,6 @@ def main(): governor.run() finally: governor.postgresql.stop() - governor.etcd.delete_member(governor.postgresql.name) governor.etcd.delete_leader(governor.postgresql.name) diff --git a/helpers/etcd.py b/helpers/etcd.py index 4ea9ba12..e808f8d4 100644 --- a/helpers/etcd.py +++ b/helpers/etcd.py @@ -9,13 +9,19 @@ logger = logging.getLogger(__name__) Member = namedtuple('Member', 'hostname,address,ttl') -Cluster = namedtuple('Cluster', 'leader,last_leader_operation,members') + + +class Cluster(namedtuple('Cluster', 'leader,last_leader_operation,members')): + + def is_unlocked(self): + return not (self.leader and self.leader.hostname) class Etcd: def __init__(self, config): self.ttl = config['ttl'] + self.member_ttl = config.get('member_ttl', 3600) self.base_client_url = 'http://{host}/v2/keys/service/{scope}'.format(**config) self.postgres_cluster = None @@ -115,14 +121,12 @@ class Etcd: def current_leader(self): try: cluster = self.get_cluster() - if not cluster.leader or not cluster.leader.address: - return None - return cluster.leader + return None if cluster.is_unlocked() else cluster.leader except: raise CurrentLeaderError("Etcd is not responding properly") def touch_member(self, member, connection_string): - return self.put_client_path('/members/' + member, value=connection_string, ttl=self.ttl) + return self.put_client_path('/members/' + member, value=connection_string, ttl=self.member_ttl) def take_leader(self, value): return self.put_client_path('/leader', value=value, ttl=self.ttl) diff --git a/helpers/ha.py b/helpers/ha.py index 89467bf4..b3e97d82 100644 --- a/helpers/ha.py +++ b/helpers/ha.py @@ -22,9 +22,6 @@ class Ha: def update_lock(self): return self.etcd.update_leader(self.state_handler) - def is_unlocked(self): - return not (self.cluster.leader and self.cluster.leader.hostname) - def has_lock(self): lock_owner = self.cluster.leader and self.cluster.leader.hostname logger.info('Lock owner: %s; I am %s', lock_owner, self.state_handler.name) @@ -48,7 +45,7 @@ class Ha: logging.info('started as readonly because i had the session lock') self.load_cluster_from_etcd() - if self.is_unlocked(): + if self.cluster.is_unlocked(): if self.state_handler.is_healthiest_node(self.cluster): if self.acquire_lock(): if not self.state_handler.is_leader(): diff --git a/helpers/postgresql.py b/helpers/postgresql.py index e1041d71..75ce0998 100644 --- a/helpers/postgresql.py +++ b/helpers/postgresql.py @@ -54,8 +54,9 @@ class Postgresql: self.config = config + connect_address = config.get('connect_address', config['listen']) or config['listen'] self.connection_string = 'postgres://{username}:{password}@{connect_address}/postgres'.format( - connect_address=self.config['connect_address'], **self.replication) + connect_address=connect_address, **self.replication) self.conn = None self.cursor_holder = None @@ -271,11 +272,12 @@ class Postgresql: def write_pg_hba(self): with open(os.path.join(self.data_dir, 'pg_hba.conf'), 'a') as f: - f.write('host replication {username} {network} md5'.format(**self.replication)) + f.write('\nhost replication {username} {network} md5\n'.format(**self.replication)) # allow TCP connections from the host's own address f.write("\nhost postgres postgres samehost trust\n") # allow TCP connections from the rest of the world with a password, prefer ssl - f.write("\nhostssl all all 0.0.0.0/0 md5\n") + if self.config['parameters'].get('ssl', 'off').lower() == 'on': + f.write("\nhostssl all all 0.0.0.0/0 md5\n") f.write("\nhost all all 0.0.0.0/0 md5\n") @staticmethod diff --git a/tests/test_ha.py b/tests/test_ha.py index 741b674f..c262c9ee 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -1,7 +1,7 @@ import unittest import requests -from helpers.etcd import Etcd +from helpers.etcd import Cluster, Etcd from helpers.ha import Ha from test_etcd import requests_get, requests_put, requests_delete @@ -50,6 +50,10 @@ class MockPostgresql: return 0 +def nop(*args, **kwargs): + pass + + class TestHa(unittest.TestCase): def __init__(self, method_name='runTest'): @@ -63,59 +67,60 @@ class TestHa(unittest.TestCase): self.p = MockPostgresql() self.e = Etcd({'ttl': 30, 'host': 'remotehost', 'scope': 'test'}) self.ha = Ha(self.p, self.e) + self.ha.cluster = Cluster(None, None, []) + self.ha.load_cluster_from_etcd = nop def test_start_as_slave(self): self.p.is_healthy = false self.assertEquals(self.ha.run_cycle(), 'started as a secondary') def test_start_as_readonly(self): + self.ha.cluster.is_unlocked = false self.p.is_leader = self.p.is_healthy = false self.ha.has_lock = true self.assertEquals(self.ha.run_cycle(), 'promoted self to leader because i had the session lock') def test_acquire_lock_as_master(self): - self.ha.is_unlocked = true self.assertEquals(self.ha.run_cycle(), 'acquired session lock as a leader') def test_promoted_by_acquiring_lock(self): - self.ha.is_unlocked = true self.p.is_leader = false self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') def test_demote_after_failing_to_obtain_lock(self): - self.ha.is_unlocked = true self.ha.acquire_lock = false self.assertEquals(self.ha.run_cycle(), 'demoted self due after trying and failing to obtain lock') def test_follow_new_leader_after_failing_to_obtain_lock(self): - self.ha.is_unlocked = true self.ha.acquire_lock = false self.p.is_leader = false self.assertEquals(self.ha.run_cycle(), 'following new leader after trying and failing to obtain lock') def test_demote_because_not_healthiest(self): - self.ha.is_unlocked = true self.p.is_healthiest_node = false self.assertEquals(self.ha.run_cycle(), 'demoting self because i am not the healthiest node') def test_follow_new_leader_because_not_healthiest(self): - self.ha.is_unlocked = true self.p.is_healthiest_node = false self.p.is_leader = false self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node') def test_promote_because_have_lock(self): + self.ha.cluster.is_unlocked = false self.ha.has_lock = true self.p.is_leader = false self.assertEquals(self.ha.run_cycle(), 'promoted self to leader because i had the session lock') def test_leader_with_lock(self): + self.ha.cluster.is_unlocked = false self.ha.has_lock = true self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock') def test_demote_because_not_having_lock(self): + self.ha.cluster.is_unlocked = false self.assertEquals(self.ha.run_cycle(), 'demoting self because i do not have the lock and i was a leader') def test_follow_the_leader(self): + self.ha.cluster.is_unlocked = false self.p.is_leader = false self.assertEquals(self.ha.run_cycle(), 'no action. i am a secondary and i am following a leader')