From b7b47ffd7967581e46ae68e99b78bac3ea879c95 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Fri, 23 Oct 2015 10:11:38 +0200 Subject: [PATCH] Add support for the nofailover tag. --- patroni/ha.py | 39 +++++++++++++++++++++++++++------------ tests/test_etcd.py | 2 +- tests/test_ha.py | 10 +++++----- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/patroni/ha.py b/patroni/ha.py index d8ff4756..7a632673 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -140,7 +140,9 @@ class Ha: reachable - `!False` if the node is not reachable or is not responding with correct JSON in_recovery - `!True` if pg_is_in_recovery() == true - xlog_location - value of `replayed_location` or `location` from JSON, dependin on its role.""" + xlog_location - value of `replayed_location` or `location` from JSON, dependin on its role. + tags - dictionary with values of different tags (i.e. nofailover) + """ try: response = requests.get(member.api_url, timeout=2, verify=False) @@ -148,10 +150,11 @@ class Ha: json = response.json() is_master = json['role'] == 'master' xlog_location = json['xlog']['location' if is_master else 'replayed_location'] - return (member, True, not is_master, xlog_location) + tags = json.get('tags', dict()) + return (member, True, not is_master, xlog_location, tags) except: logging.exception('request failed: GET %s', member.api_url) - return (member, False, None, 0) + return (member, False, None, 0, {}) def fetch_nodes_statuses(self, members): pool = ThreadPool(len(members)) @@ -166,16 +169,20 @@ class Ha: if self.state_handler.is_leader(): return True + if self.patroni.tags.get('nofailover') is True: + return False + if check_replication_lag and not self.state_handler.check_replication_lag(self.cluster.last_leader_operation): return False # Too far behind last reported xlog location on master # Prepare list of nodes to run check against - members = [m for m in members if m.name != self.state_handler.name and m.api_url] + members = [m for m in members if m.name != self.state_handler.name + and not m.data.get('tags', {}).get('nofailover', None) and m.api_url] if members: my_xlog_location = self.state_handler.xlog_position() - for member, reachable, in_recovery, xlog_location in self.fetch_nodes_statuses(members): - if reachable: # If the node is unreachable it's not healhy + for member, reachable, in_recovery, xlog_location, tags in self.fetch_nodes_statuses(members): + if reachable and not tags.get('nofailover'): # If the node is unreachable it's not healhy if not in_recovery: logger.warning('Master (%s) is still alive', member.name) return False @@ -187,11 +194,13 @@ class Ha: ret = False members = [m for m in members if m.name != self.state_handler.name and m.api_url] if members: - for member, reachable, in_recovery, xlog_location in self.fetch_nodes_statuses(members): - if reachable: + for member, reachable, in_recovery, xlog_location, tags in self.fetch_nodes_statuses(members): + if reachable and not tags.get('nofailover'): ret = True # TODO: check xlog_location - else: + elif not reachable: logger.info('Member %s is not reachable', member.name) + elif tags.get('nofailover'): + logger.info('Member %s is not allowed to promote', member.name) else: logger.warning('manual failover: members list is empty') return ret @@ -205,12 +214,15 @@ class Ha: # find specific node and check that it is healthy members = [m for m in self.cluster.members if m.name == failover.member] if members: - member, reachable, in_recovery, xlog_location = self.fetch_node_status(members[0]) - if reachable: # node is healthy + member, reachable, in_recovery, xlog_location, tags = self.fetch_node_status(members[0]) + if reachable and not tags.get('nofailover'): # node is healthy logger.info('manual failover: to %s, i am %s', member.name, self.state_handler.name) return False # we wanted to failover to specific member but it is not healthy - logger.warning('manual failover: member %s is unhealthy', member.name) + if not reachable: + logger.warning('manual failover: member %s is unhealthy', member.name) + elif tags.get('nofailover'): + logger.warning('manual failover: member %s is not allowed to promote', member.name) # at this point we should consider all members as a candidates for failover # i.e. we assume that failover.member is None @@ -278,6 +290,9 @@ class Ha: return self.follow_the_leader('demoted self due after trying and failing to obtain lock', 'following new leader after trying and failing to obtain lock') else: + if self.patroni.tags.get('nofailover'): + return self.follow_the_leader('demoting self because I am not allowed to become master', + 'following a different leader because I am not allowed to promote') return self.follow_the_leader('demoting self because i am not the healthiest node', 'following a different leader because i am not the healthiest node') diff --git a/tests/test_etcd.py b/tests/test_etcd.py index 53c054e5..6b9df83f 100644 --- a/tests/test_etcd.py +++ b/tests/test_etcd.py @@ -50,7 +50,7 @@ def requests_get(url, **kwargs): if url.startswith('http://local'): raise requests.exceptions.RequestException() elif ':8011/patroni' in url: - response.content = '{"role": "replica", "xlog": {"replayed_location": 0}}' + response.content = '{"role": "replica", "xlog": {"replayed_location": 0}, "tags": {}}' elif url.endswith('/members'): if url.startswith('http://error'): response.content = '[{}]' diff --git a/tests/test_ha.py b/tests/test_ha.py index d149746c..32248c61 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -275,21 +275,21 @@ class TestHa(unittest.TestCase): self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader')) self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') - self.ha.fetch_node_status = lambda e: (e, True, True, 0) # accessible, in_recovery + self.ha.fetch_node_status = lambda e: (e, True, True, 0, {}) # accessible, in_recovery self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node') self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, MockPostgresql.name, '')) self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node') - self.ha.fetch_node_status = lambda e: (e, False, True, 0) # accessible, in_recovery + self.ha.fetch_node_status = lambda e: (e, False, True, 0, {}) # accessible, in_recovery self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') def test__is_healthiest_node(self): self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members)) self.p.is_leader = false - self.ha.fetch_node_status = lambda e: (e, True, True, 0) # accessible, in_recovery + self.ha.fetch_node_status = lambda e: (e, True, True, 0, {}) # accessible, in_recovery self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members)) - self.ha.fetch_node_status = lambda e: (e, True, False, 0) # accessible, not in_recovery + self.ha.fetch_node_status = lambda e: (e, True, False, 0, {}) # accessible, not in_recovery self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members)) - self.ha.fetch_node_status = lambda e: (e, True, True, 1) # accessible, in_recovery, xlog location ahead + self.ha.fetch_node_status = lambda e: (e, True, True, 1, {}) # accessible, in_recovery, xlog location ahead self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members)) self.p.check_replication_lag = false self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))