gitops/patroni
1
0
Fork 0
mirror of https://github.com/outbackdingo/patroni.git synced 2026-08-26 15:40:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add support for the nofailover tag.

Browse Source
This commit is contained in:
Oleksii Kliukin
2015-10-23 10:11:38 +02:00
parent 2200a4ce8c
commit b7b47ffd79
3 changed files with 33 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
patroni/ha.py
+27 -12
View File
@@ -140,7 +140,9 @@ class Ha:
reachable - `!False` if the node is not reachable or is not responding with correct JSON
in_recovery - `!True` if pg_is_in_recovery() == true
xlog_location - value of `replayed_location` or `location` from JSON, dependin on its role."""
xlog_location - value of `replayed_location` or `location` from JSON, dependin on its role.
tags - dictionary with values of different tags (i.e. nofailover)
"""
try:
response = requests.get(member.api_url, timeout=2, verify=False)
@@ -148,10 +150,11 @@ class Ha:
json = response.json()
is_master = json['role'] == 'master'
xlog_location = json['xlog']['location' if is_master else 'replayed_location']
return (member, True, not is_master, xlog_location)
tags = json.get('tags', dict())
return (member, True, not is_master, xlog_location, tags)
except:
logging.exception('request failed: GET %s', member.api_url)
return (member, False, None, 0)
return (member, False, None, 0, {})
def fetch_nodes_statuses(self, members):
pool = ThreadPool(len(members))
@@ -166,16 +169,20 @@ class Ha:
if self.state_handler.is_leader():
return True
if self.patroni.tags.get('nofailover') is True:
return False
if check_replication_lag and not self.state_handler.check_replication_lag(self.cluster.last_leader_operation):
return False # Too far behind last reported xlog location on master
# Prepare list of nodes to run check against
members = [m for m in members if m.name != self.state_handler.name and m.api_url]
members = [m for m in members if m.name != self.state_handler.name
and not m.data.get('tags', {}).get('nofailover', None) and m.api_url]
if members:
my_xlog_location = self.state_handler.xlog_position()
for member, reachable, in_recovery, xlog_location in self.fetch_nodes_statuses(members):
if reachable: # If the node is unreachable it's not healhy
for member, reachable, in_recovery, xlog_location, tags in self.fetch_nodes_statuses(members):
if reachable and not tags.get('nofailover'): # If the node is unreachable it's not healhy
if not in_recovery:
logger.warning('Master (%s) is still alive', member.name)
return False
@@ -187,11 +194,13 @@ class Ha:
ret = False
members = [m for m in members if m.name != self.state_handler.name and m.api_url]
if members:
for member, reachable, in_recovery, xlog_location in self.fetch_nodes_statuses(members):
if reachable:
for member, reachable, in_recovery, xlog_location, tags in self.fetch_nodes_statuses(members):
if reachable and not tags.get('nofailover'):
ret = True # TODO: check xlog_location
else:
elif not reachable:
logger.info('Member %s is not reachable', member.name)
elif tags.get('nofailover'):
logger.info('Member %s is not allowed to promote', member.name)
else:
logger.warning('manual failover: members list is empty')
return ret
@@ -205,12 +214,15 @@ class Ha:
# find specific node and check that it is healthy
members = [m for m in self.cluster.members if m.name == failover.member]
if members:
member, reachable, in_recovery, xlog_location = self.fetch_node_status(members[0])
if reachable: # node is healthy
member, reachable, in_recovery, xlog_location, tags = self.fetch_node_status(members[0])
if reachable and not tags.get('nofailover'): # node is healthy
logger.info('manual failover: to %s, i am %s', member.name, self.state_handler.name)
return False
# we wanted to failover to specific member but it is not healthy
logger.warning('manual failover: member %s is unhealthy', member.name)
if not reachable:
logger.warning('manual failover: member %s is unhealthy', member.name)
elif tags.get('nofailover'):
logger.warning('manual failover: member %s is not allowed to promote', member.name)
# at this point we should consider all members as a candidates for failover
# i.e. we assume that failover.member is None
@@ -278,6 +290,9 @@ class Ha:
return self.follow_the_leader('demoted self due after trying and failing to obtain lock',
'following new leader after trying and failing to obtain lock')
else:
if self.patroni.tags.get('nofailover'):
return self.follow_the_leader('demoting self because I am not allowed to become master',
'following a different leader because I am not allowed to promote')
return self.follow_the_leader('demoting self because i am not the healthiest node',
'following a different leader because i am not the healthiest node')
tests/test_etcd.py
+1 -1
View File
@@ -50,7 +50,7 @@ def requests_get(url, **kwargs):
if url.startswith('http://local'):
raise requests.exceptions.RequestException()
elif ':8011/patroni' in url:
response.content = '{"role": "replica", "xlog": {"replayed_location": 0}}'
response.content = '{"role": "replica", "xlog": {"replayed_location": 0}, "tags": {}}'
elif url.endswith('/members'):
if url.startswith('http://error'):
response.content = '[{}]'
tests/test_ha.py
+5 -5
View File
@@ -275,21 +275,21 @@ class TestHa(unittest.TestCase):
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader'))
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
self.ha.fetch_node_status = lambda e: (e, True, True, 0) # accessible, in_recovery
self.ha.fetch_node_status = lambda e: (e, True, True, 0, {}) # accessible, in_recovery
self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, MockPostgresql.name, ''))
self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
self.ha.fetch_node_status = lambda e: (e, False, True, 0) # accessible, in_recovery
self.ha.fetch_node_status = lambda e: (e, False, True, 0, {}) # accessible, in_recovery
self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
def test__is_healthiest_node(self):
self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
self.p.is_leader = false
self.ha.fetch_node_status = lambda e: (e, True, True, 0) # accessible, in_recovery
self.ha.fetch_node_status = lambda e: (e, True, True, 0, {}) # accessible, in_recovery
self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
self.ha.fetch_node_status = lambda e: (e, True, False, 0) # accessible, not in_recovery
self.ha.fetch_node_status = lambda e: (e, True, False, 0, {}) # accessible, not in_recovery
self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
self.ha.fetch_node_status = lambda e: (e, True, True, 1) # accessible, in_recovery, xlog location ahead
self.ha.fetch_node_status = lambda e: (e, True, True, 1, {}) # accessible, in_recovery, xlog location ahead
self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
self.p.check_replication_lag = false
self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))