Speed up reattach of former leader to the cluster

Instead of starting it up in "read-only", it wil wait 2 seconds, to give
a time to somebody to prompte and after it will execute normal `recover`
procedure.
This commit is contained in:
Alexander Kukushkin
2016-03-24 12:29:30 +01:00
parent 0d3dca56ff
commit 9dc588231f
2 changed files with 12 additions and 9 deletions
+11 -9
View File
@@ -110,14 +110,12 @@ class Ha(object):
def recover(self):
# try to see if we are the former master that crashed. If so - we likely need to run pg_rewind
# in order to join the former standby being promoted.
pg_controldata = self.state_handler.controldata()
if (self.state_handler.role == 'master') and pg_controldata and\
pg_controldata.get('Database cluster state', '') == 'in production': # crashed master
self.state_handler.require_rewind()
if self.state_handler.role == 'master':
pg_controldata = self.state_handler.controldata()
if pg_controldata and pg_controldata.get('Database cluster state', '') == 'in production': # crashed master
self.state_handler.require_rewind()
self.recovering = True
return self.follow("started as readonly because i had the session lock",
"started as a secondary",
refresh=True, recovery=True)
return self.follow("started as readonly because i had the session lock", "started as a secondary", True, True)
def follow(self, demote_reason, follow_reason, refresh=True, recovery=False):
if refresh:
@@ -279,7 +277,11 @@ class Ha(object):
self.dcs.delete_leader()
self.touch_member()
self.dcs.reset_cluster()
self.state_handler.follow(None)
self.state_handler.set_role('replica')
sleep(2) # Give a time to somebody to promote
self.recover()
else:
self.state_handler.follow(None)
def process_manual_failover_from_leader(self):
failover = self.cluster.failover
@@ -331,7 +333,7 @@ class Ha(object):
if self.cluster.failover:
logger.info('Cleaning up failover key after acquiring leader lock...')
self.dcs.manual_failover('', '')
self.dcs.get_cluster()
self.load_cluster_from_dcs()
return self.enforce_master_role('acquired session lock as a leader',
'promoted self to leader by acquiring session lock')
else:
+1
View File
@@ -274,6 +274,7 @@ class TestHa(unittest.TestCase):
self.assertEquals(self.ha.run_cycle(), 'failed to update leader lock during restart')
@patch('requests.get', requests_get)
@patch('time.sleep', Mock())
def test_manual_failover_from_leader(self):
self.ha.has_lock = true
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None))