gitops/patroni
1
0
Fork 0
mirror of https://github.com/outbackdingo/patroni.git synced 2026-08-25 14:53:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #164 from zalando/feature/fast-reattach-former-leader

Browse Source 
Speed up reattach of former leader to the cluster
This commit is contained in:
Alexander Kukushkin
2016-03-31 21:57:24 +02:00
parent e76f820cb5 d4cb15179f
commit 727021306d
5 changed files with 31 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
features/patroni_api.feature
+5 -2
View File
@@ -29,20 +29,23 @@ Scenario: check API requests for the primary-replica pair
And I receive a response role replica
When I issue an empty POST request to http://127.0.0.1:8009/reinitialize
Then I receive a response code 200
Given replication works from postgres0 to postgres1 after 10 seconds
When I issue an empty POST request to http://127.0.0.1:8008/restart
Then I receive a response code 200
And postgres0 is a leader after 5 seconds
When I sleep for 10 seconds
Then postgres1 role is the secondary after 15 seconds
Scenario: check the failover via the API
Given I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0,candidate=postgres1
Then I receive a response code 200
And postgres1 is a leader after 5 seconds
And postgres1 role is the primary after 5 seconds
And postgres0 role is the secondary after 5 seconds
And replication works from postgres1 to postgres0 after 15 seconds
Scenario: check the scheduled failover
Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgres0 in 10 seconds
Then I receive a response code 200
And postgres0 is a leader after 15 seconds
And postgres0 is a leader after 20 seconds
And replication works from postgres0 to postgres1 after 25 seconds
patroni/__init__.py
+1 -1
View File
@@ -85,5 +85,5 @@ def main():
pass
finally:
patroni.api.shutdown()
patroni.postgresql.stop()
patroni.postgresql.stop(checkpoint=False)
patroni.dcs.delete_leader()
patroni/ha.py
+15 -14
View File
@@ -110,32 +110,30 @@ class Ha(object):
def recover(self):
# try to see if we are the former master that crashed. If so - we likely need to run pg_rewind
# in order to join the former standby being promoted.
pg_controldata = self.state_handler.controldata()
if (self.state_handler.role == 'master') and pg_controldata and\
pg_controldata.get('Database cluster state', '') == 'in production': # crashed master
self.state_handler.require_rewind()
if self.state_handler.role == 'master':
pg_controldata = self.state_handler.controldata()
if pg_controldata and pg_controldata.get('Database cluster state', '') == 'in production': # crashed master
self.state_handler.require_rewind()
self.recovering = True
return self.follow("started as readonly because i had the session lock",
"started as a secondary",
refresh=True, recovery=True)
return self.follow("starting as readonly because i had the session lock", "starting as a secondary", True, True)
def follow(self, demote_reason, follow_reason, refresh=True, recovery=False):
if refresh:
self.load_cluster_from_dcs()
if not recovery and self.state_handler.is_leader() or recovery and self.state_handler.role == 'master':
ret = demote_reason
else:
ret = follow_reason
ret = demote_reason if not recovery and self.state_handler.is_leader() else follow_reason
# determine the node to follow. If replicatefrom tag is set,
# try to follow the node mentioned there, otherwise, follow the leader.
if self.patroni.replicatefrom:
node_to_follow = [m for m in self.cluster.members if m.name == self.patroni.replicatefrom]
node_to_follow = node_to_follow[0] if node_to_follow else self.cluster.leader
else:
node_to_follow = self.cluster.leader
node_to_follow = None if node_to_follow and node_to_follow.name == self.state_handler.name else node_to_follow
if node_to_follow and node_to_follow.name == self.state_handler.name:
ret = demote_reason
node_to_follow = None
if not self.state_handler.check_recovery_conf(node_to_follow) or recovery:
self._async_executor.schedule('changing primary_conninfo and restarting')
self._async_executor.run_async(self.state_handler.follow, (node_to_follow, recovery))
@@ -279,7 +277,10 @@ class Ha(object):
self.dcs.delete_leader()
self.touch_member()
self.dcs.reset_cluster()
self.state_handler.follow(None)
sleep(2) # Give a time to somebody to promote
self.recover()
else:
self.state_handler.follow(None)
def process_manual_failover_from_leader(self):
failover = self.cluster.failover
@@ -331,7 +332,7 @@ class Ha(object):
if self.cluster.failover:
logger.info('Cleaning up failover key after acquiring leader lock...')
self.dcs.manual_failover('', '')
self.dcs.get_cluster()
self.load_cluster_from_dcs()
return self.enforce_master_role('acquired session lock as a leader',
'promoted self to leader by acquiring session lock')
else:
patroni/postgresql.py
+4 -3
View File
@@ -388,7 +388,7 @@ class Postgresql(object):
except psycopg2.Error:
logging.exception('Exception during CHECKPOINT')
def stop(self, mode='fast', block_callbacks=False):
def stop(self, mode='fast', block_callbacks=False, checkpoint=True):
# make sure we close all connections established against
# the former node, otherwise, we might get a stalled one
# after kill -9, which would report incorrect data to
@@ -400,9 +400,10 @@ class Postgresql(object):
self.set_state('stopped')
return True
if block_callbacks:
if checkpoint:
self.checkpoint()
else:
if not block_callbacks:
self.set_state('stopping')
ret = subprocess.call(self._pg_ctl + ['stop', '-m', mode]) == 0
tests/test_ha.py
+6 -5
View File
@@ -110,25 +110,25 @@ class TestHa(unittest.TestCase):
def test_start_as_replica(self):
self.p.is_healthy = false
self.assertEquals(self.ha.run_cycle(), 'started as a secondary')
self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
def test_recover_replica_failed(self):
self.p.controldata = lambda: {'Database cluster state': 'in production'}
self.p.is_healthy = false
self.p.is_running = false
self.p.follow = false
self.assertEquals(self.ha.run_cycle(), 'started as a secondary')
self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
self.assertEquals(self.ha.run_cycle(), 'failed to start postgres')
def test_recover_master_failed(self):
self.p.follow = false
self.p.is_healthy = false
self.p.is_running = false
self.ha.has_lock = true
self.p.name = 'leader'
self.p.set_role('master')
self.p.controldata = lambda: {'Database cluster state': 'in production'}
self.assertEquals(self.ha.run_cycle(), 'started as readonly because i had the session lock')
self.assertEquals(self.ha.run_cycle(), 'removed leader key after trying and failing to start postgres')
self.ha.cluster = get_cluster_initialized_with_leader()
self.assertEquals(self.ha.run_cycle(), 'starting as readonly because i had the session lock')
@patch('sys.exit', return_value=1)
@patch('patroni.ha.Ha.sysid_valid', MagicMock(return_value=True))
@@ -274,6 +274,7 @@ class TestHa(unittest.TestCase):
self.assertEquals(self.ha.run_cycle(), 'failed to update leader lock during restart')
@patch('requests.get', requests_get)
@patch('time.sleep', Mock())
def test_manual_failover_from_leader(self):
self.ha.has_lock = true
self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None))