Implement manual failover

Implementation is done on top of feature/is-healthiest-via-api and
feature/api branches.
In order to trigger manual failover one has to create 'failover' key in
a configuration store with the value in following format:
'leader_name:member_name'
leader_name can be empty or should match with the name of current leader
member_name can be empty or should match with the name one of cluster
nodes
Leader always checks that either desired member (if specified) or one of
the memners is accessible and healthy before demote.
After leader has deomted himself other nodes are performig checks that
desired node is healthy. If it is not they are participating in a leader
race. In some cases (when accidently there is no healthy nodes) former
leader can also participate in a leader race.

Current implementation does not provide REST API endpoint for a manual
failover.
This commit is contained in:
Alexander Kukushkin
2015-09-28 17:00:42 +02:00
parent 3b1b6ff448
commit c218054d05
12 changed files with 320 additions and 108 deletions
+7 -22
View File
@@ -25,18 +25,9 @@ class MockCursor:
raise RetryFailedError('retry')
elif sql.startswith('SELECT slot_name'):
self.results = [('blabla',), ('foobar',)]
elif sql.startswith('SELECT pg_current_xlog_location()'):
self.results = [(0,)]
elif sql.startswith('SELECT pg_is_in_recovery(), %s'):
if params[0][0] == 1:
raise psycopg2.OperationalError()
elif params[0][0] == 2:
self.results = [(True, -1)]
else:
self.results = [(False, 0)]
elif sql.startswith('SELECT pg_xlog_location_diff'):
self.results = [(0,)]
elif sql.startswith('SELECT pg_is_in_recovery()'):
elif sql == 'SELECT pg_is_in_recovery()':
self.results = [(False, )]
elif sql.startswith('SELECT to_char(pg_postmaster_start_time'):
self.results = [('', True, '', '', '', False)]
@@ -164,7 +155,7 @@ class TestPostgresql(unittest.TestCase):
def test_sync_replication_slots(self):
self.p.start()
cluster = Cluster(True, self.leader, 0, [self.me, self.other, self.leadermem])
cluster = Cluster(True, self.leader, 0, [self.me, self.other, self.leadermem], None)
self.p.sync_replication_slots(cluster)
@patch.object(MockConnect, 'closed', 2)
@@ -178,17 +169,8 @@ class TestPostgresql(unittest.TestCase):
self.assertRaises(PostgresConnectionException, self.p.query, 'RetryFailedError')
self.assertRaises(psycopg2.OperationalError, self.p.query, 'blabla')
def test_is_healthiest_node(self):
cluster = Cluster(True, self.leader, 0, [self.me, self.other, self.leadermem])
self.assertTrue(self.p.is_healthiest_node(cluster))
self.p.is_leader = false
self.assertFalse(self.p.is_healthiest_node(cluster))
self.p.xlog_position = lambda: 1
self.assertTrue(self.p.is_healthiest_node(cluster))
self.p.xlog_position = lambda: 2
self.assertFalse(self.p.is_healthiest_node(cluster))
self.p.config['maximum_lag_on_failover'] = -3
self.assertFalse(self.p.is_healthiest_node(cluster))
def test_is_leader(self):
self.assertTrue(self.p.is_leader())
def test_reload(self):
self.assertTrue(self.p.reload())
@@ -218,6 +200,9 @@ class TestPostgresql(unittest.TestCase):
self.p.query = Mock(side_effect=psycopg2.OperationalError("not supported"))
self.assertTrue(self.p.stop())
def test_check_replication_lag(self):
self.assertTrue(self.p.check_replication_lag(0))
@patch('os.rename', Mock())
@patch('os.path.isdir', Mock(return_value=True))
def test_move_data_directory(self):