diff --git a/tests/test_api.py b/tests/test_api.py index 837cdbaf..67db9ba7 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -500,87 +500,156 @@ class TestRestApiHandler(unittest.TestCase): post = 'POST /switchover HTTP/1.0' + self._authorization + '\nContent-Length: ' - MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}') + # Invalid content + with patch.object(RestApiHandler, 'write_response') as response_mock: + MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}') + response_mock.assert_called_with(400, 'Switchover could be performed only from a specific leader') + # Empty content request = post + '0\n\n' MockRestApiServer(RestApiHandler, request) - cluster.leader.name = 'postgresql1' - MockRestApiServer(RestApiHandler, request) + # [Switchover without a candidate] - request = post + '25\n\n{"leader": "postgresql1"}' - - with patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)): + # Cluster with only a leader + with patch.object(RestApiHandler, 'write_response') as response_mock: + cluster.leader.name = 'postgresql1' + request = post + '25\n\n{"leader": "postgresql1"}' MockRestApiServer(RestApiHandler, request) + response_mock.assert_called_with( + 412, 'switchover is not possible: cluster does not have members except leader') - for is_synchronous_mode in (True, False): - with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)): + # Switchover in pause mode + with patch.object(RestApiHandler, 'write_response') as response_mock, \ + patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)): + MockRestApiServer(RestApiHandler, request) + response_mock.assert_called_with( + 400, 'Switchover is possible only to a specific candidate in a paused state') + + # Switchover without a candidate specified + # No healthy nodes to promote in both sync and async mode + for is_synchronous_mode, response in ( + (True, 'switchover is not possible: can not find sync_standby'), + (False, 'switchover is not possible: cluster does not have members except leader')): + with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \ + patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, request) + response_mock.assert_called_with(412, response) - cluster.leader.name = 'postgresql2' - request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}' - MockRestApiServer(RestApiHandler, request) + # [Switchover to the candidate specified] + # Current leader is different from the one specified + with patch.object(RestApiHandler, 'write_response') as response_mock: + cluster.leader.name = 'postgresql2' + request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}' + MockRestApiServer(RestApiHandler, request) + response_mock.assert_called_with(412, 'leader name does not match') + + # Candidate to promote is not a member of the cluster cluster.leader.name = 'postgresql1' cluster.sync.matches.return_value = False - for is_synchronous_mode in (True, False): - with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)): + for is_synchronous_mode, response in ( + (True, 'candidate name does not match with sync_standby'), (False, 'candidate does not exists')): + with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \ + patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, request) + response_mock.assert_called_with(412, response) cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}), Member(0, 'postgresql2', 30, {'api_url': 'http'})] - MockRestApiServer(RestApiHandler, request) - cluster.failover = None - MockRestApiServer(RestApiHandler, request) + # Failover key is empty in DCS + with patch.object(RestApiHandler, 'write_response') as response_mock: + cluster.failover = None + MockRestApiServer(RestApiHandler, request) + response_mock.assert_called_with(503, 'Switchover failed') - dcs.get_cluster.side_effect = [cluster] - MockRestApiServer(RestApiHandler, request) + # Result polling failed + with patch.object(RestApiHandler, 'write_response') as response_mock: + dcs.get_cluster.side_effect = [cluster] + MockRestApiServer(RestApiHandler, request) + response_mock.assert_called_with(503, 'Switchover status unknown') - cluster2 = cluster.copy() - cluster2.leader.name = 'postgresql0' - cluster2.is_unlocked.return_value = False - dcs.get_cluster.side_effect = [cluster, cluster2] - MockRestApiServer(RestApiHandler, request) + # Switchover to a node different from the candidate specified + with patch.object(RestApiHandler, 'write_response') as response_mock: + cluster2 = cluster.copy() + cluster2.leader.name = 'postgresql0' + cluster2.is_unlocked.return_value = False + dcs.get_cluster.side_effect = [cluster, cluster2] + MockRestApiServer(RestApiHandler, request) + response_mock.assert_called_with(200, 'Switched over to "postgresql0" instead of "postgresql2"') - cluster2.leader.name = 'postgresql2' - dcs.get_cluster.side_effect = [cluster, cluster2] - MockRestApiServer(RestApiHandler, request) + # Successful switchover to the candidate + with patch.object(RestApiHandler, 'write_response') as response_mock: + cluster2.leader.name = 'postgresql2' + dcs.get_cluster.side_effect = [cluster, cluster2] + MockRestApiServer(RestApiHandler, request) + response_mock.assert_called_with(200, 'Successfully switched over to "postgresql2"') + + with patch.object(RestApiHandler, 'write_response') as response_mock: + dcs.manual_failover.return_value = False + dcs.get_cluster.side_effect = None + MockRestApiServer(RestApiHandler, request) + response_mock.assert_called_with(503, 'failed to write failover key into DCS') - dcs.get_cluster.side_effect = None - dcs.manual_failover.return_value = False - MockRestApiServer(RestApiHandler, request) dcs.manual_failover.return_value = True - with patch.object(MockHa, 'fetch_nodes_statuses', Mock(return_value=[])): + # Candidate is not healthy to be promoted + with patch.object(MockHa, 'fetch_nodes_statuses', Mock(return_value=[])), \ + patch.object(RestApiHandler, 'write_response') as response_mock: MockRestApiServer(RestApiHandler, request) + response_mock.assert_called_with(412, 'switchover is not possible: no good candidates have been found') + + # [Scheduled switchover] # Valid future date - request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2",' +\ - ' "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}' - MockRestApiServer(RestApiHandler, request) - with patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)),\ - patch.object(MockPatroni, 'dcs') as d: - d.manual_failover.return_value = False + with patch.object(RestApiHandler, 'write_response') as response_mock: + request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2",' + \ + ' "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}' MockRestApiServer(RestApiHandler, request) + response_mock.assert_called_with(202, 'Switchover scheduled') - # Exception: No timezone specified - request = post + '97\n\n{"leader": "postgresql1", "member": "postgresql2",' +\ - ' "scheduled_at": "6016-02-15T18:13:30.568224"}' - MockRestApiServer(RestApiHandler, request) + # Schedule in paused mode + with patch.object(RestApiHandler, 'write_response') as response_mock, \ + patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)): + dcs.manual_failover.return_value = False + MockRestApiServer(RestApiHandler, request) + response_mock.assert_called_with(400, "Can't schedule switchover in the paused state") + + # No timezone specified + with patch.object(RestApiHandler, 'write_response') as response_mock: + request = post + '97\n\n{"leader": "postgresql1", "member": "postgresql2",' + \ + ' "scheduled_at": "6016-02-15T18:13:30.568224"}' + MockRestApiServer(RestApiHandler, request) + response_mock.assert_called_with(400, 'Timezone information is mandatory for the scheduled switchover') - # Exception: Scheduled in the past request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "' - MockRestApiServer(RestApiHandler, request + '1016-02-15T18:13:30.568224+01:00"}') + + # Scheduled in the past + with patch.object(RestApiHandler, 'write_response') as response_mock: + MockRestApiServer(RestApiHandler, request + '1016-02-15T18:13:30.568224+01:00"}') + response_mock.assert_called_with(422, 'Cannot schedule switchover in the past') # Invalid date - self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}')) + with patch.object(RestApiHandler, 'write_response') as response_mock: + MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}') + response_mock.assert_called_with( + 422, 'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601') def test_do_POST_failover(self): post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: ' - MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}') - MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}') - MockRestApiServer(RestApiHandler, post + '30\n\n{"leader":"1","candidate":"2"}') + + with patch.object(RestApiHandler, 'write_response') as response_mock: + MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}') + response_mock.assert_called_once_with(400, 'Failover could be performed only to a specific candidate') + + with patch.object(RestApiHandler, 'write_response') as response_mock: + MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}') + response_mock.assert_called_once_with(400, "Failover can't be scheduled") + + with patch.object(RestApiHandler, 'write_response') as response_mock: + MockRestApiServer(RestApiHandler, post + '30\n\n{"leader":"1","candidate":"2"}') + response_mock.assert_called_once_with(412, 'leader name does not match') @patch.object(MockHa, 'is_leader', Mock(return_value=True)) def test_do_POST_citus(self): diff --git a/tests/test_ctl.py b/tests/test_ctl.py index c541edb0..e846f4bf 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -92,103 +92,123 @@ class TestCtl(unittest.TestCase): mock_get_dcs.return_value = self.e mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader mock_get_dcs.return_value.set_failover_value = Mock() - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') - assert 'leader' in result.output + # Confirm + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') + self.assertEqual(result.exit_code, 0) + + # Abort + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\nN') + self.assertEqual(result.exit_code, 1) + + # Without a candidate with --force option + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force']) + self.assertEqual(result.exit_code, 0) + + # Scheduled (confirm) result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n2300-01-01T12:23:00\ny') - assert result.exit_code == 0 + self.assertEqual(result.exit_code, 0) + # Scheduled (abort) + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', + '--scheduled', '2015-01-01T12:00:00+01:00'], input='leader\nother\n\nN') + self.assertEqual(result.exit_code, 1) + + # Scheduled with --force option + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', + '--force', '--scheduled', '2015-01-01T12:00:00+01:00']) + self.assertEqual(result.exit_code, 0) + + # Scheduled in pause mode with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)): result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force', '--scheduled', '2015-01-01T12:00:00']) - assert result.exit_code == 1 - - # Aborting switchover, as we answer NO to the confirmation - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\nN') - assert result.exit_code == 1 - - # Aborting scheduled switchover, as we answer NO to the confirmation - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', - '--scheduled', '2015-01-01T12:00:00+01:00'], input='leader\nother\n\nN') - assert result.exit_code == 1 + self.assertEqual(result.exit_code, 1) + self.assertIn("Can't schedule switchover in the paused state", result.output) # Target and source are equal result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nleader\n\ny') - assert result.exit_code == 1 + self.assertEqual(result.exit_code, 1) + self.assertIn('Switchover target and source are the same', result.output) - # Reality is not part of this cluster + # Candidate is not a member of the cluster result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nReality\n\ny') - assert result.exit_code == 1 - - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force']) - assert 'Member' in result.output - - result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', - '--force', '--scheduled', '2015-01-01T12:00:00+01:00']) - assert result.exit_code == 0 + self.assertEqual(result.exit_code, 1) + self.assertIn('Member Reality does not exist in cluster dummy or is tagged as nofailover', result.output) # Invalid timestamp result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force', '--scheduled', 'invalid']) - assert result.exit_code != 0 + self.assertEqual(result.exit_code, 1) + self.assertIn('Unable to parse scheduled timestamp', result.output) # Invalid timestamp result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0', '--force', '--scheduled', '2115-02-30T12:00:00+01:00']) - assert result.exit_code != 0 + self.assertEqual(result.exit_code, 1) + self.assertIn('Unable to parse scheduled timestamp', result.output) # Specifying wrong leader result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='dummy') - assert result.exit_code == 1 + self.assertEqual(result.exit_code, 1) + self.assertIn('Member dummy is not the leader of cluster dummy', result.output) + # Errors while sending Patroni REST API request with patch.object(PoolManager, 'request', Mock(side_effect=Exception)): - # Non-responding patroni result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n2300-01-01T12:23:00\ny') - assert 'falling back to DCS' in result.output + self.assertIn('falling back to DCS', result.output) - with patch.object(PoolManager, 'request') as mocked: - mocked.return_value.status = 500 + with patch.object(PoolManager, 'request') as mock_api_request: + mock_api_request.return_value.status = 500 result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') - assert 'Switchover failed' in result.output + self.assertIn('Switchover failed', result.output) - mocked.return_value.status = 501 - mocked.return_value.data = b'Server does not support this operation' + mock_api_request.return_value.status = 501 + mock_api_request.return_value.data = b'Server does not support this operation' result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') - assert 'Switchover failed' in result.output + self.assertIn('Switchover failed', result.output) # No members available mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') - assert result.exit_code == 1 + self.assertEqual(result.exit_code, 1) + self.assertIn('No candidates found to switchover to', result.output) # No leader available mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny') - assert result.exit_code == 1 + self.assertEqual(result.exit_code, 1) + self.assertIn('This cluster has no leader', result.output) + + # Citus cluster, no group number specified + result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force'], input='\n') + self.assertEqual(result.exit_code, 1) + self.assertIn('For Citus clusters the --group must me specified', result.output) @patch('patroni.ctl.get_dcs') @patch.object(PoolManager, 'request', Mock(return_value=MockResponse())) def test_failover(self, mock_get_dcs): mock_get_dcs.return_value = self.e - mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader mock_get_dcs.return_value.set_failover_value = Mock() - result = self.runner.invoke(ctl, ['failover', 'dummy', '--force'], input='\n') - assert 'For Citus clusters the --group must me specified' in result.output - result = self.runner.invoke(ctl, ['failover', 'dummy'], input='0\n') - assert 'Failover could be performed only to a specific candidate' in result.output - # failover to an async member in sync mode (confirm) + # No candidate specified + mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader + result = self.runner.invoke(ctl, ['failover', 'dummy'], input='0\n') + self.assertIn('Failover could be performed only to a specific candidate', result.output) + + # Failover to an async member in sync mode (confirm) cluster = get_cluster_initialized_with_leader(sync=('leader', 'other')) cluster.members.append(Member(0, 'async', 28, {'api_url': 'http://127.0.0.1:8012/patroni'})) cluster.config.data['synchronous_mode'] = True mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster) result = self.runner.invoke(ctl, ['failover', 'dummy', '--group', '0', '--candidate', 'async'], input='y\ny') - assert 'Are you sure you want to failover to the asynchronous node async' in result.output - # failover to an async member in sync mode (abort) + self.assertIn('Are you sure you want to failover to the asynchronous node async', result.output) + + # Failover to an async member in sync mode (abort) mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster) result = self.runner.invoke(ctl, ['failover', 'dummy', '--group', '0', '--candidate', 'async'], input='N') - assert result.exit_code == 1 + self.assertEqual(result.exit_code, 1) @patch('patroni.dcs.dcs_modules', Mock(return_value=['patroni.dcs.dummy', 'patroni.dcs.etcd'])) def test_get_dcs(self):