diff --git a/patroni/api.py b/patroni/api.py index 9367496e..71257b68 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -199,6 +199,11 @@ class RestApiHandler(BaseHTTPRequestHandler): return if request: logger.debug("received restart request: {0}".format(request)) + + if self.server.patroni.ha.is_paused() and 'schedule' in request and self: + self._write_response(status_code, "Can't schedule restart in the paused state") + return + for k in request: if k == 'schedule': (_, data, request[k]) = self.parse_schedule(request[k], "restart") @@ -239,18 +244,27 @@ class RestApiHandler(BaseHTTPRequestHandler): @check_auth def do_DELETE_restart(self): - if self.server.patroni.ha.delete_future_restart(): - data = "scheduled restart deleted" - code = 200 + if self.server.patroni.ha.is_paused(): + data = "Can't delete scheduled restart in the paused state" + code = 500 else: - data = "no restarts are scheduled" - code = 404 + if self.server.patroni.ha.delete_future_restart(): + data = "scheduled restart deleted" + code = 200 + else: + data = "no restarts are scheduled" + code = 404 self._write_response(code, data) @check_auth def do_POST_reinitialize(self): patroni = self.server.patroni cluster = patroni.dcs.get_cluster() + status_code = 500 + if self.server.patroni.ha.is_paused(): + self._write_response(status_code, "Can't do reinitialize in the paused state") + return + if cluster.is_unlocked(): status_code = 503 data = 'Cluster has no leader, can not reinitialize' @@ -303,14 +317,17 @@ class RestApiHandler(BaseHTTPRequestHandler): @check_auth def do_POST_failover(self): request = self._read_json_content() + status_code = 500 if not request: return leader = request.get('leader') candidate = request.get('candidate') or request.get('member') scheduled_at = request.get('scheduled_at') + if scheduled_at and self.server.patroni.ha.is_paused(): + self._write_response(status_code, "Can't schedule failover in the paused state") + cluster = self.server.patroni.dcs.get_cluster() - status_code = 500 logger.info("received failover request with leader=%s candidate=%s scheduled_at=%s", leader, candidate, scheduled_at) diff --git a/patroni/ctl.py b/patroni/ctl.py index 07ef8a64..b5dbbc28 100644 --- a/patroni/ctl.py +++ b/patroni/ctl.py @@ -499,6 +499,8 @@ def restart(cluster_name, member_names, config_file, dcs, force, role, p_any, sc scheduled_at = parse_scheduled(scheduled) if scheduled_at: + if is_paused(cluster): + raise PatroniCtlException("Can't schedule restart in the paused state") content['schedule'] = scheduled_at.isoformat() for member in members: @@ -554,17 +556,17 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs) - if cluster.leader is None: + if cluster.leader is None and not is_paused(cluster): raise PatroniCtlException('This cluster has no master') - if master is None: + if master is None and (not is_paused(cluster) or cluster.leader): if force: master = cluster.leader.member.name else: master = click.prompt('Master', type=str, default=cluster.leader.member.name) - if cluster.leader.member.name != master: - raise PatroniCtlException('Member {0} is not the leader of cluster {1}'.format(master, cluster_name)) + if not is_paused(cluster) and cluster.leader.member.name != master: + raise PatroniCtlException('Member {0} is not the leader of cluster {1}'.format(master, cluster_name)) candidate_names = [str(m.name) for m in cluster.members if m.name != master] # We sort the names for consistent output to the client @@ -589,9 +591,14 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled scheduled_at = parse_scheduled(scheduled) if scheduled_at: + if is_paused(cluster): + raise PatroniCtlException("Can't schedule failover in the paused state") scheduled_at = scheduled_at.isoformat() - failover_value = {'leader': master, 'candidate': candidate, 'scheduled_at': scheduled_at} + failover_value = {'candidate': candidate, 'scheduled_at': scheduled_at} + if master: + failover_value['leader'] = master + logging.debug(failover_value) # By now we have established that the leader exists and the candidate exists diff --git a/tests/test_api.py b/tests/test_api.py index 3e90346a..cc5f590e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -39,6 +39,10 @@ class MockHa(object): state_handler = MockPostgresql() + @staticmethod + def is_paused(): + return False + @staticmethod def schedule_reinitialize(): return 'reinitialize' @@ -221,11 +225,18 @@ class TestRestApiHandler(unittest.TestCase): request = make_request(role='master', postgres_version='9.5.2') MockRestApiServer(RestApiHandler, request) + with patch.object(MockHa, 'is_paused', Mock(return_value=True)): + request = make_request(schedule='2016-08-42 12:45TZ+1', role='master') + MockRestApiServer(RestApiHandler, request) + def test_do_DELETE_restart(self): for retval in (True, False): with patch.object(MockHa, 'delete_future_restart', Mock(return_value=retval)): request = 'DELETE /restart HTTP/1.0' + self._authorization self.assertIsNotNone(MockRestApiServer(RestApiHandler, request)) + with patch.object(MockHa, 'is_paused', Mock(return_value=True)): + request = 'DELETE /restart HTTP/1.0' + self._authorization + self.assertIsNotNone(MockRestApiServer(RestApiHandler, request)) @patch.object(MockPatroni, 'dcs') def test_do_POST_reinitialize(self, dcs): @@ -239,6 +250,9 @@ class TestRestApiHandler(unittest.TestCase): cluster.leader.name = 'test' self.assertIsNotNone(MockRestApiServer(RestApiHandler, request)) + with patch.object(MockHa, 'is_paused', Mock(return_value=True)): + self.assertIsNotNone(MockRestApiServer(RestApiHandler, request)) + @patch('time.sleep', Mock()) def test_RestApiServer_query(self): with patch.object(MockCursor, 'execute', Mock(side_effect=psycopg2.OperationalError)): @@ -306,6 +320,9 @@ class TestRestApiHandler(unittest.TestCase): d.manual_failover.return_value = False MockRestApiServer(RestApiHandler, request) + with patch.object(MockHa, 'is_paused', Mock(return_value=True)): + MockRestApiServer(RestApiHandler, request) + # Exception: No timezone specified request = post + '97\n\n{"leader": "postgresql1", "member": "postgresql2",' +\ ' "scheduled_at": "6016-02-15T18:13:30.568224"}' diff --git a/tests/test_ctl.py b/tests/test_ctl.py index 81a2261e..e9a1bc82 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -82,6 +82,11 @@ class TestCtl(unittest.TestCase): result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2030-01-01T12:23:00\ny') assert result.exit_code == 0 + with patch('patroni.ctl.is_paused', Mock(return_value=True)): + result = self.runner.invoke(ctl, + ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00+01:00']) + assert result.exit_code == 1 + # Aborting failover,as we anser NO to the confirmation result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\nN') assert result.exit_code == 1 @@ -241,6 +246,12 @@ class TestCtl(unittest.TestCase): '--scheduled', '2300-10-01T14:30']) assert 'Failed: flush scheduled restart' in result.output + with patch('patroni.ctl.is_paused', Mock(return_value=True)): + result = self.runner.invoke(ctl, + ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30']) + assert result.exit_code == 1 + + with patch('requests.post', Mock(return_value=MockResponse())): # normal restart, the schedule is actually parsed, but not validated in patronictl result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0.0', diff --git a/tests/test_ha.py b/tests/test_ha.py index 0b5ed96a..69293cec 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -499,3 +499,8 @@ class TestHa(unittest.TestCase): self.assertEquals(self.ha.run_cycle(), 'PAUSE: postgres is not running') self.ha.has_lock = true self.assertEquals(self.ha.run_cycle(), 'PAUSE: removed leader lock because postgres is not running') + + def test_no_etcd_connection_in_pause(self): + self.ha.is_paused = true + self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly')) + self.assertEquals(self.ha.run_cycle(), 'PAUSE: DCS is not accessible')