diff --git a/patroni/api.py b/patroni/api.py index dc83249f..1fa5ca62 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -3,6 +3,7 @@ import fcntl import json import logging import psycopg2 +import time from patroni.exceptions import PostgresConnectionException from patroni.utils import Retry, RetryFailedError @@ -121,6 +122,56 @@ class RestApiHandler(BaseHTTPRequestHandler): self.end_headers() self.wfile.write(data) + def poll_failover_result(self, leader, member): + for a in range(0, 15): + time.sleep(1) + try: + cluster = self.server.patroni.dcs.get_cluster() + if cluster.leader and cluster.leader.name != leader: + return 200, ('Successfully failed over to ' + cluster.leader.name).encode('utf-8') + if not cluster.failover: + return 503, b'Failover failed' + except: + pass + return 503, b'Failover status unknown' + + def is_failover_possible(self, cluster, leader, member): + if leader and not cluster.leader or cluster.leader.name != leader: + return b'leader name does not match' + if member: + members = [m for m in cluster.members if m.name == member] + if not members: + return b'member does not exists' + else: + members = [m for m in cluster.members if m.name != cluster.leader.name and m.api_url] + if not members: + return b'failover is not possible: cluster does not have members except leader' + for member, reachable, in_recovery, xlog_location in self.server.patroni.ha.fetch_nodes_statuses(members): + if reachable: + return None + return b'failover is not possible: no good candidates have been found' + + @check_auth + def do_POST_failover(self): + content_length = int(self.headers.get('content-length', 0)) + request = json.loads(self.rfile.read(content_length).decode('utf-8')) + leader = request.get('leader', None) + member = request.get('member', None) + cluster = self.server.patroni.ha.dcs.get_cluster() + status_code = 503 + data = self.is_failover_possible(cluster, leader, member) + if not data: + if not self.server.patroni.dcs.manual_failover(leader, member): + data = b'failed to write failover key into DCS' + else: + self.server.patroni.dcs.event.set() + status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, member) + + self.send_response(status_code) + self.send_header('Content-Type', 'text/html') + self.end_headers() + self.wfile.write(data) + def parse_request(self): """Override parse_request method to enrich basic functionality of `BaseHTTPRequestHandler` class diff --git a/patroni/postgresql.py b/patroni/postgresql.py index 9ac0d69a..7e1d2712 100644 --- a/patroni/postgresql.py +++ b/patroni/postgresql.py @@ -490,19 +490,23 @@ recovery_target_timeline = 'latest' def save_configuration_files(self): """ - copy postgresql.conf to postgresql.conf.backup to preserve it in the WAL-e backup. - see http://comments.gmane.org/gmane.comp.db.postgresql.wal-e/239 + copy postgresql.conf to postgresql.conf.backup to be able to retrive configuration files + - originally stored as symlinks, those are normally skipped by pg_basebackup + - in case of WAL-E basebackup (see http://comments.gmane.org/gmane.comp.db.postgresql.wal-e/239) """ - for f in self.configuration_to_save: - shutil.copy(f, f + '.backup') + try: + for f in self.configuration_to_save: + os.path.isfile(f) and shutil.copy(f, f + '.backup') + except: + logger.exception('unable to create backup copies of configuration files') def restore_configuration_files(self): """ restore a previously saved postgresql.conf """ try: for f in self.configuration_to_save: - shutil.copy(f + '.backup', f) + not os.path.isfile(f) and os.path.isfile(f+'.backup') and shutil.copy(f + '.backup', f) except: - logger.exception('unable to restore configuration from WAL-E backup') + logger.exception('unable to restore configuration files from backup') def promote(self): if self.role == 'master': @@ -590,6 +594,7 @@ recovery_target_timeline = 'latest' raise PostgresException("Could not bootstrap master PostgreSQL") else: if self.sync_from_leader(current_leader): + self.restore_configuration_files() self.write_recovery_conf(current_leader) ret = self.start() return ret diff --git a/patroni/zookeeper.py b/patroni/zookeeper.py index 4cb73e0f..1fa2cca3 100644 --- a/patroni/zookeeper.py +++ b/patroni/zookeeper.py @@ -198,7 +198,7 @@ class ZooKeeper(AbstractDCS): self.client.retry(self.client.set, self.failover_path, value.encode('utf-8'), version=index or -1) return True except NoNodeError: - return value == '' or (not index and self._create(self.failover_path, value.encode('utf-8'))) + return value == '' or (not index and self._create(self.failover_path, value)) except: logging.exception('set_failover_value') return False diff --git a/tests/test_api.py b/tests/test_api.py index e4b8f87e..72d5c2b0 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -3,6 +3,7 @@ import unittest from mock import Mock, patch from patroni.api import RestApiHandler, RestApiServer +from patroni.dcs import Member from six import BytesIO as IO from six.moves import BaseHTTPServer from test_postgresql import psycopg2_connect, MockCursor @@ -38,6 +39,9 @@ class MockHa(Mock): def restart_scheduled(self): return False + def fetch_nodes_statuses(self, members): + return [[None, True, None, None]] + class MockPatroni: @@ -117,3 +121,31 @@ class TestRestApiHandler(unittest.TestCase): MockRestApiServer(RestApiHandler, b'GET /patroni') with patch.object(MockPostgresql, 'connection', Mock(side_effect=psycopg2.OperationalError)): MockRestApiServer(RestApiHandler, b'GET /patroni') + + @patch('time.sleep', Mock()) + @patch.object(MockHa, 'dcs') + def test_do_POST_failover(self, dcs): + cluster = dcs.get_cluster.return_value + request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\ + b'Content-Length: 25\n\n{"leader": "postgresql1"}' + MockRestApiServer(RestApiHandler, request) + cluster.leader.name = 'postgresql1' + MockRestApiServer(RestApiHandler, request) + cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'})] + MockRestApiServer(RestApiHandler, request) + with patch.object(MockPatroni, 'dcs') as d: + cluster = d.get_cluster.return_value + cluster.leader.name = 'postgresql0' + MockRestApiServer(RestApiHandler, request) + cluster.leader.name = 'postgresql1' + cluster.failover = None + MockRestApiServer(RestApiHandler, request) + d.get_cluster = Mock(side_effect=Exception()) + MockRestApiServer(RestApiHandler, request) + d.manual_failover.return_value = False + MockRestApiServer(RestApiHandler, request) + with patch.object(MockHa, 'fetch_nodes_statuses', Mock(return_value=[])): + MockRestApiServer(RestApiHandler, request) + request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\ + b'Content-Length: 50\n\n{"leader": "postgresql1", "member": "postgresql2"}' + MockRestApiServer(RestApiHandler, request) diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py index c4628135..4d2f2dd0 100644 --- a/tests/test_postgresql.py +++ b/tests/test_postgresql.py @@ -19,6 +19,11 @@ from test_ha import false import subprocess +def is_file_raise_on_backup(*args, **kwargs): + if args[0].endswith('.backup'): + raise Exception("foo") + + class MockCursor: def __init__(self, connection): @@ -433,3 +438,15 @@ class TestPostgresql(unittest.TestCase): @patch('subprocess.check_output', MagicMock(return_value=0, side_effect=pg_controldata_string)) def test_sysid(self): self.assertEqual(self.p.sysid, "6200971513092291716") + + @patch('os.path.isfile', MagicMock(return_value=True)) + @patch('shutil.copy', side_effect=Exception) + def test_save_configuration_files(self, mock_copy): + shutil.copy = mock_copy + self.p.save_configuration_files() + + @patch('os.path.isfile', MagicMock(side_effect=is_file_raise_on_backup)) + @patch('shutil.copy', side_effect=Exception) + def test_restore_configuration_files(self, mock_copy): + shutil.copy = mock_copy + self.p.restore_configuration_files()