From c2d46a084efcbf1bc3b1b48f6ba306f5c1df8e7e Mon Sep 17 00:00:00 2001 From: Feike Steenbergen Date: Fri, 5 Feb 2016 09:22:38 +0100 Subject: [PATCH 01/89] Include timestamp of last replayed location in api call. --- patroni/api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/patroni/api.py b/patroni/api.py index 2318fbf0..3137a829 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -229,6 +229,7 @@ class RestApiHandler(BaseHTTPRequestHandler): END, pg_xlog_location_diff(pg_last_xlog_receive_location(), '0/0')::bigint, pg_xlog_location_diff(pg_last_xlog_replay_location(), '0/0')::bigint, + to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'), pg_is_in_recovery() AND pg_is_xlog_replay_paused()""", retry=retry)[0] return { 'state': self.server.patroni.postgresql.state, @@ -238,7 +239,8 @@ class RestApiHandler(BaseHTTPRequestHandler): 'xlog': ({ 'received_location': row[3], 'replayed_location': row[4], - 'paused': row[5]} if row[1] else { + 'replayed_timestamp': row[5], + 'paused': row[6]} if row[1] else { 'location': row[2] }) } From 38bd037d99e7b1d6ac137ea2d4d2f740fbdce3f6 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Fri, 5 Feb 2016 13:30:42 +0100 Subject: [PATCH 02/89] Add the 1st lettuce test for the basic replication. Basically check that the table inserted on the primary will get its way to the secondary. --- features/basic_replication.feature | 12 ++ features/basic_replication.py | 41 +++++++ features/terrain.py | 184 +++++++++++++++++++++++++++++ 3 files changed, 237 insertions(+) create mode 100644 features/basic_replication.feature create mode 100644 features/basic_replication.py create mode 100644 features/terrain.py diff --git a/features/basic_replication.feature b/features/basic_replication.feature new file mode 100644 index 00000000..5e61963d --- /dev/null +++ b/features/basic_replication.feature @@ -0,0 +1,12 @@ +Feature: basic replication + In order to check that basic replication is working + As observers + We'll start 2 nodes of a new cluster, + add a table to the primary + and check that it gets replicated to the other over time. + + Scenario: check replication of a single table + Given I have started postgres0 + And I have started postgres1 + When I add the table foo to postgres0 + Then table foo is present on postgres1 diff --git a/features/basic_replication.py b/features/basic_replication.py new file mode 100644 index 00000000..26c3f0d2 --- /dev/null +++ b/features/basic_replication.py @@ -0,0 +1,41 @@ +import psycopg2 as pg +from time import sleep + +from lettuce import world, steps + +PATRONI_CONFIG = '{}.yml' + + +@steps +class BasicReplicationSteps(object): + + def __init__(self, environ): + self.env = environ + self.processes = {} + self.connstring = {} + self.cwd = None + self.max_replication_delay = 10 + + def start_patroni(self, step, pg_name): + '''I have started (\w+)''' + return world.pctl.start_patroni(pg_name) + + def add_table(self, step, table_name, pg_name): + '''I add the table (\w+) to (\w+)''' + # parse the configuration file and get the port + try: + world.pctl.query(pg_name, "CREATE TABLE {0}()".format(table_name)) + except pg.Error as e: + assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e) + + def table_is_present_on(self, step, table_name, pg_name): + '''Then table (\w+) is present on (\w+)''' + for i in range(self.max_replication_delay): + if world.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None: + break + sleep(1) + else: + assert False,\ + "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, self.max_replication_delay) + +BasicReplicationSteps(world) diff --git a/features/terrain.py b/features/terrain.py new file mode 100644 index 00000000..d970d1d0 --- /dev/null +++ b/features/terrain.py @@ -0,0 +1,184 @@ +from lettuce import * +import os.path +import psycopg2 +import requests +import subprocess +import shutil +import tempfile +from time import sleep +import yaml + + +ETCD_VERSION_URL = 'http://127.0.0.1:2379/version' +ETCD_CLEANUP_URL = 'http://127.0.0.1:2379/v2/keys/service/batman?recursive=true' +PATRONI_CONFIG = '{}.yml' +etcd_handle = None +etcd_dir = None +pctl = None + + +@world.absorb +class PatroniController(object): + """ starts and stops individual patronis""" + + def __init__(self): + self.processes = {} + self.patroni_path = None + self.cwd = None + self.connstring = {} + self.connections = {} + self.cursors = {} + self.availability_check_time_limit = 10 + pass + + def get_patroni_path(self): + if self.patroni_path is None: + cwd = os.path.realpath(__file__) + while True: + path, entry = os.path.split(cwd) + cwd = path + if entry == 'features' or cwd == '/': + break + self.patroni_path = cwd + return self.patroni_path + + def patroni_is_running(self, pg_name): + return pg_name in self.processes and self.processes[pg_name].pid and (self.processes[pg_name].poll() is None) + + def stop_patroni(self, pg_name): + if pg_name in self.processes and self.processes[pg_name].pid and (self.processes[pg_name].poll() is None): + self.processes[pg_name].terminate() + while self.patroni_is_running(pg_name): + self.processes[pg_name].terminate() + sleep(1) + del self.processes[pg_name] + + def start_patroni(self, pg_name): + if not self.patroni_is_running(pg_name): + if pg_name in self.processes: + del self.processes[pg_name] + self.cwd = self.cwd or self.get_patroni_path() + p = subprocess.Popen(['python', 'patroni.py', PATRONI_CONFIG.format(pg_name)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.cwd) + if not (p and p.pid and p.poll() is None): + assert False, "PostgreSQL {0} is not running after being started".format(pg_name) + self.processes[pg_name] = p + # wait while patroni is available for queries, but not more than 10 seconds. + for tick in range(self.availability_check_time_limit): + if self.query(pg_name, "SELECT 1", fail_ok=True) is not None: + break + sleep(1) + else: + assert False,\ + "Patroni instance is not available for queries after {0} seconds".format(self.availability_check_time_limit) + + def make_connstring(self, pg_name): + if pg_name in self.connstring: + return self.connstring[pg_name] + try: + patroni_path = self.get_patroni_path() + with open(os.path.join(patroni_path, world.PATRONI_CONFIG.format(pg_name)), 'r') as f: + config = yaml.load(f) + except OSError: + return None + connstring = config['postgresql']['connect_address'] + if ':' in connstring: + address, port = connstring.split(':') + else: + address = connstring + port = '5432' + user = "postgres" + dbname = "postgres" + self.connstring[pg_name] = "host={0} port={1} dbname={2} user={3}".format(address, port, dbname, user) + return self.connstring[pg_name] + + def connection(self, pg_name): + if pg_name not in self.connections or self.connections[pg_name].closed: + conn = psycopg2.connect(self.make_connstring(pg_name)) + conn.autocommit = True + self.connections[pg_name] = conn + return self.connections[pg_name] + + def cursor(self, pg_name): + if pg_name not in self.cursors or self.cursors[pg_name].closed: + cursor = self.connection(pg_name).cursor() + self.cursors[pg_name] = cursor + return self.cursors[pg_name] + + def query(self, pg_name, query, fail_ok=False): + try: + cursor = self.cursor(pg_name) + cursor.execute(query) + return cursor + except psycopg2.Error: + if fail_ok: + return None + else: + raise + + def stop_all(self): + for patroni in self.processes.copy(): + self.stop_patroni(patroni) + +pctl = PatroniController() +world.pctl = pctl +patroni_path = pctl.get_patroni_path() +world.patroni_path = patroni_path +world.PATRONI_CONFIG = PATRONI_CONFIG + + +def etcd_is_running(): + # if we have already started etcd + if etcd_handle and etcd_handle.pid and (etcd_handle.poll() is None): + return True + # if etcd is running, but we didn't start it + try: + r = requests.get(ETCD_VERSION_URL) + if r and r.ok and 'etcdserver' in r.content: + return True + except requests.ConnectionError: + pass + return False + + +@before.all +def start_etcd(): + if not etcd_is_running(): + global etcd_handle + global etcd_dir + etcd_dir = tempfile.mkdtemp() + etcd_handle = subprocess.Popen(["etcd", "--data-dir", etcd_dir], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if not etcd_is_running(): + assert False, "Failed to start etcd" + + +@after.all +def stop_etcd(total): + global etcd_handle + global etcd_dir + if etcd_is_running() and etcd_handle: + etcd_handle.terminate() + etcd_handle = None + shutil.rmtree(etcd_dir) + etcd_dir = None + + +def patroni_cleanup_all(): + pctl.stop_all() + # remove the data directory + shutil.rmtree(os.path.join(patroni_path, 'data')) + + +def etcd_cleanup(): + try: + r = requests.delete(ETCD_CLEANUP_URL) + if not r.ok: + raise Exception('{}'.format(r.reason)) + except Exception as e: + assert False, "Unable to cleanup etcd: {0}".format(e) + + +@after.each_scenario +def cleanup(scenario): + patroni_cleanup_all() + etcd_cleanup() From 37315903fa0f84c7d26380ad86f4c681f2b007af Mon Sep 17 00:00:00 2001 From: Feike Steenbergen Date: Wed, 10 Feb 2016 13:52:39 +0100 Subject: [PATCH 03/89] Implement scheduled failover. Scheduled failover allows scheduling of a failover in the future. It does this by writing a failover key in the DCS which contains the scheduled failover time. The reason to allow a scheduled failover, is that it does not require one to use a scheduler (e.g. cron) to schedule such a failover. One of the issues with using a scheduler is that it may need to authenticate itself. With scheduled failover the authentication takes place during the scheduling, not during the actual failover. To allow the time of failover to be expressed, the failover key has changed its format; the old format however can still be used. The new format expects the failover key to be a json-document with relevant keys set. We need the timestamp specified to be time zone aware and to be expressed unambigiously, e.g. ISO 8601. --- patroni/api.py | 35 +++++++++++++++++++++++++------- patroni/ctl.py | 53 +++++++++++++++++++++++++++++------------------- patroni/dcs.py | 53 +++++++++++++++++++++++++++++++++++++++++++----- patroni/ha.py | 25 +++++++++++++++++++++++ patroni/utils.py | 33 ++++++++---------------------- 5 files changed, 142 insertions(+), 57 deletions(-) diff --git a/patroni/api.py b/patroni/api.py index 2318fbf0..677c8f85 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -5,6 +5,9 @@ import logging import psycopg2 import socket import time +import dateutil +import datetime +import pytz from patroni.exceptions import PostgresConnectionException from patroni.utils import Retry, RetryFailedError @@ -176,13 +179,31 @@ class RestApiHandler(BaseHTTPRequestHandler): member = request.get('member', None) cluster = self.server.patroni.ha.dcs.get_cluster() status_code = 503 - data = self.is_failover_possible(cluster, leader, member) - if not data: - if not self.server.patroni.dcs.manual_failover(leader, member): - data = b'failed to write failover key into DCS' - else: - self.server.patroni.dcs.event.set() - status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, member) + + data = b'' + if request.get('scheduled_at'): + try: + scheduled_at = dateutil.parser.parse(request['scheduled_at']) + if scheduled_at.tzinfo is None: + data = b'Timezone information is mandatory for scheduled_at' + status_code = 400 + elif scheduled_at < datetime.datetime.now(pytz.utc): + data = b'Cannot schedule failover in the past' + status_code = 422 + elif self.server.patroni.dcs.manual_failover(leader, member, scheduled_at): + data = b'Failover scheduled' + status_code = 200 + except (ValueError, TypeError): + logger.exception('Invalid scheduled failover time: {}'.format(request['scheduled_at'])) + data = b'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601' + else: + data = self.is_failover_possible(cluster, leader, member) + if not data: + if not self.server.patroni.dcs.manual_failover(leader, member): + data = b'failed to write failover key into DCS' + else: + self.server.patroni.dcs.event.set() + status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, member) self.send_response(status_code) self.send_header('Content-Type', 'text/html') diff --git a/patroni/ctl.py b/patroni/ctl.py index 635b0a89..1d923420 100644 --- a/patroni/ctl.py +++ b/patroni/ctl.py @@ -14,6 +14,8 @@ import datetime from prettytable import PrettyTable from six.moves.urllib_parse import urlparse import logging +import dateutil +import tzlocal from .etcd import Etcd from .exceptions import PatroniCtlException @@ -473,10 +475,12 @@ def reinit(cluster_name, member_names, config_file, dcs, force): @click.argument('cluster_name') @click.option('--master', help='The name of the current master', default=None) @click.option('--candidate', help='The name of the candidate', default=None) +@click.option('--scheduled', help='Timestamp of a scheduled failover in unambiguous format (e.g. ISO 8601)', + default=None) @click.option('--force', is_flag=True) @option_config_file @option_dcs -def failover(config_file, cluster_name, master, candidate, force, dcs): +def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled): """ We want to trigger a failover for the specified cluster name. @@ -514,6 +518,25 @@ def failover(config_file, cluster_name, master, candidate, force, dcs): if candidate and candidate not in candidate_names: raise PatroniCtlException('Member {} does not exist in cluster {}'.format(candidate, cluster_name)) + if scheduled is None and not force: + scheduled = click.prompt('When should the failover take place (e.g. 2015-10-01T14:30) ', type=str, + default='now') + + if (scheduled or 'now') == 'now': + scheduled_at = None + else: + try: + scheduled_at = dateutil.parser.parse(scheduled) + if scheduled_at.tzinfo is None: + scheduled_at = tzlocal.get_localzone().localize(scheduled_at) + except (ValueError, TypeError): + message = 'Unable to parse scheduled timestamp ({}). It should be in an unambiguous format (e.g. ISO 8601)' + raise PatroniCtlException(message.format(scheduled)) + scheduled_at = scheduled_at.isoformat() + + failover_value = {'leader': master, 'member': candidate, 'scheduled_at': scheduled_at} + logging.debug(failover_value) + # By now we have established that the leader exists and the candidate exists click.echo('Current cluster topology') output_members(dcs.get_cluster(), name=cluster_name) @@ -525,17 +548,14 @@ def failover(config_file, cluster_name, master, candidate, force, dcs): if not a: raise PatroniCtlException('Aborting failover') - failover_value = '{}:{}'.format(master, candidate or '') - - t_started = time.time() r = None try: - r = post_patroni(cluster.leader.member, 'failover', {'leader': master, 'member': candidate or ''}) + r = post_patroni(cluster.leader.member, 'failover', failover_value) if r.status_code == 200: logging.debug(r) - logging.debug(r.text) cluster = dcs.get_cluster() - click.echo(timestamp() + ' Failing over to new leader: {}'.format(cluster.leader.member.name)) + logging.debug(cluster) + click.echo('{} {}'.format(timestamp(), r.text)) else: click.echo('Failover failed, details: {}, {}'.format(r.status_code, r.text)) return @@ -543,17 +563,9 @@ def failover(config_file, cluster_name, master, candidate, force, dcs): logging.exception(r) logging.warning('Failing over to DCS') click.echo(timestamp() + ' Could not failover using Patroni api, falling back to DCS') - dcs.set_failover_value(failover_value) - click.echo(timestamp() + ' Initialized failover from master {}'.format(master)) - # The failover process should within a minute update the failover key, we will keep watching it until it changes - # or we timeout - cluster = wait_for_leader(dcs, timeout=60) - if cluster.leader.member.name == master: - click.echo('Failover failed, master did not change after {:0.1f} seconds'.format(time.time() - t_started)) - return + click.echo(timestamp() + ' Initializing failover from master {}'.format(master)) + dcs.manual_failover(leader=master, member=candidate, scheduled_at=failover_value) - click.echo(timestamp() + ' Failover completed in {:0.1f} seconds, new leader is {}'.format(time.time() - t_started, - str(cluster.leader.member.name))) output_members(cluster, name=cluster_name) @@ -577,10 +589,9 @@ def output_members(cluster, name=None, format='pretty'): host = build_connect_parameters(m.conn_url)['host'] - xlog_location = m.data.get('xlog_location') - if xlog_location is None or (xlog_location_cluster < xlog_location): - lag = '' - else: + xlog_location = m.data.get('xlog_location') or 0 + lag = '' + if (xlog_location_cluster >= xlog_location): lag = round((xlog_location_cluster - xlog_location)/1024/1024) rows.append([ diff --git a/patroni/dcs.py b/patroni/dcs.py index a6b9d856..c689c4a9 100644 --- a/patroni/dcs.py +++ b/patroni/dcs.py @@ -1,5 +1,6 @@ import abc import json +import dateutil from collections import namedtuple from patroni.exceptions import DCSError @@ -89,12 +90,44 @@ class Leader(namedtuple('Leader', 'index,session,member')): return self.member.conn_url -class Failover(namedtuple('Failover', 'index,leader,member')): +class Failover(namedtuple('Failover', 'index,leader,member,scheduled_at')): + """ + >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader"}')) + True + >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader", "member": "cluster:member"}')) + True + >>> Failover.from_node(1, 'null') is None + True + >>> n = '{"leader": "cluster_leader", "member": "cluster:member", "scheduled_at": "2016-01-14T10:09:57.1394Z"}' + >>> 'tzinfo=' in str(Failover.from_node(1, n)) + True + >>> Failover.from_node(1, None) is None + True + >>> Failover.from_node(1, '{}') is None + True + >>> 'abc' in Failover.from_node(1, 'abc:def') + True + """ @staticmethod def from_node(index, value): - t = [a.strip() for a in value.split(':')] + [''] - return Failover(index, t[0], t[1]) if t[0] or t[1] else None + if not value: + return None + + try: + data = json.loads(value) + if not data: + return None + except ValueError: + t = [a.strip() for a in value.split(':')] + leader = t[0] + candidate = t[1] if len(t) > 1 else None + return Failover(index, leader, candidate, None) if leader or candidate else None + + if data.get('scheduled_at'): + data['scheduled_at'] = dateutil.parser.parse(data['scheduled_at']) + + return Failover(index, data.get('leader'), data.get('member'), data.get('scheduled_at')) class Cluster(namedtuple('Cluster', 'initialize,leader,last_leader_operation,members,failover')): @@ -223,8 +256,18 @@ class AbstractDCS: def set_failover_value(self, value, index=None): """Create or update `/failover` key""" - def manual_failover(self, leader, member, index=None): - return self.set_failover_value(leader + (':' + member if member else ''), index) + def manual_failover(self, leader, member, scheduled_at=None, index=None): + failover_value = dict() + if leader: + failover_value['leader'] = leader + + if member: + failover_value['member'] = member + + if scheduled_at: + failover_value['scheduled_at'] = scheduled_at.isoformat() + + return self.set_failover_value(json.dumps(failover_value), index) def current_leader(self): try: diff --git a/patroni/ha.py b/patroni/ha.py index 7e78eb77..dc038da9 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -3,6 +3,9 @@ import logging import psycopg2 import requests import sys +import time +import datetime +import pytz from patroni.async_executor import AsyncExecutor from patroni.exceptions import DCSError, PostgresConnectionException @@ -268,6 +271,28 @@ class Ha: def process_manual_failover_from_leader(self): failover = self.cluster.failover + + if failover.scheduled_at: + # If the failover is in the far future, we shouldn't do anything and just return. + # If the failover is in the past, we consider the value to be stale and we remove + # the value. + # If the value is close to now, we initiate the failover + now = datetime.datetime.now(pytz.utc) + delta = (failover.scheduled_at - now).total_seconds() + + if delta > 10: + logging.info('Awaiting failover at {0} (in {1:.0f} seconds)'.format(failover.scheduled_at.isoformat(), + delta)) + return + elif delta < -15: + logger.warning('Found a stale failover value, cleaning up: {}'.format(failover.scheduled_at)) + self.dcs.manual_failover('', '', self.cluster.failover.index) + return + + # The value is very close to now + time.sleep(max(delta, 0)) + logger.info('Manual scheduled failover at {}'.format(failover.scheduled_at.isoformat())) + if not failover.leader or failover.leader == self.state_handler.name: if not failover.member or failover.member != self.state_handler.name: members = [m for m in self.cluster.members if not failover.member or m.name == failover.member] diff --git a/patroni/utils.py b/patroni/utils.py index 9b040294..7e827222 100644 --- a/patroni/utils.py +++ b/patroni/utils.py @@ -1,10 +1,11 @@ import datetime import os import random -import re import signal import sys import time +import pytz +import dateutil.parser from patroni.exceptions import PatroniException @@ -12,39 +13,23 @@ ignore_sigterm = False interrupted_sleep = False reap_children = False -_DATE_TIME_RE = re.compile(r'''^ -(?P\d{4})\-(?P\d{2})\-(?P\d{2}) # date -T -(?P\d{2}):(?P\d{2}):(?P\d{2})\.(?P\d{6}) # time -\d*Z$''', re.X) - - -def parse_datetime(time_str): - """ - >>> parse_datetime('2015-06-10T12:56:30.552539016Z') - datetime.datetime(2015, 6, 10, 12, 56, 30, 552539) - >>> parse_datetime('2015-06-10 12:56:30.552539016Z') - """ - m = _DATE_TIME_RE.match(time_str) - if not m: - return None - p = dict((n, int(m.group(n))) for n in 'year month day hour minute second microsecond'.split(' ')) - return datetime.datetime(**p) - def calculate_ttl(expiration): """ >>> calculate_ttl(None) - >>> calculate_ttl('2015-06-10 12:56:30.552539016Z') + >>> calculate_ttl('2015-06-10 12:56:30.552539016Z') < 0 + True >>> calculate_ttl('2015-06-10T12:56:30.552539016Z') < 0 True + >>> calculate_ttl('fail-06-10T12:56:30.552539016Z') """ if not expiration: return None - expiration = parse_datetime(expiration) - if not expiration: + try: + expiration = dateutil.parser.parse(expiration) + except (ValueError, TypeError): return None - now = datetime.datetime.utcnow() + now = datetime.datetime.now(pytz.utc) return int((expiration - now).total_seconds()) From 1e2fdac8919ea78224921759d81b82f54084b652 Mon Sep 17 00:00:00 2001 From: Feike Steenbergen Date: Wed, 10 Feb 2016 14:19:41 +0100 Subject: [PATCH 04/89] Scheduled Failover tests Add tests for the scheduled failover feature, also add more and better tests for patronictl. --- tests/test_api.py | 20 +++++++ tests/test_ctl.py | 134 +++++++++++++++++++++++++++++---------------- tests/test_etcd.py | 2 + tests/test_ha.py | 43 ++++++++++++--- 4 files changed, 142 insertions(+), 57 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 4b24704d..a9e9398b 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -169,3 +169,23 @@ class TestRestApiHandler(unittest.TestCase): request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\ b'Content-Length: 50\n\n{"leader": "postgresql1", "member": "postgresql2"}' MockRestApiServer(RestApiHandler, request) + + ## Valid future date + request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\ + b'Content-Length: 103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}' + MockRestApiServer(RestApiHandler, request) + + ## Exception: No timezone specified + request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\ + b'Content-Length: 97\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "6016-02-15T18:13:30.568224"}' + MockRestApiServer(RestApiHandler, request) + + ## Exception: Scheduled in the past + request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\ + b'Content-Length: 103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "1016-02-15T18:13:30.568224+01:00"}' + MockRestApiServer(RestApiHandler, request) + + ## Invalid date + request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\ + b'Content-Length: 103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "2010-02-29T18:13:30.568224+01:00"}' + MockRestApiServer(RestApiHandler, request) diff --git a/tests/test_ctl.py b/tests/test_ctl.py index 81f825aa..985b526f 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -84,6 +84,7 @@ class TestCtl(unittest.TestCase): output_members(cluster, name='abc', format='json') output_members(cluster, name='abc', format='tsv') + @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None)) @patch('patroni.etcd.Etcd.set_failover_value', Mock(return_value=None)) @@ -97,73 +98,99 @@ class TestCtl(unittest.TestCase): with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())): result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader other -y''') - assert 'Failing over to new leader' in result.output +y''') + assert 'leader' in result.output + + result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader +other +2100-01-01T12:23:00 +y''') + assert result.exit_code == 0 + + result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader +other +2030-01-01T12:23:00 +y''') + assert result.exit_code == 0 + + ## Aborting failover,as we anser NO to the confirmation + result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader +other +2030-01-01T12:23:00 +y''') + assert result.exit_code == 0 + + ## Aborting failover,as we anser NO to the confirmation result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader other N''') - assert 'Aborting failover' in str(result.output) + assert result.exit_code == 1 + ## Target and source are equal result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader leader -y''') - assert 'target and source are the same' in str(result.output) +y''') + assert result.exit_code == 1 + + ## Reality is not part of this cluster result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader Reality + y''') - assert 'Reality does not exist' in str(result.output) + assert result.exit_code == 1 result = runner.invoke(ctl, ['failover', 'dummy', '--force']) - assert 'Failing over to new leader' in result.output + assert 'Member' in result.output + result = runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00+01:00']) + assert result.exit_code == 0 + + ## Invalid timestamp + result = runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', 'invalid']) + assert result.exit_code != 0 + + ## Invalid timestamp + result = runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2115-02-30T12:00:00+01:00']) + assert result.exit_code != 0 + + ## Specifying wrong leader result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='dummy') - assert 'is not the leader of cluster' in str(result.output) + assert result.exit_code == 1 with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_only_leader())): + ## No members available result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader other y''') - assert 'No candidates found to failover to' in str(result.output) + assert result.exit_code == 1 with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_without_leader())): + ## No master available result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader other + y''') - assert 'This cluster has no master' in str(result.output) + assert result.exit_code == 1 with patch('patroni.ctl.post_patroni', Mock(side_effect=Exception())): + ## Non-responding patroni result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader other + y''') assert 'falling back to DCS' in result.output - assert 'Failover failed' in result.output mocked = Mock() mocked.return_value.status_code = 500 with patch('patroni.ctl.post_patroni', Mock(return_value=mocked)): result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader other + y''') - assert 'Failover failed, details' in result.output + assert 'Failover failed' in result.output -# with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())): -# result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='nonsense') -# assert 'is not the leader of cluster' in str(result.output) - - # result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8', '--master', 'nonsense']) - # assert 'is not the leader of cluster' in str(result.output) - - # result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='leader\nother\nn') - # assert 'Aborting failover' in str(result.output) - - # with patch('patroni.ctl.wait_for_leader', Mock(return_value = get_cluster_initialized_with_leader())): - # result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='leader\nother\nY') - # assert 'master did not change after' in result.output - - # result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='leader\nother\nY') - # assert 'Failover failed' in result.output def test_(self): self.assertRaises(patroni.exceptions.PatroniCtlException, get_dcs, {'scheme': 'dummy'}, 'dummy') @@ -174,6 +201,7 @@ y''') runner = CliRunner() with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)): + ## Mutually exclusive result = runner.invoke(ctl, [ 'query', 'alpha', @@ -182,19 +210,14 @@ y''') '--role', 'master', ]) - assert 'mutually exclusive' in str(result.output) + assert result.exit_code == 1 with runner.isolated_filesystem(): dummy_file = open('dummy', 'w') dummy_file.write('SELECT 1') dummy_file.close() - result = runner.invoke(ctl, [ - 'query', - 'alpha' - ]) - assert 'You need to specify' in str(result.output) - + ## Mutually exclusive result = runner.invoke(ctl, [ 'query', 'alpha', @@ -203,7 +226,7 @@ y''') '--command', 'dummy', ]) - assert 'mutually exclusive' in str(result.output) + assert result.exit_code == 1 result = runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy']) @@ -212,7 +235,12 @@ y''') result = runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1']) assert 'mock column' in result.output - result = runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1', '--dbname', 'dummy', '--password', '--username', 'dummy'], input='password\n') + ## --command or --file is mandatory + result = runner.invoke(ctl, ['query', 'alpha']) + assert result.exit_code == 1 + + result = runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1', + '--username', 'root', '--password', '--dbname', 'postgres'], input='ab\nab') assert 'mock column' in result.output @patch('patroni.ctl.get_cursor', Mock(return_value=MockConnect().cursor())) @@ -244,6 +272,7 @@ y''') result = runner.invoke(ctl, ['dsn', 'alpha', '--dcs', '8.8.8.8']) assert 'host=127.0.0.1 port=5435' in result.output + ## Mutually exclusive options result = runner.invoke(ctl, [ 'dsn', 'alpha', @@ -252,13 +281,11 @@ y''') '--member', 'dummy', ]) - assert 'mutually exclusive' in str(result.output) + assert result.exit_code == 1 + ## Non-existing member result = runner.invoke(ctl, ['dsn', 'alpha', '--member', 'dummy']) - assert 'Can not find' in str(result.output) - - # result = runner.invoke(ctl, ['dsn', 'alpha', '--dcs', '8.8.8.8', '--role', 'replica']) - # assert 'host=127.0.0.1 port=5436' in result.output + assert result.exit_code == 1 @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None)) @@ -268,9 +295,16 @@ y''') runner = CliRunner() result = runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='y') - result = runner.invoke(ctl, ['reinit', 'alpha', '--dcs', '8.8.8.8'], input='y') + assert result.exit_code == 0 + result = runner.invoke(ctl, ['reinit', 'alpha', '--dcs', '8.8.8.8'], input='y') + assert result.exit_code == 1 + + # Aborted restart result = runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='N') + assert result.exit_code == 1 + + ## Not a member result = runner.invoke(ctl, [ 'restart', 'alpha', @@ -279,7 +313,7 @@ y''') 'dummy', '--any', ], input='y') - assert 'not a member' in str(result.output) + assert result.exit_code == 1 with patch('requests.post', Mock(return_value=MockResponse())): result = runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='y') @@ -292,15 +326,18 @@ y''') result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='alpha\nslave') assert 'Please confirm' in result.output assert 'You are about to remove all' in result.output - assert 'You did not exactly type' in str(result.output) + ## Not typing an exact confirmation + assert result.exit_code == 1 + ## master specified does not match master of cluster result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='''alpha Yes I am aware slave''') - assert 'You did not specify the current master of the cluster' in str(result.output) + assert result.exit_code == 1 + ## cluster specified on cmdline does not match verification prompt result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='beta\nleader') - assert 'Cluster names specified do not match' in str(result.output) + assert result.exit_code == 1 with patch('patroni.etcd.Etcd.get_cluster', get_cluster_initialized_with_leader): result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], @@ -310,11 +347,12 @@ leader''') assert 'object has no attribute' in str(result.exception) with patch('patroni.ctl.get_dcs', Mock(return_value=Mock())): + ## Not implemented DCS result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='''alpha Yes I am aware leader''') - assert 'We have not implemented this for DCS of type' in str(result.output) + assert result.exit_code == 1 @patch('patroni.etcd.Etcd.watch', Mock(return_value=None)) @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) diff --git a/tests/test_etcd.py b/tests/test_etcd.py index 443dc2d8..33043936 100644 --- a/tests/test_etcd.py +++ b/tests/test_etcd.py @@ -60,6 +60,8 @@ def requests_get(url, **kwargs): response.content = '[{}]' else: response.content = members + elif url.endswith('/members'): + response.content = '{"action":"set","node":{"key":"/service/alpha/failover","value":"{\"leader\": \"f1410e163b6a\"}","modifiedIndex":257,"createdIndex":257},"prevNode":{"key":"/service/alpha/failover","value":"{\"scheduled_at\": \"2016-01-15T17:50:00+01:00\", \"leader\": \"f1410e163b6a\"}","modifiedIndex":241,"createdIndex":241}}' elif url.startswith('http://exhibitor'): response.content = '{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}' else: diff --git a/tests/test_ha.py b/tests/test_ha.py index 1ac164cf..5b4104b9 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -1,5 +1,7 @@ import etcd import unittest +import datetime +import pytz from mock import Mock, MagicMock, patch from patroni.dcs import Cluster, Failover, Leader, Member @@ -284,40 +286,63 @@ class TestHa(unittest.TestCase): self.ha.update_lock = false self.assertEquals(self.ha.run_cycle(), 'failed to update leader lock during restart') + @patch('requests.get', requests_get) def test_manual_failover_from_leader(self): self.ha.has_lock = true - self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '')) + self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None)) self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock') - self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', MockPostgresql.name)) + self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', MockPostgresql.name, None)) self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock') - self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'blabla')) + self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'blabla', None)) self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock') - f = Failover(0, MockPostgresql.name, '') + f = Failover(0, MockPostgresql.name, '', None) self.ha.cluster = get_cluster_initialized_with_leader(f) self.assertEquals(self.ha.run_cycle(), 'manual failover: demoting myself') self.ha.fetch_node_status = lambda e: (e, True, True, 0, {'nofailover': 'True'}) self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock') # manual failover from the previous leader to us won't happen if we hold the nofailover flag - self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name)) + self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name, None)) self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock') + ## Failover scheduled time must include timezone + scheduled = datetime.datetime.now() + self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name, scheduled)) + + self.assertRaises(TypeError, self.ha.run_cycle) + + scheduled = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC) + self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name, scheduled)) + self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle()) + + scheduled = scheduled + datetime.timedelta(seconds=30) + self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name, scheduled)) + self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle()) + + scheduled = scheduled + datetime.timedelta(seconds=-600) + self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name, scheduled)) + self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle()) + + scheduled = None + self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name, scheduled)) + self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle()) + @patch('requests.get', requests_get) def test_manual_failover_process_no_leader(self): self.p.is_leader = false - self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', MockPostgresql.name)) + self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', MockPostgresql.name, None)) self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') - self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader')) + self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader', None)) self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') self.ha.fetch_node_status = lambda e: (e, True, True, 0, {}) # accessible, in_recovery self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node') - self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, MockPostgresql.name, '')) + self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, MockPostgresql.name, '', None)) self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node') self.ha.fetch_node_status = lambda e: (e, False, True, 0, {}) # inaccessible, in_recovery self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') # set failover flag to True for all members of the cluster # this should elect the current member, as we are not going to call the API for it. - self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other')) + self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None)) self.ha.fetch_node_status = lambda e: (e, True, True, 0, {'nofailover': 'True'}) # accessible, in_recovery self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') # same as previous, but set the current member to nofailover. In no case it should be elected as a leader From 854ad293c57d8daadd832a5f9f7e1bfa02cf1164 Mon Sep 17 00:00:00 2001 From: Feike Steenbergen Date: Wed, 10 Feb 2016 14:26:25 +0100 Subject: [PATCH 05/89] Scheduled failover: Add requirements --- requirements-py2.txt | 2 ++ requirements-py3.txt | 2 ++ 2 files changed, 4 insertions(+) diff --git a/requirements-py2.txt b/requirements-py2.txt index 1e194bc0..26c0892d 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -9,3 +9,5 @@ kazoo>=2.2.1 python-etcd==0.4.2 click>=4.1 prettytable>=0.7 +tzlocal +python-dateutil diff --git a/requirements-py3.txt b/requirements-py3.txt index 13c3010c..a827efb3 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -9,3 +9,5 @@ kazoo>=2.2.1 python-etcd==0.4.2 click>=4.1 prettytable>=0.7 +tzlocal +python-dateutil From 0c2efeb7a7c1b571a654d907c7a8e74f4c19f5c2 Mon Sep 17 00:00:00 2001 From: Feike Steenbergen Date: Thu, 11 Feb 2016 09:01:44 +0100 Subject: [PATCH 06/89] Change default http status code to 500. Instead of returning 503 (Service Unavailable) we no default to returning 500 (Internal Server Error). --- patroni/api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/patroni/api.py b/patroni/api.py index 677c8f85..9f715882 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -104,7 +104,7 @@ class RestApiHandler(BaseHTTPRequestHandler): @check_auth def do_POST_restart(self): - status_code = 503 + status_code = 500 data = b'restart failed' try: status, msg = self.server.patroni.ha.restart() @@ -178,7 +178,7 @@ class RestApiHandler(BaseHTTPRequestHandler): leader = request.get('leader', None) member = request.get('member', None) cluster = self.server.patroni.ha.dcs.get_cluster() - status_code = 503 + status_code = 500 data = b'' if request.get('scheduled_at'): @@ -196,11 +196,13 @@ class RestApiHandler(BaseHTTPRequestHandler): except (ValueError, TypeError): logger.exception('Invalid scheduled failover time: {}'.format(request['scheduled_at'])) data = b'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601' + status_code = 422 else: data = self.is_failover_possible(cluster, leader, member) if not data: if not self.server.patroni.dcs.manual_failover(leader, member): data = b'failed to write failover key into DCS' + status_code = 503 else: self.server.patroni.dcs.event.set() status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, member) From f781d0b9feec4aeb2828ce4ab24e245cb885ed45 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Tue, 16 Feb 2016 16:50:50 +0100 Subject: [PATCH 07/89] Address the code review by Alex Shulgin. --- features/basic_replication.py | 2 +- features/terrain.py | 32 ++++++++++++++------------------ 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/features/basic_replication.py b/features/basic_replication.py index 26c3f0d2..69121223 100644 --- a/features/basic_replication.py +++ b/features/basic_replication.py @@ -29,7 +29,7 @@ class BasicReplicationSteps(object): assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e) def table_is_present_on(self, step, table_name, pg_name): - '''Then table (\w+) is present on (\w+)''' + '''Table (\w+) is present on (\w+)''' for i in range(self.max_replication_delay): if world.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None: break diff --git a/features/terrain.py b/features/terrain.py index d970d1d0..7395deba 100644 --- a/features/terrain.py +++ b/features/terrain.py @@ -23,43 +23,40 @@ class PatroniController(object): def __init__(self): self.processes = {} - self.patroni_path = None - self.cwd = None + self._patroni_path = None self.connstring = {} self.connections = {} self.cursors = {} self.availability_check_time_limit = 10 - pass - def get_patroni_path(self): - if self.patroni_path is None: + @property + def patroni_path(self): + if self._patroni_path is None: cwd = os.path.realpath(__file__) while True: path, entry = os.path.split(cwd) cwd = path if entry == 'features' or cwd == '/': break - self.patroni_path = cwd - return self.patroni_path + self._patroni_path = cwd + return self._patroni_path def patroni_is_running(self, pg_name): return pg_name in self.processes and self.processes[pg_name].pid and (self.processes[pg_name].poll() is None) def stop_patroni(self, pg_name): - if pg_name in self.processes and self.processes[pg_name].pid and (self.processes[pg_name].poll() is None): + while self.patroni_is_running(pg_name): self.processes[pg_name].terminate() - while self.patroni_is_running(pg_name): - self.processes[pg_name].terminate() - sleep(1) - del self.processes[pg_name] + sleep(1) + del self.processes[pg_name] def start_patroni(self, pg_name): if not self.patroni_is_running(pg_name): if pg_name in self.processes: del self.processes[pg_name] - self.cwd = self.cwd or self.get_patroni_path() + cwd = self.patroni_path p = subprocess.Popen(['python', 'patroni.py', PATRONI_CONFIG.format(pg_name)], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.cwd) + stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd) if not (p and p.pid and p.poll() is None): assert False, "PostgreSQL {0} is not running after being started".format(pg_name) self.processes[pg_name] = p @@ -76,7 +73,7 @@ class PatroniController(object): if pg_name in self.connstring: return self.connstring[pg_name] try: - patroni_path = self.get_patroni_path() + patroni_path = self.patroni_path with open(os.path.join(patroni_path, world.PATRONI_CONFIG.format(pg_name)), 'r') as f: config = yaml.load(f) except OSError: @@ -122,8 +119,7 @@ class PatroniController(object): pctl = PatroniController() world.pctl = pctl -patroni_path = pctl.get_patroni_path() -world.patroni_path = patroni_path +world.patroni_path = pctl.patroni_path world.PATRONI_CONFIG = PATRONI_CONFIG @@ -166,7 +162,7 @@ def stop_etcd(total): def patroni_cleanup_all(): pctl.stop_all() # remove the data directory - shutil.rmtree(os.path.join(patroni_path, 'data')) + shutil.rmtree(os.path.join(pctl.patroni_path, 'data')) def etcd_cleanup(): From 1b14229da480e36ea36d662b1f1b92e5a46f8269 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 17 Feb 2016 12:18:50 +0100 Subject: [PATCH 08/89] Catch TypeError within ha loop not in the unit test In addition to that use sleep function from patroni.utils instead of time.sleep which is interruptable --- patroni/ha.py | 30 ++++++++++++++++-------------- tests/test_ha.py | 6 ++---- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/patroni/ha.py b/patroni/ha.py index bf002a51..4aead262 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -3,13 +3,13 @@ import logging import psycopg2 import requests import sys -import time import datetime import pytz +from multiprocessing.pool import ThreadPool from patroni.async_executor import AsyncExecutor from patroni.exceptions import DCSError, PostgresConnectionException -from multiprocessing.pool import ThreadPool +from patroni.utils import sleep logger = logging.getLogger(__name__) @@ -283,20 +283,22 @@ class Ha(object): # the value. # If the value is close to now, we initiate the failover now = datetime.datetime.now(pytz.utc) - delta = (failover.scheduled_at - now).total_seconds() + try: + delta = (failover.scheduled_at - now).total_seconds() - if delta > 10: - logging.info('Awaiting failover at {0} (in {1:.0f} seconds)'.format(failover.scheduled_at.isoformat(), - delta)) - return - elif delta < -15: - logger.warning('Found a stale failover value, cleaning up: {}'.format(failover.scheduled_at)) - self.dcs.manual_failover('', '', self.cluster.failover.index) - return + if delta > 10: + logging.info('Awaiting failover at %s (in %.0f seconds)', failover.scheduled_at.isoformat(), delta) + return + elif delta < -15: + logger.warning('Found a stale failover value, cleaning up: %s', failover.scheduled_at) + self.dcs.manual_failover('', '', self.cluster.failover.index) + return - # The value is very close to now - time.sleep(max(delta, 0)) - logger.info('Manual scheduled failover at {}'.format(failover.scheduled_at.isoformat())) + # The value is very close to now + sleep(max(delta, 0)) + logger.info('Manual scheduled failover at {}'.format(failover.scheduled_at.isoformat())) + except TypeError: + logger.warning('Incorrect value in of scheduled_at: %s', failover.scheduled_at) if not failover.leader or failover.leader == self.state_handler.name: if not failover.member or failover.member != self.state_handler.name: diff --git a/tests/test_ha.py b/tests/test_ha.py index c6b98712..3589e952 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -297,7 +297,6 @@ class TestHa(unittest.TestCase): self.ha.update_lock = false self.assertEquals(self.ha.run_cycle(), 'failed to update leader lock during restart') - @patch('requests.get', requests_get) def test_manual_failover_from_leader(self): self.ha.has_lock = true @@ -316,11 +315,10 @@ class TestHa(unittest.TestCase): self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name, None)) self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock') - ## Failover scheduled time must include timezone + # Failover scheduled time must include timezone scheduled = datetime.datetime.now() self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name, scheduled)) - - self.assertRaises(TypeError, self.ha.run_cycle) + self.ha.run_cycle() scheduled = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC) self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name, scheduled)) From 1b9e77fe8320f375367a0b7eac22e70d689b8399 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 17 Feb 2016 12:34:04 +0100 Subject: [PATCH 09/89] pep8 formatting --- tests/test_api.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index f06b0bb0..82a76f34 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -177,22 +177,22 @@ class TestRestApiHandler(unittest.TestCase): b'Content-Length: 50\n\n{"leader": "postgresql1", "member": "postgresql2"}' MockRestApiServer(RestApiHandler, request) - ## Valid future date - request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\ - b'Content-Length: 103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}' + # Valid future date + request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\ + b'"postgresql1", "member": "postgresql2", "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}' MockRestApiServer(RestApiHandler, request) - ## Exception: No timezone specified - request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\ - b'Content-Length: 97\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "6016-02-15T18:13:30.568224"}' + # Exception: No timezone specified + request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 97\n\n{"leader": ' +\ + b'"postgresql1", "member": "postgresql2", "scheduled_at": "6016-02-15T18:13:30.568224"}' MockRestApiServer(RestApiHandler, request) - ## Exception: Scheduled in the past - request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\ - b'Content-Length: 103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "1016-02-15T18:13:30.568224+01:00"}' + # Exception: Scheduled in the past + request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\ + b'"postgresql1", "member": "postgresql2", "scheduled_at": "1016-02-15T18:13:30.568224+01:00"}' MockRestApiServer(RestApiHandler, request) - ## Invalid date - request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\ - b'Content-Length: 103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "2010-02-29T18:13:30.568224+01:00"}' + # Invalid date + request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\ + b'"postgresql1", "member": "postgresql2", "scheduled_at": "2010-02-29T18:13:30.568224+01:00"}' MockRestApiServer(RestApiHandler, request) From f079a9f308a7159c9c85c5d56ef3595835c854be Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 17 Feb 2016 12:34:22 +0100 Subject: [PATCH 10/89] remove unused code --- tests/test_etcd.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/test_etcd.py b/tests/test_etcd.py index f81885dd..601a0cd6 100644 --- a/tests/test_etcd.py +++ b/tests/test_etcd.py @@ -58,12 +58,7 @@ def requests_get(url, **kwargs): elif ':8011/patroni' in url: response.content = '{"role": "replica", "xlog": {"replayed_location": 0}, "tags": {}}' elif url.endswith('/members'): - if url.startswith('http://error'): - response.content = '[{}]' - else: - response.content = members - elif url.endswith('/members'): - response.content = '{"action":"set","node":{"key":"/service/alpha/failover","value":"{\"leader\": \"f1410e163b6a\"}","modifiedIndex":257,"createdIndex":257},"prevNode":{"key":"/service/alpha/failover","value":"{\"scheduled_at\": \"2016-01-15T17:50:00+01:00\", \"leader\": \"f1410e163b6a\"}","modifiedIndex":241,"createdIndex":241}}' + response.content = '[{}]' if url.startswith('http://error') else members elif url.startswith('http://exhibitor'): response.content = '{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}' else: From de129b733d27f2704371f054eb90350ba17adb30 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 17 Feb 2016 12:46:32 +0100 Subject: [PATCH 11/89] Fix unit tests --- tests/test_postgresql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py index 4c701c60..a63a90cc 100644 --- a/tests/test_postgresql.py +++ b/tests/test_postgresql.py @@ -38,7 +38,7 @@ class MockCursor(object): elif sql == 'SELECT pg_is_in_recovery()': self.results = [(False, )] elif sql.startswith('SELECT to_char(pg_postmaster_start_time'): - self.results = [('', True, '', '', '', False)] + self.results = [('', True, '', '', '', '', False)] else: self.results = [( None, From f7d60c61b6cd107c9e81fa9fd05fdb692067e64e Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 17 Feb 2016 14:09:00 +0100 Subject: [PATCH 12/89] remove unused code --- patroni/postgresql.py | 3 +-- tests/test_postgresql.py | 22 +++++++--------------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/patroni/postgresql.py b/patroni/postgresql.py index f5f3d37c..893503e2 100644 --- a/patroni/postgresql.py +++ b/patroni/postgresql.py @@ -526,8 +526,7 @@ recovery_target_timeline = 'latest' result[name] = val except IOError: logger.exception('Error when reading postmaster.opts') - finally: - return result + return result def single_user_mode(self, command=None, options=None): """ run a given command in a single-user mode. If the command is empty - then just start and stop """ diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py index 4c701c60..5558437b 100644 --- a/tests/test_postgresql.py +++ b/tests/test_postgresql.py @@ -2,21 +2,16 @@ import mock # for the mock.call method, importing it without a namespace breaks import os import psycopg2 import shutil +import subprocess import unittest -from six.moves import builtins from mock import Mock, MagicMock, PropertyMock, patch, mock_open from patroni.dcs import Cluster, Leader, Member from patroni.exceptions import PostgresException, PostgresConnectionException from patroni.postgresql import Postgresql from patroni.utils import RetryFailedError +from six.moves import builtins from test_ha import false -import subprocess - - -def is_file_raise_on_backup(*args, **kwargs): - if args[0].endswith('.backup'): - raise Exception("foo") class MockCursor(object): @@ -283,7 +278,7 @@ class TestPostgresql(unittest.TestCase): self.p.pg_rewind = tmp with mock.patch('subprocess.call', MagicMock(return_value=1)): self.assertFalse(self.p.can_rewind) - with mock.patch('subprocess.call', side_effect=OSError("foo")): + with mock.patch('subprocess.call', side_effect=OSError): self.assertFalse(self.p.can_rewind) tmp = self.p.controldata self.p.controldata = lambda: {'wal_log_hints setting': 'on'} @@ -292,7 +287,7 @@ class TestPostgresql(unittest.TestCase): @patch('time.sleep', Mock()) def test_create_replica(self): - self.p.delete_trigger_file = Mock(side_effect=OSError()) + self.p.delete_trigger_file = Mock(side_effect=OSError) with patch('subprocess.call', Mock(side_effect=[1, 0])): self.assertEquals(self.p.create_replica(self.leader, ''), 0) with patch('subprocess.call', Mock(side_effect=[Exception(), 0])): @@ -349,7 +344,7 @@ class TestPostgresql(unittest.TestCase): def test_last_operation(self): self.assertEquals(self.p.last_operation(), '0') - @patch('subprocess.Popen', Mock(side_effect=OSError())) + @patch('subprocess.Popen', Mock(side_effect=OSError)) def test_call_nowait(self): self.assertFalse(self.p.call_nowait('on_start')) @@ -369,7 +364,7 @@ class TestPostgresql(unittest.TestCase): def test_move_data_directory(self): self.p.is_running = false self.p.move_data_directory() - with patch('os.rename', Mock(side_effect=OSError())): + with patch('os.rename', Mock(side_effect=OSError)): self.p.move_data_directory() @patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict())) @@ -411,13 +406,10 @@ class TestPostgresql(unittest.TestCase): self.assertEquals(int(data['max_replication_slots']), 5) self.assertEqual(data.get('D'), None) - m.side_effect = IOError("foo") + m.side_effect = IOError data = self.p.read_postmaster_opts() self.assertEqual(data, dict()) - m.side_effect = Exception("foo") - self.assertRaises(Exception, self.p.read_postmaster_opts()) - @patch('subprocess.Popen') @patch.object(builtins, 'open', MagicMock(return_value=42)) def test_single_user_mode(self, subprocess_popen_mock): From a210cfd1abde26a56f6fae93c13c7c72d720a4f7 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 17 Feb 2016 14:51:59 +0100 Subject: [PATCH 13/89] Fix more codacy issues --- tests/test_api.py | 16 ++++++++-------- tests/test_ctl.py | 43 ++++++++++++++++++------------------------- tests/test_utils.py | 6 +++--- 3 files changed, 29 insertions(+), 36 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 2faee523..265394e8 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -87,7 +87,7 @@ class MockRestApiServer(RestApiServer): @patch('ssl.wrap_socket', Mock(return_value=0)) class TestRestApiHandler(unittest.TestCase): - def test_do_GET(self): + def test_do_GET(*args): MockRestApiServer(RestApiHandler, b'GET /replica') with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={})): MockRestApiServer(RestApiHandler, b'GET /replica') @@ -103,7 +103,7 @@ class TestRestApiHandler(unittest.TestCase): MockRestApiServer(RestApiHandler, b'GET /master') MockRestApiServer(RestApiHandler, b'GET /master') - def test_do_OPTIONS(self): + def test_do_OPTIONS(*args): MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0') with patch.object(BaseHTTPRequestHandler, 'handle_one_request') as mock_handle_request: @@ -117,14 +117,14 @@ class TestRestApiHandler(unittest.TestCase): makefile.return_value.flush = Mock(side_effect=socket.error("foo")) MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0') - def test_do_GET_patroni(self): + def test_do_GET_patroni(*args): MockRestApiServer(RestApiHandler, b'GET /patroni') - def test_basicauth(self): + def test_basicauth(*args): MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0') MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0\nAuthorization:') - def test_do_POST_restart(self): + def test_do_POST_restart(*args): request = b'POST /restart HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0' MockRestApiServer(RestApiHandler, request) with patch.object(MockHa, 'restart', Mock(side_effect=Exception)): @@ -140,10 +140,10 @@ class TestRestApiHandler(unittest.TestCase): with patch.object(MockHa, 'schedule_reinitialize', Mock(return_value=None)): MockRestApiServer(RestApiHandler, request) cluster.leader.name = 'test' - MockRestApiServer(RestApiHandler, request) + self.assertIsNotNone(MockRestApiServer(RestApiHandler, request)) @patch('time.sleep', Mock()) - def test_RestApiServer_query(self): + def test_RestApiServer_query(*args): with patch.object(MockCursor, 'execute', Mock(side_effect=psycopg2.OperationalError)): MockRestApiServer(RestApiHandler, b'GET /patroni') with patch.object(MockPostgresql, 'connection', Mock(side_effect=psycopg2.OperationalError)): @@ -175,4 +175,4 @@ class TestRestApiHandler(unittest.TestCase): MockRestApiServer(RestApiHandler, request) request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\ b'Content-Length: 50\n\n{"leader": "postgresql1", "member": "postgresql2"}' - MockRestApiServer(RestApiHandler, request) + self.assertIsNotNone(MockRestApiServer(RestApiHandler, request)) diff --git a/tests/test_ctl.py b/tests/test_ctl.py index fb63aebd..12ab4ad8 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -69,20 +69,16 @@ class TestCtl(unittest.TestCase): @patch('psycopg2.connect', psycopg2_connect) def test_get_cursor(self): - c = get_cursor(get_cluster_initialized_without_leader(), role='master') - assert c is None + self.assertIsNone(get_cursor(get_cluster_initialized_without_leader(), role='master')) - c = get_cursor(get_cluster_initialized_with_leader(), role='master') - assert c is not None + self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), role='master')) - c = get_cursor(get_cluster_initialized_with_leader(), role='replica') - # # MockCursor returns pg_is_in_recovery as false - assert c is None + # MockCursor returns pg_is_in_recovery as false + self.assertIsNone(get_cursor(get_cluster_initialized_with_leader(), role='replica')) - c = get_cursor(get_cluster_initialized_with_leader(), role='any') - assert c is not None + self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), role='any')) - def test_output_members(self): + def test_output_members(*args): cluster = get_cluster_initialized_with_leader() output_members(cluster, name='abc', fmt='pretty') output_members(cluster, name='abc', fmt='json') @@ -224,17 +220,17 @@ y''') @patch('patroni.ctl.get_cursor', Mock(return_value=MockConnect().cursor())) def test_query_member(self): rows = query_member(None, None, None, 'master', 'SELECT pg_is_in_recovery()') - assert 'False' in str(rows) + self.assertTrue('False' in str(rows)) rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()') - assert rows == (None, None) + self.assertEquals(rows, (None, None)) with patch('patroni.ctl.get_cursor', Mock(return_value=None)): rows = query_member(None, None, None, None, 'SELECT pg_is_in_recovery()') - assert 'No connection to' in str(rows) + self.assertTrue('No connection to' in str(rows)) rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()') - assert 'No connection to' in str(rows) + self.assertTrue('No connection to' in str(rows)) with patch('patroni.ctl.get_cursor', Mock(side_effect=psycopg2.OperationalError('bla'))): rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()') @@ -337,26 +333,23 @@ leader''') assert 'Usage:' in result.output def test_get_any_member(self): - m = get_any_member(get_cluster_initialized_without_leader(), role='master') - assert m is None + self.assertIsNone(get_any_member(get_cluster_initialized_without_leader(), role='master')) m = get_any_member(get_cluster_initialized_with_leader(), role='master') - assert m.name == 'leader' + self.assertEquals(m.name, 'leader') def test_get_all_members(self): - r = list(get_all_members(get_cluster_initialized_without_leader(), role='master')) - assert len(r) == 0 + self.assertEquals(list(get_all_members(get_cluster_initialized_without_leader(), role='master')), []) r = list(get_all_members(get_cluster_initialized_with_leader(), role='master')) - assert len(r) == 1 - assert r[0].name == 'leader' + self.assertEquals(len(r), 1) + self.assertEquals(r[0].name, 'leader') r = list(get_all_members(get_cluster_initialized_with_leader(), role='replica')) - assert len(r) == 1 - assert r[0].name == 'other' + self.assertEquals(len(r), 1) + self.assertEquals(r[0].name, 'other') - r = list(get_all_members(get_cluster_initialized_without_leader(), role='replica')) - assert len(r) == 2 + self.assertEquals(len(list(get_all_members(get_cluster_initialized_without_leader(), role='replica'))), 2) @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None)) diff --git a/tests/test_utils.py b/tests/test_utils.py index 6f66f4c3..740fef03 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -16,14 +16,14 @@ class TestUtils(unittest.TestCase): @patch('time.sleep', Mock()) def test_reap_children(self): - reap_children() + self.assertIsNone(reap_children()) with patch('os.waitpid', Mock(return_value=(0, 0))): sigchld_handler(None, None) - reap_children() + self.assertIsNone(reap_children()) @patch('time.sleep', time_sleep) def test_sleep(self): - sleep(0.01) + self.assertIsNone(sleep(0.01)) @patch('time.sleep', Mock()) From 4038d94c5ac61b345c5248d72a640e7bfd975073 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 17 Feb 2016 14:59:17 +0100 Subject: [PATCH 14/89] Fix more codacy issues --- tests/test_api.py | 26 +++++++++++++------------- tests/test_ctl.py | 8 ++++---- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 265394e8..dd0f3dfa 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -87,7 +87,7 @@ class MockRestApiServer(RestApiServer): @patch('ssl.wrap_socket', Mock(return_value=0)) class TestRestApiHandler(unittest.TestCase): - def test_do_GET(*args): + def test_do_GET(self): MockRestApiServer(RestApiHandler, b'GET /replica') with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={})): MockRestApiServer(RestApiHandler, b'GET /replica') @@ -101,10 +101,10 @@ class TestRestApiHandler(unittest.TestCase): MockRestApiServer(RestApiHandler, b'GET /master') with patch.object(MockHa, 'restart_scheduled', Mock(return_value=True)): MockRestApiServer(RestApiHandler, b'GET /master') - MockRestApiServer(RestApiHandler, b'GET /master') + self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /master')) - def test_do_OPTIONS(*args): - MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0') + def test_do_OPTIONS(self): + self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0')) with patch.object(BaseHTTPRequestHandler, 'handle_one_request') as mock_handle_request: mock_handle_request.side_effect = socket.error("foo") @@ -117,16 +117,16 @@ class TestRestApiHandler(unittest.TestCase): makefile.return_value.flush = Mock(side_effect=socket.error("foo")) MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0') - def test_do_GET_patroni(*args): - MockRestApiServer(RestApiHandler, b'GET /patroni') + def test_do_GET_patroni(self): + self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /patroni')) - def test_basicauth(*args): - MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0') + def test_basicauth(self): + self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0')) MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0\nAuthorization:') - def test_do_POST_restart(*args): + def test_do_POST_restart(self): request = b'POST /restart HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0' - MockRestApiServer(RestApiHandler, request) + self.assertIsNotNone(MockRestApiServer(RestApiHandler, request)) with patch.object(MockHa, 'restart', Mock(side_effect=Exception)): MockRestApiServer(RestApiHandler, request) @@ -143,11 +143,11 @@ class TestRestApiHandler(unittest.TestCase): self.assertIsNotNone(MockRestApiServer(RestApiHandler, request)) @patch('time.sleep', Mock()) - def test_RestApiServer_query(*args): + def test_RestApiServer_query(self): with patch.object(MockCursor, 'execute', Mock(side_effect=psycopg2.OperationalError)): - MockRestApiServer(RestApiHandler, b'GET /patroni') + self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /patroni')) with patch.object(MockPostgresql, 'connection', Mock(side_effect=psycopg2.OperationalError)): - MockRestApiServer(RestApiHandler, b'GET /patroni') + self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /patroni')) @patch('time.sleep', Mock()) @patch.object(MockHa, 'dcs') diff --git a/tests/test_ctl.py b/tests/test_ctl.py index 12ab4ad8..8559e793 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -78,11 +78,11 @@ class TestCtl(unittest.TestCase): self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), role='any')) - def test_output_members(*args): + def test_output_members(self): cluster = get_cluster_initialized_with_leader() - output_members(cluster, name='abc', fmt='pretty') - output_members(cluster, name='abc', fmt='json') - output_members(cluster, name='abc', fmt='tsv') + self.assertIsNone(output_members(cluster, name='abc', fmt='pretty')) + self.assertIsNone(output_members(cluster, name='abc', fmt='json')) + self.assertIsNone(output_members(cluster, name='abc', fmt='tsv')) @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None)) From 26e15862884933fced7694e240605d99e02da4f6 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Mon, 22 Feb 2016 12:20:20 +0100 Subject: [PATCH 15/89] Make the patronictl test provide an input for the schedule, even if it's empty. --- tests/test_ctl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_ctl.py b/tests/test_ctl.py index 16814839..9ae7e3f0 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -118,6 +118,7 @@ y''') # Aborting failover,as we anser NO to the confirmation result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader other + N''') assert result.exit_code == 1 @@ -159,6 +160,7 @@ y''') # No members available result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader other + y''') assert result.exit_code == 1 From 287c0b312522e4dd25b613311e6c34cb61245d9d Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Mon, 22 Feb 2016 14:31:29 +0100 Subject: [PATCH 16/89] Fix the call to the function that was forgotten to be renamed. --- patroni/ha.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/patroni/ha.py b/patroni/ha.py index 4aead262..72414d89 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -272,7 +272,7 @@ class Ha(object): self.dcs.delete_leader() self.touch_member() self.dcs.reset_cluster() - self.state_handler.follow_the_leader(None) + self.state_handler.follow(None) def process_manual_failover_from_leader(self): failover = self.cluster.failover From 641cc4013e76156dc27b22642499341a8cc703e9 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Tue, 23 Feb 2016 11:46:49 +0100 Subject: [PATCH 17/89] Mock a few of methods in Postgresql class instead of the whole class --- tests/test_ha.py | 100 +++++++++++++++++------------------------------ 1 file changed, 36 insertions(+), 64 deletions(-) diff --git a/tests/test_ha.py b/tests/test_ha.py index 3589e952..cdf1156f 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -1,13 +1,14 @@ -import etcd import unittest import datetime import pytz +from etcd import EtcdException from mock import Mock, MagicMock, patch from patroni.dcs import Cluster, Failover, Leader, Member from patroni.etcd import Client, Etcd from patroni.exceptions import DCSError, PostgresException from patroni.ha import Ha +from patroni.postgresql import Postgresql from test_etcd import socket_getaddrinfo, etcd_read, etcd_write, requests_get @@ -45,56 +46,6 @@ def get_cluster_initialized_with_only_leader(failover=None): return get_cluster(True, l, [l], failover) -class MockPostgresql(Mock): - - name = 'postgresql0' - role = 'replica' - state = 'running' - connection_string = 'postgres://foo@bar/postgres' - server_version = '999999' - scope = 'dummy' - - @staticmethod - def is_healthy(): - return True - - @staticmethod - def start(): - return True - - @staticmethod - def is_healthiest_node(members): - return True - - @staticmethod - def is_leader(): - return True - - @staticmethod - def xlog_position(): - return 0 - - @staticmethod - def last_operation(): - return 0 - - @staticmethod - def data_directory_empty(): - return False - - @staticmethod - def bootstrap(*args, **kwargs): - return True - - @staticmethod - def check_replication_lag(last_leader_operation): - return True - - @staticmethod - def check_recovery_conf(leader): - return False - - class MockPatroni(object): def __init__(self, p, d): @@ -111,18 +62,36 @@ def run_async(func, args=()): return func(*args) if args else func() +@patch.object(Postgresql, 'is_running', Mock(return_value=True)) +@patch.object(Postgresql, 'is_leader', Mock(return_value=True)) +@patch.object(Postgresql, 'xlog_position', Mock(return_value=0)) +@patch.object(Postgresql, 'call_nowait', Mock(return_value=True)) +@patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False)) +@patch.object(Postgresql, 'controldata', Mock(return_value={})) +@patch.object(Postgresql, 'sync_replication_slots', Mock()) +@patch.object(Postgresql, 'write_pg_hba', Mock()) +@patch.object(Postgresql, 'write_pgpass', Mock()) +@patch.object(Postgresql, 'write_recovery_conf', Mock()) +@patch.object(Postgresql, 'query', Mock()) +@patch.object(Postgresql, 'checkpoint', Mock()) +@patch('subprocess.call', Mock(return_value=0)) class TestHa(unittest.TestCase): @patch('socket.getaddrinfo', socket_getaddrinfo) def setUp(self): with patch.object(Client, 'machines') as mock_machines: mock_machines.__get__ = Mock(return_value=['http://remotehost:2379']) - self.p = MockPostgresql() + self.p = Postgresql({'name': 'postgresql0', 'scope': 'dummy', 'listen': '127.0.0.1:5432', + 'data_dir': 'data/postgresql0', 'superuser': {}, 'admin': {}, + 'replication': {'username': '', 'password': '', 'network': ''}}) + self.p._state = 'running' + self.p._sysid = '1234567890' + self.p.check_replication_lag = true self.p.can_create_replica_without_leader = MagicMock(return_value=False) self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'}) self.e.client.read = etcd_read self.e.client.write = etcd_write - self.e.client.delete = Mock(side_effect=etcd.EtcdException()) + self.e.client.delete = Mock(side_effect=EtcdException()) self.ha = Ha(MockPatroni(self.p, self.e)) self.ha._async_executor.run_async = run_async self.ha.old_cluster = self.e.get_cluster() @@ -154,7 +123,7 @@ class TestHa(unittest.TestCase): self.p.is_healthy = false self.p.is_running = false self.ha.has_lock = true - self.p.role = 'master' + self.p._role = 'master' self.p.controldata = lambda: {'Database cluster state': 'in production'} self.assertEquals(self.ha.run_cycle(), 'started as readonly because i had the session lock') self.assertEquals(self.ha.run_cycle(), 'removed leader key after trying and failing to start postgres') @@ -302,57 +271,60 @@ class TestHa(unittest.TestCase): self.ha.has_lock = true self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None)) self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock') - self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', MockPostgresql.name, None)) + self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', self.p.name, None)) self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock') self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'blabla', None)) self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock') - f = Failover(0, MockPostgresql.name, '', None) + f = Failover(0, self.p.name, '', None) self.ha.cluster = get_cluster_initialized_with_leader(f) self.assertEquals(self.ha.run_cycle(), 'manual failover: demoting myself') self.ha.fetch_node_status = lambda e: (e, True, True, 0, {'nofailover': 'True'}) self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock') # manual failover from the previous leader to us won't happen if we hold the nofailover flag - self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name, None)) + self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, None)) self.assertEquals(self.ha.run_cycle(), 'no action. i am the leader with the lock') # Failover scheduled time must include timezone scheduled = datetime.datetime.now() - self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name, scheduled)) + self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled)) self.ha.run_cycle() scheduled = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC) - self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name, scheduled)) + self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled)) self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle()) scheduled = scheduled + datetime.timedelta(seconds=30) - self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name, scheduled)) + self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled)) self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle()) scheduled = scheduled + datetime.timedelta(seconds=-600) - self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name, scheduled)) + self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled)) self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle()) scheduled = None - self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name, scheduled)) + self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled)) self.assertEquals('no action. i am the leader with the lock', self.ha.run_cycle()) @patch('requests.get', requests_get) def test_manual_failover_process_no_leader(self): self.p.is_leader = false - self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', MockPostgresql.name, None)) + self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', self.p.name, None)) self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader', None)) + self.p._role = 'replica' self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') self.ha.fetch_node_status = lambda e: (e, True, True, 0, {}) # accessible, in_recovery self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node') - self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, MockPostgresql.name, '', None)) + self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, self.p.name, '', None)) self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node') self.ha.fetch_node_status = lambda e: (e, False, True, 0, {}) # inaccessible, in_recovery + self.p._role = 'replica' self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') # set failover flag to True for all members of the cluster # this should elect the current member, as we are not going to call the API for it. self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None)) self.ha.fetch_node_status = lambda e: (e, True, True, 0, {'nofailover': 'True'}) # accessible, in_recovery + self.p._role = 'replica' self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') # same as previous, but set the current member to nofailover. In no case it should be elected as a leader self.ha.patroni.nofailover = True From dd20fc7e71ad05423b65fa6d2c24e77a25888d44 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Tue, 23 Feb 2016 11:47:47 +0100 Subject: [PATCH 18/89] Refactor follow method --- patroni/postgresql.py | 76 +++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/patroni/postgresql.py b/patroni/postgresql.py index 893503e2..a1cedb58 100644 --- a/patroni/postgresql.py +++ b/patroni/postgresql.py @@ -559,47 +559,45 @@ recovery_target_timeline = 'latest' logger.exception("Unable to list %s", status_dir) def follow(self, leader, recovery=False): - if not self.check_recovery_conf(leader) or recovery: - change_role = (self.role == 'master') - - self._need_rewind = (self._need_rewind or change_role) and self.can_rewind - if self._need_rewind: - logger.info("set the rewind flag after demote") - self.write_recovery_conf(leader) - if not leader or not self._need_rewind: # do not rewind until the leader becomes available - ret = self.restart() - else: # we have a leader and need to rewind - if self.is_running(): - self.stop() - # at present, pg_rewind only runs when the cluster is shut down cleanly - # and not shutdown in recovery. We have to remove the recovery.conf if present - # and start/shutdown in a single user mode to emulate this. - # XXX: if recovery.conf is linked, it will be written anew as a normal file. - if os.path.islink(self.recovery_conf): - os.unlink(self.recovery_conf) - else: - os.remove(self.recovery_conf) - # Archived segments might be useful to pg_rewind, - # clean the flags that tell we should remove them. - self.cleanup_archive_status() - # Start in a single user mode and stop to produce a clean shutdown - opts = self.read_postmaster_opts() - opts['archive_mode'] = 'on' - opts['archive_command'] = 'false' - self.single_user_mode(options=opts) - if self.rewind(leader): - ret = self.start() - else: - logger.error("unable to rewind the former master") - self.remove_data_directory() - ret = True - self._need_rewind = False - if change_role and ret: - self.call_nowait(ACTION_ON_ROLE_CHANGE) - return ret - else: + if self.check_recovery_conf(leader) and not recovery: return True + change_role = self.role == 'master' + self._need_rewind = (self._need_rewind or change_role) and self.can_rewind + if self._need_rewind: + logger.info("set the rewind flag after demote") + self.write_recovery_conf(leader) + if leader and self._need_rewind: # we have a leader and need to rewind + if self.is_running(): + self.stop() + # at present, pg_rewind only runs when the cluster is shut down cleanly + # and not shutdown in recovery. We have to remove the recovery.conf if present + # and start/shutdown in a single user mode to emulate this. + # XXX: if recovery.conf is linked, it will be written anew as a normal file. + if os.path.islink(self.recovery_conf): + os.unlink(self.recovery_conf) + else: + os.remove(self.recovery_conf) + # Archived segments might be useful to pg_rewind, + # clean the flags that tell we should remove them. + self.cleanup_archive_status() + # Start in a single user mode and stop to produce a clean shutdown + opts = self.read_postmaster_opts() + opts.update({'archive_mode': 'on', 'archive_command': 'false'}) + self.single_user_mode(options=opts) + if self.rewind(leader): + ret = self.start() + else: + logger.error("unable to rewind the former master") + self.remove_data_directory() + ret = True + self._need_rewind = False + else: # do not rewind until the leader becomes available + ret = self.restart() + if change_role and ret: + self.call_nowait(ACTION_ON_ROLE_CHANGE) + return ret + def save_configuration_files(self): """ copy postgresql.conf to postgresql.conf.backup to be able to retrive configuration files From ce33090c0d4d0665ab80788e106b4059dec5bd99 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Tue, 23 Feb 2016 11:48:52 +0100 Subject: [PATCH 19/89] Mock dcs.watch directly instead of using wraper --- tests/test_patroni.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_patroni.py b/tests/test_patroni.py index aaefdf1b..2698de58 100644 --- a/tests/test_patroni.py +++ b/tests/test_patroni.py @@ -15,10 +15,6 @@ from test_postgresql import Postgresql, psycopg2_connect from test_zookeeper import MockKazooClient -def time_sleep(*args): - raise SleepException() - - @patch('time.sleep', Mock()) @patch('subprocess.call', Mock(return_value=0)) @patch('psycopg2.connect', psycopg2_connect) @@ -62,7 +58,7 @@ class TestPatroni(unittest.TestCase): @patch('time.sleep', Mock(side_effect=SleepException())) def test_run(self): - self.p.ha.dcs.watch = time_sleep + self.p.ha.dcs.watch = Mock(side_effect=SleepException()) self.assertRaises(SleepException, self.p.run) self.p.ha.state_handler.is_leader = Mock(return_value=False) From 6b3c4697fc36409280911e378d7c146854bbb08e Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Tue, 23 Feb 2016 11:49:22 +0100 Subject: [PATCH 20/89] Remove unused code --- tests/test_ctl.py | 40 ++++++++++++++-------------------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/tests/test_ctl.py b/tests/test_ctl.py index a877ed4b..b2590da5 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -1,25 +1,19 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - import os import pytest import unittest -import psycopg2 -import requests -import patroni.exceptions -import etcd -from mock import patch, Mock, MagicMock - from click.testing import CliRunner +from etcd import EtcdException +from mock import patch, Mock, MagicMock from patroni.ctl import ctl, members, store_config, load_config, output_members, post_patroni, get_dcs, \ wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure -from patroni.ha import Ha from patroni.etcd import Etcd, Client -from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \ - get_cluster_initialized_with_only_leader, MockPostgresql, MockPatroni, run_async, \ - get_cluster_not_initialized_without_leader +from patroni.exceptions import PatroniCtlException +from psycopg2 import OperationalError +from requests.exceptions import ConnectionError from test_etcd import etcd_read, etcd_write, requests_get, socket_getaddrinfo, MockResponse +from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \ + get_cluster_initialized_with_only_leader from test_postgresql import MockConnect, psycopg2_connect CONFIG_FILE_PATH = './test-ctl.yaml' @@ -56,16 +50,10 @@ class TestCtl(unittest.TestCase): self.runner = CliRunner() with patch.object(Client, 'machines') as mock_machines: mock_machines.__get__ = Mock(return_value=['http://remotehost:2379']) - self.p = MockPostgresql() self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'}) self.e.client.read = etcd_read self.e.client.write = etcd_write - self.e.client.delete = Mock(side_effect=etcd.EtcdException()) - self.ha = Ha(MockPatroni(self.p, self.e)) - self.ha._async_executor.run_async = run_async - self.ha.old_cluster = self.e.get_cluster() - self.ha.cluster = get_cluster_not_initialized_without_leader() - self.ha.load_cluster_from_dcs = Mock() + self.e.client.delete = Mock(side_effect=EtcdException) @patch('psycopg2.connect', psycopg2_connect) def test_get_cursor(self): @@ -186,7 +174,7 @@ y''') assert 'Failover failed' in result.output def test_(self): - self.assertRaises(patroni.exceptions.PatroniCtlException, get_dcs, {'scheme': 'dummy'}, 'dummy') + self.assertRaises(PatroniCtlException, get_dcs, {'scheme': 'dummy'}, 'dummy') @patch('psycopg2.connect', psycopg2_connect) @patch('patroni.ctl.query_member', Mock(return_value=([['mock column']], None))) @@ -248,10 +236,10 @@ y''') rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()') self.assertTrue('No connection to' in str(rows)) - with patch('patroni.ctl.get_cursor', Mock(side_effect=psycopg2.OperationalError('bla'))): + with patch('patroni.ctl.get_cursor', Mock(side_effect=OperationalError('bla'))): rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()') - with patch('test_postgresql.MockCursor.execute', Mock(side_effect=psycopg2.OperationalError('bla'))): + with patch('test_postgresql.MockCursor.execute', Mock(side_effect=OperationalError('bla'))): rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()') @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) @@ -341,15 +329,15 @@ leader''') @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) def test_wait_for_leader(self): dcs = self.e - self.assertRaises(patroni.exceptions.PatroniCtlException, wait_for_leader, dcs, 0) + self.assertRaises(PatroniCtlException, wait_for_leader, dcs, 0) cluster = wait_for_leader(dcs=dcs, timeout=2) assert cluster.leader.member.name == 'leader' def test_post_patroni(self): - with patch('requests.post', MagicMock(side_effect=requests.exceptions.ConnectionError('foo'))): + with patch('requests.post', MagicMock(side_effect=ConnectionError('foo'))): member = get_cluster_initialized_with_leader().leader.member - self.assertRaises(requests.exceptions.ConnectionError, post_patroni, member, 'dummy', {}) + self.assertRaises(ConnectionError, post_patroni, member, 'dummy', {}) def test_ctl(self): self.runner.invoke(ctl, ['list']) From 756158a735efd3cbb127e5749fc82b8b232a1d74 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Tue, 23 Feb 2016 11:59:02 +0100 Subject: [PATCH 21/89] make codacy and quantifiedcode happier --- tests/test_ctl.py | 6 +++--- tests/test_ha.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_ctl.py b/tests/test_ctl.py index b2590da5..3987619d 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -1,5 +1,6 @@ import os import pytest +import requests.exceptions import unittest from click.testing import CliRunner @@ -10,7 +11,6 @@ from patroni.ctl import ctl, members, store_config, load_config, output_members, from patroni.etcd import Etcd, Client from patroni.exceptions import PatroniCtlException from psycopg2 import OperationalError -from requests.exceptions import ConnectionError from test_etcd import etcd_read, etcd_write, requests_get, socket_getaddrinfo, MockResponse from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \ get_cluster_initialized_with_only_leader @@ -335,9 +335,9 @@ leader''') assert cluster.leader.member.name == 'leader' def test_post_patroni(self): - with patch('requests.post', MagicMock(side_effect=ConnectionError('foo'))): + with patch('requests.post', MagicMock(side_effect=requests.exceptions.ConnectionError('foo'))): member = get_cluster_initialized_with_leader().leader.member - self.assertRaises(ConnectionError, post_patroni, member, 'dummy', {}) + self.assertRaises(requests.exceptions.ConnectionError, post_patroni, member, 'dummy', {}) def test_ctl(self): self.runner.invoke(ctl, ['list']) diff --git a/tests/test_ha.py b/tests/test_ha.py index cdf1156f..57754aa6 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -84,7 +84,7 @@ class TestHa(unittest.TestCase): self.p = Postgresql({'name': 'postgresql0', 'scope': 'dummy', 'listen': '127.0.0.1:5432', 'data_dir': 'data/postgresql0', 'superuser': {}, 'admin': {}, 'replication': {'username': '', 'password': '', 'network': ''}}) - self.p._state = 'running' + self.p.set_state('running') self.p._sysid = '1234567890' self.p.check_replication_lag = true self.p.can_create_replica_without_leader = MagicMock(return_value=False) @@ -123,7 +123,7 @@ class TestHa(unittest.TestCase): self.p.is_healthy = false self.p.is_running = false self.ha.has_lock = true - self.p._role = 'master' + self.p.set_role('master') self.p.controldata = lambda: {'Database cluster state': 'in production'} self.assertEquals(self.ha.run_cycle(), 'started as readonly because i had the session lock') self.assertEquals(self.ha.run_cycle(), 'removed leader key after trying and failing to start postgres') @@ -311,20 +311,20 @@ class TestHa(unittest.TestCase): self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', self.p.name, None)) self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader', None)) - self.p._role = 'replica' + self.p.set_role('replica') self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') self.ha.fetch_node_status = lambda e: (e, True, True, 0, {}) # accessible, in_recovery self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node') self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, self.p.name, '', None)) self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node') self.ha.fetch_node_status = lambda e: (e, False, True, 0, {}) # inaccessible, in_recovery - self.p._role = 'replica' + self.p.set_role('replica') self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') # set failover flag to True for all members of the cluster # this should elect the current member, as we are not going to call the API for it. self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None)) self.ha.fetch_node_status = lambda e: (e, True, True, 0, {'nofailover': 'True'}) # accessible, in_recovery - self.p._role = 'replica' + self.p.set_role('replica') self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock') # same as previous, but set the current member to nofailover. In no case it should be elected as a leader self.ha.patroni.nofailover = True From ec85e2eb4908a7fa1b50257e1de8cfd2d81f9a23 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Tue, 23 Feb 2016 12:05:02 +0100 Subject: [PATCH 22/89] make quantifiedcode happier --- tests/test_ha.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_ha.py b/tests/test_ha.py index 57754aa6..a65a444a 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -67,7 +67,7 @@ def run_async(func, args=()): @patch.object(Postgresql, 'xlog_position', Mock(return_value=0)) @patch.object(Postgresql, 'call_nowait', Mock(return_value=True)) @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False)) -@patch.object(Postgresql, 'controldata', Mock(return_value={})) +@patch.object(Postgresql, 'controldata', Mock(return_value={'Database system identifier': '1234567890'})) @patch.object(Postgresql, 'sync_replication_slots', Mock()) @patch.object(Postgresql, 'write_pg_hba', Mock()) @patch.object(Postgresql, 'write_pgpass', Mock()) @@ -85,7 +85,6 @@ class TestHa(unittest.TestCase): 'data_dir': 'data/postgresql0', 'superuser': {}, 'admin': {}, 'replication': {'username': '', 'password': '', 'network': ''}}) self.p.set_state('running') - self.p._sysid = '1234567890' self.p.check_replication_lag = true self.p.can_create_replica_without_leader = MagicMock(return_value=False) self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'}) From 2f84e9f4ec38bec637b86b5b229d52c163658346 Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Tue, 23 Feb 2016 16:44:09 +0100 Subject: [PATCH 23/89] Add support for ZooKeeper/Exhibitor DCS URI in patronictl ... -d --- patroni/ctl.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/patroni/ctl.py b/patroni/ctl.py index fb4c241b..48a0c413 100644 --- a/patroni/ctl.py +++ b/patroni/ctl.py @@ -16,6 +16,7 @@ from six.moves.urllib_parse import urlparse import logging from .etcd import Etcd +from .zookeeper import ZooKeeper from .exceptions import PatroniCtlException from .postgresql import parseurl @@ -44,12 +45,12 @@ def parse_dcs(dcs): parsed = urlparse('//' + dcs) if scheme == '': - default_schemes = {'2181': 'zookeeper', '8500': 'consul'} + default_schemes = {'2181': 'zookeeper', '8181': 'exhibitor', '8500': 'consul'} scheme = default_schemes.get(str(parsed.port), 'etcd') port = parsed.port if port is None: - default_ports = {'consul': 8500, 'zookeeper': 2181} + default_ports = {'consul': 8500, 'zookeeper': 2181, 'exhibitor': 8181} port = default_ports.get(str(scheme), 4001) return {'scheme': str(scheme), 'hostname': str(parsed.hostname), 'port': int(port)} @@ -103,6 +104,12 @@ def get_dcs(config, scope): if scheme == 'etcd': return Etcd(name=scope, config={'scope': scope, 'host': '{0}:{1}'.format(hostname, port)}) + if scheme == 'zookeeper': + return ZooKeeper(name=scope, config={'scope': scope, 'hosts': [hostname], 'port': port}) + + if scheme == 'exhibitor': + return ZooKeeper(name=scope, config={'scope': scope, 'exhibitor': {'hosts': [hostname], 'port': port}}) + raise PatroniCtlException('Can not find suitable configuration of distributed configuration store') From a1417875a2e2e223f3b8a309c4e19047db644a93 Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Tue, 23 Feb 2016 17:04:16 +0100 Subject: [PATCH 24/89] Add dummy patronictl tests with ZooKeeper --- tests/test_ctl.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_ctl.py b/tests/test_ctl.py index fb63aebd..e0e5fc99 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -16,10 +16,12 @@ from patroni.ctl import ctl, members, store_config, load_config, output_members, wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure from patroni.ha import Ha from patroni.etcd import Etcd, Client +from patroni.zookeeper import ZooKeeper from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \ get_cluster_initialized_with_only_leader, MockPostgresql, MockPatroni, run_async, \ get_cluster_not_initialized_without_leader from test_etcd import etcd_read, etcd_write, requests_get, socket_getaddrinfo, MockResponse +from test_zookeeper import MockKazooClient from test_postgresql import MockConnect, psycopg2_connect CONFIG_FILE_PATH = './test-ctl.yaml' @@ -52,6 +54,7 @@ def test_rw_config(): class TestCtl(unittest.TestCase): @patch('socket.getaddrinfo', socket_getaddrinfo) + @patch('patroni.zookeeper.KazooClient', MockKazooClient) def setUp(self): self.runner = CliRunner() with patch.object(Client, 'machines') as mock_machines: @@ -61,6 +64,7 @@ class TestCtl(unittest.TestCase): self.e.client.read = etcd_read self.e.client.write = etcd_write self.e.client.delete = Mock(side_effect=etcd.EtcdException()) + self.zk = ZooKeeper('foo', {'ttl': 30, 'hosts': ['ok:2181'], 'scope': 'test'}) self.ha = Ha(MockPatroni(self.p, self.e)) self.ha._async_executor.run_async = run_async self.ha.old_cluster = self.e.get_cluster() @@ -377,3 +381,13 @@ leader''') ]) assert result.exit_code == 0 + + @patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'zookeeper', 'hostname': 'localhost', 'port': 2181}})) + def test_zookeeper(self): + with patch('patroni.ctl.get_dcs', Mock(return_value=self.zk)): + self.runner.invoke(ctl, ['list']) + + @patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'exhibitor', 'hostname': 'localhost', 'port': 8181}})) + def test_exhibitor(self): + with patch('patroni.ctl.get_dcs', Mock(return_value=self.zk)): + self.runner.invoke(ctl, ['list']) From 524cfafbbe07a1fe6a471245e5255edf803be7b5 Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Tue, 23 Feb 2016 17:18:16 +0100 Subject: [PATCH 25/89] Don't mock get_dcs() for ZK, we are trying to test it --- tests/test_ctl.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_ctl.py b/tests/test_ctl.py index 55612a94..f8f2783a 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -390,10 +390,8 @@ leader''') @patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'zookeeper', 'hostname': 'localhost', 'port': 2181}})) def test_zookeeper(self): - with patch('patroni.ctl.get_dcs', Mock(return_value=self.zk)): - self.runner.invoke(ctl, ['list']) + self.runner.invoke(ctl, ['list']) @patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'exhibitor', 'hostname': 'localhost', 'port': 8181}})) def test_exhibitor(self): - with patch('patroni.ctl.get_dcs', Mock(return_value=self.zk)): - self.runner.invoke(ctl, ['list']) + self.runner.invoke(ctl, ['list']) From 16b321e0a53494d70e5c606f023d78aef02c9d94 Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Tue, 23 Feb 2016 17:33:15 +0100 Subject: [PATCH 26/89] Add dummy cluster name to test_ctl / zookeeper --- tests/test_ctl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_ctl.py b/tests/test_ctl.py index f8f2783a..4092a669 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -390,8 +390,8 @@ leader''') @patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'zookeeper', 'hostname': 'localhost', 'port': 2181}})) def test_zookeeper(self): - self.runner.invoke(ctl, ['list']) + self.runner.invoke(ctl, ['list', 'foo']) @patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'exhibitor', 'hostname': 'localhost', 'port': 8181}})) def test_exhibitor(self): - self.runner.invoke(ctl, ['list']) + self.runner.invoke(ctl, ['list', 'foo']) From 9c12eb671da4c83e7f3f2dae35bb2eed2b644741 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Wed, 24 Feb 2016 12:10:58 +0100 Subject: [PATCH 27/89] Fix unit tests --- tests/test_ctl.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/tests/test_ctl.py b/tests/test_ctl.py index 4092a669..ed3aa1b9 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -9,7 +9,6 @@ from mock import patch, Mock, MagicMock from patroni.ctl import ctl, members, store_config, load_config, output_members, post_patroni, get_dcs, \ wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure from patroni.etcd import Etcd, Client -from patroni.zookeeper import ZooKeeper from patroni.exceptions import PatroniCtlException from psycopg2 import OperationalError from test_etcd import etcd_read, etcd_write, requests_get, socket_getaddrinfo, MockResponse @@ -48,7 +47,6 @@ def test_rw_config(): class TestCtl(unittest.TestCase): @patch('socket.getaddrinfo', socket_getaddrinfo) - @patch('patroni.zookeeper.KazooClient', MockKazooClient) def setUp(self): self.runner = CliRunner() with patch.object(Client, 'machines') as mock_machines: @@ -57,7 +55,6 @@ class TestCtl(unittest.TestCase): self.e.client.read = etcd_read self.e.client.write = etcd_write self.e.client.delete = Mock(side_effect=EtcdException) - self.zk = ZooKeeper('foo', {'ttl': 30, 'hosts': ['ok:2181'], 'scope': 'test'}) @patch('psycopg2.connect', psycopg2_connect) def test_get_cursor(self): @@ -177,7 +174,11 @@ other y''') assert 'Failover failed' in result.output - def test_(self): + @patch('patroni.zookeeper.KazooClient', MockKazooClient) + @patch('requests.get', requests_get) + def test_get_dcs(self): + self.assertIsNotNone(get_dcs({'dcs': {'scheme': 'zookeeper', 'hostname': 'foo', 'port': 2181}}, 'dummy')) + self.assertIsNotNone(get_dcs({'dcs': {'scheme': 'exhibitor', 'hostname': 'exhibitor', 'port': 8181}}, 'dummy')) self.assertRaises(PatroniCtlException, get_dcs, {'scheme': 'dummy'}, 'dummy') @patch('psycopg2.connect', psycopg2_connect) @@ -387,11 +388,3 @@ leader''') ]) assert result.exit_code == 0 - - @patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'zookeeper', 'hostname': 'localhost', 'port': 2181}})) - def test_zookeeper(self): - self.runner.invoke(ctl, ['list', 'foo']) - - @patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'exhibitor', 'hostname': 'localhost', 'port': 8181}})) - def test_exhibitor(self): - self.runner.invoke(ctl, ['list', 'foo']) From 6ec3523748a7214f111bff32298ad2c689492383 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Wed, 24 Feb 2016 16:30:52 +0100 Subject: [PATCH 28/89] Collect test output, add basic failover test. --- features/basic_failover.feature | 13 +++++++ features/basic_failover.py | 33 ++++++++++++++++ features/basic_replication.feature | 14 +++---- features/basic_replication.py | 14 +++---- features/terrain.py | 60 +++++++++++++++++++++++++++--- 5 files changed, 113 insertions(+), 21 deletions(-) create mode 100644 features/basic_failover.feature create mode 100644 features/basic_failover.py diff --git a/features/basic_failover.feature b/features/basic_failover.feature new file mode 100644 index 00000000..95b7d646 --- /dev/null +++ b/features/basic_failover.feature @@ -0,0 +1,13 @@ +Feature: basic failover + In order to check that failover works + As observers + We start the primary and the replica, + shut down the primary + and check that the replica assumed the primary role. + +Scenario: check the basic failover + Given basic replication + When I shut down postgres0 + Then postgres1 role is the primary after 10 seconds + When I start postgres0 + Then postgres0 role is the secondary after 10 seconds \ No newline at end of file diff --git a/features/basic_failover.py b/features/basic_failover.py new file mode 100644 index 00000000..661cfb28 --- /dev/null +++ b/features/basic_failover.py @@ -0,0 +1,33 @@ +from lettuce import world, steps + +PATRONI_CONFIG = '{}.yml' + + +@steps +class BasicFailoverSteps(object): + + def __init__(self, environ): + self.env = environ + + def basic_replication(self, step): + '''Basic replication''' + step.behave_as(""" + Given I start postgres0 + And I start postgres1 + When I add the table foo to postgres0 + Then table foo is present on postgres1 after 10 seconds + """) + + def start_patroni(self, step, pg_name): + '''I start (\w+)''' + return world.pctl.start_patroni(pg_name) + + def stop_patroni(self, step, pg_name): + '''I shut down (\w+)''' + return world.pctl.stop_patroni(pg_name) + + def check_role(self, step, pg_name, pg_role, max_promotion_timeout): + '''(\w+) role is the (\w+) after (\d+) seconds''' + return world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)) + +BasicFailoverSteps(world) diff --git a/features/basic_replication.feature b/features/basic_replication.feature index 5e61963d..4306bc10 100644 --- a/features/basic_replication.feature +++ b/features/basic_replication.feature @@ -1,12 +1,12 @@ Feature: basic replication - In order to check that basic replication is working + In order to check that basic replication works As observers - We'll start 2 nodes of a new cluster, - add a table to the primary - and check that it gets replicated to the other over time. + We start the primary and the replica + add a table on the primary + and check that it gets replicated to the replica over time. Scenario: check replication of a single table - Given I have started postgres0 - And I have started postgres1 + Given I start postgres0 + And I start postgres1 When I add the table foo to postgres0 - Then table foo is present on postgres1 + Then table foo is present on postgres1 after 10 seconds diff --git a/features/basic_replication.py b/features/basic_replication.py index 69121223..d856c6fb 100644 --- a/features/basic_replication.py +++ b/features/basic_replication.py @@ -11,13 +11,9 @@ class BasicReplicationSteps(object): def __init__(self, environ): self.env = environ - self.processes = {} - self.connstring = {} - self.cwd = None - self.max_replication_delay = 10 def start_patroni(self, step, pg_name): - '''I have started (\w+)''' + '''I start (\w+)''' return world.pctl.start_patroni(pg_name) def add_table(self, step, table_name, pg_name): @@ -28,14 +24,14 @@ class BasicReplicationSteps(object): except pg.Error as e: assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e) - def table_is_present_on(self, step, table_name, pg_name): - '''Table (\w+) is present on (\w+)''' - for i in range(self.max_replication_delay): + def table_is_present_on(self, step, table_name, pg_name, max_replication_delay): + '''Table (\w+) is present on (\w+) after (\d+) seconds''' + for i in range(int(max_replication_delay)): if world.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None: break sleep(1) else: assert False,\ - "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, self.max_replication_delay) + "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay) BasicReplicationSteps(world) diff --git a/features/terrain.py b/features/terrain.py index 7395deba..3c7a0609 100644 --- a/features/terrain.py +++ b/features/terrain.py @@ -5,7 +5,7 @@ import requests import subprocess import shutil import tempfile -from time import sleep +import time import yaml @@ -27,7 +27,10 @@ class PatroniController(object): self.connstring = {} self.connections = {} self.cursors = {} + self.log = {} + self.config = {} self.availability_check_time_limit = 10 + self.output_dir = None @property def patroni_path(self): @@ -47,16 +50,40 @@ class PatroniController(object): def stop_patroni(self, pg_name): while self.patroni_is_running(pg_name): self.processes[pg_name].terminate() - sleep(1) + time.sleep(1) + self.log.get('pg_name') and self.log[pg_name].close() del self.processes[pg_name] + def make_patroni_test_config(self, pg_name, output_dir): + patroni_config_name = PATRONI_CONFIG.format(pg_name) + patroni_config_path = os.path.join(output_dir, patroni_config_name) + + with open(patroni_config_name) as f: + config = yaml.load(f) + postgresql = config['postgresql']['parameters'] + postgresql['logging_collector'] = 'on' + postgresql['log_destination'] = 'csvlog' + postgresql['log_directory'] = output_dir + postgresql['log_filename'] = '{0}.log'.format(pg_name) + postgresql['log_statement'] = 'all' + postgresql['log_min_messages'] = 'debug1' + + with open(patroni_config_path, 'w') as f: + yaml.dump(config, f, default_flow_style=False) + + return patroni_config_path + def start_patroni(self, pg_name): if not self.patroni_is_running(pg_name): if pg_name in self.processes: del self.processes[pg_name] cwd = self.patroni_path - p = subprocess.Popen(['python', 'patroni.py', PATRONI_CONFIG.format(pg_name)], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd) + self.log[pg_name] = open(os.path.join(self.output_dir, 'patroni_{0}.log'.format(pg_name)), 'a') + + self.config[pg_name] = self.make_patroni_test_config(pg_name, self.output_dir) + + p = subprocess.Popen(['python', 'patroni.py', self.config[pg_name]], + stdout=self.log[pg_name], stderr=subprocess.STDOUT, cwd=cwd) if not (p and p.pid and p.poll() is None): assert False, "PostgreSQL {0} is not running after being started".format(pg_name) self.processes[pg_name] = p @@ -64,7 +91,7 @@ class PatroniController(object): for tick in range(self.availability_check_time_limit): if self.query(pg_name, "SELECT 1", fail_ok=True) is not None: break - sleep(1) + time.sleep(1) else: assert False,\ "Patroni instance is not available for queries after {0} seconds".format(self.availability_check_time_limit) @@ -113,6 +140,20 @@ class PatroniController(object): else: raise + def check_role_has_changed_to(self, pg_name, new_role, timeout=10): + bound_time = time.time() + timeout + current_role = 't' if new_role == 'primary' else 'f' + role_has_changed = False + while not role_has_changed: + cur = self.query(pg_name, "SELECT pg_is_in_recovery()", fail_ok=True) + if cur: + row = cur.fetchone() + if row and len(row) > 0 and row[0] != current_role: + role_has_changed = True + if time.time() > bound_time: + break + return role_has_changed + def stop_all(self): for patroni in self.processes.copy(): self.stop_patroni(patroni) @@ -159,6 +200,15 @@ def stop_etcd(total): etcd_dir = None +@before.each_feature +def make_test_output_dir(feature): + feature_dir = os.path.join(pctl.patroni_path, "features", "output", feature.name.encode('utf-8').replace(' ', '_')) + if os.path.exists(feature_dir): + shutil.rmtree(feature_dir) + os.makedirs(feature_dir) + pctl.output_dir = feature_dir + + def patroni_cleanup_all(): pctl.stop_all() # remove the data directory From 6f039532680fd5cc42c81a0c1b357c528f431727 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Wed, 24 Feb 2016 17:12:45 +0100 Subject: [PATCH 29/89] Merge basic failover and basic replication scenarios in one feature. --- features/basic_failover.feature | 13 ------------ features/basic_failover.py | 33 ------------------------------ features/basic_replication.feature | 10 +++++++++ features/basic_replication.py | 5 +++++ features/terrain.py | 2 +- 5 files changed, 16 insertions(+), 47 deletions(-) delete mode 100644 features/basic_failover.feature delete mode 100644 features/basic_failover.py diff --git a/features/basic_failover.feature b/features/basic_failover.feature deleted file mode 100644 index 95b7d646..00000000 --- a/features/basic_failover.feature +++ /dev/null @@ -1,13 +0,0 @@ -Feature: basic failover - In order to check that failover works - As observers - We start the primary and the replica, - shut down the primary - and check that the replica assumed the primary role. - -Scenario: check the basic failover - Given basic replication - When I shut down postgres0 - Then postgres1 role is the primary after 10 seconds - When I start postgres0 - Then postgres0 role is the secondary after 10 seconds \ No newline at end of file diff --git a/features/basic_failover.py b/features/basic_failover.py deleted file mode 100644 index 661cfb28..00000000 --- a/features/basic_failover.py +++ /dev/null @@ -1,33 +0,0 @@ -from lettuce import world, steps - -PATRONI_CONFIG = '{}.yml' - - -@steps -class BasicFailoverSteps(object): - - def __init__(self, environ): - self.env = environ - - def basic_replication(self, step): - '''Basic replication''' - step.behave_as(""" - Given I start postgres0 - And I start postgres1 - When I add the table foo to postgres0 - Then table foo is present on postgres1 after 10 seconds - """) - - def start_patroni(self, step, pg_name): - '''I start (\w+)''' - return world.pctl.start_patroni(pg_name) - - def stop_patroni(self, step, pg_name): - '''I shut down (\w+)''' - return world.pctl.stop_patroni(pg_name) - - def check_role(self, step, pg_name, pg_role, max_promotion_timeout): - '''(\w+) role is the (\w+) after (\d+) seconds''' - return world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)) - -BasicFailoverSteps(world) diff --git a/features/basic_replication.feature b/features/basic_replication.feature index 4306bc10..140e0cf2 100644 --- a/features/basic_replication.feature +++ b/features/basic_replication.feature @@ -4,9 +4,19 @@ Feature: basic replication We start the primary and the replica add a table on the primary and check that it gets replicated to the replica over time. + We stop the primary and check that the replica promotes itself to primary + We start the old primary and check that it rejoins as a replica. Scenario: check replication of a single table Given I start postgres0 And I start postgres1 When I add the table foo to postgres0 Then table foo is present on postgres1 after 10 seconds + + Scenario: check the basic failover + When I shut down postgres0 + Then postgres1 role is the primary after 10 seconds + When I start postgres0 + Then postgres0 role is the secondary after 10 seconds + When I add the table bar to postgres1 + Then table bar is present on postgres1 after 10 seconds \ No newline at end of file diff --git a/features/basic_replication.py b/features/basic_replication.py index d856c6fb..4c26da37 100644 --- a/features/basic_replication.py +++ b/features/basic_replication.py @@ -34,4 +34,9 @@ class BasicReplicationSteps(object): assert False,\ "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay) + def check_role(self, step, pg_name, pg_role, max_promotion_timeout): + '''(\w+) role is the (\w+) after (\d+) seconds''' + return world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)) + + BasicReplicationSteps(world) diff --git a/features/terrain.py b/features/terrain.py index 3c7a0609..2a9b23cc 100644 --- a/features/terrain.py +++ b/features/terrain.py @@ -224,7 +224,7 @@ def etcd_cleanup(): assert False, "Unable to cleanup etcd: {0}".format(e) -@after.each_scenario +@after.each_feature def cleanup(scenario): patroni_cleanup_all() etcd_cleanup() From c9b8c2d3a91dc1ac275f367c81ccad42fce6fef8 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Wed, 24 Feb 2016 19:22:42 +0100 Subject: [PATCH 30/89] Bugfixes, add a function to kill patroni daemon, make the feature description more concise. --- features/basic_replication.feature | 14 ++++---------- features/basic_replication.py | 15 ++++++++++++++- features/terrain.py | 12 ++++++++---- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/features/basic_replication.feature b/features/basic_replication.feature index 140e0cf2..fe766c11 100644 --- a/features/basic_replication.feature +++ b/features/basic_replication.feature @@ -1,11 +1,5 @@ Feature: basic replication - In order to check that basic replication works - As observers - We start the primary and the replica - add a table on the primary - and check that it gets replicated to the replica over time. - We stop the primary and check that the replica promotes itself to primary - We start the old primary and check that it rejoins as a replica. + We should check that the basic bootstrapping, replication and failover works. Scenario: check replication of a single table Given I start postgres0 @@ -14,9 +8,9 @@ Feature: basic replication Then table foo is present on postgres1 after 10 seconds Scenario: check the basic failover - When I shut down postgres0 - Then postgres1 role is the primary after 10 seconds + When I kill postgres0 + Then postgres1 role is the primary after 30 seconds When I start postgres0 Then postgres0 role is the secondary after 10 seconds When I add the table bar to postgres1 - Then table bar is present on postgres1 after 10 seconds \ No newline at end of file + Then table bar is present on postgres0 after 10 seconds diff --git a/features/basic_replication.py b/features/basic_replication.py index 4c26da37..176f1f30 100644 --- a/features/basic_replication.py +++ b/features/basic_replication.py @@ -16,6 +16,18 @@ class BasicReplicationSteps(object): '''I start (\w+)''' return world.pctl.start_patroni(pg_name) + def stop_patroni(self, step, pg_name): + '''I shut down (\w+)''' + return world.pctl.stop_patroni(pg_name) + + def kill_patroni(self, step, pg_name): + '''I kill (\w+)''' + return world.pctl.stop_patroni(pg_name, kill=True) + + def do_sleep(self, step, sleep_seconds): + '''I sleep (\w+)''' + sleep(int(sleep_seconds)) + def add_table(self, step, table_name, pg_name): '''I add the table (\w+) to (\w+)''' # parse the configuration file and get the port @@ -36,7 +48,8 @@ class BasicReplicationSteps(object): def check_role(self, step, pg_name, pg_role, max_promotion_timeout): '''(\w+) role is the (\w+) after (\d+) seconds''' - return world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)) + if not world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)): + assert False, "pg_name role didn't change to {0} after {1} seconds".format(pg_role, max_promotion_timeout) BasicReplicationSteps(world) diff --git a/features/terrain.py b/features/terrain.py index 2a9b23cc..5f66e780 100644 --- a/features/terrain.py +++ b/features/terrain.py @@ -47,9 +47,12 @@ class PatroniController(object): def patroni_is_running(self, pg_name): return pg_name in self.processes and self.processes[pg_name].pid and (self.processes[pg_name].poll() is None) - def stop_patroni(self, pg_name): + def stop_patroni(self, pg_name, kill=False): while self.patroni_is_running(pg_name): - self.processes[pg_name].terminate() + if not kill: + self.processes[pg_name].terminate() + else: + self.processes[pg_name].kill() time.sleep(1) self.log.get('pg_name') and self.log[pg_name].close() del self.processes[pg_name] @@ -142,16 +145,17 @@ class PatroniController(object): def check_role_has_changed_to(self, pg_name, new_role, timeout=10): bound_time = time.time() + timeout - current_role = 't' if new_role == 'primary' else 'f' + recovery_status = False if new_role == 'primary' else True role_has_changed = False while not role_has_changed: cur = self.query(pg_name, "SELECT pg_is_in_recovery()", fail_ok=True) if cur: row = cur.fetchone() - if row and len(row) > 0 and row[0] != current_role: + if row and len(row) > 0 and row[0] == recovery_status: role_has_changed = True if time.time() > bound_time: break + time.sleep(1) return role_has_changed def stop_all(self): From 53b5dfe39e6635d6aff29cbd82a1f92c5d082331 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Wed, 24 Feb 2016 19:24:46 +0100 Subject: [PATCH 31/89] Remove an unused function. --- features/basic_replication.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/features/basic_replication.py b/features/basic_replication.py index 176f1f30..02927405 100644 --- a/features/basic_replication.py +++ b/features/basic_replication.py @@ -24,10 +24,6 @@ class BasicReplicationSteps(object): '''I kill (\w+)''' return world.pctl.stop_patroni(pg_name, kill=True) - def do_sleep(self, step, sleep_seconds): - '''I sleep (\w+)''' - sleep(int(sleep_seconds)) - def add_table(self, step, table_name, pg_name): '''I add the table (\w+) to (\w+)''' # parse the configuration file and get the port From 4986db5c6a1f7c28771152018351269c39bff2fc Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Thu, 25 Feb 2016 12:52:45 +0100 Subject: [PATCH 32/89] Code refactoring, no functional changes. Move etcd code into a separate class. Reduce the number of global interdependencies. Clearly define private members of PatroniController and EtcdController. It would not make the QuantifiedCode entirely happy, since lettuce passes the step argument to the step definition, that is not used in the client code, but internally (via the @steps decorator on the steps class), but that's the issue of the tool used. --- features/basic_replication.py | 16 +- features/terrain.py | 319 ++++++++++++++++++---------------- 2 files changed, 176 insertions(+), 159 deletions(-) diff --git a/features/basic_replication.py b/features/basic_replication.py index 02927405..77351428 100644 --- a/features/basic_replication.py +++ b/features/basic_replication.py @@ -3,8 +3,6 @@ from time import sleep from lettuce import world, steps -PATRONI_CONFIG = '{}.yml' - @steps class BasicReplicationSteps(object): @@ -12,17 +10,17 @@ class BasicReplicationSteps(object): def __init__(self, environ): self.env = environ - def start_patroni(self, step, pg_name): + def start_patroni(self, step, name): '''I start (\w+)''' - return world.pctl.start_patroni(pg_name) + return world.pctl.start(name) - def stop_patroni(self, step, pg_name): + def stop_patroni(self, step, name): '''I shut down (\w+)''' - return world.pctl.stop_patroni(pg_name) + return world.pctl.stop(name) - def kill_patroni(self, step, pg_name): + def kill_patroni(self, step, name): '''I kill (\w+)''' - return world.pctl.stop_patroni(pg_name, kill=True) + return world.pctl.stop(name, kill=True) def add_table(self, step, table_name, pg_name): '''I add the table (\w+) to (\w+)''' @@ -45,7 +43,7 @@ class BasicReplicationSteps(object): def check_role(self, step, pg_name, pg_role, max_promotion_timeout): '''(\w+) role is the (\w+) after (\d+) seconds''' if not world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)): - assert False, "pg_name role didn't change to {0} after {1} seconds".format(pg_role, max_promotion_timeout) + assert False, "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout) BasicReplicationSteps(world) diff --git a/features/terrain.py b/features/terrain.py index 5f66e780..750c1316 100644 --- a/features/terrain.py +++ b/features/terrain.py @@ -1,4 +1,4 @@ -from lettuce import * +from lettuce import world, before, after import os.path import psycopg2 import requests @@ -9,28 +9,19 @@ import time import yaml -ETCD_VERSION_URL = 'http://127.0.0.1:2379/version' -ETCD_CLEANUP_URL = 'http://127.0.0.1:2379/v2/keys/service/batman?recursive=true' -PATRONI_CONFIG = '{}.yml' -etcd_handle = None -etcd_dir = None -pctl = None - - -@world.absorb class PatroniController(object): + PATRONI_CONFIG = '{}.yml' """ starts and stops individual patronis""" def __init__(self): - self.processes = {} + self._output_dir = None self._patroni_path = None - self.connstring = {} - self.connections = {} - self.cursors = {} - self.log = {} - self.config = {} - self.availability_check_time_limit = 10 - self.output_dir = None + self._connections = {} + self._config = {} + self._connstring = {} + self._cursors = {} + self._log = {} + self._processes = {} @property def patroni_path(self): @@ -44,97 +35,43 @@ class PatroniController(object): self._patroni_path = cwd return self._patroni_path - def patroni_is_running(self, pg_name): - return pg_name in self.processes and self.processes[pg_name].pid and (self.processes[pg_name].poll() is None) - - def stop_patroni(self, pg_name, kill=False): - while self.patroni_is_running(pg_name): - if not kill: - self.processes[pg_name].terminate() - else: - self.processes[pg_name].kill() - time.sleep(1) - self.log.get('pg_name') and self.log[pg_name].close() - del self.processes[pg_name] - - def make_patroni_test_config(self, pg_name, output_dir): - patroni_config_name = PATRONI_CONFIG.format(pg_name) - patroni_config_path = os.path.join(output_dir, patroni_config_name) - - with open(patroni_config_name) as f: - config = yaml.load(f) - postgresql = config['postgresql']['parameters'] - postgresql['logging_collector'] = 'on' - postgresql['log_destination'] = 'csvlog' - postgresql['log_directory'] = output_dir - postgresql['log_filename'] = '{0}.log'.format(pg_name) - postgresql['log_statement'] = 'all' - postgresql['log_min_messages'] = 'debug1' - - with open(patroni_config_path, 'w') as f: - yaml.dump(config, f, default_flow_style=False) - - return patroni_config_path - - def start_patroni(self, pg_name): - if not self.patroni_is_running(pg_name): - if pg_name in self.processes: - del self.processes[pg_name] + def start(self, pg_name, max_wait_limit=15): + if not self._is_running(pg_name): + if pg_name in self._processes: + del self._processes[pg_name] cwd = self.patroni_path - self.log[pg_name] = open(os.path.join(self.output_dir, 'patroni_{0}.log'.format(pg_name)), 'a') + self._log[pg_name] = open(os.path.join(self._output_dir, 'patroni_{0}.log'.format(pg_name)), 'a') - self.config[pg_name] = self.make_patroni_test_config(pg_name, self.output_dir) + self._config[pg_name] = self._make_patroni_test_config(pg_name, self._output_dir) - p = subprocess.Popen(['python', 'patroni.py', self.config[pg_name]], - stdout=self.log[pg_name], stderr=subprocess.STDOUT, cwd=cwd) + p = subprocess.Popen(['python', 'patroni.py', self._config[pg_name]], + stdout=self._log[pg_name], stderr=subprocess.STDOUT, cwd=cwd) if not (p and p.pid and p.poll() is None): assert False, "PostgreSQL {0} is not running after being started".format(pg_name) - self.processes[pg_name] = p + self._processes[pg_name] = p # wait while patroni is available for queries, but not more than 10 seconds. - for tick in range(self.availability_check_time_limit): + for tick in range(max_wait_limit): if self.query(pg_name, "SELECT 1", fail_ok=True) is not None: break time.sleep(1) else: assert False,\ - "Patroni instance is not available for queries after {0} seconds".format(self.availability_check_time_limit) + "Patroni instance is not available for queries after {0} seconds".format(max_wait_limit) - def make_connstring(self, pg_name): - if pg_name in self.connstring: - return self.connstring[pg_name] - try: - patroni_path = self.patroni_path - with open(os.path.join(patroni_path, world.PATRONI_CONFIG.format(pg_name)), 'r') as f: - config = yaml.load(f) - except OSError: - return None - connstring = config['postgresql']['connect_address'] - if ':' in connstring: - address, port = connstring.split(':') - else: - address = connstring - port = '5432' - user = "postgres" - dbname = "postgres" - self.connstring[pg_name] = "host={0} port={1} dbname={2} user={3}".format(address, port, dbname, user) - return self.connstring[pg_name] - - def connection(self, pg_name): - if pg_name not in self.connections or self.connections[pg_name].closed: - conn = psycopg2.connect(self.make_connstring(pg_name)) - conn.autocommit = True - self.connections[pg_name] = conn - return self.connections[pg_name] - - def cursor(self, pg_name): - if pg_name not in self.cursors or self.cursors[pg_name].closed: - cursor = self.connection(pg_name).cursor() - self.cursors[pg_name] = cursor - return self.cursors[pg_name] + def stop(self, pg_name, kill=False): + while self._is_running(pg_name): + if not kill: + self._processes[pg_name].terminate() + else: + self._processes[pg_name].kill() + time.sleep(1) + self._log.get('pg_name') and self._log[pg_name].close() + if pg_name in self._processes: + del self._processes[pg_name] def query(self, pg_name, query, fail_ok=False): try: - cursor = self.cursor(pg_name) + cursor = self._cursor(pg_name) cursor.execute(query) return cursor except psycopg2.Error: @@ -159,76 +96,158 @@ class PatroniController(object): return role_has_changed def stop_all(self): - for patroni in self.processes.copy(): - self.stop_patroni(patroni) + for patroni in self._processes.copy(): + self.stop(patroni) + + def create_and_set_output_directory(self, feature_name): + feature_dir = os.path.join(pctl.patroni_path, "features", "output", feature_name.encode('utf-8').replace(' ', '_')) + if os.path.exists(feature_dir): + shutil.rmtree(feature_dir) + os.makedirs(feature_dir) + self._output_dir = feature_dir + + def _is_running(self, pg_name): + return pg_name in self._processes and self._processes[pg_name].pid and (self._processes[pg_name].poll() is None) + + def _make_patroni_test_config(self, pg_name, output_dir): + patroni_config_name = PatroniController.PATRONI_CONFIG.format(pg_name) + patroni_config_path = os.path.join(output_dir, patroni_config_name) + + with open(patroni_config_name) as f: + config = yaml.load(f) + postgresql = config['postgresql']['parameters'] + postgresql['logging_collector'] = 'on' + postgresql['log_destination'] = 'csvlog' + postgresql['log_directory'] = output_dir + postgresql['log_filename'] = '{0}.log'.format(pg_name) + postgresql['log_statement'] = 'all' + postgresql['log_min_messages'] = 'debug1' + + with open(patroni_config_path, 'w') as f: + yaml.dump(config, f, default_flow_style=False) + + return patroni_config_path + + def _make_connstring(self, pg_name): + if pg_name in self._connstring: + return self._connstring[pg_name] + try: + patroni_path = self.patroni_path + with open(os.path.join(patroni_path, PatroniController.PATRONI_CONFIG.format(pg_name)), 'r') as f: + config = yaml.load(f) + except OSError: + return None + connstring = config['postgresql']['connect_address'] + if ':' in connstring: + address, port = connstring.split(':') + else: + address = connstring + port = '5432' + user = "postgres" + dbname = "postgres" + self._connstring[pg_name] = "host={0} port={1} dbname={2} user={3}".format(address, port, dbname, user) + return self._connstring[pg_name] + + def _connection(self, pg_name): + if pg_name not in self._connections or self._connections[pg_name].closed: + conn = psycopg2.connect(self._make_connstring(pg_name)) + conn.autocommit = True + self._connections[pg_name] = conn + return self._connections[pg_name] + + def _cursor(self, pg_name): + if pg_name not in self._cursors or self._cursors[pg_name].closed: + cursor = self._connection(pg_name).cursor() + self._cursors[pg_name] = cursor + return self._cursors[pg_name] + + +class EtcdController(object): + """ handles all etcd related tasks, used for the tests setup and cleanup """ + ETCD_VERSION_URL = 'http://127.0.0.1:2379/version' + ETCD_CLEANUP_URL = 'http://127.0.0.1:2379/v2/keys/service/batman?recursive=true' + + def __init__(self): + self.handle = None + self.work_directory = None + self.pid = None + self.start_timeot = 5 + + def start(self): + """ start etcd if it's not already running """ + if self._is_running(): + return True + self.work_directory = tempfile.mkdtemp() + self.handle =\ + subprocess.Popen(["etcd", "--data-dir", self.work_directory], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + start_time = time.time() + while (not self._is_running()): + if time.time() - start_time > self.start_timeout: + assert False, "Failed to start etcd" + time.sleep(1) + return True + + def stop_and_remove_work_directory(self): + """ terminate etcd and wipe out the temp work directory, but only if we actually started it""" + if self._is_running() and self.handle: + self.handle.terminate() + self.handle = None + if self.work_directory: + shutil.rmtree(self.work_directory) + self.work_directory = None + + @staticmethod + def cleanup_service_tree(): + """ clean all contents stored in the tree used for the tests """ + r = None + try: + r = requests.delete(EtcdController.ETCD_CLEANUP_URL) + if r and not r.ok: + assert False,\ + "request to cleanup the etcd contents was not successfull: status code {0}".format(r.status_code) + except requests.exceptions.RequestException as e: + assert False, "exception when cleanin up etcd contents: {0}".format(e) + + def _is_running(self): + # if we have already started etcd + if self.handle and self.handle.pid and (self.handle.poll() is None): + return True + # if etcd is running, but we didn't start it + try: + r = requests.get(EtcdController.ETCD_VERSION_URL) + if r and r.ok and 'etcdserver' in r.content: + return True + except requests.ConnectionError: + pass + return False + pctl = PatroniController() +etcd_ctl = EtcdController() +# export pctl to manage patroni from scenario files world.pctl = pctl -world.patroni_path = pctl.patroni_path -world.PATRONI_CONFIG = PATRONI_CONFIG - - -def etcd_is_running(): - # if we have already started etcd - if etcd_handle and etcd_handle.pid and (etcd_handle.poll() is None): - return True - # if etcd is running, but we didn't start it - try: - r = requests.get(ETCD_VERSION_URL) - if r and r.ok and 'etcdserver' in r.content: - return True - except requests.ConnectionError: - pass - return False +# actions to execute on start/stop of the tests and before running invidual features @before.all def start_etcd(): - if not etcd_is_running(): - global etcd_handle - global etcd_dir - etcd_dir = tempfile.mkdtemp() - etcd_handle = subprocess.Popen(["etcd", "--data-dir", etcd_dir], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - if not etcd_is_running(): - assert False, "Failed to start etcd" + etcd_ctl.start() @after.all -def stop_etcd(total): - global etcd_handle - global etcd_dir - if etcd_is_running() and etcd_handle: - etcd_handle.terminate() - etcd_handle = None - shutil.rmtree(etcd_dir) - etcd_dir = None +def stop_etcd(*args, **kwargs): + etcd_ctl.stop_and_remove_work_directory() @before.each_feature def make_test_output_dir(feature): - feature_dir = os.path.join(pctl.patroni_path, "features", "output", feature.name.encode('utf-8').replace(' ', '_')) - if os.path.exists(feature_dir): - shutil.rmtree(feature_dir) - os.makedirs(feature_dir) - pctl.output_dir = feature_dir - - -def patroni_cleanup_all(): - pctl.stop_all() - # remove the data directory - shutil.rmtree(os.path.join(pctl.patroni_path, 'data')) - - -def etcd_cleanup(): - try: - r = requests.delete(ETCD_CLEANUP_URL) - if not r.ok: - raise Exception('{}'.format(r.reason)) - except Exception as e: - assert False, "Unable to cleanup etcd: {0}".format(e) + """ create per-feature output directory to collect Patroni and PostgreSQL logs """ + pctl.create_and_set_output_directory(feature.name) @after.each_feature -def cleanup(scenario): - patroni_cleanup_all() - etcd_cleanup() +def cleanup(*args, **kwargs): + """ stop all Patronis, remove their data directory and cleanup the keys in etcd """ + pctl.stop_all() + shutil.rmtree(os.path.join(pctl.patroni_path, 'data')) + etcd_ctl.cleanup_service_tree() From 481a80a3cecb44d6d109a8b52d135e7e9f05cae8 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Thu, 25 Feb 2016 14:39:22 +0100 Subject: [PATCH 33/89] Fix another couple of warnings from the QuantifiedCode and Co. --- features/terrain.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/features/terrain.py b/features/terrain.py index 750c1316..03489c5e 100644 --- a/features/terrain.py +++ b/features/terrain.py @@ -42,7 +42,7 @@ class PatroniController(object): cwd = self.patroni_path self._log[pg_name] = open(os.path.join(self._output_dir, 'patroni_{0}.log'.format(pg_name)), 'a') - self._config[pg_name] = self._make_patroni_test_config(pg_name, self._output_dir) + self._config[pg_name] = self._make_patroni_test_config(pg_name) p = subprocess.Popen(['python', 'patroni.py', self._config[pg_name]], stdout=self._log[pg_name], stderr=subprocess.STDOUT, cwd=cwd) @@ -109,16 +109,16 @@ class PatroniController(object): def _is_running(self, pg_name): return pg_name in self._processes and self._processes[pg_name].pid and (self._processes[pg_name].poll() is None) - def _make_patroni_test_config(self, pg_name, output_dir): + def _make_patroni_test_config(self, pg_name): patroni_config_name = PatroniController.PATRONI_CONFIG.format(pg_name) - patroni_config_path = os.path.join(output_dir, patroni_config_name) + patroni_config_path = os.path.join(self._output_dir, patroni_config_name) with open(patroni_config_name) as f: config = yaml.load(f) postgresql = config['postgresql']['parameters'] postgresql['logging_collector'] = 'on' postgresql['log_destination'] = 'csvlog' - postgresql['log_directory'] = output_dir + postgresql['log_directory'] = self._output_dir postgresql['log_filename'] = '{0}.log'.format(pg_name) postgresql['log_statement'] = 'all' postgresql['log_min_messages'] = 'debug1' @@ -215,11 +215,10 @@ class EtcdController(object): # if etcd is running, but we didn't start it try: r = requests.get(EtcdController.ETCD_VERSION_URL) - if r and r.ok and 'etcdserver' in r.content: - return True + running = (r and r.ok and 'etcdserver' in r.content) except requests.ConnectionError: - pass - return False + running = False + return running pctl = PatroniController() From 4a8edf44e649a167961c885f22700b7d3250ece9 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Thu, 25 Feb 2016 14:55:29 +0100 Subject: [PATCH 34/89] Convert normal methods to static methods when possible. --- features/basic_replication.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/features/basic_replication.py b/features/basic_replication.py index 77351428..3a39b8b7 100644 --- a/features/basic_replication.py +++ b/features/basic_replication.py @@ -10,19 +10,23 @@ class BasicReplicationSteps(object): def __init__(self, environ): self.env = environ - def start_patroni(self, step, name): + @staticmethod + def start_patroni(step, name): '''I start (\w+)''' return world.pctl.start(name) - def stop_patroni(self, step, name): + @staticmethod + def stop_patroni(step, name): '''I shut down (\w+)''' return world.pctl.stop(name) - def kill_patroni(self, step, name): + @staticmethod + def kill_patroni(step, name): '''I kill (\w+)''' return world.pctl.stop(name, kill=True) - def add_table(self, step, table_name, pg_name): + @staticmethod + def add_table(step, table_name, pg_name): '''I add the table (\w+) to (\w+)''' # parse the configuration file and get the port try: @@ -30,7 +34,8 @@ class BasicReplicationSteps(object): except pg.Error as e: assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e) - def table_is_present_on(self, step, table_name, pg_name, max_replication_delay): + @staticmethod + def table_is_present_on(step, table_name, pg_name, max_replication_delay): '''Table (\w+) is present on (\w+) after (\d+) seconds''' for i in range(int(max_replication_delay)): if world.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None: @@ -40,7 +45,8 @@ class BasicReplicationSteps(object): assert False,\ "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay) - def check_role(self, step, pg_name, pg_role, max_promotion_timeout): + @staticmethod + def check_role(step, pg_name, pg_role, max_promotion_timeout): '''(\w+) role is the (\w+) after (\d+) seconds''' if not world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)): assert False, "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout) From 67f55b460668d4868e1b60f74689b16759b65b04 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Thu, 25 Feb 2016 15:11:20 +0100 Subject: [PATCH 35/89] Stylistic issues: clearly mark unused variables. --- features/basic_replication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/basic_replication.py b/features/basic_replication.py index 3a39b8b7..0d2b7172 100644 --- a/features/basic_replication.py +++ b/features/basic_replication.py @@ -37,7 +37,7 @@ class BasicReplicationSteps(object): @staticmethod def table_is_present_on(step, table_name, pg_name, max_replication_delay): '''Table (\w+) is present on (\w+) after (\d+) seconds''' - for i in range(int(max_replication_delay)): + for _ in range(int(max_replication_delay)): if world.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None: break sleep(1) From a84a3fc5e1a3dde792d6e9dc05634caf2ac26545 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Thu, 25 Feb 2016 15:16:35 +0100 Subject: [PATCH 36/89] Changeset missing in the previous commit. --- features/terrain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/terrain.py b/features/terrain.py index 03489c5e..8a8396bb 100644 --- a/features/terrain.py +++ b/features/terrain.py @@ -50,7 +50,7 @@ class PatroniController(object): assert False, "PostgreSQL {0} is not running after being started".format(pg_name) self._processes[pg_name] = p # wait while patroni is available for queries, but not more than 10 seconds. - for tick in range(max_wait_limit): + for _ in range(max_wait_limit): if self.query(pg_name, "SELECT 1", fail_ok=True) is not None: break time.sleep(1) From 83b7c34b003f47181d68f526b531f55be2ae0256 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Thu, 25 Feb 2016 15:35:15 +0100 Subject: [PATCH 37/89] Do not try to close an already closed file. --- features/terrain.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/features/terrain.py b/features/terrain.py index 8a8396bb..76fd345e 100644 --- a/features/terrain.py +++ b/features/terrain.py @@ -65,7 +65,8 @@ class PatroniController(object): else: self._processes[pg_name].kill() time.sleep(1) - self._log.get('pg_name') and self._log[pg_name].close() + if self._log.get('pg_name') and not self._log['pg_name'].closed: + self._log[pg_name].close() if pg_name in self._processes: del self._processes[pg_name] From cb38e50ac1d90b860767e6e914fe6a61270f0f31 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 26 Feb 2016 08:50:53 +0100 Subject: [PATCH 38/89] Remove unused code --- patroni/dcs.py | 8 -------- tests/test_etcd.py | 8 +++----- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/patroni/dcs.py b/patroni/dcs.py index 4c2dddad..8cda0620 100644 --- a/patroni/dcs.py +++ b/patroni/dcs.py @@ -3,7 +3,6 @@ import json import dateutil from collections import namedtuple -from patroni.exceptions import DCSError from six.moves.urllib_parse import urlparse, urlunparse, parse_qsl from threading import Event, Lock @@ -269,13 +268,6 @@ class AbstractDCS(object): return self.set_failover_value(json.dumps(failover_value), index) - def current_leader(self): - try: - cluster = self.get_cluster() - return None if cluster.is_unlocked() else cluster.leader - except DCSError: - return None - @abc.abstractmethod def touch_member(self, connection_string, ttl=None): """Update member key in DCS. diff --git a/tests/test_etcd.py b/tests/test_etcd.py index 601a0cd6..c14d3cac 100644 --- a/tests/test_etcd.py +++ b/tests/test_etcd.py @@ -7,8 +7,9 @@ import unittest from dns.exception import DNSException from mock import Mock, patch -from patroni.dcs import Cluster, DCSError, Leader +from patroni.dcs import Cluster from patroni.etcd import Client, Etcd, EtcdError +from patroni.exceptions import DCSError class MockResponse(object): @@ -229,11 +230,8 @@ class TestEtcd(unittest.TestCase): cluster = self.etcd.get_cluster() self.assertIsInstance(cluster, Cluster) self.assertIsNone(cluster.leader) - - def test_current_leader(self): - self.assertIsInstance(self.etcd.current_leader(), Leader) self.etcd._base_path = '/service/noleader' - self.assertIsNone(self.etcd.current_leader()) + self.assertRaises(EtcdError, self.etcd.get_cluster) def test_touch_member(self): self.assertFalse(self.etcd.touch_member('', '')) From e564fa7f083aab2018d7b095ebd9309a5164267d Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Fri, 26 Feb 2016 10:53:00 +0100 Subject: [PATCH 39/89] Update DCS status right after acquiring the lock. This commit only handles the initial bootstrap case, uncovered by the upcoming lettuce tests --- patroni/ha.py | 1 + 1 file changed, 1 insertion(+) diff --git a/patroni/ha.py b/patroni/ha.py index 9e9f88aa..3f179b14 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -88,6 +88,7 @@ class Ha(object): self.state_handler.move_data_directory() raise self.dcs.take_leader() + self.load_cluster_from_dcs() return 'initialized a new cluster' else: return 'failed to acquire initialize lock' From 4e9ebf48a80b862e27ebdb879d871b481a3d97ce Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Fri, 26 Feb 2016 17:37:37 +0100 Subject: [PATCH 40/89] Add API tests for a stand-alone node. Bugfixes. Add tests for patroni API. Fix test failures when an already running etcd is used. --- features/patroni_api.feature | 21 +++++++++ features/patroni_api.py | 83 ++++++++++++++++++++++++++++++++++++ features/terrain.py | 27 +++++++++--- 3 files changed, 124 insertions(+), 7 deletions(-) create mode 100644 features/patroni_api.feature create mode 100644 features/patroni_api.py diff --git a/features/patroni_api.feature b/features/patroni_api.feature new file mode 100644 index 00000000..3ff28584 --- /dev/null +++ b/features/patroni_api.feature @@ -0,0 +1,21 @@ +Feature: patroni api + We should check that patroni correctly responds to valid and not-valid API requests. + +Scenario: check API requests on a stand-alone server + Given I start postgres0 + And postgres0 is a leader after 10 seconds + When I issue a GET request to http://127.0.0.1:8008/ + Then I receive a response code 200 + And I receive a response state running + And I receive a response role master + When I issue a GET request to http://127.0.0.1:8008/replica + Then I receive a response code 503 + When I issue an empty POST request to http://127.0.0.1:8008/reinitialize + Then I receive a response code 503 + And I receive a response text "I am the leader, can not reinitialize" + When I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0 + Then I receive a response code 503 + And I receive a response text "failover is not possible: cluster does not have members except leader" + When I issue an empty POST request to http://127.0.0.1:8008/failover + Then I receive a response code 503 + And I receive a response text "No values given for required parameters leader and member" \ No newline at end of file diff --git a/features/patroni_api.py b/features/patroni_api.py new file mode 100644 index 00000000..96d6e04a --- /dev/null +++ b/features/patroni_api.py @@ -0,0 +1,83 @@ +from lettuce import world, steps +import time +import requests + + +@steps +class PatroniAPISteps(object): + + def __init__(self, environ): + self.env = environ + self.response = None + self.status_code = None + + # there is no way we can find out if the node has already + # started as a leader without checking the DCS. We cannot + # just rely on the database availability, since there is + # a short gap between the time PostgreSQL becomes available + # and Patroni assuming the leader role. + @staticmethod + def is_a_leader(step, name, time_limit): + '''(\w+) is a leader after (\d+) seconds''' + max_time = time.time() + int(time_limit) + while (world.etcd_ctl.query("leader") != name): + time.sleep(1) + if time.time() > max_time: + assert False, "{0} is not a leader in etcd after {1} seconds".format(name, time_limit) + + @staticmethod + def sleep_for_n_seconds(step, value): + '''I sleep for (\d+) seconds''' + time.sleep(int(value)) + + def do_get(self, step, url): + '''I issue a GET request to (https?://(?:\w|\.|:|/)+)''' + try: + r = requests.get(url) + except requests.exceptions.RequestException: + self.code = None + self.response = None + else: + self.status_code = r.status_code + try: + self.response = r.json() + except ValueError: + self.response = r.content + + def do_post_empty(self, step, url): + '''I issue an empty POST request to (https?://(?:\w|\.|:|/)+)''' + self.do_post(step, url, None) + + def do_post(self, step, url, data): + '''I issue a POST request to (https?://(?:\w|\.|:|/)+) with (\s*\w+\s*=\s*\w+\s*,?)+''' + post_data = {} + if data: + post_components = data.split(',') + for pc in post_components: + if '=' in pc: + k, v = pc.split('=', 2) + post_data[k.strip()] = v.strip() + try: + r = requests.post(url, json=post_data) + except requests.exceptions.RequestException: + self.code = None + self.response = None + else: + self.status_code = r.status_code + try: + self.response = r.json() + except ValueError: + self.response = r.content + + def check_response(self, step, component, data): + '''I receive a response (\w+) (.*)''' + if component == 'code': + assert self.status_code == int(data), "status code {0} != {1}".format(self.status_code, int(data)) + elif component == 'text': + assert self.response == data.strip('"'), "response {0} does not contain {1}".format(self.response, data) + else: + assert component in self.response, "{0} is not part of the response".format(component) + assert self.response[component] == data, "{0} does not contain {1}".format(component, data) + + +PatroniAPISteps(world) diff --git a/features/terrain.py b/features/terrain.py index 76fd345e..dd1aeade 100644 --- a/features/terrain.py +++ b/features/terrain.py @@ -116,13 +116,15 @@ class PatroniController(object): with open(patroni_config_name) as f: config = yaml.load(f) - postgresql = config['postgresql']['parameters'] - postgresql['logging_collector'] = 'on' - postgresql['log_destination'] = 'csvlog' - postgresql['log_directory'] = self._output_dir - postgresql['log_filename'] = '{0}.log'.format(pg_name) - postgresql['log_statement'] = 'all' - postgresql['log_min_messages'] = 'debug1' + postgresql = config['postgresql'] + postgresql['name'] = pg_name.encode('utf-8') + postgresql_params = postgresql['parameters'] + postgresql_params['logging_collector'] = 'on' + postgresql_params['log_destination'] = 'csvlog' + postgresql_params['log_directory'] = self._output_dir + postgresql_params['log_filename'] = '{0}.log'.format(pg_name) + postgresql_params['log_statement'] = 'all' + postgresql_params['log_min_messages'] = 'debug1' with open(patroni_config_path, 'w') as f: yaml.dump(config, f, default_flow_style=False) @@ -188,6 +190,15 @@ class EtcdController(object): time.sleep(1) return True + def query(self, key): + """ query etcd for a value of a given key """ + r = requests.get("http://127.0.0.1:2379/v2/keys/service/batman/{0}".format(key)) + if r.ok: + content = r.json() + if content: + return content.get('node', {}).get('value', None) + return None + def stop_and_remove_work_directory(self): """ terminate etcd and wipe out the temp work directory, but only if we actually started it""" if self._is_running() and self.handle: @@ -226,12 +237,14 @@ pctl = PatroniController() etcd_ctl = EtcdController() # export pctl to manage patroni from scenario files world.pctl = pctl +world.etcd_ctl = etcd_ctl # actions to execute on start/stop of the tests and before running invidual features @before.all def start_etcd(): etcd_ctl.start() + etcd_ctl.cleanup_service_tree() @after.all From 0d44e3eb7cc2e71f8449903640343dcfbb9ac165 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Fri, 26 Feb 2016 18:00:11 +0100 Subject: [PATCH 41/89] Add simple API tests for 2 nodes, to be extended. --- features/patroni_api.feature | 12 +++++++++++- features/patroni_api.py | 7 +++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/features/patroni_api.feature b/features/patroni_api.feature index 3ff28584..3649337f 100644 --- a/features/patroni_api.feature +++ b/features/patroni_api.feature @@ -18,4 +18,14 @@ Scenario: check API requests on a stand-alone server And I receive a response text "failover is not possible: cluster does not have members except leader" When I issue an empty POST request to http://127.0.0.1:8008/failover Then I receive a response code 503 - And I receive a response text "No values given for required parameters leader and member" \ No newline at end of file + And I receive a response text "No values given for required parameters leader and member" + +Scenario: check API requests for the primary-replica pair + Given I start postgres1 + And replication works after 10 seconds + When I issue a GET request to http://127.0.0.1:8009/replica + Then I receive a response code 200 + And I receive a response state running + And I receive a response role replica + When I issue an empty POST request to http://127.0.0.1:8009/reinitialize + Then I receive a response code 200 diff --git a/features/patroni_api.py b/features/patroni_api.py index 96d6e04a..188f93cb 100644 --- a/features/patroni_api.py +++ b/features/patroni_api.py @@ -79,5 +79,12 @@ class PatroniAPISteps(object): assert component in self.response, "{0} is not part of the response".format(component) assert self.response[component] == data, "{0} does not contain {1}".format(component, data) + def replication_works(self, step, time_limit): + '''And replication works after (\d+) seconds''' + step.behave_as(""" + When I add the table foo to postgres0 + Then table foo is present on postgres1 after {0} seconds + """.format(time_limit)) + PatroniAPISteps(world) From fa1a7687e5f1c5bf1ea450221501ddd9ff56d5b1 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Tue, 1 Mar 2016 22:00:30 +0100 Subject: [PATCH 42/89] Correct the step definition, randomize the table. Make sure the step definition does not include "command" worlds. Use the table name that includes current timestamp in the tests. --- features/patroni_api.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/features/patroni_api.py b/features/patroni_api.py index 188f93cb..44de0f56 100644 --- a/features/patroni_api.py +++ b/features/patroni_api.py @@ -49,7 +49,7 @@ class PatroniAPISteps(object): self.do_post(step, url, None) def do_post(self, step, url, data): - '''I issue a POST request to (https?://(?:\w|\.|:|/)+) with (\s*\w+\s*=\s*\w+\s*,?)+''' + '''I issue a POST request to (https?://(?:\w|\.|:|/)+) with ((?:\s*\w+\s*=\s*\w+\s*,?)+)''' post_data = {} if data: post_components = data.split(',') @@ -80,11 +80,11 @@ class PatroniAPISteps(object): assert self.response[component] == data, "{0} does not contain {1}".format(component, data) def replication_works(self, step, time_limit): - '''And replication works after (\d+) seconds''' + '''replication works after (\d+) seconds''' step.behave_as(""" - When I add the table foo to postgres0 - Then table foo is present on postgres1 after {0} seconds - """.format(time_limit)) + When I add the table test_{0} to postgres0 + Then table test_{0} is present on postgres1 after {1} seconds + """.format(int(time.time()), time_limit)) PatroniAPISteps(world) From ed15f7cd730e60c4784ae6ad2048454a054340cb Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Tue, 1 Mar 2016 22:03:38 +0100 Subject: [PATCH 43/89] Improve tests start/stop, add etcd logging. Toggle the etcd debug logging and write the log to the test dir. Make sure etcd and patroni are terminated when the tests finish by sending SIGKILL in case SIGTERM does not work. Make sure before.all code does the proper cleanup when the exception is thrown. --- features/terrain.py | 47 +++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/features/terrain.py b/features/terrain.py index dd1aeade..19841a19 100644 --- a/features/terrain.py +++ b/features/terrain.py @@ -58,13 +58,16 @@ class PatroniController(object): assert False,\ "Patroni instance is not available for queries after {0} seconds".format(max_wait_limit) - def stop(self, pg_name, kill=False): + def stop(self, pg_name, kill=False, timeout=15): + start_time = time.time() while self._is_running(pg_name): if not kill: self._processes[pg_name].terminate() else: self._processes[pg_name].kill() time.sleep(1) + if not kill and time.time() - start_time > timeout: + kill = True if self._log.get('pg_name') and not self._log['pg_name'].closed: self._log[pg_name].close() if pg_name in self._processes: @@ -170,19 +173,24 @@ class EtcdController(object): ETCD_VERSION_URL = 'http://127.0.0.1:2379/version' ETCD_CLEANUP_URL = 'http://127.0.0.1:2379/v2/keys/service/batman?recursive=true' - def __init__(self): + def __init__(self, log_directory): self.handle = None self.work_directory = None + self.log_directory = log_directory + self.log_file = None self.pid = None - self.start_timeot = 5 + self.start_timeout = 5 def start(self): """ start etcd if it's not already running """ if self._is_running(): return True self.work_directory = tempfile.mkdtemp() + # etcd is running throughout the tests, no need to append to the log + self.log_file = open(os.path.join(self.log_directory, "features", "output", 'etcd.log'), 'w') self.handle =\ - subprocess.Popen(["etcd", "--data-dir", self.work_directory], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + subprocess.Popen(["etcd", "--debug", "--data-dir", self.work_directory], + stdout=self.log_file, stderr=subprocess.STDOUT) start_time = time.time() while (not self._is_running()): if time.time() - start_time > self.start_timeout: @@ -199,11 +207,21 @@ class EtcdController(object): return content.get('node', {}).get('value', None) return None - def stop_and_remove_work_directory(self): + def stop_and_remove_work_directory(self, timeout=15): """ terminate etcd and wipe out the temp work directory, but only if we actually started it""" - if self._is_running() and self.handle: - self.handle.terminate() - self.handle = None + kill = False + start_time = time.time() + while self._is_running() and self.handle: + if not kill: + self.handle.terminate() + else: + self.handle.kill() + time.sleep(1) + if not kill and time.time() - start_time > timeout: + kill = True + self.handle = None + if self.log_file and not self.log_file.closed: + self.log_file.close() if self.work_directory: shutil.rmtree(self.work_directory) self.work_directory = None @@ -218,12 +236,9 @@ class EtcdController(object): assert False,\ "request to cleanup the etcd contents was not successfull: status code {0}".format(r.status_code) except requests.exceptions.RequestException as e: - assert False, "exception when cleanin up etcd contents: {0}".format(e) + assert False, "exception when cleaning up etcd contents: {0}".format(e) def _is_running(self): - # if we have already started etcd - if self.handle and self.handle.pid and (self.handle.poll() is None): - return True # if etcd is running, but we didn't start it try: r = requests.get(EtcdController.ETCD_VERSION_URL) @@ -234,7 +249,7 @@ class EtcdController(object): pctl = PatroniController() -etcd_ctl = EtcdController() +etcd_ctl = EtcdController(pctl.patroni_path) # export pctl to manage patroni from scenario files world.pctl = pctl world.etcd_ctl = etcd_ctl @@ -244,7 +259,11 @@ world.etcd_ctl = etcd_ctl @before.all def start_etcd(): etcd_ctl.start() - etcd_ctl.cleanup_service_tree() + try: + etcd_ctl.cleanup_service_tree() + except AssertionError: # after.all handlers won't be executed in before.all + etcd_ctl.stop_and_remove_work_directory() + raise @after.all From 24ebcc72f6a25bb8f5cf53d6efe00bdd4e27c890 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Tue, 1 Mar 2016 22:07:18 +0100 Subject: [PATCH 44/89] Add more tests for the restart and promotion. --- features/patroni_api.feature | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/features/patroni_api.feature b/features/patroni_api.feature index 3649337f..c7779af1 100644 --- a/features/patroni_api.feature +++ b/features/patroni_api.feature @@ -29,3 +29,12 @@ Scenario: check API requests for the primary-replica pair And I receive a response role replica When I issue an empty POST request to http://127.0.0.1:8009/reinitialize Then I receive a response code 200 + Given replication works after 10 seconds + When I issue an empty POST request to http://127.0.0.1:8008/restart + Then I receive a response code 200 + And postgres0 is a leader after 5 seconds + +Scenario: check promotion via the API + Given I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0,candidate=postgres1 + Then I receive a response code 200 + And postgres1 is a leader after 10 seconds From 069440be15c4834316c13806b38879cd99e65086 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Wed, 2 Mar 2016 15:43:44 +0100 Subject: [PATCH 45/89] Improve the "replication work" sentence definition. Add an ability to specify the origin and the destination for the replication works clause. Use this ability in the API promotion test to ensure the replication from the former replica to the former master. --- features/patroni_api.feature | 7 ++++--- features/patroni_api.py | 10 +++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/features/patroni_api.feature b/features/patroni_api.feature index c7779af1..78f014e2 100644 --- a/features/patroni_api.feature +++ b/features/patroni_api.feature @@ -22,14 +22,14 @@ Scenario: check API requests on a stand-alone server Scenario: check API requests for the primary-replica pair Given I start postgres1 - And replication works after 10 seconds + And replication works from postgres0 to postgres1 after 15 seconds When I issue a GET request to http://127.0.0.1:8009/replica Then I receive a response code 200 And I receive a response state running And I receive a response role replica When I issue an empty POST request to http://127.0.0.1:8009/reinitialize Then I receive a response code 200 - Given replication works after 10 seconds + Given replication works from postgres0 to postgres1 after 10 seconds When I issue an empty POST request to http://127.0.0.1:8008/restart Then I receive a response code 200 And postgres0 is a leader after 5 seconds @@ -37,4 +37,5 @@ Scenario: check API requests for the primary-replica pair Scenario: check promotion via the API Given I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0,candidate=postgres1 Then I receive a response code 200 - And postgres1 is a leader after 10 seconds + And postgres1 is a leader after 5 seconds + And replication works from postgres1 to postgres0 after 15 seconds diff --git a/features/patroni_api.py b/features/patroni_api.py index 44de0f56..37650d99 100644 --- a/features/patroni_api.py +++ b/features/patroni_api.py @@ -79,12 +79,12 @@ class PatroniAPISteps(object): assert component in self.response, "{0} is not part of the response".format(component) assert self.response[component] == data, "{0} does not contain {1}".format(component, data) - def replication_works(self, step, time_limit): - '''replication works after (\d+) seconds''' + def replication_works(self, step, master, replica, time_limit): + '''replication works from (\w+) to (\w+) after (\d+) seconds''' step.behave_as(""" - When I add the table test_{0} to postgres0 - Then table test_{0} is present on postgres1 after {1} seconds - """.format(int(time.time()), time_limit)) + When I add the table test_{0} to {1} + Then table test_{0} is present on {2} after {3} seconds + """.format(int(time.time()), master, replica, time_limit)) PatroniAPISteps(world) From 3f1c34f5570d1e0f4165e8d765f3def73c2fd4d9 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Wed, 2 Mar 2016 19:39:12 +0100 Subject: [PATCH 46/89] Add tests for the scheduled failover. The actual amount of time to establish the master and the replication after the scheduled failover seems sufficient (15 seconds with the failover in 10 seconds), but occasionally leads to test failures. This is unlikely the test issue and should be investigated inside the patroni. --- features/patroni_api.feature | 13 ++++++++++--- features/patroni_api.py | 13 +++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/features/patroni_api.feature b/features/patroni_api.feature index 78f014e2..6eedc26c 100644 --- a/features/patroni_api.feature +++ b/features/patroni_api.feature @@ -14,10 +14,10 @@ Scenario: check API requests on a stand-alone server Then I receive a response code 503 And I receive a response text "I am the leader, can not reinitialize" When I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0 - Then I receive a response code 503 + Then I receive a response code 500 And I receive a response text "failover is not possible: cluster does not have members except leader" When I issue an empty POST request to http://127.0.0.1:8008/failover - Then I receive a response code 503 + Then I receive a response code 500 And I receive a response text "No values given for required parameters leader and member" Scenario: check API requests for the primary-replica pair @@ -34,8 +34,15 @@ Scenario: check API requests for the primary-replica pair Then I receive a response code 200 And postgres0 is a leader after 5 seconds -Scenario: check promotion via the API +Scenario: check the failover via the API Given I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0,candidate=postgres1 Then I receive a response code 200 And postgres1 is a leader after 5 seconds And replication works from postgres1 to postgres0 after 15 seconds + +Scenario: check the scheduled failover + Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgresq0 in 10 seconds + Then I receive a response code 200 + And postgres0 is a leader after 15 seconds + And replication works from postgres0 to postgres1 after 25 seconds + diff --git a/features/patroni_api.py b/features/patroni_api.py index 37650d99..0659bc2c 100644 --- a/features/patroni_api.py +++ b/features/patroni_api.py @@ -1,5 +1,7 @@ +from datetime import datetime, timedelta from lettuce import world, steps import time +import pytz import requests @@ -49,7 +51,7 @@ class PatroniAPISteps(object): self.do_post(step, url, None) def do_post(self, step, url, data): - '''I issue a POST request to (https?://(?:\w|\.|:|/)+) with ((?:\s*\w+\s*=\s*\w+\s*,?)+)''' + '''I issue a POST request to (https?://(?:\w|\.|:|/)+) with ((?:\w+=(?:\w|\.|:|-|\+|\s)+,?)+)''' post_data = {} if data: post_components = data.split(',') @@ -72,7 +74,8 @@ class PatroniAPISteps(object): def check_response(self, step, component, data): '''I receive a response (\w+) (.*)''' if component == 'code': - assert self.status_code == int(data), "status code {0} != {1}".format(self.status_code, int(data)) + assert self.status_code == int(data),\ + "status code {0} != {1}, response: {2}".format(self.status_code, int(data), self.response) elif component == 'text': assert self.response == data.strip('"'), "response {0} does not contain {1}".format(self.response, data) else: @@ -86,5 +89,11 @@ class PatroniAPISteps(object): Then table test_{0} is present on {2} after {3} seconds """.format(int(time.time()), master, replica, time_limit)) + def scheduld_failover(self, step, at_url, from_host, to_host, in_seconds): + '''I issue a scheduled failover at (https?://(?:\w|\.|:|/)+) from (\w+) to (\w+) in (\d+) seconds''' + step.behave_as(""" + Given I issue a POST request to {0}/failover with leader={1},candidate={2},scheduled_at={3} + """.format(at_url, from_host, to_host, datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds)))) + PatroniAPISteps(world) From aa844b63d0bb82c39b08e5b834a07b966a41c3da Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Fri, 4 Mar 2016 19:21:14 +0100 Subject: [PATCH 47/89] Avoid an unhandled exception in the API thread. When receiving a failover request with no data or non-JSON data, emit a message to the client instead of crashing. --- patroni/api.py | 58 +++++++++++++++++++++++++++-------------------- tests/test_api.py | 4 ++++ 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/patroni/api.py b/patroni/api.py index ba18b333..1186321f 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -174,38 +174,48 @@ class RestApiHandler(BaseHTTPRequestHandler): @check_auth def do_POST_failover(self): content_length = int(self.headers.get('content-length', 0)) - request = json.loads(self.rfile.read(content_length).decode('utf-8')) + try: + request = json.loads(self.rfile.read(content_length).decode('utf-8')) + except ValueError: + request = {} leader = request.get('leader') member = request.get('member') cluster = self.server.patroni.ha.dcs.get_cluster() status_code = 500 + logger.info("received failover request with leader {0} member {1} scheduled_at {2}". + format(leader, member, request.get("scheduled_at"))) + data = b'' - if request.get('scheduled_at'): - try: - scheduled_at = dateutil.parser.parse(request['scheduled_at']) - if scheduled_at.tzinfo is None: - data = b'Timezone information is mandatory for scheduled_at' - status_code = 400 - elif scheduled_at < datetime.datetime.now(pytz.utc): - data = b'Cannot schedule failover in the past' + if leader or member: + if request.get('scheduled_at'): + try: + scheduled_at = dateutil.parser.parse(request['scheduled_at']) + if scheduled_at.tzinfo is None: + data = b'Timezone information is mandatory for scheduled_at' + status_code = 400 + elif scheduled_at < datetime.datetime.now(pytz.utc): + data = b'Cannot schedule failover in the past' + status_code = 422 + elif self.server.patroni.dcs.manual_failover(leader, member, scheduled_at): + data = b'Failover scheduled' + status_code = 200 + except (ValueError, TypeError): + logger.exception('Invalid scheduled failover time: {}'.format(request['scheduled_at'])) + data = b'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601' status_code = 422 - elif self.server.patroni.dcs.manual_failover(leader, member, scheduled_at): - data = b'Failover scheduled' - status_code = 200 - except (ValueError, TypeError): - logger.exception('Invalid scheduled failover time: {}'.format(request['scheduled_at'])) - data = b'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601' - status_code = 422 + else: + data = self.is_failover_possible(cluster, leader, member) + if not data: + if not self.server.patroni.dcs.manual_failover(leader, member): + data = b'failed to write failover key into DCS' + status_code = 503 + else: + self.server.patroni.dcs.event.set() + status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, member) else: - data = self.is_failover_possible(cluster, leader, member) - if not data: - if not self.server.patroni.dcs.manual_failover(leader, member): - data = b'failed to write failover key into DCS' - status_code = 503 - else: - self.server.patroni.dcs.event.set() - status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, member) + status_code = 400 + data = b'No values given for required parameters leader and member' self.send_response(status_code) self.send_header('Content-Type', 'text/html') diff --git a/tests/test_api.py b/tests/test_api.py index 1be0f96c..0d6dd151 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -153,6 +153,10 @@ class TestRestApiHandler(unittest.TestCase): @patch.object(MockHa, 'dcs') def test_do_POST_failover(self, dcs): cluster = dcs.get_cluster.return_value + + request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\ + b'Content-Length: 0\n\n' + MockRestApiServer(RestApiHandler, request) request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\ b'Content-Length: 25\n\n{"leader": "postgresql1"}' MockRestApiServer(RestApiHandler, request) From ede5661da3b472366385bf8747c51dec26c047d0 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Fri, 4 Mar 2016 19:21:48 +0100 Subject: [PATCH 48/89] Remove hard-coded values from the failover code. --- patroni/ha.py | 4 ++-- tests/test_ha.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/patroni/ha.py b/patroni/ha.py index 8f5d772d..98ba4398 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -287,10 +287,10 @@ class Ha(object): try: delta = (failover.scheduled_at - now).total_seconds() - if delta > 10: + if delta > self.patroni.nap_time: logging.info('Awaiting failover at %s (in %.0f seconds)', failover.scheduled_at.isoformat(), delta) return - elif delta < -15: + elif delta < - int(self.patroni.nap_time * 1.5): logger.warning('Found a stale failover value, cleaning up: %s', failover.scheduled_at) self.dcs.manual_failover('', '', self.cluster.failover.index) return diff --git a/tests/test_ha.py b/tests/test_ha.py index a65a444a..98b1ad1c 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -54,6 +54,7 @@ class MockPatroni(object): self.api = Mock() self.tags = {} self.nofailover = None + self.nap_time = 10 self.replicatefrom = None self.api.connection_string = 'http://127.0.0.1:8008' From 998f0da3d8c71f9ddd770549747a7e179f5e5985 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Thu, 10 Mar 2016 16:05:06 +0100 Subject: [PATCH 49/89] Add cascading replication (backup from the replica) tests. --- features/cascading_replication.feature | 12 +++++++++++ features/cascading_replication.py | 26 ++++++++++++++++++++++ features/terrain.py | 30 ++++++++++++++++++++++---- 3 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 features/cascading_replication.feature create mode 100644 features/cascading_replication.py diff --git a/features/cascading_replication.feature b/features/cascading_replication.feature new file mode 100644 index 00000000..31500ae9 --- /dev/null +++ b/features/cascading_replication.feature @@ -0,0 +1,12 @@ +Feature: cascading replication + We should check that patroni can do base backup and streaming from the replica + +Scenario: check a base backup from the replica + Given I start postgres0 + And I start postgres1 + And replication works from postgres0 to postgres1 after 15 seconds + And I create label with "postgres0" in postgres0 data directory + And I create label with "postgres1" in postgres1 data directory + And I configure and start postgres2 with a tag clonefrom postgres1 + Then replication works from postgres0 to postgres2 after 15 seconds + And there is a label with "postgres1" in postgres2 data directory \ No newline at end of file diff --git a/features/cascading_replication.py b/features/cascading_replication.py new file mode 100644 index 00000000..a5733aa7 --- /dev/null +++ b/features/cascading_replication.py @@ -0,0 +1,26 @@ +from lettuce import world, steps + + +@steps +class CascadingReplicationSteps(object): + + def __init__(self, environ): + self.env = environ + + @staticmethod + def start_patroni_with_a_name_value_tag(step, name, tag_name, tag_value): + '''I configure and start (\w+) with a tag (\w+) (\w+)''' + return world.pctl.start(name, tags={tag_name: tag_value}) + + @staticmethod + def check_label(step, content, name): + '''There is a label with "(\w+)" in (\w+) data directory''' + label = world.pctl.read_label(name) + assert label == content, "{0} is not equal to {1}".format(label, content) + + @staticmethod + def write_label(step, content, name): + '''I create label with "(\w+)" in (\w+) data directory''' + world.pctl.write_label(name, content) + +CascadingReplicationSteps(world) diff --git a/features/terrain.py b/features/terrain.py index 19841a19..01450070 100644 --- a/features/terrain.py +++ b/features/terrain.py @@ -35,14 +35,30 @@ class PatroniController(object): self._patroni_path = cwd return self._patroni_path - def start(self, pg_name, max_wait_limit=15): + def data_dir(self, pg_name): + return os.path.join(self.patroni_path, 'data', pg_name) + + def write_label(self, pg_name, content): + with open(os.path.join(self.data_dir(pg_name.encode('utf-8')), 'label'), 'w') as f: + f.write(content.encode('utf-8')) + + def read_label(self, pg_name): + content = None + try: + with open(os.path.join(self.data_dir(pg_name.encode('utf-8')), 'label'), 'r') as f: + content = f.read() + except IOError: + return None + return content.strip() + + def start(self, pg_name, max_wait_limit=15, tags=None): if not self._is_running(pg_name): if pg_name in self._processes: del self._processes[pg_name] cwd = self.patroni_path self._log[pg_name] = open(os.path.join(self._output_dir, 'patroni_{0}.log'.format(pg_name)), 'a') - self._config[pg_name] = self._make_patroni_test_config(pg_name) + self._config[pg_name] = self._make_patroni_test_config(pg_name, tags=tags) p = subprocess.Popen(['python', 'patroni.py', self._config[pg_name]], stdout=self._log[pg_name], stderr=subprocess.STDOUT, cwd=cwd) @@ -113,7 +129,7 @@ class PatroniController(object): def _is_running(self, pg_name): return pg_name in self._processes and self._processes[pg_name].pid and (self._processes[pg_name].poll() is None) - def _make_patroni_test_config(self, pg_name): + def _make_patroni_test_config(self, pg_name, tags=None): patroni_config_name = PatroniController.PATRONI_CONFIG.format(pg_name) patroni_config_path = os.path.join(self._output_dir, patroni_config_name) @@ -121,6 +137,7 @@ class PatroniController(object): config = yaml.load(f) postgresql = config['postgresql'] postgresql['name'] = pg_name.encode('utf-8') + postgresql['data_dir'] = 'data/{0}'.format(pg_name.encode('utf-8')) postgresql_params = postgresql['parameters'] postgresql_params['logging_collector'] = 'on' postgresql_params['log_destination'] = 'csvlog' @@ -129,6 +146,11 @@ class PatroniController(object): postgresql_params['log_statement'] = 'all' postgresql_params['log_min_messages'] = 'debug1' + if tags: + config['tags'] = {} + for tag_name in tags: + config['tags'][tag_name.encode('utf-8')] = tags[tag_name].encode('utf-8') + with open(patroni_config_path, 'w') as f: yaml.dump(config, f, default_flow_style=False) @@ -141,7 +163,7 @@ class PatroniController(object): patroni_path = self.patroni_path with open(os.path.join(patroni_path, PatroniController.PATRONI_CONFIG.format(pg_name)), 'r') as f: config = yaml.load(f) - except OSError: + except IOError: return None connstring = config['postgresql']['connect_address'] if ':' in connstring: From 9057ddeb7c80fb2fd54af83a09d62018beabc137 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Thu, 10 Mar 2016 16:06:31 +0100 Subject: [PATCH 50/89] First implementation of cloning from the replica. At the moment we just replace the master with the node at the 'clonefrom' tag if it's present. Master should be available anyway, otherwise, it will not even try to do cloning. Acceptance tests: https://github.com/zalando/patroni/pull/144/commits --- patroni/__init__.py | 4 ++++ patroni/dcs.py | 7 +++++++ patroni/ha.py | 4 +++- patroni/postgresql.py | 13 +++++++------ tests/test_ha.py | 1 + 5 files changed, 22 insertions(+), 7 deletions(-) diff --git a/patroni/__init__.py b/patroni/__init__.py index 36c8feac..85b6c5df 100644 --- a/patroni/__init__.py +++ b/patroni/__init__.py @@ -35,6 +35,10 @@ class Patroni(object): def replicatefrom(self): return self.tags.get('replicatefrom') + @property + def clonefrom(self): + return self.tags.get('clonefrom') + @staticmethod def get_dcs(name, config): if 'etcd' in config: diff --git a/patroni/dcs.py b/patroni/dcs.py index 4c2dddad..b705105d 100644 --- a/patroni/dcs.py +++ b/patroni/dcs.py @@ -72,6 +72,10 @@ class Member(namedtuple('Member', 'index,name,session,data')): def replicatefrom(self): return self.data.get('tags', {}).get('replicatefrom') + @property + def clonefrom(self): + return self.data.get('tags', {}).get('clonefrom') + class Leader(namedtuple('Leader', 'index,session,member')): @@ -147,6 +151,9 @@ class Cluster(namedtuple('Cluster', 'initialize,leader,last_leader_operation,mem def has_member(self, member_name): return any(m for m in self.members if m.name == member_name) + def get_member(self, member_name): + return ([m for m in self.members if m.name == member_name] or [None])[0] + class AbstractDCS(object): diff --git a/patroni/ha.py b/patroni/ha.py index 98ba4398..61e27814 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -76,7 +76,9 @@ class Ha(object): def bootstrap(self): if not self.cluster.is_unlocked(): # cluster already has leader self._async_executor.schedule('bootstrap from leader') - self._async_executor.run_async(self.clone, args=(self.cluster.leader, )) + clonefrom = self.patroni.clonefrom + source = self.cluster.get_member(clonefrom) if self.cluster.has_member(clonefrom) else self.cluster.leader + self._async_executor.run_async(self.clone, args=(source,)) return 'trying to bootstrap from leader' elif not self.cluster.initialize and not self.patroni.nofailover: # no initialize key if self.dcs.initialize(create_new=True): # race for initialization diff --git a/patroni/postgresql.py b/patroni/postgresql.py index a1cedb58..da561f8a 100644 --- a/patroni/postgresql.py +++ b/patroni/postgresql.py @@ -234,6 +234,7 @@ class Postgresql(object): return env def sync_replica(self, leader): + # add either the leader's or replica's credentials to pgpass env = self.write_pgpass(parseurl(leader.conn_url)) if leader else os.environ.copy() if self.create_replica(leader, env) == 0: self.delete_trigger_file() @@ -258,23 +259,23 @@ class Postgresql(object): replica_methods = self.config.get('create_replica_method', []) return any(self.replica_method_can_work_without_leader(replica_method) for replica_method in replica_methods) - def create_replica(self, leader, env): + def create_replica(self, source, env): # create the replica according to the replica_method # defined by the user. this is a list, so we need to # loop through all methods the user supplies - connstring = leader.conn_url if leader else "" + connstring = source.conn_url if source else "" # get list of replica methods from config. # If there is no configuration key, or no value is specified, use basebackup replica_methods = self.config.get('create_replica_method') or ['basebackup'] - # if we don't have any leader, leave only replica methods that work without it - replica_methods = [r for r in replica_methods if self.replica_method_can_work_without_leader(r)] if not leader \ + # if we don't have any source, leave only replica methods that work without it + replica_methods = [r for r in replica_methods if self.replica_method_can_work_without_leader(r)] if not source \ else replica_methods # go through them in priority order ret = 1 for replica_method in replica_methods: # if the method is basebackup, then use the built-in if replica_method == "basebackup": - ret = self.basebackup(leader, env) + ret = self.basebackup(source, env) if ret == 0: logger.info("replica has been created using basebackup") # if basebackup succeeds, exit with success @@ -702,7 +703,7 @@ $$""".format(name, options), name, password, password) """ Populate PostgreSQL data directory by doing one of the following: - create with initdb if there is no master. - - initialize the replica from an existing master + - initialize the replica from an existing master or replica - initialize the replica using the replica creation method that works without the master (i.e. restore from on-disk base backup) diff --git a/tests/test_ha.py b/tests/test_ha.py index 98b1ad1c..2b5e0ada 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -57,6 +57,7 @@ class MockPatroni(object): self.nap_time = 10 self.replicatefrom = None self.api.connection_string = 'http://127.0.0.1:8008' + self.clonefrom = None def run_async(func, args=()): From 42d798a3de965b8f9ead9e7a3b64b3b7a252a2bc Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 10 Mar 2016 17:19:10 +0100 Subject: [PATCH 51/89] acceptance tests on travis --- .travis.yml | 13 +++++++++++++ acceptance_tests.sh | 16 ++++++++++++++++ features/basic_replication.feature | 2 +- features/basic_replication.py | 3 ++- features/patroni_api.feature | 2 +- features/patroni_api.py | 2 +- features/terrain.py | 9 +++++++-- patroni/ha.py | 4 ++-- patroni/postgresql.py | 4 ++-- 9 files changed, 45 insertions(+), 10 deletions(-) create mode 100644 acceptance_tests.sh diff --git a/.travis.yml b/.travis.yml index 23c69358..c1ccd74e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,28 @@ +sudo: required language: python +addons: + postgresql: "9.5" python: - "2.7" - "3.4" - "3.5" install: + - sudo /etc/init.d/postgresql stop + - sudo apt-get -y remove --purge postgresql-9.1 postgresql-9.2 postgresql-9.3 postgresql-9.4 + - sudo apt-get -y autoremove + - sudo apt-key adv --keyserver keys.gnupg.net --recv-keys 7FCC7D46ACCC4CF8 + - sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main 9.5" >> /etc/apt/sources.list.d/postgresql.list' + - sudo apt-get update + - sudo apt-get -y install postgresql-9.5 + - sudo /etc/init.d/postgresql stop + - pip uninstall boto - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py2.txt --use-mirrors; fi - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r requirements-py3.txt; fi - pip install coveralls codacy-coverage script: - python setup.py test - python setup.py flake8 + - bash -x acceptance_tests.sh after_success: - coveralls - python-codacy-coverage -r coverage.xml diff --git a/acceptance_tests.sh b/acceptance_tests.sh new file mode 100644 index 00000000..2ccc233d --- /dev/null +++ b/acceptance_tests.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +ETCDVERSION=2.2.5 + +BINDIR=bin +[ -d $BINDIR ] || mkdir $BINDIR + +export PATH=$BINDIR:$PATH + +# Add etcd +curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C $BINDIR --strip=1 --wildcards --no-anchored etcd etcdctl + +sudo pip2.7 install lettuce python-Levenshtein +sudo pip2.7 install -r requirements-py2.txt + +lettuce diff --git a/features/basic_replication.feature b/features/basic_replication.feature index fe766c11..aa07a250 100644 --- a/features/basic_replication.feature +++ b/features/basic_replication.feature @@ -11,6 +11,6 @@ Feature: basic replication When I kill postgres0 Then postgres1 role is the primary after 30 seconds When I start postgres0 - Then postgres0 role is the secondary after 10 seconds + Then postgres0 role is the secondary after 15 seconds When I add the table bar to postgres1 Then table bar is present on postgres0 after 10 seconds diff --git a/features/basic_replication.py b/features/basic_replication.py index 0d2b7172..97913fc6 100644 --- a/features/basic_replication.py +++ b/features/basic_replication.py @@ -49,7 +49,8 @@ class BasicReplicationSteps(object): def check_role(step, pg_name, pg_role, max_promotion_timeout): '''(\w+) role is the (\w+) after (\d+) seconds''' if not world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)): - assert False, "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout) + assert False,\ + "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout) BasicReplicationSteps(world) diff --git a/features/patroni_api.feature b/features/patroni_api.feature index 6eedc26c..f97e059f 100644 --- a/features/patroni_api.feature +++ b/features/patroni_api.feature @@ -17,7 +17,7 @@ Scenario: check API requests on a stand-alone server Then I receive a response code 500 And I receive a response text "failover is not possible: cluster does not have members except leader" When I issue an empty POST request to http://127.0.0.1:8008/failover - Then I receive a response code 500 + Then I receive a response code 400 And I receive a response text "No values given for required parameters leader and member" Scenario: check API requests for the primary-replica pair diff --git a/features/patroni_api.py b/features/patroni_api.py index 0659bc2c..632bafa3 100644 --- a/features/patroni_api.py +++ b/features/patroni_api.py @@ -75,7 +75,7 @@ class PatroniAPISteps(object): '''I receive a response (\w+) (.*)''' if component == 'code': assert self.status_code == int(data),\ - "status code {0} != {1}, response: {2}".format(self.status_code, int(data), self.response) + "status code {0} != {1}, response: {2}".format(self.status_code, int(data), self.response) elif component == 'text': assert self.response == data.strip('"'), "response {0} does not contain {1}".format(self.response, data) else: diff --git a/features/terrain.py b/features/terrain.py index 01450070..3a845ec5 100644 --- a/features/terrain.py +++ b/features/terrain.py @@ -120,7 +120,8 @@ class PatroniController(object): self.stop(patroni) def create_and_set_output_directory(self, feature_name): - feature_dir = os.path.join(pctl.patroni_path, "features", "output", feature_name.encode('utf-8').replace(' ', '_')) + feature_dir = os.path.join(pctl.patroni_path, "features", "output", + feature_name.encode('utf-8').replace(' ', '_')) if os.path.exists(feature_dir): shutil.rmtree(feature_dir) os.makedirs(feature_dir) @@ -145,6 +146,7 @@ class PatroniController(object): postgresql_params['log_filename'] = '{0}.log'.format(pg_name) postgresql_params['log_statement'] = 'all' postgresql_params['log_min_messages'] = 'debug1' + postgresql_params['unix_socket_directories'] = '.' if tags: config['tags'] = {} @@ -209,7 +211,10 @@ class EtcdController(object): return True self.work_directory = tempfile.mkdtemp() # etcd is running throughout the tests, no need to append to the log - self.log_file = open(os.path.join(self.log_directory, "features", "output", 'etcd.log'), 'w') + output_dir = os.path.join(self.log_directory, "features", "output") + if not os.path.exists(output_dir): + os.makedirs(output_dir) + self.log_file = open(os.path.join(output_dir, 'etcd.log'), 'w') self.handle =\ subprocess.Popen(["etcd", "--debug", "--data-dir", self.work_directory], stdout=self.log_file, stderr=subprocess.STDOUT) diff --git a/patroni/ha.py b/patroni/ha.py index 98ba4398..f9550e81 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -457,8 +457,8 @@ class Ha(object): else: # check if we are allowed to join if self.sysid_valid(self.cluster.initialize) and self.cluster.initialize != self.state_handler.sysid: - logger.fatal("system ID mismatch, node {0} belongs to a different cluster". - format(self.state_handler.name)) + logger.fatal("system ID mismatch, node %s belongs to a different cluster: %s != %s", + self.state_handler.name, self.cluster.initialize, self.state_handler.sysid) sys.exit(1) # try to start dead postgres diff --git a/patroni/postgresql.py b/patroni/postgresql.py index a1cedb58..d875057b 100644 --- a/patroni/postgresql.py +++ b/patroni/postgresql.py @@ -207,7 +207,7 @@ class Postgresql(object): options.append('--username={0}'.format(self.superuser['username'])) if 'password' in self.superuser: (fd, pwfile) = tempfile.mkstemp() - os.write(fd, self.superuser['password'].encode()) + os.write(fd, self.superuser['password'].encode('utf-8')) os.close(fd) options.append('--pwfile={0}'.format(pwfile)) @@ -506,7 +506,7 @@ recovery_target_timeline = 'latest' try: data = subprocess.check_output(['pg_controldata', self.data_dir]) if data: - data = data.decode().splitlines() + data = data.decode('utf-8').splitlines() result = {l.split(':')[0].replace('Current ', '', 1): l.split(':')[1].strip() for l in data if l} except subprocess.CalledProcessError: logger.exception("Error when calling pg_controldata") From c2d1eea7d05d3c009239761aefbd0dc44f383a19 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 10 Mar 2016 17:19:43 +0100 Subject: [PATCH 52/89] disable clonefrom test --- features/cascading_replication.feature | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/features/cascading_replication.feature b/features/cascading_replication.feature index 31500ae9..37693311 100644 --- a/features/cascading_replication.feature +++ b/features/cascading_replication.feature @@ -8,5 +8,5 @@ Scenario: check a base backup from the replica And I create label with "postgres0" in postgres0 data directory And I create label with "postgres1" in postgres1 data directory And I configure and start postgres2 with a tag clonefrom postgres1 - Then replication works from postgres0 to postgres2 after 15 seconds - And there is a label with "postgres1" in postgres2 data directory \ No newline at end of file + Then replication works from postgres0 to postgres2 after 30 seconds + And there is a label with "postgres0" in postgres2 data directory From 33a1de7828cf97298f972329c78ec7d302dc5a95 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 10 Mar 2016 17:23:44 +0100 Subject: [PATCH 53/89] Fix .travis.yml --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c1ccd74e..58d2b988 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,6 @@ install: - sudo apt-get update - sudo apt-get -y install postgresql-9.5 - sudo /etc/init.d/postgresql stop - - pip uninstall boto - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py2.txt --use-mirrors; fi - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r requirements-py3.txt; fi - pip install coveralls codacy-coverage From c955e298057d87a8fee4d5dd9a42104a1b567d0f Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 10 Mar 2016 20:02:40 +0100 Subject: [PATCH 54/89] Disable gce boto plugins by overriding BOTO_CONFIG These plugins are not compatible with python 3 and breaking unit tests --- .travis.yml | 2 ++ acceptance_tests.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 58d2b988..3bf864f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,8 @@ sudo: required language: python addons: postgresql: "9.5" +env: + - BOTO_CONFIG='' python: - "2.7" - "3.4" diff --git a/acceptance_tests.sh b/acceptance_tests.sh index 2ccc233d..d1fbf8d9 100644 --- a/acceptance_tests.sh +++ b/acceptance_tests.sh @@ -13,4 +13,4 @@ curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v$ sudo pip2.7 install lettuce python-Levenshtein sudo pip2.7 install -r requirements-py2.txt -lettuce +exec lettuce From 805716ed689268f6be771cddf59d576434ad0994 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Fri, 11 Mar 2016 10:19:00 +0100 Subject: [PATCH 55/89] Variables and parameters renaming. Previously, "without_leader" suffix was used in the name of methods and functions that initialize a replica without an active replication connection, and leader was part of the name for parameters and messages that require an active replication conneciton. Since we support init from the members other than the leader, those conventions have to be changed. --- patroni/ha.py | 22 ++++++++++-------- patroni/postgresql.py | 49 +++++++++++++++++++++------------------- tests/test_ha.py | 4 ++-- tests/test_postgresql.py | 16 ++++++------- 4 files changed, 49 insertions(+), 42 deletions(-) diff --git a/patroni/ha.py b/patroni/ha.py index 61e27814..f9995d31 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -65,21 +65,25 @@ class Ha(object): pass self.dcs.touch_member(json.dumps(data, separators=(',', ':'))) - def clone(self, leader): - if self.state_handler.bootstrap(cluster_initialized=True, current_leader=leader): - logger.info('bootstrapped from leader' if leader else 'bootstrapped without leader') + def clone(self, clone_member, clone_member_name="leader"): + if self.state_handler.bootstrap(cluster_initialized=True, clone_member=clone_member): + logger.info('bootstrapped from {0}'.format(clone_member_name) + if clone_member else 'bootstrapped without leader') else: self.state_handler.stop('immediate') self.state_handler.remove_data_directory() - logger.error('failed to bootstrap from leader' if leader else 'failed to bootstrap (without leader)') + logger.error('failed to bootstrap from {0}'.format(clone_member_name) + if clone_member else 'failed to bootstrap (without leader)') def bootstrap(self): if not self.cluster.is_unlocked(): # cluster already has leader - self._async_executor.schedule('bootstrap from leader') clonefrom = self.patroni.clonefrom - source = self.cluster.get_member(clonefrom) if self.cluster.has_member(clonefrom) else self.cluster.leader - self._async_executor.run_async(self.clone, args=(source,)) - return 'trying to bootstrap from leader' + clone_member = self.cluster.get_member(clonefrom)\ + if self.cluster.has_member(clonefrom) else self.cluster.leader + clone_member_name = 'leader' if clone_member == self.cluster.leader else 'replica {0}'.format(clonefrom) + self._async_executor.schedule('bootstrap from {0}'.format(clone_member_name)) + self._async_executor.run_async(self.clone, args=(clone_member, clone_member_name)) + return 'trying to bootstrap from {0}'.format(clone_member_name) elif not self.cluster.initialize and not self.patroni.nofailover: # no initialize key if self.dcs.initialize(create_new=True): # race for initialization try: @@ -98,7 +102,7 @@ class Ha(object): else: return 'failed to acquire initialize lock' else: - if self.state_handler.can_create_replica_without_leader(): + if self.state_handler.can_create_replica_without_replication_connection(): self._async_executor.run_async(self.clone, args=(None, )) return "trying to bootstrap without leader" return 'waiting for leader to bootstrap' diff --git a/patroni/postgresql.py b/patroni/postgresql.py index da561f8a..d8f649c7 100644 --- a/patroni/postgresql.py +++ b/patroni/postgresql.py @@ -233,10 +233,10 @@ class Postgresql(object): env['PGPASSFILE'] = self.pgpass return env - def sync_replica(self, leader): - # add either the leader's or replica's credentials to pgpass - env = self.write_pgpass(parseurl(leader.conn_url)) if leader else os.environ.copy() - if self.create_replica(leader, env) == 0: + def sync_replica(self, clone_member): + # add the credentials to connect to the replica origin to pgpass. + env = self.write_pgpass(parseurl(clone_member.conn_url)) if clone_member else os.environ.copy() + if self.create_replica(clone_member, env) == 0: self.delete_trigger_file() return True return False @@ -249,33 +249,35 @@ class Postgresql(object): """ return ' '.join('{0}={1}'.format(param, val) for param, val in sorted(conn.items())) - def replica_method_can_work_without_leader(self, method): + def replica_method_can_work_without_replication_connection(self, method): return method != 'basebackup' and self.config and self.config.get(method, {}).get('no_master') - def can_create_replica_without_leader(self): + def can_create_replica_without_replication_connection(self): """ go through the replication methods to see if there are ones - that does not require a running leader to create the replica. + that does not require a working replication connection. """ replica_methods = self.config.get('create_replica_method', []) - return any(self.replica_method_can_work_without_leader(replica_method) for replica_method in replica_methods) + return any(self.replica_method_can_work_without_replication_connection(replica_method) + for replica_method in replica_methods) - def create_replica(self, source, env): + def create_replica(self, clone_member, env): # create the replica according to the replica_method # defined by the user. this is a list, so we need to # loop through all methods the user supplies - connstring = source.conn_url if source else "" + connstring = clone_member.conn_url if clone_member else "" # get list of replica methods from config. # If there is no configuration key, or no value is specified, use basebackup replica_methods = self.config.get('create_replica_method') or ['basebackup'] # if we don't have any source, leave only replica methods that work without it - replica_methods = [r for r in replica_methods if self.replica_method_can_work_without_leader(r)] if not source \ - else replica_methods + replica_methods = \ + [r for r in replica_methods if self.replica_method_can_work_without_replication_connection(r)]\ + if not clone_member else replica_methods # go through them in priority order ret = 1 for replica_method in replica_methods: # if the method is basebackup, then use the built-in if replica_method == "basebackup": - ret = self.basebackup(source, env) + ret = self.basebackup(clone_member, env) if ret == 0: logger.info("replica has been created using basebackup") # if basebackup succeeds, exit with success @@ -699,18 +701,19 @@ $$""".format(name, options), name, password, password) def last_operation(self): return str(self.xlog_position()) - def bootstrap(self, cluster_initialized=False, current_leader=None): + def bootstrap(self, cluster_initialized=False, clone_member=None): """ Populate PostgreSQL data directory by doing one of the following: - create with initdb if there is no master. - - initialize the replica from an existing master or replica + - initialize the replica from an existing member (master or replica) - initialize the replica using the replica creation method that - works without the master (i.e. restore from on-disk base backup) + works without the replication connection (i.e. restore from on-disk + base backup) The choice between the last 2 is triggered by the initialize flag. We should never try to initdb an already initialized cluster, nor - try to bootstrap the cluster that lacks the initialize key from from - the master-less replica creation method (in the latter case, there is + try to bootstrap the cluster that lacks the initialize key using the + master-less replica creation method (in the latter case, there is no clear inidicator of the moment we should abandon our attempts and swich to initdb). @@ -720,7 +723,7 @@ $$""".format(name, options), name, password, password) that should be retried in the future. """ ret = False - if not (cluster_initialized or current_leader): + if not (cluster_initialized or clone_member): ret = self.initialize() and self.start() if ret: self.create_replication_user() @@ -728,9 +731,9 @@ $$""".format(name, options), name, password, password) else: raise PostgresException("Could not bootstrap master PostgreSQL") else: - if self.sync_replica(current_leader): + if self.sync_replica(clone_member): self.restore_configuration_files() - self.write_recovery_conf(current_leader, True) + self.write_recovery_conf(clone_member, True) ret = self.start() return ret @@ -758,12 +761,12 @@ $$""".format(name, options), name, password, password) logger.exception('Could not remove data directory %s', self.data_dir) self.move_data_directory() - def basebackup(self, leader, env): + def basebackup(self, clone_member, env): # creates a replica data dir using pg_basebackup. # this is the default, built-in create_replica_method # tries twice, then returns failure (as 1) # uses "stream" as the xlog-method to avoid sync issues - master_connection = leader.conn_url + master_connection = clone_member.conn_url maxfailures = 2 ret = 1 for bbfailures in range(0, maxfailures): diff --git a/tests/test_ha.py b/tests/test_ha.py index 2b5e0ada..7eaf3808 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -88,7 +88,7 @@ class TestHa(unittest.TestCase): 'replication': {'username': '', 'password': '', 'network': ''}}) self.p.set_state('running') self.p.check_replication_lag = true - self.p.can_create_replica_without_leader = MagicMock(return_value=False) + self.p.can_create_replica_without_replication_connection = MagicMock(return_value=False) self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'}) self.e.client.read = etcd_read self.e.client.write = etcd_write @@ -212,7 +212,7 @@ class TestHa(unittest.TestCase): def test_bootstrap_without_leader(self): self.ha.cluster = get_cluster_initialized_without_leader() - self.p.can_create_replica_without_leader = MagicMock(return_value=True) + self.p.can_create_replica_without_replication_connection = MagicMock(return_value=True) self.assertEquals(self.ha.bootstrap(), "trying to bootstrap without leader") def test_bootstrap_initialize_lock_failed(self): diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py index 5558437b..1f5380d9 100644 --- a/tests/test_postgresql.py +++ b/tests/test_postgresql.py @@ -471,17 +471,17 @@ class TestPostgresql(unittest.TestCase): def test_restore_configuration_files(self): self.p.restore_configuration_files() - def test_can_create_replica_without_leader(self): + def test_can_create_replica_without_replication_connection(self): self.p.config['create_replica_method'] = [] - self.assertFalse(self.p.can_create_replica_without_leader()) + self.assertFalse(self.p.can_create_replica_without_replication_connection()) self.p.config['create_replica_method'] = ['wale', 'basebackup'] self.p.config['wale'] = {'command': 'foo', 'no_master': 1} - self.assertTrue(self.p.can_create_replica_without_leader()) + self.assertTrue(self.p.can_create_replica_without_replication_connection()) - def test_replica_method_can_work_without_leader(self): - self.assertFalse(self.p.replica_method_can_work_without_leader('basebackup')) - self.assertFalse(self.p.replica_method_can_work_without_leader('foobar')) + def test_replica_method_can_work_without_replication_connection(self): + self.assertFalse(self.p.replica_method_can_work_without_replication_connection('basebackup')) + self.assertFalse(self.p.replica_method_can_work_without_replication_connection('foobar')) self.p.config['foo'] = {'command': 'bar', 'no_master': 1} - self.assertTrue(self.p.replica_method_can_work_without_leader('foo')) + self.assertTrue(self.p.replica_method_can_work_without_replication_connection('foo')) self.p.config['foo'] = {'command': 'bar'} - self.assertFalse(self.p.replica_method_can_work_without_leader('foo')) + self.assertFalse(self.p.replica_method_can_work_without_replication_connection('foo')) From d965d21ada4b4b6f21896a8bfe4553e1bf2765d7 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Fri, 11 Mar 2016 10:48:58 +0100 Subject: [PATCH 56/89] Unit-tests for clone from the replica. Remove clonefrom function from dcs, since it's not used. --- patroni/dcs.py | 4 ---- patroni/ha.py | 2 +- tests/test_ha.py | 17 +++++++++++------ 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/patroni/dcs.py b/patroni/dcs.py index b705105d..66e07aae 100644 --- a/patroni/dcs.py +++ b/patroni/dcs.py @@ -72,10 +72,6 @@ class Member(namedtuple('Member', 'index,name,session,data')): def replicatefrom(self): return self.data.get('tags', {}).get('replicatefrom') - @property - def clonefrom(self): - return self.data.get('tags', {}).get('clonefrom') - class Leader(namedtuple('Leader', 'index,session,member')): diff --git a/patroni/ha.py b/patroni/ha.py index f9995d31..f6669e9d 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -80,7 +80,7 @@ class Ha(object): clonefrom = self.patroni.clonefrom clone_member = self.cluster.get_member(clonefrom)\ if self.cluster.has_member(clonefrom) else self.cluster.leader - clone_member_name = 'leader' if clone_member == self.cluster.leader else 'replica {0}'.format(clonefrom) + clone_member_name = 'leader' if clone_member == self.cluster.leader else 'replica \'{0}\''.format(clonefrom) self._async_executor.schedule('bootstrap from {0}'.format(clone_member_name)) self._async_executor.run_async(self.clone, args=(clone_member, clone_member_name)) return 'trying to bootstrap from {0}'.format(clone_member_name) diff --git a/tests/test_ha.py b/tests/test_ha.py index 7eaf3808..2f9e5478 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -29,12 +29,12 @@ def get_cluster_not_initialized_without_leader(): def get_cluster_initialized_without_leader(leader=False, failover=None): - m = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:rep-pass@127.0.0.1:5435/postgres', - 'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location': 4}) - l = Leader(0, 0, m) if leader else None - o = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:rep-pass@127.0.0.1:5436/postgres', - 'api_url': 'http://127.0.0.1:8011/patroni'}) - return get_cluster(True, l, [m, o], failover) + m1 = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:rep-pass@127.0.0.1:5435/postgres', + 'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location': 4}) + l = Leader(0, 0, m1) if leader else None + m2 = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:rep-pass@127.0.0.1:5436/postgres', + 'api_url': 'http://127.0.0.1:8011/patroni'}) + return get_cluster(True, l, [m1, m2], failover) def get_cluster_initialized_with_leader(failover=None): @@ -206,6 +206,11 @@ class TestHa(unittest.TestCase): self.p.bootstrap = false self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap from leader') + def test_bootstrap_from_another_member(self): + self.ha.cluster = get_cluster_initialized_with_leader() + self.ha.patroni.clonefrom = 'other' + self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap from replica \'other\'') + def test_bootstrap_waiting_for_leader(self): self.ha.cluster = get_cluster_initialized_without_leader() self.assertEquals(self.ha.bootstrap(), 'waiting for leader to bootstrap') From 30d3982d25d63946ea1c375360f620e2db062eaa Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 11 Mar 2016 12:56:29 +0100 Subject: [PATCH 57/89] Acceptance tests with behave --- .travis.yml | 7 +- acceptance_tests.sh | 16 ---- features/basic_replication.py | 56 -------------- features/cascading_replication.py | 26 ------- features/{terrain.py => environment.py} | 58 ++++++--------- features/patroni_api.py | 99 ------------------------- features/steps/basic_replication.py | 55 ++++++++++++++ features/steps/cascading_replication.py | 17 +++++ features/steps/patroni_api.py | 87 ++++++++++++++++++++++ 9 files changed, 186 insertions(+), 235 deletions(-) delete mode 100644 acceptance_tests.sh delete mode 100644 features/basic_replication.py delete mode 100644 features/cascading_replication.py rename features/{terrain.py => environment.py} (87%) delete mode 100644 features/patroni_api.py create mode 100644 features/steps/basic_replication.py create mode 100644 features/steps/cascading_replication.py create mode 100644 features/steps/patroni_api.py diff --git a/.travis.yml b/.travis.yml index 3bf864f3..073b6572 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ language: python addons: postgresql: "9.5" env: - - BOTO_CONFIG='' + - BOTO_CONFIG='' ETCDVERSION=2.2.5 python: - "2.7" - "3.4" @@ -19,11 +19,12 @@ install: - sudo /etc/init.d/postgresql stop - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py2.txt --use-mirrors; fi - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r requirements-py3.txt; fi - - pip install coveralls codacy-coverage + - curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd + - pip install behave coveralls codacy-coverage script: - python setup.py test - python setup.py flake8 - - bash -x acceptance_tests.sh + - PATH=.:$PATH behave after_success: - coveralls - python-codacy-coverage -r coverage.xml diff --git a/acceptance_tests.sh b/acceptance_tests.sh deleted file mode 100644 index d1fbf8d9..00000000 --- a/acceptance_tests.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -ETCDVERSION=2.2.5 - -BINDIR=bin -[ -d $BINDIR ] || mkdir $BINDIR - -export PATH=$BINDIR:$PATH - -# Add etcd -curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C $BINDIR --strip=1 --wildcards --no-anchored etcd etcdctl - -sudo pip2.7 install lettuce python-Levenshtein -sudo pip2.7 install -r requirements-py2.txt - -exec lettuce diff --git a/features/basic_replication.py b/features/basic_replication.py deleted file mode 100644 index 97913fc6..00000000 --- a/features/basic_replication.py +++ /dev/null @@ -1,56 +0,0 @@ -import psycopg2 as pg -from time import sleep - -from lettuce import world, steps - - -@steps -class BasicReplicationSteps(object): - - def __init__(self, environ): - self.env = environ - - @staticmethod - def start_patroni(step, name): - '''I start (\w+)''' - return world.pctl.start(name) - - @staticmethod - def stop_patroni(step, name): - '''I shut down (\w+)''' - return world.pctl.stop(name) - - @staticmethod - def kill_patroni(step, name): - '''I kill (\w+)''' - return world.pctl.stop(name, kill=True) - - @staticmethod - def add_table(step, table_name, pg_name): - '''I add the table (\w+) to (\w+)''' - # parse the configuration file and get the port - try: - world.pctl.query(pg_name, "CREATE TABLE {0}()".format(table_name)) - except pg.Error as e: - assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e) - - @staticmethod - def table_is_present_on(step, table_name, pg_name, max_replication_delay): - '''Table (\w+) is present on (\w+) after (\d+) seconds''' - for _ in range(int(max_replication_delay)): - if world.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None: - break - sleep(1) - else: - assert False,\ - "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay) - - @staticmethod - def check_role(step, pg_name, pg_role, max_promotion_timeout): - '''(\w+) role is the (\w+) after (\d+) seconds''' - if not world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)): - assert False,\ - "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout) - - -BasicReplicationSteps(world) diff --git a/features/cascading_replication.py b/features/cascading_replication.py deleted file mode 100644 index a5733aa7..00000000 --- a/features/cascading_replication.py +++ /dev/null @@ -1,26 +0,0 @@ -from lettuce import world, steps - - -@steps -class CascadingReplicationSteps(object): - - def __init__(self, environ): - self.env = environ - - @staticmethod - def start_patroni_with_a_name_value_tag(step, name, tag_name, tag_value): - '''I configure and start (\w+) with a tag (\w+) (\w+)''' - return world.pctl.start(name, tags={tag_name: tag_value}) - - @staticmethod - def check_label(step, content, name): - '''There is a label with "(\w+)" in (\w+) data directory''' - label = world.pctl.read_label(name) - assert label == content, "{0} is not equal to {1}".format(label, content) - - @staticmethod - def write_label(step, content, name): - '''I create label with "(\w+)" in (\w+) data directory''' - world.pctl.write_label(name, content) - -CascadingReplicationSteps(world) diff --git a/features/terrain.py b/features/environment.py similarity index 87% rename from features/terrain.py rename to features/environment.py index 3a845ec5..90207ab0 100644 --- a/features/terrain.py +++ b/features/environment.py @@ -1,4 +1,3 @@ -from lettuce import world, before, after import os.path import psycopg2 import requests @@ -39,13 +38,13 @@ class PatroniController(object): return os.path.join(self.patroni_path, 'data', pg_name) def write_label(self, pg_name, content): - with open(os.path.join(self.data_dir(pg_name.encode('utf-8')), 'label'), 'w') as f: - f.write(content.encode('utf-8')) + with open(os.path.join(self.data_dir(pg_name), 'label'), 'w') as f: + f.write(content) def read_label(self, pg_name): content = None try: - with open(os.path.join(self.data_dir(pg_name.encode('utf-8')), 'label'), 'r') as f: + with open(os.path.join(self.data_dir(pg_name), 'label'), 'r') as f: content = f.read() except IOError: return None @@ -120,8 +119,8 @@ class PatroniController(object): self.stop(patroni) def create_and_set_output_directory(self, feature_name): - feature_dir = os.path.join(pctl.patroni_path, "features", "output", - feature_name.encode('utf-8').replace(' ', '_')) + feature_dir = os.path.join(self.patroni_path, "features", "output", + feature_name.replace(' ', '_')) if os.path.exists(feature_dir): shutil.rmtree(feature_dir) os.makedirs(feature_dir) @@ -137,8 +136,8 @@ class PatroniController(object): with open(patroni_config_name) as f: config = yaml.load(f) postgresql = config['postgresql'] - postgresql['name'] = pg_name.encode('utf-8') - postgresql['data_dir'] = 'data/{0}'.format(pg_name.encode('utf-8')) + postgresql['name'] = pg_name + postgresql['data_dir'] = 'data/{0}'.format(pg_name) postgresql_params = postgresql['parameters'] postgresql_params['logging_collector'] = 'on' postgresql_params['log_destination'] = 'csvlog' @@ -149,9 +148,7 @@ class PatroniController(object): postgresql_params['unix_socket_directories'] = '.' if tags: - config['tags'] = {} - for tag_name in tags: - config['tags'][tag_name.encode('utf-8')] = tags[tag_name].encode('utf-8') + config['tags'] = tags with open(patroni_config_path, 'w') as f: yaml.dump(config, f, default_flow_style=False) @@ -269,44 +266,35 @@ class EtcdController(object): # if etcd is running, but we didn't start it try: r = requests.get(EtcdController.ETCD_VERSION_URL) - running = (r and r.ok and 'etcdserver' in r.content) + running = (r and r.ok and b'etcdserver' in r.content) except requests.ConnectionError: running = False return running -pctl = PatroniController() -etcd_ctl = EtcdController(pctl.patroni_path) -# export pctl to manage patroni from scenario files -world.pctl = pctl -world.etcd_ctl = etcd_ctl - - # actions to execute on start/stop of the tests and before running invidual features -@before.all -def start_etcd(): - etcd_ctl.start() +def before_all(context): + context.pctl = PatroniController() + context.etcd_ctl = EtcdController(context.pctl.patroni_path) + context.etcd_ctl.start() try: - etcd_ctl.cleanup_service_tree() + context.etcd_ctl.cleanup_service_tree() except AssertionError: # after.all handlers won't be executed in before.all - etcd_ctl.stop_and_remove_work_directory() + context.etcd_ctl.stop_and_remove_work_directory() raise -@after.all -def stop_etcd(*args, **kwargs): - etcd_ctl.stop_and_remove_work_directory() +def after_all(context): + context.etcd_ctl.stop_and_remove_work_directory() -@before.each_feature -def make_test_output_dir(feature): +def before_feature(context, feature): """ create per-feature output directory to collect Patroni and PostgreSQL logs """ - pctl.create_and_set_output_directory(feature.name) + context.pctl.create_and_set_output_directory(feature.name) -@after.each_feature -def cleanup(*args, **kwargs): +def after_feature(context, feature): """ stop all Patronis, remove their data directory and cleanup the keys in etcd """ - pctl.stop_all() - shutil.rmtree(os.path.join(pctl.patroni_path, 'data')) - etcd_ctl.cleanup_service_tree() + context.pctl.stop_all() + shutil.rmtree(os.path.join(context.pctl.patroni_path, 'data')) + context.etcd_ctl.cleanup_service_tree() diff --git a/features/patroni_api.py b/features/patroni_api.py deleted file mode 100644 index 632bafa3..00000000 --- a/features/patroni_api.py +++ /dev/null @@ -1,99 +0,0 @@ -from datetime import datetime, timedelta -from lettuce import world, steps -import time -import pytz -import requests - - -@steps -class PatroniAPISteps(object): - - def __init__(self, environ): - self.env = environ - self.response = None - self.status_code = None - - # there is no way we can find out if the node has already - # started as a leader without checking the DCS. We cannot - # just rely on the database availability, since there is - # a short gap between the time PostgreSQL becomes available - # and Patroni assuming the leader role. - @staticmethod - def is_a_leader(step, name, time_limit): - '''(\w+) is a leader after (\d+) seconds''' - max_time = time.time() + int(time_limit) - while (world.etcd_ctl.query("leader") != name): - time.sleep(1) - if time.time() > max_time: - assert False, "{0} is not a leader in etcd after {1} seconds".format(name, time_limit) - - @staticmethod - def sleep_for_n_seconds(step, value): - '''I sleep for (\d+) seconds''' - time.sleep(int(value)) - - def do_get(self, step, url): - '''I issue a GET request to (https?://(?:\w|\.|:|/)+)''' - try: - r = requests.get(url) - except requests.exceptions.RequestException: - self.code = None - self.response = None - else: - self.status_code = r.status_code - try: - self.response = r.json() - except ValueError: - self.response = r.content - - def do_post_empty(self, step, url): - '''I issue an empty POST request to (https?://(?:\w|\.|:|/)+)''' - self.do_post(step, url, None) - - def do_post(self, step, url, data): - '''I issue a POST request to (https?://(?:\w|\.|:|/)+) with ((?:\w+=(?:\w|\.|:|-|\+|\s)+,?)+)''' - post_data = {} - if data: - post_components = data.split(',') - for pc in post_components: - if '=' in pc: - k, v = pc.split('=', 2) - post_data[k.strip()] = v.strip() - try: - r = requests.post(url, json=post_data) - except requests.exceptions.RequestException: - self.code = None - self.response = None - else: - self.status_code = r.status_code - try: - self.response = r.json() - except ValueError: - self.response = r.content - - def check_response(self, step, component, data): - '''I receive a response (\w+) (.*)''' - if component == 'code': - assert self.status_code == int(data),\ - "status code {0} != {1}, response: {2}".format(self.status_code, int(data), self.response) - elif component == 'text': - assert self.response == data.strip('"'), "response {0} does not contain {1}".format(self.response, data) - else: - assert component in self.response, "{0} is not part of the response".format(component) - assert self.response[component] == data, "{0} does not contain {1}".format(component, data) - - def replication_works(self, step, master, replica, time_limit): - '''replication works from (\w+) to (\w+) after (\d+) seconds''' - step.behave_as(""" - When I add the table test_{0} to {1} - Then table test_{0} is present on {2} after {3} seconds - """.format(int(time.time()), master, replica, time_limit)) - - def scheduld_failover(self, step, at_url, from_host, to_host, in_seconds): - '''I issue a scheduled failover at (https?://(?:\w|\.|:|/)+) from (\w+) to (\w+) in (\d+) seconds''' - step.behave_as(""" - Given I issue a POST request to {0}/failover with leader={1},candidate={2},scheduled_at={3} - """.format(at_url, from_host, to_host, datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds)))) - - -PatroniAPISteps(world) diff --git a/features/steps/basic_replication.py b/features/steps/basic_replication.py new file mode 100644 index 00000000..98f5879f --- /dev/null +++ b/features/steps/basic_replication.py @@ -0,0 +1,55 @@ +import psycopg2 as pg + +from behave import step, then +from time import sleep, time + + +@step('I start {name}') +def start_patroni(context, name): + return context.pctl.start(name) + + +@step('I shut down {name}') +def stop_patroni(context, name): + return context.pctl.stop(name) + + +@step('I kill {name}') +def kill_patroni(context, name): + return context.pctl.stop(name, kill=True) + + +@step('I add the table {table_name} to {pg_name}') +def add_table(context, table_name, pg_name): + # parse the configuration file and get the port + try: + context.pctl.query(pg_name, "CREATE TABLE {0}()".format(table_name)) + except pg.Error as e: + assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e) + + +@then('Table {table_name} is present on {pg_name} after {max_replication_delay} seconds') +def table_is_present_on(context, table_name, pg_name, max_replication_delay): + for _ in range(int(max_replication_delay)): + if context.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None: + break + sleep(1) + else: + assert False,\ + "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay) + + +@then('{pg_name} role is the {pg_role} after {max_promotion_timeout} seconds') +def check_role(context, pg_name, pg_role, max_promotion_timeout): + if not context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)): + assert False,\ + "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout) + + +@step('replication works from {master} to {replica} after {time_limit} seconds') +@then('replication works from {master} to {replica} after {time_limit} seconds') +def replication_works(context, master, replica, time_limit): + context.execute_steps(""" + When I add the table test_{0} to {1} + Then table test_{0} is present on {2} after {3} seconds + """.format(int(time()), master, replica, time_limit)) diff --git a/features/steps/cascading_replication.py b/features/steps/cascading_replication.py new file mode 100644 index 00000000..07d8fbd2 --- /dev/null +++ b/features/steps/cascading_replication.py @@ -0,0 +1,17 @@ +from behave import step, then + + +@step('I configure and start {name} with a tag {tag_name} {tag_value}') +def start_patroni_with_a_name_value_tag(context, name, tag_name, tag_value): + return context.pctl.start(name, tags={tag_name: tag_value}) + + +@then('There is a label with "{content}" in {name} data directory') +def check_label(context, content, name): + label = context.pctl.read_label(name) + assert label == content, "{0} is not equal to {1}".format(label, content) + + +@step('I create label with "{content}" in {name} data directory') +def write_label(context, content, name): + context.pctl.write_label(name, content) diff --git a/features/steps/patroni_api.py b/features/steps/patroni_api.py new file mode 100644 index 00000000..309a20af --- /dev/null +++ b/features/steps/patroni_api.py @@ -0,0 +1,87 @@ +import time +import pytz +import requests + +from datetime import datetime, timedelta +from behave import step, then + + +# there is no way we can find out if the node has already +# started as a leader without checking the DCS. We cannot +# just rely on the database availability, since there is +# a short gap between the time PostgreSQL becomes available +# and Patroni assuming the leader role. +@step('{name} is a leader after {time_limit} seconds') +@then('{name} is a leader after {time_limit} seconds') +def is_a_leader(context, name, time_limit): + max_time = time.time() + int(time_limit) + while (context.etcd_ctl.query("leader") != name): + time.sleep(1) + if time.time() > max_time: + assert False, "{0} is not a leader in etcd after {1} seconds".format(name, time_limit) + + +@step('I sleep for {value} seconds') +def sleep_for_n_seconds(context, value): + time.sleep(int(value)) + + +@step('I issue a GET request to {url}') +def do_get(context, url): + try: + r = requests.get(url) + except requests.exceptions.RequestException: + context.status_code = None + context.response = None + else: + context.status_code = r.status_code + try: + context.response = r.json() + except ValueError: + context.response = r.content.decode('utf-8') + + +@step('I issue an empty POST request to {url}') +def do_post_empty(context, url): + do_post(context, url, None) + + +@step('I issue a POST request to {url} with {data}') +def do_post(context, url, data): + post_data = {} + if data: + post_components = data.split(',') + for pc in post_components: + if '=' in pc: + k, v = pc.split('=', 2) + post_data[k.strip()] = v.strip() + try: + r = requests.post(url, json=post_data) + except requests.exceptions.RequestException: + context.status_code = None + context.response = None + else: + context.status_code = r.status_code + try: + context.response = r.json() + except ValueError: + context.response = r.content.decode('utf-8') + + +@then('I receive a response {component} {data}') +def check_response(context, component, data): + if component == 'code': + assert context.status_code == int(data),\ + "status code {0} != {1}, response: {2}".format(context.status_code, int(data), context.response) + elif component == 'text': + assert context.response == data.strip('"'), "response {0} does not contain {1}".format(context.response, data) + else: + assert component in context.response, "{0} is not part of the response".format(component) + assert context.response[component] == data, "{0} does not contain {1}".format(component, data) + + +@step('I issue a scheduled failover at {at_url} from {from_host} to {to_host} in {in_seconds} seconds') +def scheduld_failover(context, at_url, from_host, to_host, in_seconds): + context.execute_steps(""" + Given I issue a POST request to {0}/failover with leader={1},candidate={2},scheduled_at={3} + """.format(at_url, from_host, to_host, datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds)))) From 8b81d270bcbdc4acfec09716cc021002f41f330b Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 11 Mar 2016 13:47:57 +0100 Subject: [PATCH 58/89] BUGFIX: Assertion Failed: Steps must be unicode --- features/steps/basic_replication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/steps/basic_replication.py b/features/steps/basic_replication.py index 98f5879f..26af60e9 100644 --- a/features/steps/basic_replication.py +++ b/features/steps/basic_replication.py @@ -49,7 +49,7 @@ def check_role(context, pg_name, pg_role, max_promotion_timeout): @step('replication works from {master} to {replica} after {time_limit} seconds') @then('replication works from {master} to {replica} after {time_limit} seconds') def replication_works(context, master, replica, time_limit): - context.execute_steps(""" + context.execute_steps(u""" When I add the table test_{0} to {1} Then table test_{0} is present on {2} after {3} seconds """.format(int(time()), master, replica, time_limit)) From 5f6beae22f6fe4383b3ae79c8a0fd409c6083073 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 11 Mar 2016 14:46:14 +0100 Subject: [PATCH 59/89] Enforce data-type checks for step matcher and increase default timeout for patroni start --- features/environment.py | 6 ++-- features/steps/basic_replication.py | 16 +++++------ features/steps/cascading_replication.py | 6 ++-- features/steps/patroni_api.py | 38 +++++++++++++++++-------- 4 files changed, 40 insertions(+), 26 deletions(-) diff --git a/features/environment.py b/features/environment.py index 90207ab0..f031f19b 100644 --- a/features/environment.py +++ b/features/environment.py @@ -1,8 +1,8 @@ -import os.path +import os import psycopg2 import requests -import subprocess import shutil +import subprocess import tempfile import time import yaml @@ -50,7 +50,7 @@ class PatroniController(object): return None return content.strip() - def start(self, pg_name, max_wait_limit=15, tags=None): + def start(self, pg_name, max_wait_limit=20, tags=None): if not self._is_running(pg_name): if pg_name in self._processes: del self._processes[pg_name] diff --git a/features/steps/basic_replication.py b/features/steps/basic_replication.py index 26af60e9..364de61f 100644 --- a/features/steps/basic_replication.py +++ b/features/steps/basic_replication.py @@ -4,22 +4,22 @@ from behave import step, then from time import sleep, time -@step('I start {name}') +@step('I start {name:w}') def start_patroni(context, name): return context.pctl.start(name) -@step('I shut down {name}') +@step('I shut down {name:w}') def stop_patroni(context, name): return context.pctl.stop(name) -@step('I kill {name}') +@step('I kill {name:w}') def kill_patroni(context, name): return context.pctl.stop(name, kill=True) -@step('I add the table {table_name} to {pg_name}') +@step('I add the table {table_name:w} to {pg_name:w}') def add_table(context, table_name, pg_name): # parse the configuration file and get the port try: @@ -28,7 +28,7 @@ def add_table(context, table_name, pg_name): assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e) -@then('Table {table_name} is present on {pg_name} after {max_replication_delay} seconds') +@then('Table {table_name:w} is present on {pg_name:w} after {max_replication_delay:d} seconds') def table_is_present_on(context, table_name, pg_name, max_replication_delay): for _ in range(int(max_replication_delay)): if context.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None: @@ -39,15 +39,15 @@ def table_is_present_on(context, table_name, pg_name, max_replication_delay): "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay) -@then('{pg_name} role is the {pg_role} after {max_promotion_timeout} seconds') +@then('{pg_name:w} role is the {pg_role:w} after {max_promotion_timeout:d} seconds') def check_role(context, pg_name, pg_role, max_promotion_timeout): if not context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)): assert False,\ "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout) -@step('replication works from {master} to {replica} after {time_limit} seconds') -@then('replication works from {master} to {replica} after {time_limit} seconds') +@step('replication works from {master:w} to {replica:w} after {time_limit:d} seconds') +@then('replication works from {master:w} to {replica:w} after {time_limit:d} seconds') def replication_works(context, master, replica, time_limit): context.execute_steps(u""" When I add the table test_{0} to {1} diff --git a/features/steps/cascading_replication.py b/features/steps/cascading_replication.py index 07d8fbd2..59399c97 100644 --- a/features/steps/cascading_replication.py +++ b/features/steps/cascading_replication.py @@ -1,17 +1,17 @@ from behave import step, then -@step('I configure and start {name} with a tag {tag_name} {tag_value}') +@step('I configure and start {name:w} with a tag {tag_name:w} {tag_value:w}') def start_patroni_with_a_name_value_tag(context, name, tag_name, tag_value): return context.pctl.start(name, tags={tag_name: tag_value}) -@then('There is a label with "{content}" in {name} data directory') +@then('There is a label with "{content:w}" in {name:w} data directory') def check_label(context, content, name): label = context.pctl.read_label(name) assert label == content, "{0} is not equal to {1}".format(label, content) -@step('I create label with "{content}" in {name} data directory') +@step('I create label with "{content:w}" in {name:w} data directory') def write_label(context, content, name): context.pctl.write_label(name, content) diff --git a/features/steps/patroni_api.py b/features/steps/patroni_api.py index 309a20af..0f802dee 100644 --- a/features/steps/patroni_api.py +++ b/features/steps/patroni_api.py @@ -1,9 +1,23 @@ -import time +import parse import pytz import requests +import time +from behave import register_type, step, then from datetime import datetime, timedelta -from behave import step, then + + +@parse.with_pattern(r'https?://(?:\w|\.|:|/)+') +def parse_url(text): + return text + + +@parse.with_pattern(r'(?:\w+=(?:\w|\.|:|-|\+|\s)+,?)+') +def parse_data(text): + return text + + +register_type(url=parse_url, data=parse_data) # there is no way we can find out if the node has already @@ -11,8 +25,8 @@ from behave import step, then # just rely on the database availability, since there is # a short gap between the time PostgreSQL becomes available # and Patroni assuming the leader role. -@step('{name} is a leader after {time_limit} seconds') -@then('{name} is a leader after {time_limit} seconds') +@step('{name:w} is a leader after {time_limit:d} seconds') +@then('{name:w} is a leader after {time_limit:d} seconds') def is_a_leader(context, name, time_limit): max_time = time.time() + int(time_limit) while (context.etcd_ctl.query("leader") != name): @@ -21,12 +35,12 @@ def is_a_leader(context, name, time_limit): assert False, "{0} is not a leader in etcd after {1} seconds".format(name, time_limit) -@step('I sleep for {value} seconds') +@step('I sleep for {value:d} seconds') def sleep_for_n_seconds(context, value): time.sleep(int(value)) -@step('I issue a GET request to {url}') +@step('I issue a GET request to {url:url}') def do_get(context, url): try: r = requests.get(url) @@ -41,12 +55,12 @@ def do_get(context, url): context.response = r.content.decode('utf-8') -@step('I issue an empty POST request to {url}') +@step('I issue an empty POST request to {url:url}') def do_post_empty(context, url): do_post(context, url, None) -@step('I issue a POST request to {url} with {data}') +@step('I issue a POST request to {url:url} with {data:data}') def do_post(context, url, data): post_data = {} if data: @@ -68,7 +82,7 @@ def do_post(context, url, data): context.response = r.content.decode('utf-8') -@then('I receive a response {component} {data}') +@then('I receive a response {component:w} {data}') def check_response(context, component, data): if component == 'code': assert context.status_code == int(data),\ @@ -80,8 +94,8 @@ def check_response(context, component, data): assert context.response[component] == data, "{0} does not contain {1}".format(component, data) -@step('I issue a scheduled failover at {at_url} from {from_host} to {to_host} in {in_seconds} seconds') -def scheduld_failover(context, at_url, from_host, to_host, in_seconds): - context.execute_steps(""" +@step('I issue a scheduled failover at {at_url:url} from {from_host:w} to {to_host:w} in {in_seconds:d} seconds') +def scheduled_failover(context, at_url, from_host, to_host, in_seconds): + context.execute_steps(u""" Given I issue a POST request to {0}/failover with leader={1},candidate={2},scheduled_at={3} """.format(at_url, from_host, to_host, datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds)))) From d3c2b8b2aa28a09f4c2c30169ba061fae70268c7 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 11 Mar 2016 15:18:58 +0100 Subject: [PATCH 60/89] replace unused variables with _ --- patroni/api.py | 4 ++-- patroni/ctl.py | 2 +- patroni/ha.py | 4 ++-- patroni/scripts/wale_restore.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/patroni/api.py b/patroni/api.py index 1186321f..f2877b05 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -143,7 +143,7 @@ class RestApiHandler(BaseHTTPRequestHandler): self.wfile.write(data) def poll_failover_result(self, leader, member): - for a in range(0, 15): + for _ in range(0, 15): time.sleep(1) try: cluster = self.server.patroni.dcs.get_cluster() @@ -166,7 +166,7 @@ class RestApiHandler(BaseHTTPRequestHandler): members = [m for m in cluster.members if m.name != cluster.leader.name and m.api_url] if not members: return b'failover is not possible: cluster does not have members except leader' - for member, reachable, in_recovery, xlog_location, tags in self.server.patroni.ha.fetch_nodes_statuses(members): + for member, reachable, _, xlog_location, tags in self.server.patroni.ha.fetch_nodes_statuses(members): if reachable and not tags.get('nofailover', False): return None return b'failover is not possible: no good candidates have been found' diff --git a/patroni/ctl.py b/patroni/ctl.py index c4cc9fe0..9d07267b 100644 --- a/patroni/ctl.py +++ b/patroni/ctl.py @@ -240,7 +240,7 @@ def dsn(cluster_name, config_file, dcs, role, member): if member is None and role is None: role = 'master' - config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs) + _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs) m = get_any_member(cluster=cluster, role=role, member=member) if m is None: raise PatroniCtlException('Can not find a suitable member') diff --git a/patroni/ha.py b/patroni/ha.py index 98ba4398..d4697fe0 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -203,7 +203,7 @@ class Ha(object): ret = False members = [m for m in members if m.name != self.state_handler.name and not m.nofailover and m.api_url] if members: - for member, reachable, in_recovery, xlog_location, tags in self.fetch_nodes_statuses(members): + for member, reachable, _, xlog_location, tags in self.fetch_nodes_statuses(members): if reachable and not tags.get('nofailover', False): ret = True # TODO: check xlog_location elif not reachable: @@ -223,7 +223,7 @@ class Ha(object): # find specific node and check that it is healthy members = [m for m in self.cluster.members if m.name == failover.member] if members: - member, reachable, in_recovery, xlog_location, tags = self.fetch_node_status(members[0]) + member, reachable, _, xlog_location, tags = self.fetch_node_status(members[0]) if reachable and not tags.get('nofailover', False): # node is healthy logger.info('manual failover: to %s, i am %s', member.name, self.state_handler.name) return False diff --git a/patroni/scripts/wale_restore.py b/patroni/scripts/wale_restore.py index c80cdfae..f3691907 100755 --- a/patroni/scripts/wale_restore.py +++ b/patroni/scripts/wale_restore.py @@ -154,7 +154,7 @@ def main(): args = parser.parse_args() # retry cloning in a loop - for retry in range(0, args.retries + 1): + for _ in range(0, args.retries + 1): restore = WALERestore(scope=args.scope, datadir=args.datadir, connstring=args.connstring, env_dir=args.envdir, threshold_mb=args.threshold_megabytes, threshold_pct=args.threshold_backup_size_percentage, use_iam=args.use_iam, From ba444adb67674438fdf04e089396df68ae07a137 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 11 Mar 2016 15:32:16 +0100 Subject: [PATCH 61/89] make codacy and quantifiedcode happier --- features/environment.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/features/environment.py b/features/environment.py index f031f19b..f822007b 100644 --- a/features/environment.py +++ b/features/environment.py @@ -222,13 +222,14 @@ class EtcdController(object): time.sleep(1) return True - def query(self, key): + @staticmethod + def query(key): """ query etcd for a value of a given key """ r = requests.get("http://127.0.0.1:2379/v2/keys/service/batman/{0}".format(key)) if r.ok: content = r.json() if content: - return content.get('node', {}).get('value', None) + return content.get('node', {}).get('value') return None def stop_and_remove_work_directory(self, timeout=15): @@ -262,7 +263,8 @@ class EtcdController(object): except requests.exceptions.RequestException as e: assert False, "exception when cleaning up etcd contents: {0}".format(e) - def _is_running(self): + @staticmethod + def _is_running(): # if etcd is running, but we didn't start it try: r = requests.get(EtcdController.ETCD_VERSION_URL) From 3319c3eeea7e3996b56d8e2d7eafc7ba2455674b Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 11 Mar 2016 15:36:24 +0100 Subject: [PATCH 62/89] replace unused variables with _ --- patroni/ha.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/patroni/ha.py b/patroni/ha.py index d4697fe0..5a7e50a9 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -203,7 +203,7 @@ class Ha(object): ret = False members = [m for m in members if m.name != self.state_handler.name and not m.nofailover and m.api_url] if members: - for member, reachable, _, xlog_location, tags in self.fetch_nodes_statuses(members): + for member, reachable, _, _, tags in self.fetch_nodes_statuses(members): if reachable and not tags.get('nofailover', False): ret = True # TODO: check xlog_location elif not reachable: @@ -223,7 +223,7 @@ class Ha(object): # find specific node and check that it is healthy members = [m for m in self.cluster.members if m.name == failover.member] if members: - member, reachable, _, xlog_location, tags = self.fetch_node_status(members[0]) + member, reachable, _, _, tags = self.fetch_node_status(members[0]) if reachable and not tags.get('nofailover', False): # node is healthy logger.info('manual failover: to %s, i am %s', member.name, self.state_handler.name) return False From 9ca1b754a5d846e0772bec7ed4fe5c0280f5d05c Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Fri, 11 Mar 2016 16:33:26 +0100 Subject: [PATCH 63/89] Add zappr configuration. --- .zappr.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .zappr.yaml diff --git a/.zappr.yaml b/.zappr.yaml new file mode 100644 index 00000000..4d628636 --- /dev/null +++ b/.zappr.yaml @@ -0,0 +1,12 @@ +approvals: + # PR needs at least 4 approvals + minimum: 1 + # approval = comment that matches this regex + pattern: "^:?\\+1:?$" + from: + # commenter must be either one of: + # a public zalando org member + orgs: + - zalando + # a collaborator of the repo + collaborators: true From 6c63d32253e81d7ef26b801f9f69610abe8e18b5 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Fri, 11 Mar 2016 16:39:57 +0100 Subject: [PATCH 64/89] zappr config must be .yml. --- .zappr.yaml => .zappr.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .zappr.yaml => .zappr.yml (100%) diff --git a/.zappr.yaml b/.zappr.yml similarity index 100% rename from .zappr.yaml rename to .zappr.yml From 6985df3aca1f20a306aebfb17c77c973432f2b8b Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Fri, 11 Mar 2016 16:59:35 +0100 Subject: [PATCH 65/89] Restore the test for the clone from the replica. --- features/cascading_replication.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/cascading_replication.feature b/features/cascading_replication.feature index 37693311..7d297e41 100644 --- a/features/cascading_replication.feature +++ b/features/cascading_replication.feature @@ -9,4 +9,4 @@ Scenario: check a base backup from the replica And I create label with "postgres1" in postgres1 data directory And I configure and start postgres2 with a tag clonefrom postgres1 Then replication works from postgres0 to postgres2 after 30 seconds - And there is a label with "postgres0" in postgres2 data directory + And there is a label with "postgres1" in postgres2 data directory From 01afd09ca27f89deac9aab737738cae402377d23 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Sat, 12 Mar 2016 15:49:42 +0100 Subject: [PATCH 66/89] Migrate to python-etcd 0.4.3 Despite this release was very buggy it has really nice features: * EtcdWatchTimedOut exception is raised when `watch` call timed out * it supports SRV autodiscovery Since we already implemented our own SRV discovery this feature is not really interesting for us, but it solves the problem of having two requirements files for different python versions, because python-etcd will install dnspython or dnspython3 as a dependency. In order to fix https://github.com/jplana/python-etcd/issues/152 and https://github.com/jplana/python-etcd/pull/154 I had to override `api_execute` method. --- .travis.yml | 3 +- patroni/etcd.py | 62 +++++++++++++++++++----- requirements-py2.txt | 13 ----- requirements-py3.txt => requirements.txt | 0 setup.py | 11 ++--- tests/test_etcd.py | 26 ++++------ 6 files changed, 65 insertions(+), 50 deletions(-) delete mode 100644 requirements-py2.txt rename requirements-py3.txt => requirements.txt (100%) diff --git a/.travis.yml b/.travis.yml index 23c69358..ba4733d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,7 @@ python: - "3.4" - "3.5" install: - - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py2.txt --use-mirrors; fi - - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r requirements-py3.txt; fi + - pip install -r requirements.txt - pip install coveralls codacy-coverage script: - python setup.py test diff --git a/patroni/etcd.py b/patroni/etcd.py index 79842ca4..31c0fd37 100644 --- a/patroni/etcd.py +++ b/patroni/etcd.py @@ -14,6 +14,7 @@ from patroni.dcs import AbstractDCS, Cluster, Failover, Leader, Member from patroni.exceptions import DCSError from patroni.utils import Retry, RetryFailedError, sleep from requests.exceptions import RequestException +from six.moves.http_client import HTTPException logger = logging.getLogger(__name__) @@ -49,12 +50,57 @@ class Client(etcd.Client): self._update_machines_cache = True return [self._base_uri] - def api_execute(self, path, method, **kwargs): + def _do_http_request(self, request_executor, method, url, fields=None, **kwargs): + try: + response = request_executor(method, url, fields=fields, **kwargs) + response.data.decode('utf-8') +# self._check_cluster_id(response) + except (urllib3.exceptions.HTTPError, HTTPException, socket.error) as e: + if (isinstance(fields, dict) and fields.get("wait") == "true" and + isinstance(e, urllib3.exceptions.ReadTimeoutError)): + logger.debug("Watch timed out.") + raise etcd.EtcdWatchTimedOut("Watch timed out: {0}".format(e), cause=e) + logger.error("Request to server %s failed: %r", self._base_uri, e) + logger.info("Reconnection allowed, looking for another server.") + self._base_uri = self._next_server(cause=e) + response = False + return response + + def api_execute(self, path, method, params=None, timeout=None): + if not path.startswith('/'): + raise ValueError('Path does not start with /') + + if timeout is None: + timeout = self.read_timeout + + if timeout == 0: + timeout = None + + kwargs = {'timeout': timeout, 'fields': params, 'redirect': self.allow_redirect, + 'headers': self._get_headers(), 'preload_content': False} + + if method in [self._MGET, self._MDELETE]: + request_executor = self.http.request + elif method in [self._MPUT, self._MPOST]: + request_executor = self.http.request_encode_body + kwargs['encode_multipart'] = False + else: + raise etcd.EtcdException('HTTP method {} not supported'.format(method)) + # Update machines_cache if previous attempt of update has failed if self._update_machines_cache: self._load_machines_cache() + + response = False + try: - return super(Client, self).api_execute(path, method, **kwargs) + while not response: + response = self._do_http_request(request_executor, method, self._base_uri + path, **kwargs) + + if response is False and not self._use_proxies: + self._machines_cache = self.machines + self._machines_cache.remove(self._base_uri) + return self._handle_server_response(response) except etcd.EtcdConnectionFailed: self._update_machines_cache = True raise @@ -67,16 +113,6 @@ class Client(etcd.Client): logger.exception('Can not resolve SRV for %s', host) return [] - # try to workarond bug in python-etcd: https://github.com/jplana/python-etcd/issues/81 - def _result_from_response(self, response): - try: - response.data.decode('utf-8') - except urllib3.exceptions.TimeoutError: - raise - except Exception as e: - raise etcd.EtcdException('Unable to decode server response: {0}'.format(e)) - return super(Client, self)._result_from_response(response) - def _get_machines_cache_from_srv(self, discovery_srv): """Fetch list of etcd-cluster member by resolving _etcd-server._tcp. SRV record. This record should contain list of host and peer ports which could be used to run @@ -272,7 +308,7 @@ class Etcd(AbstractDCS): # Synchronous work of all cluster members with etcd is less expensive # than reestablishing http connection every time from every replica. return True - except urllib3.exceptions.TimeoutError: + except etcd.EtcdWatchTimedOut: self.client.http.clear() return False except etcd.EtcdException: diff --git a/requirements-py2.txt b/requirements-py2.txt deleted file mode 100644 index 26c0892d..00000000 --- a/requirements-py2.txt +++ /dev/null @@ -1,13 +0,0 @@ -boto -dnspython -mock -psycopg2>=2.6.1 -PyYAML -requests -six >= 1.7 -kazoo>=2.2.1 -python-etcd==0.4.2 -click>=4.1 -prettytable>=0.7 -tzlocal -python-dateutil diff --git a/requirements-py3.txt b/requirements.txt similarity index 100% rename from requirements-py3.txt rename to requirements.txt diff --git a/setup.py b/setup.py index 437354e2..a8936d7c 100644 --- a/setup.py +++ b/setup.py @@ -51,8 +51,8 @@ CLASSIFIERS = [ 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: Implementation :: CPython', ] @@ -76,7 +76,7 @@ class PyTest(TestCommand): def finalize_options(self): TestCommand.finalize_options(self) if self.cov_xml or self.cov_html: - self.cov = ['--cov', MAIN_PACKAGE, '--cov', MAIN_PACKAGE, '--cov-report', 'term-missing'] + self.cov = ['--cov', MAIN_PACKAGE, '--cov-report', 'term-missing'] if self.cov_xml: self.cov.extend(['--cov-report', 'xml']) if self.cov_html: @@ -116,8 +116,7 @@ def setup_package(): # Some helper variables version = os.getenv('GO_PIPELINE_LABEL', VERSION) - requirements = 'requirements-py2.txt' if sys.version_info[0] == 2 else 'requirements-py3.txt' - install_reqs = get_install_requirements(requirements) + install_reqs = get_install_requirements('requirements.txt') command_options = {'test': {'test_suite': ('setup.py', 'tests')}} if JUNIT_XML: @@ -142,9 +141,9 @@ def setup_package(): packages=setuptools.find_packages(exclude=['tests', 'tests.*']), package_data={MAIN_PACKAGE: ["*.json"]}, install_requires=install_reqs, - setup_requires=['six', 'flake8'], + setup_requires=['flake8'], cmdclass=cmdclass, - tests_require=['pytest-cov', 'pytest'], + tests_require=['mock', 'pytest-cov', 'pytest'], command_options=command_options, entry_points={'console_scripts': CONSOLE_SCRIPTS}, ) diff --git a/tests/test_etcd.py b/tests/test_etcd.py index c14d3cac..fa2538c9 100644 --- a/tests/test_etcd.py +++ b/tests/test_etcd.py @@ -25,11 +25,7 @@ class MockResponse(object): @property def data(self): - if self.content == 'TimeoutError': - raise urllib3.exceptions.TimeoutError - if self.content == 'Exception': - raise Exception - return self.content + return self.content.encode('utf-8') @property def status(self): @@ -70,7 +66,7 @@ def requests_get(url, **kwargs): def etcd_watch(key, index=None, timeout=None, recursive=None): if timeout == 2.0: - raise urllib3.exceptions.TimeoutError + raise etcd.EtcdWatchTimedOut elif timeout == 5.0: return etcd.EtcdResult('delete', {}) elif timeout == 10.0: @@ -147,6 +143,8 @@ def socket_getaddrinfo(*args): def http_request(method, url, **kwargs): + if url == 'http://localhost:2379/timeout': + raise urllib3.exceptions.ReadTimeoutError(None, None, None) if url == 'http://localhost:2379/': return MockResponse() raise socket.error @@ -164,31 +162,27 @@ class TestClient(unittest.TestCase): mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001']) self.client = Client({'discovery_srv': 'test'}) self.client.http.request = http_request + self.client.http.request_encode_body = http_request def test_api_execute(self): self.client._base_uri = 'http://localhost:4001' self.client._machines_cache = ['http://localhost:2379'] - self.client.api_execute('/', 'GET') + self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'}) + self.client._update_machines_cache = False + self.client.api_execute('/', 'POST', timeout=0) self.client._update_machines_cache = False self.client._base_uri = 'http://localhost:4001' self.client._machines_cache = [] self.assertRaises(etcd.EtcdConnectionFailed, self.client.api_execute, '/', 'GET') self.assertTrue(self.client._update_machines_cache) self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'GET') + self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '') + self.assertRaises(ValueError, self.client.api_execute, '', '') def test_get_srv_record(self): self.assertEquals(self.client.get_srv_record('blabla'), []) self.assertEquals(self.client.get_srv_record('exception'), []) - def test__result_from_response(self): - response = MockResponse() - response.content = 'TimeoutError' - self.assertRaises(urllib3.exceptions.TimeoutError, self.client._result_from_response, response) - response.content = 'Exception' - self.assertRaises(etcd.EtcdException, self.client._result_from_response, response) - response.content = b'{}' - self.assertRaises(etcd.EtcdException, self.client._result_from_response, response) - def test__get_machines_cache_from_srv(self): self.client.get_srv_record = Mock(return_value=[('localhost', 2380)]) self.client._get_machines_cache_from_srv('blabla') From cdee1c291158f899bbd1d660f5d8561d3dffbd8d Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Sat, 12 Mar 2016 16:06:35 +0100 Subject: [PATCH 67/89] update requirements.txt --- requirements.txt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index a827efb3..66b059f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,10 @@ boto -mock -dnspython3 psycopg2>=2.6.1 PyYAML requests -six +six >= 1.7 kazoo>=2.2.1 -python-etcd==0.4.2 +python-etcd==0.4.3 click>=4.1 prettytable>=0.7 tzlocal From 62f11ab747dbef7af59ff4ff96c8a7c028c9936b Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Sun, 13 Mar 2016 09:09:31 +0100 Subject: [PATCH 68/89] Attempt to export acceptance tests coverage results to coveralls --- .travis.yml | 11 ++++++----- features/basic_replication.feature | 3 ++- features/cascading_replication.feature | 1 + features/environment.py | 5 ++++- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 073b6572..f20910d1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,8 @@ language: python addons: postgresql: "9.5" env: - - BOTO_CONFIG='' ETCDVERSION=2.2.5 + - TEST_SUITE="python setup.py test" + - TEST_SUITE="behave" python: - "2.7" - "3.4" @@ -19,12 +20,12 @@ install: - sudo /etc/init.d/postgresql stop - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py2.txt --use-mirrors; fi - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r requirements-py3.txt; fi - - curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd - - pip install behave coveralls codacy-coverage + - ETCDVERSION=2.2.5 curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd + - pip install behave codacy-coverage coverage coveralls script: - python setup.py test + - BOTO_CONFIG='' PATH=.:$PATH $TEST_SUITE - python setup.py flake8 - - PATH=.:$PATH behave after_success: - coveralls - - python-codacy-coverage -r coverage.xml + - if [[ -f coverage.xml ]]; then python-codacy-coverage -r coverage.xml; fi diff --git a/features/basic_replication.feature b/features/basic_replication.feature index aa07a250..b3307e40 100644 --- a/features/basic_replication.feature +++ b/features/basic_replication.feature @@ -3,9 +3,10 @@ Feature: basic replication Scenario: check replication of a single table Given I start postgres0 + And postgres0 is a leader after 10 seconds And I start postgres1 When I add the table foo to postgres0 - Then table foo is present on postgres1 after 10 seconds + Then table foo is present on postgres1 after 15 seconds Scenario: check the basic failover When I kill postgres0 diff --git a/features/cascading_replication.feature b/features/cascading_replication.feature index 7d297e41..8a3d2f80 100644 --- a/features/cascading_replication.feature +++ b/features/cascading_replication.feature @@ -3,6 +3,7 @@ Feature: cascading replication Scenario: check a base backup from the replica Given I start postgres0 + And postgres0 is a leader after 10 seconds And I start postgres1 And replication works from postgres0 to postgres1 after 15 seconds And I create label with "postgres0" in postgres0 data directory diff --git a/features/environment.py b/features/environment.py index f822007b..bd6d5054 100644 --- a/features/environment.py +++ b/features/environment.py @@ -59,7 +59,7 @@ class PatroniController(object): self._config[pg_name] = self._make_patroni_test_config(pg_name, tags=tags) - p = subprocess.Popen(['python', 'patroni.py', self._config[pg_name]], + p = subprocess.Popen(['coverage', 'run', '--branch', '--source=patroni', '-p', 'patroni.py', self._config[pg_name]], stdout=self._log[pg_name], stderr=subprocess.STDOUT, cwd=cwd) if not (p and p.pid and p.poll() is None): assert False, "PostgreSQL {0} is not running after being started".format(pg_name) @@ -190,6 +190,7 @@ class PatroniController(object): class EtcdController(object): + """ handles all etcd related tasks, used for the tests setup and cleanup """ ETCD_VERSION_URL = 'http://127.0.0.1:2379/version' ETCD_CLEANUP_URL = 'http://127.0.0.1:2379/v2/keys/service/batman?recursive=true' @@ -300,3 +301,5 @@ def after_feature(context, feature): context.pctl.stop_all() shutil.rmtree(os.path.join(context.pctl.patroni_path, 'data')) context.etcd_ctl.cleanup_service_tree() + subprocess.call(['coverage', 'combine']) + subprocess.call(['coverage', 'report']) From 7e0723a7fc5f94e38a9d75c9a9b4acaaa0087614 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Sun, 13 Mar 2016 09:17:52 +0100 Subject: [PATCH 69/89] Attempt to export acceptance tests coverage results to coveralls --- .travis.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f20910d1..6f92756a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,11 @@ language: python addons: postgresql: "9.5" env: - - TEST_SUITE="python setup.py test" - - TEST_SUITE="behave" + global: + - BOTO_CONFIG='' ETCDVERSION=2.2.5 + matrix: + - TEST_SUITE="python setup.py test" + - TEST_SUITE="behave" python: - "2.7" - "3.4" @@ -20,11 +23,11 @@ install: - sudo /etc/init.d/postgresql stop - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py2.txt --use-mirrors; fi - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r requirements-py3.txt; fi - - ETCDVERSION=2.2.5 curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd + - curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd - pip install behave codacy-coverage coverage coveralls script: - python setup.py test - - BOTO_CONFIG='' PATH=.:$PATH $TEST_SUITE + - PATH=.:$PATH $TEST_SUITE - python setup.py flake8 after_success: - coveralls From f3a238ccbcbca7da7919ffc26b8ba39f552f401f Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Sun, 13 Mar 2016 09:24:01 +0100 Subject: [PATCH 70/89] Attempt to export acceptance tests coverage results to coveralls --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6f92756a..38dddc77 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,6 @@ install: - curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd - pip install behave codacy-coverage coverage coveralls script: - - python setup.py test - PATH=.:$PATH $TEST_SUITE - python setup.py flake8 after_success: From 79f4d9a13b63cd872f447efcbbda194aacf94f9e Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Sun, 13 Mar 2016 09:42:02 +0100 Subject: [PATCH 71/89] Attempt to export acceptance tests coverage results to coveralls --- features/environment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/features/environment.py b/features/environment.py index bd6d5054..73f77011 100644 --- a/features/environment.py +++ b/features/environment.py @@ -289,6 +289,8 @@ def before_all(context): def after_all(context): context.etcd_ctl.stop_and_remove_work_directory() + subprocess.call(['coverage', 'combine']) + subprocess.call(['coverage', 'report']) def before_feature(context, feature): @@ -301,5 +303,3 @@ def after_feature(context, feature): context.pctl.stop_all() shutil.rmtree(os.path.join(context.pctl.patroni_path, 'data')) context.etcd_ctl.cleanup_service_tree() - subprocess.call(['coverage', 'combine']) - subprocess.call(['coverage', 'report']) From 7d4ec19bcaeb002db27f7e0252a9fb45cab5848e Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Mon, 14 Mar 2016 10:51:50 +0100 Subject: [PATCH 72/89] Enable etcd cluster id check --- patroni/etcd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/patroni/etcd.py b/patroni/etcd.py index 31c0fd37..03d6259d 100644 --- a/patroni/etcd.py +++ b/patroni/etcd.py @@ -54,7 +54,7 @@ class Client(etcd.Client): try: response = request_executor(method, url, fields=fields, **kwargs) response.data.decode('utf-8') -# self._check_cluster_id(response) + self._check_cluster_id(response) except (urllib3.exceptions.HTTPError, HTTPException, socket.error) as e: if (isinstance(fields, dict) and fields.get("wait") == "true" and isinstance(e, urllib3.exceptions.ReadTimeoutError)): @@ -85,7 +85,7 @@ class Client(etcd.Client): request_executor = self.http.request_encode_body kwargs['encode_multipart'] = False else: - raise etcd.EtcdException('HTTP method {} not supported'.format(method)) + raise etcd.EtcdException('HTTP method {0} not supported'.format(method)) # Update machines_cache if previous attempt of update has failed if self._update_machines_cache: From 75d196a1f4935d6233446326d25d7c3c39650626 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Mon, 14 Mar 2016 15:57:34 +0100 Subject: [PATCH 73/89] Update the TODO list. --- TODO.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/TODO.md b/TODO.md index 17d1cb0c..91b77499 100644 --- a/TODO.md +++ b/TODO.md @@ -1,4 +1,16 @@ Failover ======== -- When determining who should become master, include the minor version of PostgreSQL in the decision -- Create a way to disable governance of a cluster, something like the existence of a "nogover" or "admin" file in PGDATA will stop governor from changing the cluster state +- When determining who should become master, include the minor version of PostgreSQL in the decision. +- Create a way to disable governance of a cluster, something like the existence of a "nogover" or "admin" file in PGDATA will stop patroni from changing the cluster state. + +Configuration +============== +- Provide a way to change postgresql.conf and pg_hba.conf of a running cluster on the Patroni level, without changing individual nodes. +- Provide hooks to store and retrieve cluster-wide passwords without exposing them in a plain-text form to unauthorized users. +- Implement patronictl command to create initial configuration of the cluster with leader and member keys fixed to the user-supplied values in order to simplify migrations. +- Implement support for consul in addtion to etcd and zookeeper +- Complete zookeeper support in patronictl + +Documentation +============== +- Document how to run cascading replication and possibly initialize the cluster without an access to the master node. From 3fdf249f4bdd178149bbd1f2c699fbe0c86e7f94 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Mon, 14 Mar 2016 16:02:49 +0100 Subject: [PATCH 74/89] Bumped version to 0.80 --- patroni/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/patroni/version.py b/patroni/version.py index b7831c97..45b09904 100644 --- a/patroni/version.py +++ b/patroni/version.py @@ -1 +1 @@ -__version__ = '0.76' +__version__ = '0.80' From 0e0c8ed8d797db1ea50267a83508ecc29d7913d7 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Tue, 15 Mar 2016 16:25:48 +0100 Subject: [PATCH 75/89] Implement `delete_cluster` interface in for all available dcs In addition to that rename confusing `Etcd.client` and `ZooKeeper.client` into `_client`. This attribute is available from AbstractDCS and people had wrong impression that it provides the same interface for different DCS implementations, which is obviously not the case. For Etcd it has type etcd.Client and for ZooKeeper - KazooClient. --- patroni/ctl.py | 5 +--- patroni/dcs.py | 4 +++ patroni/etcd.py | 30 +++++++++++--------- patroni/zookeeper.py | 62 ++++++++++++++++++++++------------------- tests/test_ctl.py | 43 +++++++++++----------------- tests/test_etcd.py | 14 +++++----- tests/test_ha.py | 8 +++--- tests/test_patroni.py | 5 ++-- tests/test_zookeeper.py | 7 +++-- 9 files changed, 92 insertions(+), 86 deletions(-) diff --git a/patroni/ctl.py b/patroni/ctl.py index 0cfd6039..5434cebb 100644 --- a/patroni/ctl.py +++ b/patroni/ctl.py @@ -363,9 +363,6 @@ def query_member(cluster, cursor, member, role, command, connect_parameters=None def remove(config_file, cluster_name, fmt, dcs): config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs) - if not isinstance(dcs, Etcd): - raise PatroniCtlException('We have not implemented this for DCS of type {0}'.format(type(dcs))) - output_members(cluster, fmt=fmt) confirm = click.prompt('Please confirm the cluster name to remove', type=str) @@ -384,7 +381,7 @@ def remove(config_file, cluster_name, fmt, dcs): if confirm != cluster.leader.name: raise PatroniCtlException('You did not specify the current master of the cluster') - dcs.client.delete(dcs.client_path(''), recursive=True) + dcs.delete_cluster() def wait_for_leader(dcs, timeout=30): diff --git a/patroni/dcs.py b/patroni/dcs.py index 2768b13e..aa7e178a 100644 --- a/patroni/dcs.py +++ b/patroni/dcs.py @@ -308,6 +308,10 @@ class AbstractDCS(object): def cancel_initialization(self): """ Removes the initialize key for a cluster """ + @abc.abstractmethod + def delete_cluster(self): + """Delete cluster from DCS""" + def watch(self, timeout): """If the current node is a master it should just sleep. Any other node should watch for changes of leader key with a given timeout diff --git a/patroni/etcd.py b/patroni/etcd.py index 03d6259d..3f798cf8 100644 --- a/patroni/etcd.py +++ b/patroni/etcd.py @@ -199,7 +199,7 @@ class Etcd(AbstractDCS): etcd.EtcdLeaderElectionInProgress, etcd.EtcdWatcherCleared, etcd.EtcdEventIndexCleared)) - self.client = self.get_etcd_client(config) + self._client = self.get_etcd_client(config) def retry(self, *args, **kwargs): return self._retry.copy()(*args, **kwargs) @@ -221,7 +221,7 @@ class Etcd(AbstractDCS): def _load_cluster(self): try: - result = self.retry(self.client.read, self.client_path(''), recursive=True) + result = self.retry(self._client.read, self.client_path(''), recursive=True) nodes = {os.path.relpath(node.key, result.key): node for node in result.leaves} # get initialize flag @@ -256,15 +256,15 @@ class Etcd(AbstractDCS): @catch_etcd_errors def touch_member(self, connection_string, ttl=None): - return self.retry(self.client.set, self.member_path, connection_string, ttl or self.ttl) + return self.retry(self._client.set, self.member_path, connection_string, ttl or self.ttl) @catch_etcd_errors def take_leader(self): - return self.retry(self.client.set, self.leader_path, self._name, self.ttl) + return self.retry(self._client.set, self.leader_path, self._name, self.ttl) def attempt_to_acquire_leader(self): try: - return bool(self.retry(self.client.write, self.leader_path, self._name, ttl=self.ttl, prevExist=False)) + return bool(self.retry(self._client.write, self.leader_path, self._name, ttl=self.ttl, prevExist=False)) except etcd.EtcdAlreadyExist: logger.info('Could not take out TTL lock') except (RetryFailedError, etcd.EtcdException): @@ -273,27 +273,31 @@ class Etcd(AbstractDCS): @catch_etcd_errors def set_failover_value(self, value, index=None): - return self.client.write(self.failover_path, value, prevIndex=index or 0) + return self._client.write(self.failover_path, value, prevIndex=index or 0) @catch_etcd_errors def write_leader_optime(self, last_operation): - return self.client.set(self.leader_optime_path, last_operation) + return self._client.set(self.leader_optime_path, last_operation) @catch_etcd_errors def update_leader(self): - return self.retry(self.client.test_and_set, self.leader_path, self._name, self._name, self.ttl) + return self.retry(self._client.test_and_set, self.leader_path, self._name, self._name, self.ttl) @catch_etcd_errors def initialize(self, create_new=True, sysid=""): - return self.retry(self.client.write, self.initialize_path, sysid, prevExist=(not create_new)) + return self.retry(self._client.write, self.initialize_path, sysid, prevExist=(not create_new)) @catch_etcd_errors def delete_leader(self): - return self.client.delete(self.leader_path, prevValue=self._name) + return self._client.delete(self.leader_path, prevValue=self._name) @catch_etcd_errors def cancel_initialization(self): - return self.retry(self.client.delete, self.initialize_path) + return self.retry(self._client.delete, self.initialize_path) + + @catch_etcd_errors + def delete_cluster(self): + return self.retry(self._client.delete, self.client_path(''), recursive=True) def watch(self, timeout): cluster = self.cluster @@ -304,12 +308,12 @@ class Etcd(AbstractDCS): while index and timeout >= 1: # when timeout is too small urllib3 doesn't have enough time to connect try: - self.client.watch(self.leader_path, index=index + 1, timeout=timeout + 0.5) + self._client.watch(self.leader_path, index=index + 1, timeout=timeout + 0.5) # Synchronous work of all cluster members with etcd is less expensive # than reestablishing http connection every time from every replica. return True except etcd.EtcdWatchTimedOut: - self.client.http.clear() + self._client.http.clear() return False except etcd.EtcdException: logging.exception('watch') diff --git a/patroni/zookeeper.py b/patroni/zookeeper.py index 79e8a72a..7d458872 100644 --- a/patroni/zookeeper.py +++ b/patroni/zookeeper.py @@ -83,20 +83,20 @@ class ZooKeeper(AbstractDCS): self.exhibitor = ExhibitorEnsembleProvider(exhibitor['hosts'], exhibitor['port'], poll_interval=interval) hosts = self.exhibitor.zookeeper_hosts - self.client = KazooClient(hosts=hosts, - timeout=(config.get('session_timeout') or 30), - command_retry={ - 'deadline': (config.get('reconnect_timeout') or 10), - 'max_delay': 1, - 'max_tries': -1}, - connection_retry={'max_delay': 1, 'max_tries': -1}) - self.client.add_listener(self.session_listener) + self._client = KazooClient(hosts=hosts, + timeout=(config.get('session_timeout') or 30), + command_retry={ + 'deadline': (config.get('reconnect_timeout') or 10), + 'max_delay': 1, + 'max_tries': -1}, + connection_retry={'max_delay': 1, 'max_tries': -1}) + self._client.add_listener(self.session_listener) self._my_member_data = None self.fetch_cluster = True self.last_leader_operation = 0 - self.client.start(None) + self._client.start(None) def session_listener(self, state): if state in [KazooState.SUSPENDED, KazooState.LOST]: @@ -108,7 +108,7 @@ class ZooKeeper(AbstractDCS): def get_node(self, key, watch=None): try: - ret = self.client.get(key, watch) + ret = self._client.get(key, watch) return (ret[0].decode('utf-8'), ret[1]) except NoNodeError: return None @@ -119,7 +119,7 @@ class ZooKeeper(AbstractDCS): def get_children(self, key, watch=None): try: - return self.client.get_children(key, watch) + return self._client.get_children(key, watch) except NoNodeError: return [] @@ -147,10 +147,10 @@ class ZooKeeper(AbstractDCS): # get leader leader = self.get_node(self.leader_path) if self._LEADER in nodes else None if leader: - client_id = self.client.client_id + client_id = self._client.client_id if leader[0] == self._name and client_id is not None and client_id[0] != leader[1].ephemeralOwner: logger.info('I am leader but not owner of the session. Removing leader node') - self.client.delete(self.leader_path) + self._client.delete(self.leader_path) leader = None if leader: @@ -171,11 +171,11 @@ class ZooKeeper(AbstractDCS): def _load_cluster(self): if self.exhibitor and self.exhibitor.poll(): - self.client.set_hosts(self.exhibitor.zookeeper_hosts) + self._client.set_hosts(self.exhibitor.zookeeper_hosts) if self.fetch_cluster: try: - self.client.retry(self._inner_load_cluster) + self._client.retry(self._inner_load_cluster) except: logger.exception('get_cluster') self.session_listener(KazooState.LOST) @@ -183,7 +183,7 @@ class ZooKeeper(AbstractDCS): def _create(self, path, value, **kwargs): try: - self.client.retry(self.client.create, path, value.encode('utf-8'), **kwargs) + self._client.retry(self._client.create, path, value.encode('utf-8'), **kwargs) return True except: return False @@ -196,7 +196,7 @@ class ZooKeeper(AbstractDCS): def set_failover_value(self, value, index=None): try: - self.client.retry(self.client.set, self.failover_path, value.encode('utf-8'), version=index or -1) + self._client.retry(self._client.set, self.failover_path, value.encode('utf-8'), version=index or -1) return True except NoNodeError: return value == '' or (not index and self._create(self.failover_path, value)) @@ -206,7 +206,7 @@ class ZooKeeper(AbstractDCS): def initialize(self, create_new=True, sysid=""): return self._create(self.initialize_path, sysid, makepath=True) if create_new \ - else self.client.retry(self.client.set, self.initialize_path, sysid.encode("utf-8")) + else self._client.retry(self._client.set, self.initialize_path, sysid.encode("utf-8")) def touch_member(self, data, ttl=None): cluster = self.cluster @@ -214,9 +214,9 @@ class ZooKeeper(AbstractDCS): path = self.member_path data = data.encode('utf-8') create = not me - if me and self.client.client_id is not None and me.session != self.client.client_id[0]: + if me and self._client.client_id is not None and me.session != self._client.client_id[0]: try: - self.client.retry(self.client.delete, path) + self._client.retry(self._client.delete, path) except NoNodeError: pass except: @@ -228,14 +228,14 @@ class ZooKeeper(AbstractDCS): try: if create: - self.client.retry(self.client.create, path, data, makepath=True, ephemeral=True) + self._client.retry(self._client.create, path, data, makepath=True, ephemeral=True) else: - self.client.retry(self.client.set, path, data) + self._client.retry(self._client.set, path, data) self._my_member_data = data return True except NodeExistsError: try: - self.client.retry(self.client.set, path, data) + self._client.retry(self._client.set, path, data) self._my_member_data = data return True except: @@ -253,10 +253,10 @@ class ZooKeeper(AbstractDCS): self.last_leader_operation = last_operation path = self.leader_optime_path try: - self.client.retry(self.client.set, path, last_operation) + self._client.retry(self._client.set, path, last_operation) except NoNodeError: try: - self.client.retry(self.client.create, path, last_operation, makepath=True) + self._client.retry(self._client.create, path, last_operation, makepath=True) except: logger.exception('Failed to create %s', path) except: @@ -266,21 +266,27 @@ class ZooKeeper(AbstractDCS): return True def delete_leader(self): - self.client.restart() + self._client.restart() self._my_member_data = None return True def _cancel_initialization(self): node = self.get_node(self.initialize_path) if node: - self.client.delete(self.initialize_path, version=node[1].version) + self._client.delete(self.initialize_path, version=node[1].version) def cancel_initialization(self): try: - self.client.retry(self._cancel_initialization) + self._client.retry(self._cancel_initialization) except: logger.exception("Unable to delete initialize key") + def delete_cluster(self): + try: + return self._client.retry(self._client.delete, self.client_path(''), recursive=True) + except NoNodeError: + return True + def watch(self, timeout): if super(ZooKeeper, self).watch(timeout): self.fetch_cluster = True diff --git a/tests/test_ctl.py b/tests/test_ctl.py index ed3aa1b9..806b567d 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -1,10 +1,10 @@ +import etcd import os import pytest import requests.exceptions import unittest from click.testing import CliRunner -from etcd import EtcdException from mock import patch, Mock, MagicMock from patroni.ctl import ctl, members, store_config, load_config, output_members, post_patroni, get_dcs, \ wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure @@ -44,6 +44,9 @@ def test_rw_config(): @patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'etcd', 'hostname': 'localhost', 'port': 4001}})) +@patch.object(etcd.Client, 'write', etcd_write) +@patch.object(etcd.Client, 'read', etcd_read) +@patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException)) class TestCtl(unittest.TestCase): @patch('socket.getaddrinfo', socket_getaddrinfo) @@ -52,9 +55,6 @@ class TestCtl(unittest.TestCase): with patch.object(Client, 'machines') as mock_machines: mock_machines.__get__ = Mock(return_value=['http://remotehost:2379']) self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'}) - self.e.client.read = etcd_read - self.e.client.write = etcd_write - self.e.client.delete = Mock(side_effect=EtcdException) @patch('psycopg2.connect', psycopg2_connect) def test_get_cursor(self): @@ -298,37 +298,28 @@ y''') result = self.runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='y') @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) - @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None)) def test_remove(self): - result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='alpha\nslave') - assert 'Please confirm' in result.output - assert 'You are about to remove all' in result.output - # Not typing an exact confirmation - assert result.exit_code == 1 + with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)): + result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='alpha\nslave') + assert 'Please confirm' in result.output + assert 'You are about to remove all' in result.output + # Not typing an exact confirmation + assert result.exit_code == 1 - # master specified does not match master of cluster - result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='''alpha + # master specified does not match master of cluster + result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='''alpha Yes I am aware slave''') - assert result.exit_code == 1 + assert result.exit_code == 1 - # cluster specified on cmdline does not match verification prompt - result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='beta\nleader') - assert result.exit_code == 1 + # cluster specified on cmdline does not match verification prompt + result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='beta\nleader') + assert result.exit_code == 1 - with patch('patroni.etcd.Etcd.get_cluster', get_cluster_initialized_with_leader): - result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], - input='''alpha -Yes I am aware -leader''') - assert 'object has no attribute' in str(result.exception) - - with patch('patroni.ctl.get_dcs', Mock(return_value=Mock())): - # Not implemented DCS result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='''alpha Yes I am aware leader''') - assert result.exit_code == 1 + assert result.exit_code == 0 @patch('patroni.etcd.Etcd.watch', Mock(return_value=None)) @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) diff --git a/tests/test_etcd.py b/tests/test_etcd.py index fa2538c9..9ed5af58 100644 --- a/tests/test_etcd.py +++ b/tests/test_etcd.py @@ -64,7 +64,7 @@ def requests_get(url, **kwargs): return response -def etcd_watch(key, index=None, timeout=None, recursive=None): +def etcd_watch(self, key, index=None, timeout=None, recursive=None): if timeout == 2.0: raise etcd.EtcdWatchTimedOut elif timeout == 5.0: @@ -77,7 +77,7 @@ def etcd_watch(key, index=None, timeout=None, recursive=None): return etcd.EtcdResult('set', {'value': 'postgresql2', 'modifiedIndex': index + 1}) -def etcd_write(key, value, **kwargs): +def etcd_write(self, key, value, **kwargs): if key == '/service/exists/leader': raise etcd.EtcdAlreadyExist if key in ['/service/test/leader', '/patroni/test/leader'] and \ @@ -86,7 +86,7 @@ def etcd_write(key, value, **kwargs): raise etcd.EtcdException -def etcd_read(key, **kwargs): +def etcd_read(self, key, **kwargs): if key == '/service/noleader/': raise DCSError('noleader') elif key == '/service/nocluster/': @@ -198,15 +198,15 @@ class TestClient(unittest.TestCase): @patch('requests.get', requests_get) +@patch.object(etcd.Client, 'write', etcd_write) +@patch.object(etcd.Client, 'read', etcd_read) +@patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException)) class TestEtcd(unittest.TestCase): def setUp(self): with patch.object(Client, 'machines') as mock_machines: mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001']) self.etcd = Etcd('foo', {'namespace': '/patroni/', 'ttl': 30, 'host': 'localhost:2379', 'scope': 'test'}) - self.etcd.client.write = etcd_write - self.etcd.client.read = etcd_read - self.etcd.client.delete = Mock(side_effect=etcd.EtcdException()) def test_base_path(self): self.assertEquals(self.etcd._base_path, '/patroni/test') @@ -254,8 +254,8 @@ class TestEtcd(unittest.TestCase): def test_delete_leader(self): self.assertFalse(self.etcd.delete_leader()) + @patch.object(etcd.Client, 'watch', etcd_watch) def test_watch(self): - self.etcd.client.watch = etcd_watch self.etcd.watch(0) self.etcd.get_cluster() self.etcd.watch(1.5) diff --git a/tests/test_ha.py b/tests/test_ha.py index 2f9e5478..9ebf55d3 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -1,8 +1,8 @@ +import etcd import unittest import datetime import pytz -from etcd import EtcdException from mock import Mock, MagicMock, patch from patroni.dcs import Cluster, Failover, Leader, Member from patroni.etcd import Client, Etcd @@ -76,10 +76,13 @@ def run_async(func, args=()): @patch.object(Postgresql, 'write_recovery_conf', Mock()) @patch.object(Postgresql, 'query', Mock()) @patch.object(Postgresql, 'checkpoint', Mock()) +@patch.object(etcd.Client, 'write', etcd_write) +@patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException)) @patch('subprocess.call', Mock(return_value=0)) class TestHa(unittest.TestCase): @patch('socket.getaddrinfo', socket_getaddrinfo) + @patch.object(etcd.Client, 'read', etcd_read) def setUp(self): with patch.object(Client, 'machines') as mock_machines: mock_machines.__get__ = Mock(return_value=['http://remotehost:2379']) @@ -90,9 +93,6 @@ class TestHa(unittest.TestCase): self.p.check_replication_lag = true self.p.can_create_replica_without_replication_connection = MagicMock(return_value=False) self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'}) - self.e.client.read = etcd_read - self.e.client.write = etcd_write - self.e.client.delete = Mock(side_effect=EtcdException()) self.ha = Ha(MockPatroni(self.p, self.e)) self.ha._async_executor.run_async = run_async self.ha.old_cluster = self.e.get_cluster() diff --git a/tests/test_patroni.py b/tests/test_patroni.py index 2698de58..d3801abb 100644 --- a/tests/test_patroni.py +++ b/tests/test_patroni.py @@ -1,3 +1,4 @@ +import etcd import sys import time import unittest @@ -22,6 +23,8 @@ from test_zookeeper import MockKazooClient @patch.object(Postgresql, 'write_recovery_conf', Mock()) @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock()) @patch.object(AsyncExecutor, 'run', Mock()) +@patch.object(etcd.Client, 'write', etcd_write) +@patch.object(etcd.Client, 'read', etcd_read) class TestPatroni(unittest.TestCase): def setUp(self): @@ -35,8 +38,6 @@ class TestPatroni(unittest.TestCase): with open('postgres0.yml', 'r') as f: config = yaml.load(f) self.p = Patroni(config) - self.p.ha.dcs.client.write = etcd_write - self.p.ha.dcs.client.read = etcd_read @patch('patroni.zookeeper.KazooClient', MockKazooClient()) def test_get_dcs(self): diff --git a/tests/test_zookeeper.py b/tests/test_zookeeper.py index f5cbdf13..339aa4bd 100644 --- a/tests/test_zookeeper.py +++ b/tests/test_zookeeper.py @@ -92,7 +92,7 @@ class MockKazooClient(Mock): raise Exception elif path == '/service/test/members/buzz': raise Exception - elif path.endswith('/initialize') or path == '/service/test/members/bar': + elif path.endswith('/') or path.endswith('/initialize') or path == '/service/test/members/bar': raise NoNodeError @@ -152,7 +152,7 @@ class TestZooKeeper(unittest.TestCase): self.zk._name = 'bar' self.zk.touch_member('new') self.zk._name = 'na' - self.zk.client.exists = 1 + self.zk._client.exists = 1 self.zk.touch_member('exists') self.zk._name = 'bar' self.zk.touch_member('retry') @@ -172,6 +172,9 @@ class TestZooKeeper(unittest.TestCase): self.zk._base_path = self.zk._base_path.replace('test', 'bla') self.zk.write_leader_optime('2') + def test_delete_cluster(self): + self.assertTrue(self.zk.delete_cluster()) + def test_watch(self): self.zk.watch(0) self.zk.event.isSet = lambda: True From 8a7d2b53e043f23f2a1df1f164ced3ce49c7dfed Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Tue, 15 Mar 2016 16:40:55 +0100 Subject: [PATCH 76/89] Add forgotten mock to the test_ha.py --- tests/test_ha.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_ha.py b/tests/test_ha.py index 9ebf55d3..3ab7442b 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -77,6 +77,7 @@ def run_async(func, args=()): @patch.object(Postgresql, 'query', Mock()) @patch.object(Postgresql, 'checkpoint', Mock()) @patch.object(etcd.Client, 'write', etcd_write) +@patch.object(etcd.Client, 'read', etcd_read) @patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException)) @patch('subprocess.call', Mock(return_value=0)) class TestHa(unittest.TestCase): From ca3414bda049a88abf36167b95d003dc02948a99 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 18 Mar 2016 13:24:30 +0100 Subject: [PATCH 77/89] Ha loop was writing into scheduled_at during "manual_failover" --- patroni/api.py | 2 +- patroni/ha.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/patroni/api.py b/patroni/api.py index 342e1bf6..ac227619 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -197,7 +197,7 @@ class RestApiHandler(BaseHTTPRequestHandler): elif scheduled_at < datetime.datetime.now(pytz.utc): data = b'Cannot schedule failover in the past' status_code = 422 - elif self.server.patroni.dcs.manual_failover(leader, member, scheduled_at): + elif self.server.patroni.dcs.manual_failover(leader, member, scheduled_at=scheduled_at): data = b'Failover scheduled' status_code = 200 except (ValueError, TypeError): diff --git a/patroni/ha.py b/patroni/ha.py index 8c841290..84fb1225 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -298,7 +298,7 @@ class Ha(object): return elif delta < - int(self.patroni.nap_time * 1.5): logger.warning('Found a stale failover value, cleaning up: %s', failover.scheduled_at) - self.dcs.manual_failover('', '', self.cluster.failover.index) + self.dcs.manual_failover('', '', index=self.cluster.failover.index) return # The value is very close to now @@ -323,7 +323,7 @@ class Ha(object): self.cluster.failover.leader, self.state_handler.name) logger.info('Trying to clean up failover key') - self.dcs.manual_failover('', '', self.cluster.failover.index) + self.dcs.manual_failover('', '', index=self.cluster.failover.index) def process_unhealthy_cluster(self): if self.is_healthiest_node(): From 54055c1ff80c4d58063d2f85c92301c49f641de9 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Fri, 18 Mar 2016 15:59:47 +0100 Subject: [PATCH 78/89] Rename ambiguous `Failover.member` to candidate But! 'member' is still accepted by REST API and also name 'member' is used to strore/read this value to/from DCS (for backward comatibility) --- features/patroni_api.feature | 4 ++-- patroni/api.py | 37 ++++++++++++++++++------------------ patroni/ctl.py | 7 +++---- patroni/dcs.py | 14 +++++++------- patroni/ha.py | 16 ++++++++-------- patroni/zookeeper.py | 2 +- tests/test_zookeeper.py | 2 ++ 7 files changed, 42 insertions(+), 40 deletions(-) diff --git a/features/patroni_api.feature b/features/patroni_api.feature index f97e059f..b1eded96 100644 --- a/features/patroni_api.feature +++ b/features/patroni_api.feature @@ -18,7 +18,7 @@ Scenario: check API requests on a stand-alone server And I receive a response text "failover is not possible: cluster does not have members except leader" When I issue an empty POST request to http://127.0.0.1:8008/failover Then I receive a response code 400 - And I receive a response text "No values given for required parameters leader and member" + And I receive a response text "No values given for required parameters leader and candidate" Scenario: check API requests for the primary-replica pair Given I start postgres1 @@ -41,7 +41,7 @@ Scenario: check the failover via the API And replication works from postgres1 to postgres0 after 15 seconds Scenario: check the scheduled failover - Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgresq0 in 10 seconds + Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgres0 in 10 seconds Then I receive a response code 200 And postgres0 is a leader after 15 seconds And replication works from postgres0 to postgres1 after 25 seconds diff --git a/patroni/api.py b/patroni/api.py index ac227619..0018e537 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -142,7 +142,7 @@ class RestApiHandler(BaseHTTPRequestHandler): self.end_headers() self.wfile.write(data) - def poll_failover_result(self, leader, member): + def poll_failover_result(self, leader, candidate): for _ in range(0, 15): time.sleep(1) try: @@ -155,18 +155,18 @@ class RestApiHandler(BaseHTTPRequestHandler): pass return 503, b'Failover status unknown' - def is_failover_possible(self, cluster, leader, member): + def is_failover_possible(self, cluster, leader, candidate): if leader and not cluster.leader or cluster.leader.name != leader: return b'leader name does not match' - if member: - members = [m for m in cluster.members if m.name == member] + if candidate: + members = [m for m in cluster.members if m.name == candidate] if not members: - return b'member does not exists' + return b'candidate does not exists' else: members = [m for m in cluster.members if m.name != cluster.leader.name and m.api_url] if not members: return b'failover is not possible: cluster does not have members except leader' - for member, reachable, _, xlog_location, tags in self.server.patroni.ha.fetch_nodes_statuses(members): + for _, reachable, _, _, tags in self.server.patroni.ha.fetch_nodes_statuses(members): if reachable and not tags.get('nofailover', False): return None return b'failover is not possible: no good candidates have been found' @@ -179,43 +179,44 @@ class RestApiHandler(BaseHTTPRequestHandler): except ValueError: request = {} leader = request.get('leader') - member = request.get('member') + candidate = request.get('candidate') or request.get('member') + scheduled_at = request.get('scheduled_at') cluster = self.server.patroni.ha.dcs.get_cluster() status_code = 500 - logger.info("received failover request with leader {0} member {1} scheduled_at {2}". - format(leader, member, request.get("scheduled_at"))) + logger.info("received failover request with leader=%s candidate=%s scheduled_at=%s", + leader, candidate, scheduled_at) data = b'' - if leader or member: - if request.get('scheduled_at'): + if leader or candidate: + if scheduled_at: try: - scheduled_at = dateutil.parser.parse(request['scheduled_at']) + scheduled_at = dateutil.parser.parse(scheduled_at) if scheduled_at.tzinfo is None: data = b'Timezone information is mandatory for scheduled_at' status_code = 400 elif scheduled_at < datetime.datetime.now(pytz.utc): data = b'Cannot schedule failover in the past' status_code = 422 - elif self.server.patroni.dcs.manual_failover(leader, member, scheduled_at=scheduled_at): + elif self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at): data = b'Failover scheduled' status_code = 200 except (ValueError, TypeError): - logger.exception('Invalid scheduled failover time: {}'.format(request['scheduled_at'])) + logger.exception('Invalid scheduled failover time: %s', request['scheduled_at']) data = b'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601' status_code = 422 else: - data = self.is_failover_possible(cluster, leader, member) + data = self.is_failover_possible(cluster, leader, candidate) if not data: - if not self.server.patroni.dcs.manual_failover(leader, member): + if not self.server.patroni.dcs.manual_failover(leader, candidate): data = b'failed to write failover key into DCS' status_code = 503 else: self.server.patroni.dcs.event.set() - status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, member) + status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, candidate) else: status_code = 400 - data = b'No values given for required parameters leader and member' + data = b'No values given for required parameters leader and candidate' self.send_response(status_code) self.send_header('Content-Type', 'text/html') diff --git a/patroni/ctl.py b/patroni/ctl.py index 5434cebb..b14db933 100644 --- a/patroni/ctl.py +++ b/patroni/ctl.py @@ -316,8 +316,7 @@ def query( cursor = None for _ in watching(w, watch, clear=False): - output, cursor = query_member(cluster=cluster, cursor=cursor, member=member, role=role, command=command, - connect_parameters=connect_parameters) + output, cursor = query_member(cluster, cursor, member, role, command, connect_parameters) print_output(None, output, fmt=fmt, delimiter=delimiter) if cursor is None: @@ -533,7 +532,7 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled raise PatroniCtlException(message.format(scheduled)) scheduled_at = scheduled_at.isoformat() - failover_value = {'leader': master, 'member': candidate, 'scheduled_at': scheduled_at} + failover_value = {'leader': master, 'candidate': candidate, 'scheduled_at': scheduled_at} logging.debug(failover_value) # By now we have established that the leader exists and the candidate exists @@ -563,7 +562,7 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled logging.warning('Failing over to DCS') click.echo(timestamp() + ' Could not failover using Patroni api, falling back to DCS') click.echo(timestamp() + ' Initializing failover from master {0}'.format(master)) - dcs.manual_failover(leader=master, member=candidate, scheduled_at=failover_value) + dcs.manual_failover(master, candidate, scheduled_at=failover_value) output_members(cluster, name=cluster_name) diff --git a/patroni/dcs.py b/patroni/dcs.py index aa7e178a..3cb76cdb 100644 --- a/patroni/dcs.py +++ b/patroni/dcs.py @@ -89,16 +89,16 @@ class Leader(namedtuple('Leader', 'index,session,member')): return self.member.conn_url -class Failover(namedtuple('Failover', 'index,leader,member,scheduled_at')): +class Failover(namedtuple('Failover', 'index,leader,candidate,scheduled_at')): """ >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader"}')) True - >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader", "member": "cluster:member"}')) + >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader", "member": "cluster_candidate"}')) True >>> Failover.from_node(1, 'null') is None True - >>> n = '{"leader": "cluster_leader", "member": "cluster:member", "scheduled_at": "2016-01-14T10:09:57.1394Z"}' + >>> n = '{"leader": "cluster_leader", "member": "cluster_candidate", "scheduled_at": "2016-01-14T10:09:57.1394Z"}' >>> 'tzinfo=' in str(Failover.from_node(1, n)) True >>> Failover.from_node(1, None) is None @@ -258,13 +258,13 @@ class AbstractDCS(object): def set_failover_value(self, value, index=None): """Create or update `/failover` key""" - def manual_failover(self, leader, member, scheduled_at=None, index=None): - failover_value = dict() + def manual_failover(self, leader, candidate, scheduled_at=None, index=None): + failover_value = {} if leader: failover_value['leader'] = leader - if member: - failover_value['member'] = member + if candidate: + failover_value['member'] = candidate if scheduled_at: failover_value['scheduled_at'] = scheduled_at.isoformat() diff --git a/patroni/ha.py b/patroni/ha.py index 84fb1225..d9916da6 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -222,12 +222,12 @@ class Ha(object): def manual_failover_process_no_leader(self): failover = self.cluster.failover - if failover.member: # manual failover to specific member - if failover.member == self.state_handler.name: # manual failover to me + if failover.candidate: # manual failover to specific member + if failover.candidate == self.state_handler.name: # manual failover to me return True # find specific node and check that it is healthy - members = [m for m in self.cluster.members if m.name == failover.member] + members = [m for m in self.cluster.members if m.name == failover.candidate] if members: member, reachable, _, _, tags = self.fetch_node_status(members[0]) if reachable and not tags.get('nofailover', False): # node is healthy @@ -240,13 +240,13 @@ class Ha(object): logger.warning('manual failover: member %s is not allowed to promote', member.name) # at this point we should consider all members as a candidates for failover - # i.e. we assume that failover.member is None + # i.e. we assume that failover.candidate is None # try to pick some other members to failover and check that they are healthy if failover.leader: if self.state_handler.name == failover.leader: # I was the leader - # exclude me and desired member which is unhealthy (failover.member can be None) - members = [m for m in self.cluster.members if m.name not in (failover.member, failover.leader)] + # exclude me and desired member which is unhealthy (failover.candidate can be None) + members = [m for m in self.cluster.members if m.name not in (failover.candidate, failover.leader)] if self.is_failover_possible(members): # check that there are healthy members return False else: # I was the leader and it looks like currently I am the only healthy member @@ -308,8 +308,8 @@ class Ha(object): logger.warning('Incorrect value in of scheduled_at: %s', failover.scheduled_at) if not failover.leader or failover.leader == self.state_handler.name: - if not failover.member or failover.member != self.state_handler.name: - members = [m for m in self.cluster.members if not failover.member or m.name == failover.member] + if not failover.candidate or failover.candidate != self.state_handler.name: + members = [m for m in self.cluster.members if not failover.candidate or m.name == failover.candidate] if self.is_failover_possible(members): # check that there are healthy members self._async_executor.schedule('manual failover: demote') self._async_executor.run_async(self.demote) diff --git a/patroni/zookeeper.py b/patroni/zookeeper.py index 7d458872..a98e4ccf 100644 --- a/patroni/zookeeper.py +++ b/patroni/zookeeper.py @@ -190,7 +190,7 @@ class ZooKeeper(AbstractDCS): def attempt_to_acquire_leader(self): ret = self._create(self.leader_path, self._name, makepath=True, ephemeral=True) - if ret: + if not ret: logger.info('Could not take out TTL lock') return ret diff --git a/tests/test_zookeeper.py b/tests/test_zookeeper.py index 339aa4bd..d671049d 100644 --- a/tests/test_zookeeper.py +++ b/tests/test_zookeeper.py @@ -162,6 +162,8 @@ class TestZooKeeper(unittest.TestCase): def test_take_leader(self): self.zk.take_leader() + with patch.object(MockKazooClient, 'create', Mock(side_effect=Exception)): + self.zk.take_leader() def test_update_leader(self): self.assertTrue(self.zk.update_leader()) From 9fec8a41e4c7b14248a70b67838a85da92ef4da1 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Sat, 19 Mar 2016 13:15:05 +0100 Subject: [PATCH 79/89] Return different status if failed over not to candidate --- patroni/api.py | 12 ++++++++---- tests/test_api.py | 25 ++++++++++++++++++------- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/patroni/api.py b/patroni/api.py index 0018e537..cb7b2d1f 100644 --- a/patroni/api.py +++ b/patroni/api.py @@ -110,7 +110,7 @@ class RestApiHandler(BaseHTTPRequestHandler): status, msg = self.server.patroni.ha.restart() status_code = 200 if status else 503 data = msg.encode('utf-8') - except: + except Exception: logger.exception('Exception during restart') self.send_response(status_code) @@ -148,11 +148,15 @@ class RestApiHandler(BaseHTTPRequestHandler): try: cluster = self.server.patroni.dcs.get_cluster() if cluster.leader and cluster.leader.name != leader: - return 200, ('Successfully failed over to ' + cluster.leader.name).encode('utf-8') + if not candidate or candidate == cluster.leader.name: + return 200, ('Successfully failed over to ' + cluster.leader.name).encode('utf-8') + else: + return 200, 'Failed over to "{0}" instead of "{1}"'.format(cluster.leader.name, + candidate).encode('utf-8') if not cluster.failover: return 503, b'Failover failed' - except: - pass + except Exception as e: + logger.debug('Exception occured during polling failover result: %s', e) return 503, b'Failover status unknown' def is_failover_possible(self, cluster, leader, candidate): diff --git a/tests/test_api.py b/tests/test_api.py index 0d6dd151..5790f8d4 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -113,8 +113,8 @@ class TestRestApiHandler(unittest.TestCase): # make sure socket.error gets propagated via wfile object in finalize() with patch.object(MockRequest, 'makefile') as makefile: makefile.return_value.closed = False - makefile.return_value.readline.side_effect = lambda x: b"foo" - makefile.return_value.flush = Mock(side_effect=socket.error("foo")) + makefile.return_value.readline = Mock(return_value=b'foo') + makefile.return_value.flush = Mock(side_effect=socket.error('foo')) MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0') def test_do_GET_patroni(self): @@ -157,29 +157,40 @@ class TestRestApiHandler(unittest.TestCase): request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\ b'Content-Length: 0\n\n' MockRestApiServer(RestApiHandler, request) + + cluster.leader.name = 'postgresql1' + MockRestApiServer(RestApiHandler, request) + request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\ b'Content-Length: 25\n\n{"leader": "postgresql1"}' MockRestApiServer(RestApiHandler, request) + + cluster.leader.name = 'postgresql2' + request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\ + b'Content-Length: 53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}' + MockRestApiServer(RestApiHandler, request) + cluster.leader.name = 'postgresql1' MockRestApiServer(RestApiHandler, request) - cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'})] + + cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}), + Member(0, 'postgresql2', 30, {'api_url': 'http'})] MockRestApiServer(RestApiHandler, request) with patch.object(MockPatroni, 'dcs') as d: cluster = d.get_cluster.return_value cluster.leader.name = 'postgresql0' MockRestApiServer(RestApiHandler, request) + cluster.leader.name = 'postgresql2' + MockRestApiServer(RestApiHandler, request) cluster.leader.name = 'postgresql1' cluster.failover = None MockRestApiServer(RestApiHandler, request) - d.get_cluster = Mock(side_effect=Exception()) + d.get_cluster = Mock(side_effect=Exception) MockRestApiServer(RestApiHandler, request) d.manual_failover.return_value = False MockRestApiServer(RestApiHandler, request) with patch.object(MockHa, 'fetch_nodes_statuses', Mock(return_value=[])): MockRestApiServer(RestApiHandler, request) - request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\ - b'Content-Length: 50\n\n{"leader": "postgresql1", "member": "postgresql2"}' - MockRestApiServer(RestApiHandler, request) # Valid future date request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\ From 3a7d2c3874633766b7749ea27281cefc7f7d5f0d Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Mon, 21 Mar 2016 20:48:17 +0100 Subject: [PATCH 80/89] Remove unused code from unit tests --- tests/test_api.py | 14 +--- tests/test_ctl.py | 157 +++++++++++++------------------------ tests/test_etcd.py | 12 --- tests/test_postgresql.py | 11 +-- tests/test_wale_restore.py | 23 +----- tests/test_zookeeper.py | 5 -- 6 files changed, 65 insertions(+), 157 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 5790f8d4..1a3d00ce 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -11,7 +11,7 @@ import socket from test_postgresql import psycopg2_connect, MockCursor -class MockPostgresql(Mock): +class MockPostgresql(object): name = 'test' state = 'running' @@ -23,20 +23,12 @@ class MockPostgresql(Mock): def connection(): return psycopg2_connect() - @staticmethod - def is_running(): - return True - -class MockHa(Mock): +class MockHa(object): dcs = Mock() state_handler = MockPostgresql() - @staticmethod - def schedule_restart(): - return 'restart' - @staticmethod def schedule_reinitialize(): return 'reinitialize' @@ -54,7 +46,7 @@ class MockHa(Mock): return [[None, True, None, None, {}]] -class MockPatroni(Mock): +class MockPatroni(object): postgresql = MockPostgresql() ha = MockHa() diff --git a/tests/test_ctl.py b/tests/test_ctl.py index 806b567d..95dd5a07 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -5,13 +5,13 @@ import requests.exceptions import unittest from click.testing import CliRunner -from mock import patch, Mock, MagicMock +from mock import patch, Mock from patroni.ctl import ctl, members, store_config, load_config, output_members, post_patroni, get_dcs, \ wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure from patroni.etcd import Etcd, Client from patroni.exceptions import PatroniCtlException from psycopg2 import OperationalError -from test_etcd import etcd_read, etcd_write, requests_get, socket_getaddrinfo, MockResponse +from test_etcd import etcd_read, requests_get, socket_getaddrinfo, MockResponse from test_zookeeper import MockKazooClient from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \ get_cluster_initialized_with_only_leader @@ -44,9 +44,6 @@ def test_rw_config(): @patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'etcd', 'hostname': 'localhost', 'port': 4001}})) -@patch.object(etcd.Client, 'write', etcd_write) -@patch.object(etcd.Client, 'read', etcd_read) -@patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException)) class TestCtl(unittest.TestCase): @patch('socket.getaddrinfo', socket_getaddrinfo) @@ -75,71 +72,64 @@ class TestCtl(unittest.TestCase): @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None)) - @patch('patroni.etcd.Etcd.set_failover_value', Mock(return_value=None)) - @patch('patroni.ctl.wait_for_leader', Mock(return_value=get_cluster_initialized_with_leader())) - @patch('requests.get', requests_get) - @patch('requests.post', requests_get) @patch('patroni.ctl.post_patroni', Mock(return_value=MockResponse())) def test_failover(self): - with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())): - result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader + result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader other y''') - assert 'leader' in result.output + assert 'leader' in result.output - result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader + result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader other 2100-01-01T12:23:00 y''') - assert result.exit_code == 0 + assert result.exit_code == 0 - result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader + result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader other 2030-01-01T12:23:00 y''') - assert result.exit_code == 0 + assert result.exit_code == 0 - # Aborting failover,as we anser NO to the confirmation - result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader + # Aborting failover,as we anser NO to the confirmation + result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader other N''') - assert result.exit_code == 1 + assert result.exit_code == 1 - # Target and source are equal - result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader + # Target and source are equal + result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader leader y''') - assert result.exit_code == 1 + assert result.exit_code == 1 - # Reality is not part of this cluster - result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader + # Reality is not part of this cluster + result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader Reality y''') - assert result.exit_code == 1 + assert result.exit_code == 1 - result = self.runner.invoke(ctl, ['failover', 'dummy', '--force']) - assert 'Member' in result.output + result = self.runner.invoke(ctl, ['failover', 'dummy', '--force']) + assert 'Member' in result.output - result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', - '--scheduled', '2015-01-01T12:00:00+01:00']) - assert result.exit_code == 0 + result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00+01:00']) + assert result.exit_code == 0 - # Invalid timestamp - result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', 'invalid']) - assert result.exit_code != 0 + # Invalid timestamp + result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', 'invalid']) + assert result.exit_code != 0 - # Invalid timestamp - result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', - '--scheduled', '2115-02-30T12:00:00+01:00']) - assert result.exit_code != 0 + # Invalid timestamp + result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2115-02-30T12:00:00+01:00']) + assert result.exit_code != 0 - # Specifying wrong leader - result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='dummy') - assert result.exit_code == 1 + # Specifying wrong leader + result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='dummy') + assert result.exit_code == 1 with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_only_leader())): # No members available @@ -157,7 +147,7 @@ other y''') assert result.exit_code == 1 - with patch('patroni.ctl.post_patroni', Mock(side_effect=Exception())): + with patch('patroni.ctl.post_patroni', Mock(side_effect=Exception)): # Non-responding patroni result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader other @@ -165,9 +155,8 @@ other y''') assert 'falling back to DCS' in result.output - mocked = Mock() - mocked.return_value.status_code = 500 - with patch('patroni.ctl.post_patroni', Mock(return_value=mocked)): + with patch('patroni.ctl.post_patroni') as mocked: + mocked.return_value.status_code = 500 result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader other @@ -183,17 +172,11 @@ y''') @patch('psycopg2.connect', psycopg2_connect) @patch('patroni.ctl.query_member', Mock(return_value=([['mock column']], None))) + @patch.object(etcd.Client, 'read', etcd_read) def test_query(self): with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)): # Mutually exclusive - result = self.runner.invoke(ctl, [ - 'query', - 'alpha', - '--member', - 'abc', - '--role', - 'master', - ]) + result = self.runner.invoke(ctl, ['query', 'alpha', '--member', 'abc', '--role', 'master']) assert result.exit_code == 1 with self.runner.isolated_filesystem(): @@ -201,17 +184,11 @@ y''') dummy_file.write('SELECT 1') # Mutually exclusive - result = self.runner.invoke(ctl, [ - 'query', - 'alpha', - '--file', - 'dummy', - '--command', - 'dummy', - ]) + result = self.runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy', '--command', 'dummy']) assert result.exit_code == 1 result = self.runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy']) + assert result.exit_code == 0 os.remove('dummy') @@ -226,13 +203,16 @@ y''') '--password', '--dbname', 'postgres'], input='ab\nab') assert 'mock column' in result.output - @patch('patroni.ctl.get_cursor', Mock(return_value=MockConnect().cursor())) def test_query_member(self): - rows = query_member(None, None, None, 'master', 'SELECT pg_is_in_recovery()') - self.assertTrue('False' in str(rows)) + with patch('patroni.ctl.get_cursor', Mock(return_value=MockConnect().cursor())): + rows = query_member(None, None, None, 'master', 'SELECT pg_is_in_recovery()') + self.assertTrue('False' in str(rows)) - rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()') - self.assertEquals(rows, (None, None)) + rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()') + self.assertEquals(rows, (None, None)) + + with patch('test_postgresql.MockCursor.execute', Mock(side_effect=OperationalError('bla'))): + rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()') with patch('patroni.ctl.get_cursor', Mock(return_value=None)): rows = query_member(None, None, None, None, 'SELECT pg_is_in_recovery()') @@ -244,9 +224,6 @@ y''') with patch('patroni.ctl.get_cursor', Mock(side_effect=OperationalError('bla'))): rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()') - with patch('test_postgresql.MockCursor.execute', Mock(side_effect=OperationalError('bla'))): - rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()') - @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) def test_dsn(self): with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)): @@ -254,14 +231,7 @@ y''') assert 'host=127.0.0.1 port=5435' in result.output # Mutually exclusive options - result = self.runner.invoke(ctl, [ - 'dsn', - 'alpha', - '--role', - 'master', - '--member', - 'dummy', - ]) + result = self.runner.invoke(ctl, ['dsn', 'alpha', '--role', 'master', '--member', 'dummy']) assert result.exit_code == 1 # Non-existing member @@ -270,7 +240,6 @@ y''') @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None)) - @patch('requests.get', requests_get) @patch('requests.post', requests_get) def test_restart_reinit(self): result = self.runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='y') @@ -284,20 +253,15 @@ y''') assert result.exit_code == 1 # Not a member - result = self.runner.invoke(ctl, [ - 'restart', - 'alpha', - '--dcs', - '8.8.8.8', - 'dummy', - '--any', - ], input='y') + result = self.runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8', 'dummy', '--any'], input='y') assert result.exit_code == 1 with patch('requests.post', Mock(return_value=MockResponse())): result = self.runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='y') + assert result.exit_code == 0 @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) + @patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException)) def test_remove(self): with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)): result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='alpha\nslave') @@ -324,16 +288,15 @@ leader''') @patch('patroni.etcd.Etcd.watch', Mock(return_value=None)) @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) def test_wait_for_leader(self): - dcs = self.e - self.assertRaises(PatroniCtlException, wait_for_leader, dcs, 0) + self.assertRaises(PatroniCtlException, wait_for_leader, self.e, 0) - cluster = wait_for_leader(dcs=dcs, timeout=2) + cluster = wait_for_leader(self.e, timeout=2) assert cluster.leader.member.name == 'leader' + @patch('requests.post', Mock(side_effect=requests.exceptions.ConnectionError('foo'))) def test_post_patroni(self): - with patch('requests.post', MagicMock(side_effect=requests.exceptions.ConnectionError('foo'))): - member = get_cluster_initialized_with_leader().leader.member - self.assertRaises(requests.exceptions.ConnectionError, post_patroni, member, 'dummy', {}) + member = get_cluster_initialized_with_leader().leader.member + self.assertRaises(requests.exceptions.ConnectionError, post_patroni, member, 'dummy', {}) def test_ctl(self): self.runner.invoke(ctl, ['list']) @@ -362,20 +325,10 @@ leader''') @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())) @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None)) - @patch('requests.get', requests_get) - @patch('requests.post', requests_get) def test_members(self): result = self.runner.invoke(members, ['alpha']) assert result.exit_code == 0 def test_configure(self): - result = self.runner.invoke(configure, [ - '--dcs', - 'abc', - '-c', - 'dummy', - '-n', - 'bla', - ]) - + result = self.runner.invoke(configure, ['--dcs', 'abc', '-c', 'dummy', '-n', 'bla']) assert result.exit_code == 0 diff --git a/tests/test_etcd.py b/tests/test_etcd.py index 9ed5af58..742c57c5 100644 --- a/tests/test_etcd.py +++ b/tests/test_etcd.py @@ -36,16 +36,6 @@ class MockResponse(object): return '' -class MockPostgresql(Mock): - - server_version = '999999' - scope = 'dummy' - - @staticmethod - def last_operation(): - return '0' - - def requests_get(url, **kwargs): members = '[{"id":14855829450254237642,"peerURLs":["http://localhost:2380","http://localhost:7001"],' +\ '"name":"default","clientURLs":["http://localhost:2379","http://localhost:4001"]}]' @@ -73,8 +63,6 @@ def etcd_watch(self, key, index=None, timeout=None, recursive=None): raise etcd.EtcdException elif index == 20729: return etcd.EtcdResult('set', {'value': 'postgresql1', 'modifiedIndex': index + 1}) - elif index == 20731: - return etcd.EtcdResult('set', {'value': 'postgresql2', 'modifiedIndex': index + 1}) def etcd_write(self, key, value, **kwargs): diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py index fdd321df..932e860d 100644 --- a/tests/test_postgresql.py +++ b/tests/test_postgresql.py @@ -54,10 +54,6 @@ class MockCursor(object): def fetchall(self): return self.results - @staticmethod - def close(): - pass - def __iter__(self): for i in self.results: yield i @@ -69,8 +65,9 @@ class MockCursor(object): pass -class MockConnect(Mock): +class MockConnect(object): + server_version = '99999' autocommit = False closed = 0 @@ -83,6 +80,10 @@ class MockConnect(Mock): def __exit__(self, *args): pass + @staticmethod + def close(): + pass + def pg_controldata_string(*args, **kwargs): return b""" diff --git a/tests/test_wale_restore.py b/tests/test_wale_restore.py index 40761051..9e15dc9b 100644 --- a/tests/test_wale_restore.py +++ b/tests/test_wale_restore.py @@ -6,22 +6,6 @@ from mock import MagicMock, patch, PropertyMock from patroni.scripts.wale_restore import WALERestore, main as _main -def fake_cursor_fetchone(*args, **kwargs): - return ('16777216',) - - -def fake_call_fail_for_wal_e(*args, **kwargs): - if len(args) > 0 and 'backup-fetch' in args[0]: - return 1 - return 0 - - -def fake_call_fail_for_base_backup(*args, **kwargs): - if len(args) > 0 and 'backup-fetch' in args[0]: - return 0 - return 1 - - def fake_backup_data(self, *args, **kwargs): """ return the fake result of WAL-E backup-list""" return """name last_modified expanded_size_bytes wal_segment_backup_start wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop @@ -52,7 +36,6 @@ base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 000 @patch('os.makedirs', MagicMock(return_value=True)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.path.isdir', MagicMock(return_value=True)) -@patch('psycopg2.extensions.cursor.fetchone', MagicMock(side_effect=fake_cursor_fetchone)) @patch('psycopg2.extensions.cursor', MagicMock(autospec=True)) @patch('psycopg2.extensions.connection', MagicMock(autospec=True)) @patch('psycopg2.connect', MagicMock(autospec=True)) @@ -60,11 +43,7 @@ base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 000 class TestWALERestore(unittest.TestCase): def setUp(self): - self.wale_restore = WALERestore("batman", "/data", - "host=batman port=5432 user=batman", "/etc", 100, 100, 1, 0) - - def tearDown(self): - pass + self.wale_restore = WALERestore("batman", "/data", "host=batman port=5432 user=batman", "/etc", 100, 100, 1, 0) def test_should_use_s3_to_create_replica(self): with patch('psycopg2.connect', MagicMock(side_effect=psycopg2.Error("foo"))): diff --git a/tests/test_zookeeper.py b/tests/test_zookeeper.py index d671049d..fe991a2c 100644 --- a/tests/test_zookeeper.py +++ b/tests/test_zookeeper.py @@ -14,7 +14,6 @@ class MockKazooClient(Mock): leader = False exists = True - handler = Mock() @property def client_id(self): @@ -34,8 +33,6 @@ class MockKazooClient(Mock): b'postgres://repuser:rep-pass@localhost:5434/postgres?application_name=http://127.0.0.1:8009/patroni', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0 if self.exists else -1, 0, 0, 0) ) - elif path.endswith('/optime/leader'): - return (b'1', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)) elif path.endswith('/leader'): if self.leader: return (b'foo', ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0)) @@ -86,8 +83,6 @@ class MockKazooClient(Mock): raise TypeError("Invalid type for 'path' (string expected)") self.exists = False if path == '/service/test/leader': - if self.leader: - return self.leader = True raise Exception elif path == '/service/test/members/buzz': From 0d3dca56ffc3c8e8c611ede2251b6b19714c0ce8 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 24 Mar 2016 12:06:39 +0100 Subject: [PATCH 81/89] In some cases Ha.cluster can be None after calling `get_cluster` Such situation is causing patroni crash. Usually it was happening during manual failover, after former master has demoted and `reset_cluster` method has been called. In this case `fetch_cluster` was `False` and `_load_cluster` method was returning value from `self._cluster`, which was `None`. --- patroni/zookeeper.py | 28 ++++++++++++++-------------- tests/test_zookeeper.py | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/patroni/zookeeper.py b/patroni/zookeeper.py index a98e4ccf..13faa4d3 100644 --- a/patroni/zookeeper.py +++ b/patroni/zookeeper.py @@ -93,8 +93,8 @@ class ZooKeeper(AbstractDCS): self._client.add_listener(self.session_listener) self._my_member_data = None - self.fetch_cluster = True - self.last_leader_operation = 0 + self._fetch_cluster = True + self._last_leader_operation = 0 self._client.start(None) @@ -103,7 +103,7 @@ class ZooKeeper(AbstractDCS): self.cluster_watcher(None) def cluster_watcher(self, event): - self.fetch_cluster = True + self._fetch_cluster = True self.event.set() def get_node(self, key, watch=None): @@ -132,11 +132,11 @@ class ZooKeeper(AbstractDCS): return members def _inner_load_cluster(self): - self.fetch_cluster = False + self._fetch_cluster = False self.event.clear() nodes = set(self.get_children(self.client_path(''), self.cluster_watcher)) if not nodes: - self.fetch_cluster = True + self._fetch_cluster = True # get initialize flag initialize = (self.get_node(self.initialize_path) or [None])[0] if self._INITIALIZE in nodes else None @@ -157,7 +157,7 @@ class ZooKeeper(AbstractDCS): member = Member(-1, leader[0], None, {}) member = ([m for m in members if m.name == leader[0]] or [member])[0] leader = Leader(leader[1].version, leader[1].ephemeralOwner, member) - self.fetch_cluster = member.index == -1 + self._fetch_cluster = member.index == -1 # failover key failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None @@ -165,15 +165,15 @@ class ZooKeeper(AbstractDCS): failover = Failover.from_node(failover[1].version, failover[0]) # get last leader operation - optime = self.get_node(self.leader_optime_path) if self._OPTIME in nodes and self.fetch_cluster else None - self.last_leader_operation = 0 if optime is None else int(optime[0]) - self._cluster = Cluster(initialize, leader, self.last_leader_operation, members, failover) + optime = self.get_node(self.leader_optime_path) if self._OPTIME in nodes and self._fetch_cluster else None + self._last_leader_operation = 0 if optime is None else int(optime[0]) + self._cluster = Cluster(initialize, leader, self._last_leader_operation, members, failover) def _load_cluster(self): if self.exhibitor and self.exhibitor.poll(): self._client.set_hosts(self.exhibitor.zookeeper_hosts) - if self.fetch_cluster: + if self._fetch_cluster or self._cluster is None: try: self._client.retry(self._inner_load_cluster) except: @@ -249,8 +249,8 @@ class ZooKeeper(AbstractDCS): def write_leader_optime(self, last_operation): last_operation = last_operation.encode('utf-8') - if last_operation != self.last_leader_operation: - self.last_leader_operation = last_operation + if last_operation != self._last_leader_operation: + self._last_leader_operation = last_operation path = self.leader_optime_path try: self._client.retry(self._client.set, path, last_operation) @@ -289,5 +289,5 @@ class ZooKeeper(AbstractDCS): def watch(self, timeout): if super(ZooKeeper, self).watch(timeout): - self.fetch_cluster = True - return self.fetch_cluster + self._fetch_cluster = True + return self._fetch_cluster diff --git a/tests/test_zookeeper.py b/tests/test_zookeeper.py index fe991a2c..5e9ebc82 100644 --- a/tests/test_zookeeper.py +++ b/tests/test_zookeeper.py @@ -151,7 +151,7 @@ class TestZooKeeper(unittest.TestCase): self.zk.touch_member('exists') self.zk._name = 'bar' self.zk.touch_member('retry') - self.zk.fetch_cluster = True + self.zk._fetch_cluster = True self.zk.get_cluster() self.zk.touch_member('retry') From 9dc588231f72117a2ad89f3655af19c03d6b65c6 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 24 Mar 2016 12:29:30 +0100 Subject: [PATCH 82/89] Speed up reattach of former leader to the cluster Instead of starting it up in "read-only", it wil wait 2 seconds, to give a time to somebody to prompte and after it will execute normal `recover` procedure. --- patroni/ha.py | 20 +++++++++++--------- tests/test_ha.py | 1 + 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/patroni/ha.py b/patroni/ha.py index d9916da6..f85064fc 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -110,14 +110,12 @@ class Ha(object): def recover(self): # try to see if we are the former master that crashed. If so - we likely need to run pg_rewind # in order to join the former standby being promoted. - pg_controldata = self.state_handler.controldata() - if (self.state_handler.role == 'master') and pg_controldata and\ - pg_controldata.get('Database cluster state', '') == 'in production': # crashed master - self.state_handler.require_rewind() + if self.state_handler.role == 'master': + pg_controldata = self.state_handler.controldata() + if pg_controldata and pg_controldata.get('Database cluster state', '') == 'in production': # crashed master + self.state_handler.require_rewind() self.recovering = True - return self.follow("started as readonly because i had the session lock", - "started as a secondary", - refresh=True, recovery=True) + return self.follow("started as readonly because i had the session lock", "started as a secondary", True, True) def follow(self, demote_reason, follow_reason, refresh=True, recovery=False): if refresh: @@ -279,7 +277,11 @@ class Ha(object): self.dcs.delete_leader() self.touch_member() self.dcs.reset_cluster() - self.state_handler.follow(None) + self.state_handler.set_role('replica') + sleep(2) # Give a time to somebody to promote + self.recover() + else: + self.state_handler.follow(None) def process_manual_failover_from_leader(self): failover = self.cluster.failover @@ -331,7 +333,7 @@ class Ha(object): if self.cluster.failover: logger.info('Cleaning up failover key after acquiring leader lock...') self.dcs.manual_failover('', '') - self.dcs.get_cluster() + self.load_cluster_from_dcs() return self.enforce_master_role('acquired session lock as a leader', 'promoted self to leader by acquiring session lock') else: diff --git a/tests/test_ha.py b/tests/test_ha.py index 3ab7442b..8ee8748b 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -274,6 +274,7 @@ class TestHa(unittest.TestCase): self.assertEquals(self.ha.run_cycle(), 'failed to update leader lock during restart') @patch('requests.get', requests_get) + @patch('time.sleep', Mock()) def test_manual_failover_from_leader(self): self.ha.has_lock = true self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None)) From e6af18f0bb8818e01719e97b07417f39ab8f79c3 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 24 Mar 2016 14:45:21 +0100 Subject: [PATCH 83/89] Former leader was not able to reattach to cluster without pg_rewind It was shutdown correctly and I expected such 'join' working, but it was not, because new leader didn't had enough time to catch up with the master before promote. --- features/patroni_api.feature | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/features/patroni_api.feature b/features/patroni_api.feature index b1eded96..3c408dfc 100644 --- a/features/patroni_api.feature +++ b/features/patroni_api.feature @@ -29,20 +29,23 @@ Scenario: check API requests for the primary-replica pair And I receive a response role replica When I issue an empty POST request to http://127.0.0.1:8009/reinitialize Then I receive a response code 200 - Given replication works from postgres0 to postgres1 after 10 seconds When I issue an empty POST request to http://127.0.0.1:8008/restart Then I receive a response code 200 And postgres0 is a leader after 5 seconds + When I sleep for 10 seconds + Then postgres1 role is the secondary after 15 seconds Scenario: check the failover via the API Given I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0,candidate=postgres1 Then I receive a response code 200 And postgres1 is a leader after 5 seconds + And postgres1 role is the primary after 5 seconds + And postgres0 role is the secondary after 5 seconds And replication works from postgres1 to postgres0 after 15 seconds Scenario: check the scheduled failover Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgres0 in 10 seconds Then I receive a response code 200 - And postgres0 is a leader after 15 seconds + And postgres0 is a leader after 20 seconds And replication works from postgres0 to postgres1 after 25 seconds From 9c41ce9f1c22ceeea0d58a8648bb6720bfe8c4a0 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 24 Mar 2016 14:50:34 +0100 Subject: [PATCH 84/89] We should run checkpoint before shutdown all the time except one case (when patroni is being shutdown) --- patroni/__init__.py | 2 +- patroni/postgresql.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/patroni/__init__.py b/patroni/__init__.py index 85b6c5df..19ef4ac9 100644 --- a/patroni/__init__.py +++ b/patroni/__init__.py @@ -85,5 +85,5 @@ def main(): pass finally: patroni.api.shutdown() - patroni.postgresql.stop() + patroni.postgresql.stop(checkpoint=False) patroni.dcs.delete_leader() diff --git a/patroni/postgresql.py b/patroni/postgresql.py index df17200d..b3e2ab00 100644 --- a/patroni/postgresql.py +++ b/patroni/postgresql.py @@ -388,7 +388,7 @@ class Postgresql(object): except psycopg2.Error: logging.exception('Exception during CHECKPOINT') - def stop(self, mode='fast', block_callbacks=False): + def stop(self, mode='fast', block_callbacks=False, checkpoint=True): # make sure we close all connections established against # the former node, otherwise, we might get a stalled one # after kill -9, which would report incorrect data to @@ -400,9 +400,10 @@ class Postgresql(object): self.set_state('stopped') return True - if block_callbacks: + if checkpoint: self.checkpoint() - else: + + if not block_callbacks: self.set_state('stopping') ret = subprocess.call(self._pg_ctl + ['stop', '-m', mode]) == 0 From d4cb15179f2f29ceb5bbdee310febf146df63c65 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 24 Mar 2016 14:52:27 +0100 Subject: [PATCH 85/89] Return "demote reason" more precise depending on cluster state --- patroni/ha.py | 13 ++++++------- tests/test_ha.py | 10 +++++----- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/patroni/ha.py b/patroni/ha.py index f85064fc..d4d7fb93 100644 --- a/patroni/ha.py +++ b/patroni/ha.py @@ -115,25 +115,25 @@ class Ha(object): if pg_controldata and pg_controldata.get('Database cluster state', '') == 'in production': # crashed master self.state_handler.require_rewind() self.recovering = True - return self.follow("started as readonly because i had the session lock", "started as a secondary", True, True) + return self.follow("starting as readonly because i had the session lock", "starting as a secondary", True, True) def follow(self, demote_reason, follow_reason, refresh=True, recovery=False): if refresh: self.load_cluster_from_dcs() - if not recovery and self.state_handler.is_leader() or recovery and self.state_handler.role == 'master': - ret = demote_reason - else: - ret = follow_reason + ret = demote_reason if not recovery and self.state_handler.is_leader() else follow_reason # determine the node to follow. If replicatefrom tag is set, # try to follow the node mentioned there, otherwise, follow the leader. + if self.patroni.replicatefrom: node_to_follow = [m for m in self.cluster.members if m.name == self.patroni.replicatefrom] node_to_follow = node_to_follow[0] if node_to_follow else self.cluster.leader else: node_to_follow = self.cluster.leader - node_to_follow = None if node_to_follow and node_to_follow.name == self.state_handler.name else node_to_follow + if node_to_follow and node_to_follow.name == self.state_handler.name: + ret = demote_reason + node_to_follow = None if not self.state_handler.check_recovery_conf(node_to_follow) or recovery: self._async_executor.schedule('changing primary_conninfo and restarting') self._async_executor.run_async(self.state_handler.follow, (node_to_follow, recovery)) @@ -277,7 +277,6 @@ class Ha(object): self.dcs.delete_leader() self.touch_member() self.dcs.reset_cluster() - self.state_handler.set_role('replica') sleep(2) # Give a time to somebody to promote self.recover() else: diff --git a/tests/test_ha.py b/tests/test_ha.py index 8ee8748b..d81d5bbd 100644 --- a/tests/test_ha.py +++ b/tests/test_ha.py @@ -110,25 +110,25 @@ class TestHa(unittest.TestCase): def test_start_as_replica(self): self.p.is_healthy = false - self.assertEquals(self.ha.run_cycle(), 'started as a secondary') + self.assertEquals(self.ha.run_cycle(), 'starting as a secondary') def test_recover_replica_failed(self): self.p.controldata = lambda: {'Database cluster state': 'in production'} self.p.is_healthy = false self.p.is_running = false self.p.follow = false - self.assertEquals(self.ha.run_cycle(), 'started as a secondary') + self.assertEquals(self.ha.run_cycle(), 'starting as a secondary') self.assertEquals(self.ha.run_cycle(), 'failed to start postgres') def test_recover_master_failed(self): self.p.follow = false self.p.is_healthy = false self.p.is_running = false - self.ha.has_lock = true + self.p.name = 'leader' self.p.set_role('master') self.p.controldata = lambda: {'Database cluster state': 'in production'} - self.assertEquals(self.ha.run_cycle(), 'started as readonly because i had the session lock') - self.assertEquals(self.ha.run_cycle(), 'removed leader key after trying and failing to start postgres') + self.ha.cluster = get_cluster_initialized_with_leader() + self.assertEquals(self.ha.run_cycle(), 'starting as readonly because i had the session lock') @patch('sys.exit', return_value=1) @patch('patroni.ha.Ha.sysid_valid', MagicMock(return_value=True)) From 0469d4374de9eaae7e44beefa47280fa189fbac0 Mon Sep 17 00:00:00 2001 From: Alexander Kukushkin Date: Thu, 24 Mar 2016 17:07:56 +0100 Subject: [PATCH 86/89] Set metaclass of AbstractDCS with `six` to be python 2&3 compatible --- patroni/dcs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/patroni/dcs.py b/patroni/dcs.py index 3cb76cdb..689470fc 100644 --- a/patroni/dcs.py +++ b/patroni/dcs.py @@ -1,6 +1,7 @@ import abc -import json import dateutil +import json +import six from collections import namedtuple from six.moves.urllib_parse import urlparse, urlunparse, parse_qsl @@ -150,10 +151,9 @@ class Cluster(namedtuple('Cluster', 'initialize,leader,last_leader_operation,mem return ([m for m in self.members if m.name == member_name] or [None])[0] +@six.add_metaclass(abc.ABCMeta) class AbstractDCS(object): - __metaclass__ = abc.ABCMeta - _INITIALIZE = 'initialize' _LEADER = 'leader' _FAILOVER = 'failover' From 9744019341608c475873d4d214299e3019cb1361 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Mon, 4 Apr 2016 09:40:30 +0200 Subject: [PATCH 87/89] Read Patroni configuration from the environment. The environment variable name is PATRONI_CONFIGURATION. If both the filename and the variable supplied, the file takes priority. --- patroni/__init__.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/patroni/__init__.py b/patroni/__init__.py index 19ef4ac9..4b2c7364 100644 --- a/patroni/__init__.py +++ b/patroni/__init__.py @@ -16,6 +16,7 @@ logger = logging.getLogger(__name__) class Patroni(object): + PATRONI_CONFIG_VARIABLE = 'PATRONI_CONFIGURATION' def __init__(self, config): self.nap_time = config['loop_wait'] @@ -71,12 +72,23 @@ def main(): logging.getLogger('requests').setLevel(logging.WARNING) setup_signal_handlers() - if len(sys.argv) < 2 or not os.path.isfile(sys.argv[1]): - print('Usage: {0} config.yml'.format(sys.argv[0])) - return + # Patroni reads the configuration from the command-line argument if it exists, and from the environment otherwise. + use_env = False + use_file = (len(sys.argv) >= 2 and os.path.isfile(sys.argv[1])) + if not use_file: + config_env = os.environ.get(Patroni.PATRONI_CONFIG_VARIABLE) + use_env = config_env is not None + if not use_env: + print('Usage: {0} config.yml'.format(sys.argv[0])) + print('\tPatroni may also read the configuration from the {} environemnt variable'. + format(Patroni.PATRONI_CONFIG_VARIABLE)) + return - with open(sys.argv[1], 'r') as f: - config = yaml.load(f) + if use_file: + with open(sys.argv[1], 'r') as f: + config = yaml.load(f) + elif use_env: + config = yaml.load(config_env) patroni = Patroni(config) try: From 60e6ed67e38918aa18b489e821eda99690291490 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Mon, 4 Apr 2016 10:06:28 +0200 Subject: [PATCH 88/89] Add unit tests. --- tests/test_patroni.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_patroni.py b/tests/test_patroni.py index d3801abb..fc806026 100644 --- a/tests/test_patroni.py +++ b/tests/test_patroni.py @@ -1,4 +1,5 @@ import etcd +import os import sys import time import unittest @@ -56,6 +57,14 @@ class TestPatroni(unittest.TestCase): self.assertRaises(SleepException, _main) with patch.object(Patroni, 'run', Mock(side_effect=KeyboardInterrupt())): _main() + sys.argv = ['patroni.py'] + # read the content of the yaml configuration file into the environment variable + # in order to test how does patroni handle the configuration passed from the environment. + with open('postgres0.yml', 'r') as f: + os.environ[Patroni.PATRONI_CONFIG_VARIABLE] = f.read() + with patch.object(Patroni, 'run', Mock(side_effect=SleepException())): + self.assertRaises(SleepException, _main) + del os.environ[Patroni.PATRONI_CONFIG_VARIABLE] @patch('time.sleep', Mock(side_effect=SleepException())) def test_run(self): From 4ad190549ad4ac7c0ae87f13229faa337f9ea8a0 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Mon, 4 Apr 2016 12:33:05 +0200 Subject: [PATCH 89/89] Fix a typo in the help message. --- patroni/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/patroni/__init__.py b/patroni/__init__.py index 4b2c7364..87f0b754 100644 --- a/patroni/__init__.py +++ b/patroni/__init__.py @@ -80,7 +80,7 @@ def main(): use_env = config_env is not None if not use_env: print('Usage: {0} config.yml'.format(sys.argv[0])) - print('\tPatroni may also read the configuration from the {} environemnt variable'. + print('\tPatroni may also read the configuration from the {} environment variable'. format(Patroni.PATRONI_CONFIG_VARIABLE)) return