From 6ec3523748a7214f111bff32298ad2c689492383 Mon Sep 17 00:00:00 2001 From: Oleksii Kliukin Date: Wed, 24 Feb 2016 16:30:52 +0100 Subject: [PATCH] Collect test output, add basic failover test. --- features/basic_failover.feature | 13 +++++++ features/basic_failover.py | 33 ++++++++++++++++ features/basic_replication.feature | 14 +++---- features/basic_replication.py | 14 +++---- features/terrain.py | 60 +++++++++++++++++++++++++++--- 5 files changed, 113 insertions(+), 21 deletions(-) create mode 100644 features/basic_failover.feature create mode 100644 features/basic_failover.py diff --git a/features/basic_failover.feature b/features/basic_failover.feature new file mode 100644 index 00000000..95b7d646 --- /dev/null +++ b/features/basic_failover.feature @@ -0,0 +1,13 @@ +Feature: basic failover + In order to check that failover works + As observers + We start the primary and the replica, + shut down the primary + and check that the replica assumed the primary role. + +Scenario: check the basic failover + Given basic replication + When I shut down postgres0 + Then postgres1 role is the primary after 10 seconds + When I start postgres0 + Then postgres0 role is the secondary after 10 seconds \ No newline at end of file diff --git a/features/basic_failover.py b/features/basic_failover.py new file mode 100644 index 00000000..661cfb28 --- /dev/null +++ b/features/basic_failover.py @@ -0,0 +1,33 @@ +from lettuce import world, steps + +PATRONI_CONFIG = '{}.yml' + + +@steps +class BasicFailoverSteps(object): + + def __init__(self, environ): + self.env = environ + + def basic_replication(self, step): + '''Basic replication''' + step.behave_as(""" + Given I start postgres0 + And I start postgres1 + When I add the table foo to postgres0 + Then table foo is present on postgres1 after 10 seconds + """) + + def start_patroni(self, step, pg_name): + '''I start (\w+)''' + return world.pctl.start_patroni(pg_name) + + def stop_patroni(self, step, pg_name): + '''I shut down (\w+)''' + return world.pctl.stop_patroni(pg_name) + + def check_role(self, step, pg_name, pg_role, max_promotion_timeout): + '''(\w+) role is the (\w+) after (\d+) seconds''' + return world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)) + +BasicFailoverSteps(world) diff --git a/features/basic_replication.feature b/features/basic_replication.feature index 5e61963d..4306bc10 100644 --- a/features/basic_replication.feature +++ b/features/basic_replication.feature @@ -1,12 +1,12 @@ Feature: basic replication - In order to check that basic replication is working + In order to check that basic replication works As observers - We'll start 2 nodes of a new cluster, - add a table to the primary - and check that it gets replicated to the other over time. + We start the primary and the replica + add a table on the primary + and check that it gets replicated to the replica over time. Scenario: check replication of a single table - Given I have started postgres0 - And I have started postgres1 + Given I start postgres0 + And I start postgres1 When I add the table foo to postgres0 - Then table foo is present on postgres1 + Then table foo is present on postgres1 after 10 seconds diff --git a/features/basic_replication.py b/features/basic_replication.py index 69121223..d856c6fb 100644 --- a/features/basic_replication.py +++ b/features/basic_replication.py @@ -11,13 +11,9 @@ class BasicReplicationSteps(object): def __init__(self, environ): self.env = environ - self.processes = {} - self.connstring = {} - self.cwd = None - self.max_replication_delay = 10 def start_patroni(self, step, pg_name): - '''I have started (\w+)''' + '''I start (\w+)''' return world.pctl.start_patroni(pg_name) def add_table(self, step, table_name, pg_name): @@ -28,14 +24,14 @@ class BasicReplicationSteps(object): except pg.Error as e: assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e) - def table_is_present_on(self, step, table_name, pg_name): - '''Table (\w+) is present on (\w+)''' - for i in range(self.max_replication_delay): + def table_is_present_on(self, step, table_name, pg_name, max_replication_delay): + '''Table (\w+) is present on (\w+) after (\d+) seconds''' + for i in range(int(max_replication_delay)): if world.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None: break sleep(1) else: assert False,\ - "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, self.max_replication_delay) + "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay) BasicReplicationSteps(world) diff --git a/features/terrain.py b/features/terrain.py index 7395deba..3c7a0609 100644 --- a/features/terrain.py +++ b/features/terrain.py @@ -5,7 +5,7 @@ import requests import subprocess import shutil import tempfile -from time import sleep +import time import yaml @@ -27,7 +27,10 @@ class PatroniController(object): self.connstring = {} self.connections = {} self.cursors = {} + self.log = {} + self.config = {} self.availability_check_time_limit = 10 + self.output_dir = None @property def patroni_path(self): @@ -47,16 +50,40 @@ class PatroniController(object): def stop_patroni(self, pg_name): while self.patroni_is_running(pg_name): self.processes[pg_name].terminate() - sleep(1) + time.sleep(1) + self.log.get('pg_name') and self.log[pg_name].close() del self.processes[pg_name] + def make_patroni_test_config(self, pg_name, output_dir): + patroni_config_name = PATRONI_CONFIG.format(pg_name) + patroni_config_path = os.path.join(output_dir, patroni_config_name) + + with open(patroni_config_name) as f: + config = yaml.load(f) + postgresql = config['postgresql']['parameters'] + postgresql['logging_collector'] = 'on' + postgresql['log_destination'] = 'csvlog' + postgresql['log_directory'] = output_dir + postgresql['log_filename'] = '{0}.log'.format(pg_name) + postgresql['log_statement'] = 'all' + postgresql['log_min_messages'] = 'debug1' + + with open(patroni_config_path, 'w') as f: + yaml.dump(config, f, default_flow_style=False) + + return patroni_config_path + def start_patroni(self, pg_name): if not self.patroni_is_running(pg_name): if pg_name in self.processes: del self.processes[pg_name] cwd = self.patroni_path - p = subprocess.Popen(['python', 'patroni.py', PATRONI_CONFIG.format(pg_name)], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd) + self.log[pg_name] = open(os.path.join(self.output_dir, 'patroni_{0}.log'.format(pg_name)), 'a') + + self.config[pg_name] = self.make_patroni_test_config(pg_name, self.output_dir) + + p = subprocess.Popen(['python', 'patroni.py', self.config[pg_name]], + stdout=self.log[pg_name], stderr=subprocess.STDOUT, cwd=cwd) if not (p and p.pid and p.poll() is None): assert False, "PostgreSQL {0} is not running after being started".format(pg_name) self.processes[pg_name] = p @@ -64,7 +91,7 @@ class PatroniController(object): for tick in range(self.availability_check_time_limit): if self.query(pg_name, "SELECT 1", fail_ok=True) is not None: break - sleep(1) + time.sleep(1) else: assert False,\ "Patroni instance is not available for queries after {0} seconds".format(self.availability_check_time_limit) @@ -113,6 +140,20 @@ class PatroniController(object): else: raise + def check_role_has_changed_to(self, pg_name, new_role, timeout=10): + bound_time = time.time() + timeout + current_role = 't' if new_role == 'primary' else 'f' + role_has_changed = False + while not role_has_changed: + cur = self.query(pg_name, "SELECT pg_is_in_recovery()", fail_ok=True) + if cur: + row = cur.fetchone() + if row and len(row) > 0 and row[0] != current_role: + role_has_changed = True + if time.time() > bound_time: + break + return role_has_changed + def stop_all(self): for patroni in self.processes.copy(): self.stop_patroni(patroni) @@ -159,6 +200,15 @@ def stop_etcd(total): etcd_dir = None +@before.each_feature +def make_test_output_dir(feature): + feature_dir = os.path.join(pctl.patroni_path, "features", "output", feature.name.encode('utf-8').replace(' ', '_')) + if os.path.exists(feature_dir): + shutil.rmtree(feature_dir) + os.makedirs(feature_dir) + pctl.output_dir = feature_dir + + def patroni_cleanup_all(): pctl.stop_all() # remove the data directory