Collect test output, add basic failover test.

features/basic_failover.feature

@@ -0,0 +1,13 @@
+Feature: basic failover
+	In order to check that failover works
+	As observers
+	We start the primary and the replica,
+	shut down the primary
+	and check that the replica assumed the primary role.
+
+Scenario: check the basic failover
+	Given basic replication
+	When I shut down postgres0
+	Then postgres1 role is the primary after 10 seconds
+	When I start postgres0
+	Then postgres0 role is the secondary after 10 seconds

features/basic_failover.py

@@ -0,0 +1,33 @@
+from lettuce import world, steps
+
+PATRONI_CONFIG = '{}.yml'
+
+
+@steps
+class BasicFailoverSteps(object):
+
+    def __init__(self, environ):
+        self.env = environ
+
+    def basic_replication(self, step):
+        '''Basic replication'''
+        step.behave_as("""
+            Given I start postgres0
+            And I start postgres1
+            When I add the table foo to postgres0
+            Then table foo is present on postgres1 after 10 seconds
+            """)
+
+    def start_patroni(self, step, pg_name):
+        '''I start (\w+)'''
+        return world.pctl.start_patroni(pg_name)
+
+    def stop_patroni(self, step, pg_name):
+        '''I shut down (\w+)'''
+        return world.pctl.stop_patroni(pg_name)
+
+    def check_role(self, step, pg_name, pg_role, max_promotion_timeout):
+        '''(\w+) role is the (\w+) after (\d+) seconds'''
+        return world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout))
+
+BasicFailoverSteps(world)

features/basic_replication.feature

@@ -1,12 +1,12 @@
 Feature: basic replication
-  In order to check that basic replication is working
+  In order to check that basic replication works
   As observers
-  We'll start 2 nodes of a new cluster,
-  add a table to the primary
-  and check that it gets replicated to the other over time.
+  We start the primary and the replica
+  add a table on the primary
+  and check that it gets replicated to the replica over time.
 
   Scenario: check replication of a single table
-    Given I have started postgres0
-    And I have started postgres1
+    Given I start postgres0
+    And I start postgres1
     When I add the table foo to postgres0
-    Then table foo is present on postgres1
+    Then table foo is present on postgres1 after 10 seconds

features/basic_replication.py

@@ -11,13 +11,9 @@ class BasicReplicationSteps(object):
 
     def __init__(self, environ):
         self.env = environ
-        self.processes = {}
-        self.connstring = {}
-        self.cwd = None
-        self.max_replication_delay = 10
 
     def start_patroni(self, step, pg_name):
-        '''I have started (\w+)'''
+        '''I start (\w+)'''
         return world.pctl.start_patroni(pg_name)
 
     def add_table(self, step, table_name, pg_name):
@@ -28,14 +24,14 @@ class BasicReplicationSteps(object):
         except pg.Error as e:
             assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e)
 
-    def table_is_present_on(self, step, table_name, pg_name):
-        '''Table (\w+) is present on (\w+)'''
-        for i in range(self.max_replication_delay):
+    def table_is_present_on(self, step, table_name, pg_name, max_replication_delay):
+        '''Table (\w+) is present on (\w+) after (\d+) seconds'''
+        for i in range(int(max_replication_delay)):
             if world.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
                 break
             sleep(1)
         else:
             assert False,\
-                "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, self.max_replication_delay)
+                "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay)
 
 BasicReplicationSteps(world)

features/terrain.py

@@ -5,7 +5,7 @@ import requests
 import subprocess
 import shutil
 import tempfile
-from time import sleep
+import time
 import yaml
 
 
@@ -27,7 +27,10 @@ class PatroniController(object):
         self.connstring = {}
         self.connections = {}
         self.cursors = {}
+        self.log = {}
+        self.config = {}
         self.availability_check_time_limit = 10
+        self.output_dir = None
 
     @property
     def patroni_path(self):
@@ -47,16 +50,40 @@ class PatroniController(object):
     def stop_patroni(self, pg_name):
         while self.patroni_is_running(pg_name):
             self.processes[pg_name].terminate()
-            sleep(1)
+            time.sleep(1)
+        self.log.get('pg_name') and self.log[pg_name].close()
         del self.processes[pg_name]
 
+    def make_patroni_test_config(self, pg_name, output_dir):
+        patroni_config_name = PATRONI_CONFIG.format(pg_name)
+        patroni_config_path = os.path.join(output_dir, patroni_config_name)
+
+        with open(patroni_config_name) as f:
+            config = yaml.load(f)
+        postgresql = config['postgresql']['parameters']
+        postgresql['logging_collector'] = 'on'
+        postgresql['log_destination'] = 'csvlog'
+        postgresql['log_directory'] = output_dir
+        postgresql['log_filename'] = '{0}.log'.format(pg_name)
+        postgresql['log_statement'] = 'all'
+        postgresql['log_min_messages'] = 'debug1'
+
+        with open(patroni_config_path, 'w') as f:
+            yaml.dump(config, f, default_flow_style=False)
+
+        return patroni_config_path
+
     def start_patroni(self, pg_name):
         if not self.patroni_is_running(pg_name):
             if pg_name in self.processes:
                 del self.processes[pg_name]
             cwd = self.patroni_path
-            p = subprocess.Popen(['python', 'patroni.py', PATRONI_CONFIG.format(pg_name)],
-                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd)
+            self.log[pg_name] = open(os.path.join(self.output_dir, 'patroni_{0}.log'.format(pg_name)), 'a')
+
+            self.config[pg_name] = self.make_patroni_test_config(pg_name, self.output_dir)
+
+            p = subprocess.Popen(['python', 'patroni.py', self.config[pg_name]],
+                                 stdout=self.log[pg_name], stderr=subprocess.STDOUT, cwd=cwd)
             if not (p and p.pid and p.poll() is None):
                 assert False, "PostgreSQL {0} is not running after being started".format(pg_name)
             self.processes[pg_name] = p
@@ -64,7 +91,7 @@ class PatroniController(object):
         for tick in range(self.availability_check_time_limit):
             if self.query(pg_name, "SELECT 1", fail_ok=True) is not None:
                 break
-            sleep(1)
+            time.sleep(1)
         else:
             assert False,\
                 "Patroni instance is not available for queries after {0} seconds".format(self.availability_check_time_limit)
@@ -113,6 +140,20 @@ class PatroniController(object):
             else:
                 raise
 
+    def check_role_has_changed_to(self, pg_name, new_role, timeout=10):
+        bound_time = time.time() + timeout
+        current_role = 't' if new_role == 'primary' else 'f'
+        role_has_changed = False
+        while not role_has_changed:
+            cur = self.query(pg_name, "SELECT pg_is_in_recovery()", fail_ok=True)
+            if cur:
+                row = cur.fetchone()
+                if row and len(row) > 0 and row[0] != current_role:
+                    role_has_changed = True
+            if time.time() > bound_time:
+                break
+        return role_has_changed
+
     def stop_all(self):
         for patroni in self.processes.copy():
             self.stop_patroni(patroni)
@@ -159,6 +200,15 @@ def stop_etcd(total):
         etcd_dir = None
 
 
+@before.each_feature
+def make_test_output_dir(feature):
+    feature_dir = os.path.join(pctl.patroni_path, "features", "output", feature.name.encode('utf-8').replace(' ', '_'))
+    if os.path.exists(feature_dir):
+        shutil.rmtree(feature_dir)
+    os.makedirs(feature_dir)
+    pctl.output_dir = feature_dir
+
+
 def patroni_cleanup_all():
     pctl.stop_all()
     # remove the data directory