From 38bd037d99e7b1d6ac137ea2d4d2f740fbdce3f6 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Fri, 5 Feb 2016 13:30:42 +0100
Subject: [PATCH 01/33] Add the 1st lettuce test for the basic replication.

Basically check that the table inserted on the primary
will get its way to the secondary.
---
 features/basic_replication.feature |  12 ++
 features/basic_replication.py      |  41 +++++++
 features/terrain.py                | 184 +++++++++++++++++++++++++++++
 3 files changed, 237 insertions(+)
 create mode 100644 features/basic_replication.feature
 create mode 100644 features/basic_replication.py
 create mode 100644 features/terrain.py

diff --git a/features/basic_replication.feature b/features/basic_replication.feature
new file mode 100644
index 00000000..5e61963d
--- /dev/null
+++ b/features/basic_replication.feature
@@ -0,0 +1,12 @@
+Feature: basic replication
+  In order to check that basic replication is working
+  As observers
+  We'll start 2 nodes of a new cluster,
+  add a table to the primary
+  and check that it gets replicated to the other over time.
+
+  Scenario: check replication of a single table
+    Given I have started postgres0
+    And I have started postgres1
+    When I add the table foo to postgres0
+    Then table foo is present on postgres1
diff --git a/features/basic_replication.py b/features/basic_replication.py
new file mode 100644
index 00000000..26c3f0d2
--- /dev/null
+++ b/features/basic_replication.py
@@ -0,0 +1,41 @@
+import psycopg2 as pg
+from time import sleep
+
+from lettuce import world, steps
+
+PATRONI_CONFIG = '{}.yml'
+
+
+@steps
+class BasicReplicationSteps(object):
+
+    def __init__(self, environ):
+        self.env = environ
+        self.processes = {}
+        self.connstring = {}
+        self.cwd = None
+        self.max_replication_delay = 10
+
+    def start_patroni(self, step, pg_name):
+        '''I have started (\w+)'''
+        return world.pctl.start_patroni(pg_name)
+
+    def add_table(self, step, table_name, pg_name):
+        '''I add the table (\w+) to (\w+)'''
+        # parse the configuration file and get the port
+        try:
+            world.pctl.query(pg_name, "CREATE TABLE {0}()".format(table_name))
+        except pg.Error as e:
+            assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e)
+
+    def table_is_present_on(self, step, table_name, pg_name):
+        '''Then table (\w+) is present on (\w+)'''
+        for i in range(self.max_replication_delay):
+            if world.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
+                break
+            sleep(1)
+        else:
+            assert False,\
+                "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, self.max_replication_delay)
+
+BasicReplicationSteps(world)
diff --git a/features/terrain.py b/features/terrain.py
new file mode 100644
index 00000000..d970d1d0
--- /dev/null
+++ b/features/terrain.py
@@ -0,0 +1,184 @@
+from lettuce import *
+import os.path
+import psycopg2
+import requests
+import subprocess
+import shutil
+import tempfile
+from time import sleep
+import yaml
+
+
+ETCD_VERSION_URL = 'http://127.0.0.1:2379/version'
+ETCD_CLEANUP_URL = 'http://127.0.0.1:2379/v2/keys/service/batman?recursive=true'
+PATRONI_CONFIG = '{}.yml'
+etcd_handle = None
+etcd_dir = None
+pctl = None
+
+
+@world.absorb
+class PatroniController(object):
+    """ starts and stops individual patronis"""
+
+    def __init__(self):
+        self.processes = {}
+        self.patroni_path = None
+        self.cwd = None
+        self.connstring = {}
+        self.connections = {}
+        self.cursors = {}
+        self.availability_check_time_limit = 10
+        pass
+
+    def get_patroni_path(self):
+        if self.patroni_path is None:
+            cwd = os.path.realpath(__file__)
+            while True:
+                path, entry = os.path.split(cwd)
+                cwd = path
+                if entry == 'features' or cwd == '/':
+                    break
+            self.patroni_path = cwd
+        return self.patroni_path
+
+    def patroni_is_running(self, pg_name):
+        return pg_name in self.processes and self.processes[pg_name].pid and (self.processes[pg_name].poll() is None)
+
+    def stop_patroni(self, pg_name):
+        if pg_name in self.processes and self.processes[pg_name].pid and (self.processes[pg_name].poll() is None):
+            self.processes[pg_name].terminate()
+            while self.patroni_is_running(pg_name):
+                self.processes[pg_name].terminate()
+                sleep(1)
+            del self.processes[pg_name]
+
+    def start_patroni(self, pg_name):
+        if not self.patroni_is_running(pg_name):
+            if pg_name in self.processes:
+                del self.processes[pg_name]
+            self.cwd = self.cwd or self.get_patroni_path()
+            p = subprocess.Popen(['python', 'patroni.py', PATRONI_CONFIG.format(pg_name)],
+                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.cwd)
+            if not (p and p.pid and p.poll() is None):
+                assert False, "PostgreSQL {0} is not running after being started".format(pg_name)
+            self.processes[pg_name] = p
+        # wait while patroni is available for queries, but not more than 10 seconds.
+        for tick in range(self.availability_check_time_limit):
+            if self.query(pg_name, "SELECT 1", fail_ok=True) is not None:
+                break
+            sleep(1)
+        else:
+            assert False,\
+                "Patroni instance is not available for queries after {0} seconds".format(self.availability_check_time_limit)
+
+    def make_connstring(self, pg_name):
+        if pg_name in self.connstring:
+            return self.connstring[pg_name]
+        try:
+            patroni_path = self.get_patroni_path()
+            with open(os.path.join(patroni_path, world.PATRONI_CONFIG.format(pg_name)), 'r') as f:
+                config = yaml.load(f)
+        except OSError:
+            return None
+        connstring = config['postgresql']['connect_address']
+        if ':' in connstring:
+            address, port = connstring.split(':')
+        else:
+            address = connstring
+            port = '5432'
+        user = "postgres"
+        dbname = "postgres"
+        self.connstring[pg_name] = "host={0} port={1} dbname={2} user={3}".format(address, port, dbname, user)
+        return self.connstring[pg_name]
+
+    def connection(self, pg_name):
+        if pg_name not in self.connections or self.connections[pg_name].closed:
+            conn = psycopg2.connect(self.make_connstring(pg_name))
+            conn.autocommit = True
+            self.connections[pg_name] = conn
+        return self.connections[pg_name]
+
+    def cursor(self, pg_name):
+        if pg_name not in self.cursors or self.cursors[pg_name].closed:
+            cursor = self.connection(pg_name).cursor()
+            self.cursors[pg_name] = cursor
+        return self.cursors[pg_name]
+
+    def query(self, pg_name, query, fail_ok=False):
+        try:
+            cursor = self.cursor(pg_name)
+            cursor.execute(query)
+            return cursor
+        except psycopg2.Error:
+            if fail_ok:
+                return None
+            else:
+                raise
+
+    def stop_all(self):
+        for patroni in self.processes.copy():
+            self.stop_patroni(patroni)
+
+pctl = PatroniController()
+world.pctl = pctl
+patroni_path = pctl.get_patroni_path()
+world.patroni_path = patroni_path
+world.PATRONI_CONFIG = PATRONI_CONFIG
+
+
+def etcd_is_running():
+    # if we have already started etcd
+    if etcd_handle and etcd_handle.pid and (etcd_handle.poll() is None):
+        return True
+    # if etcd is running, but we didn't start it
+    try:
+        r = requests.get(ETCD_VERSION_URL)
+        if r and r.ok and 'etcdserver' in r.content:
+            return True
+    except requests.ConnectionError:
+        pass
+    return False
+
+
+@before.all
+def start_etcd():
+    if not etcd_is_running():
+        global etcd_handle
+        global etcd_dir
+        etcd_dir = tempfile.mkdtemp()
+        etcd_handle = subprocess.Popen(["etcd", "--data-dir", etcd_dir], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        if not etcd_is_running():
+            assert False, "Failed to start etcd"
+
+
+@after.all
+def stop_etcd(total):
+    global etcd_handle
+    global etcd_dir
+    if etcd_is_running() and etcd_handle:
+        etcd_handle.terminate()
+        etcd_handle = None
+        shutil.rmtree(etcd_dir)
+        etcd_dir = None
+
+
+def patroni_cleanup_all():
+    pctl.stop_all()
+    # remove the data directory
+    shutil.rmtree(os.path.join(patroni_path, 'data'))
+
+
+def etcd_cleanup():
+    try:
+        r = requests.delete(ETCD_CLEANUP_URL)
+        if not r.ok:
+            raise Exception('{}'.format(r.reason))
+    except Exception as e:
+        assert False, "Unable to cleanup etcd: {0}".format(e)
+
+
+@after.each_scenario
+def cleanup(scenario):
+    patroni_cleanup_all()
+    etcd_cleanup()

From f781d0b9feec4aeb2828ce4ab24e245cb885ed45 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 16 Feb 2016 16:50:50 +0100
Subject: [PATCH 02/33] Address the code review by Alex Shulgin.

---
 features/basic_replication.py |  2 +-
 features/terrain.py           | 32 ++++++++++++++------------------
 2 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/features/basic_replication.py b/features/basic_replication.py
index 26c3f0d2..69121223 100644
--- a/features/basic_replication.py
+++ b/features/basic_replication.py
@@ -29,7 +29,7 @@ class BasicReplicationSteps(object):
             assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e)
 
     def table_is_present_on(self, step, table_name, pg_name):
-        '''Then table (\w+) is present on (\w+)'''
+        '''Table (\w+) is present on (\w+)'''
         for i in range(self.max_replication_delay):
             if world.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
                 break
diff --git a/features/terrain.py b/features/terrain.py
index d970d1d0..7395deba 100644
--- a/features/terrain.py
+++ b/features/terrain.py
@@ -23,43 +23,40 @@ class PatroniController(object):
 
     def __init__(self):
         self.processes = {}
-        self.patroni_path = None
-        self.cwd = None
+        self._patroni_path = None
         self.connstring = {}
         self.connections = {}
         self.cursors = {}
         self.availability_check_time_limit = 10
-        pass
 
-    def get_patroni_path(self):
-        if self.patroni_path is None:
+    @property
+    def patroni_path(self):
+        if self._patroni_path is None:
             cwd = os.path.realpath(__file__)
             while True:
                 path, entry = os.path.split(cwd)
                 cwd = path
                 if entry == 'features' or cwd == '/':
                     break
-            self.patroni_path = cwd
-        return self.patroni_path
+            self._patroni_path = cwd
+        return self._patroni_path
 
     def patroni_is_running(self, pg_name):
         return pg_name in self.processes and self.processes[pg_name].pid and (self.processes[pg_name].poll() is None)
 
     def stop_patroni(self, pg_name):
-        if pg_name in self.processes and self.processes[pg_name].pid and (self.processes[pg_name].poll() is None):
+        while self.patroni_is_running(pg_name):
             self.processes[pg_name].terminate()
-            while self.patroni_is_running(pg_name):
-                self.processes[pg_name].terminate()
-                sleep(1)
-            del self.processes[pg_name]
+            sleep(1)
+        del self.processes[pg_name]
 
     def start_patroni(self, pg_name):
         if not self.patroni_is_running(pg_name):
             if pg_name in self.processes:
                 del self.processes[pg_name]
-            self.cwd = self.cwd or self.get_patroni_path()
+            cwd = self.patroni_path
             p = subprocess.Popen(['python', 'patroni.py', PATRONI_CONFIG.format(pg_name)],
-                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.cwd)
+                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd)
             if not (p and p.pid and p.poll() is None):
                 assert False, "PostgreSQL {0} is not running after being started".format(pg_name)
             self.processes[pg_name] = p
@@ -76,7 +73,7 @@ class PatroniController(object):
         if pg_name in self.connstring:
             return self.connstring[pg_name]
         try:
-            patroni_path = self.get_patroni_path()
+            patroni_path = self.patroni_path
             with open(os.path.join(patroni_path, world.PATRONI_CONFIG.format(pg_name)), 'r') as f:
                 config = yaml.load(f)
         except OSError:
@@ -122,8 +119,7 @@ class PatroniController(object):
 
 pctl = PatroniController()
 world.pctl = pctl
-patroni_path = pctl.get_patroni_path()
-world.patroni_path = patroni_path
+world.patroni_path = pctl.patroni_path
 world.PATRONI_CONFIG = PATRONI_CONFIG
 
 
@@ -166,7 +162,7 @@ def stop_etcd(total):
 def patroni_cleanup_all():
     pctl.stop_all()
     # remove the data directory
-    shutil.rmtree(os.path.join(patroni_path, 'data'))
+    shutil.rmtree(os.path.join(pctl.patroni_path, 'data'))
 
 
 def etcd_cleanup():

From 6ec3523748a7214f111bff32298ad2c689492383 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Wed, 24 Feb 2016 16:30:52 +0100
Subject: [PATCH 03/33] Collect test output, add basic failover test.

---
 features/basic_failover.feature    | 13 +++++++
 features/basic_failover.py         | 33 ++++++++++++++++
 features/basic_replication.feature | 14 +++----
 features/basic_replication.py      | 14 +++----
 features/terrain.py                | 60 +++++++++++++++++++++++++++---
 5 files changed, 113 insertions(+), 21 deletions(-)
 create mode 100644 features/basic_failover.feature
 create mode 100644 features/basic_failover.py

diff --git a/features/basic_failover.feature b/features/basic_failover.feature
new file mode 100644
index 00000000..95b7d646
--- /dev/null
+++ b/features/basic_failover.feature
@@ -0,0 +1,13 @@
+Feature: basic failover
+	In order to check that failover works
+	As observers
+	We start the primary and the replica,
+	shut down the primary
+	and check that the replica assumed the primary role.
+
+Scenario: check the basic failover
+	Given basic replication
+	When I shut down postgres0
+	Then postgres1 role is the primary after 10 seconds
+	When I start postgres0
+	Then postgres0 role is the secondary after 10 seconds
\ No newline at end of file
diff --git a/features/basic_failover.py b/features/basic_failover.py
new file mode 100644
index 00000000..661cfb28
--- /dev/null
+++ b/features/basic_failover.py
@@ -0,0 +1,33 @@
+from lettuce import world, steps
+
+PATRONI_CONFIG = '{}.yml'
+
+
+@steps
+class BasicFailoverSteps(object):
+
+    def __init__(self, environ):
+        self.env = environ
+
+    def basic_replication(self, step):
+        '''Basic replication'''
+        step.behave_as("""
+            Given I start postgres0
+            And I start postgres1
+            When I add the table foo to postgres0
+            Then table foo is present on postgres1 after 10 seconds
+            """)
+
+    def start_patroni(self, step, pg_name):
+        '''I start (\w+)'''
+        return world.pctl.start_patroni(pg_name)
+
+    def stop_patroni(self, step, pg_name):
+        '''I shut down (\w+)'''
+        return world.pctl.stop_patroni(pg_name)
+
+    def check_role(self, step, pg_name, pg_role, max_promotion_timeout):
+        '''(\w+) role is the (\w+) after (\d+) seconds'''
+        return world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout))
+
+BasicFailoverSteps(world)
diff --git a/features/basic_replication.feature b/features/basic_replication.feature
index 5e61963d..4306bc10 100644
--- a/features/basic_replication.feature
+++ b/features/basic_replication.feature
@@ -1,12 +1,12 @@
 Feature: basic replication
-  In order to check that basic replication is working
+  In order to check that basic replication works
   As observers
-  We'll start 2 nodes of a new cluster,
-  add a table to the primary
-  and check that it gets replicated to the other over time.
+  We start the primary and the replica
+  add a table on the primary
+  and check that it gets replicated to the replica over time.
 
   Scenario: check replication of a single table
-    Given I have started postgres0
-    And I have started postgres1
+    Given I start postgres0
+    And I start postgres1
     When I add the table foo to postgres0
-    Then table foo is present on postgres1
+    Then table foo is present on postgres1 after 10 seconds
diff --git a/features/basic_replication.py b/features/basic_replication.py
index 69121223..d856c6fb 100644
--- a/features/basic_replication.py
+++ b/features/basic_replication.py
@@ -11,13 +11,9 @@ class BasicReplicationSteps(object):
 
     def __init__(self, environ):
         self.env = environ
-        self.processes = {}
-        self.connstring = {}
-        self.cwd = None
-        self.max_replication_delay = 10
 
     def start_patroni(self, step, pg_name):
-        '''I have started (\w+)'''
+        '''I start (\w+)'''
         return world.pctl.start_patroni(pg_name)
 
     def add_table(self, step, table_name, pg_name):
@@ -28,14 +24,14 @@ class BasicReplicationSteps(object):
         except pg.Error as e:
             assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e)
 
-    def table_is_present_on(self, step, table_name, pg_name):
-        '''Table (\w+) is present on (\w+)'''
-        for i in range(self.max_replication_delay):
+    def table_is_present_on(self, step, table_name, pg_name, max_replication_delay):
+        '''Table (\w+) is present on (\w+) after (\d+) seconds'''
+        for i in range(int(max_replication_delay)):
             if world.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
                 break
             sleep(1)
         else:
             assert False,\
-                "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, self.max_replication_delay)
+                "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay)
 
 BasicReplicationSteps(world)
diff --git a/features/terrain.py b/features/terrain.py
index 7395deba..3c7a0609 100644
--- a/features/terrain.py
+++ b/features/terrain.py
@@ -5,7 +5,7 @@ import requests
 import subprocess
 import shutil
 import tempfile
-from time import sleep
+import time
 import yaml
 
 
@@ -27,7 +27,10 @@ class PatroniController(object):
         self.connstring = {}
         self.connections = {}
         self.cursors = {}
+        self.log = {}
+        self.config = {}
         self.availability_check_time_limit = 10
+        self.output_dir = None
 
     @property
     def patroni_path(self):
@@ -47,16 +50,40 @@ class PatroniController(object):
     def stop_patroni(self, pg_name):
         while self.patroni_is_running(pg_name):
             self.processes[pg_name].terminate()
-            sleep(1)
+            time.sleep(1)
+        self.log.get('pg_name') and self.log[pg_name].close()
         del self.processes[pg_name]
 
+    def make_patroni_test_config(self, pg_name, output_dir):
+        patroni_config_name = PATRONI_CONFIG.format(pg_name)
+        patroni_config_path = os.path.join(output_dir, patroni_config_name)
+
+        with open(patroni_config_name) as f:
+            config = yaml.load(f)
+        postgresql = config['postgresql']['parameters']
+        postgresql['logging_collector'] = 'on'
+        postgresql['log_destination'] = 'csvlog'
+        postgresql['log_directory'] = output_dir
+        postgresql['log_filename'] = '{0}.log'.format(pg_name)
+        postgresql['log_statement'] = 'all'
+        postgresql['log_min_messages'] = 'debug1'
+
+        with open(patroni_config_path, 'w') as f:
+            yaml.dump(config, f, default_flow_style=False)
+
+        return patroni_config_path
+
     def start_patroni(self, pg_name):
         if not self.patroni_is_running(pg_name):
             if pg_name in self.processes:
                 del self.processes[pg_name]
             cwd = self.patroni_path
-            p = subprocess.Popen(['python', 'patroni.py', PATRONI_CONFIG.format(pg_name)],
-                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd)
+            self.log[pg_name] = open(os.path.join(self.output_dir, 'patroni_{0}.log'.format(pg_name)), 'a')
+
+            self.config[pg_name] = self.make_patroni_test_config(pg_name, self.output_dir)
+
+            p = subprocess.Popen(['python', 'patroni.py', self.config[pg_name]],
+                                 stdout=self.log[pg_name], stderr=subprocess.STDOUT, cwd=cwd)
             if not (p and p.pid and p.poll() is None):
                 assert False, "PostgreSQL {0} is not running after being started".format(pg_name)
             self.processes[pg_name] = p
@@ -64,7 +91,7 @@ class PatroniController(object):
         for tick in range(self.availability_check_time_limit):
             if self.query(pg_name, "SELECT 1", fail_ok=True) is not None:
                 break
-            sleep(1)
+            time.sleep(1)
         else:
             assert False,\
                 "Patroni instance is not available for queries after {0} seconds".format(self.availability_check_time_limit)
@@ -113,6 +140,20 @@ class PatroniController(object):
             else:
                 raise
 
+    def check_role_has_changed_to(self, pg_name, new_role, timeout=10):
+        bound_time = time.time() + timeout
+        current_role = 't' if new_role == 'primary' else 'f'
+        role_has_changed = False
+        while not role_has_changed:
+            cur = self.query(pg_name, "SELECT pg_is_in_recovery()", fail_ok=True)
+            if cur:
+                row = cur.fetchone()
+                if row and len(row) > 0 and row[0] != current_role:
+                    role_has_changed = True
+            if time.time() > bound_time:
+                break
+        return role_has_changed
+
     def stop_all(self):
         for patroni in self.processes.copy():
             self.stop_patroni(patroni)
@@ -159,6 +200,15 @@ def stop_etcd(total):
         etcd_dir = None
 
 
+@before.each_feature
+def make_test_output_dir(feature):
+    feature_dir = os.path.join(pctl.patroni_path, "features", "output", feature.name.encode('utf-8').replace(' ', '_'))
+    if os.path.exists(feature_dir):
+        shutil.rmtree(feature_dir)
+    os.makedirs(feature_dir)
+    pctl.output_dir = feature_dir
+
+
 def patroni_cleanup_all():
     pctl.stop_all()
     # remove the data directory

From 6f039532680fd5cc42c81a0c1b357c528f431727 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Wed, 24 Feb 2016 17:12:45 +0100
Subject: [PATCH 04/33] Merge basic failover and basic replication scenarios in
 one feature.

---
 features/basic_failover.feature    | 13 ------------
 features/basic_failover.py         | 33 ------------------------------
 features/basic_replication.feature | 10 +++++++++
 features/basic_replication.py      |  5 +++++
 features/terrain.py                |  2 +-
 5 files changed, 16 insertions(+), 47 deletions(-)
 delete mode 100644 features/basic_failover.feature
 delete mode 100644 features/basic_failover.py

diff --git a/features/basic_failover.feature b/features/basic_failover.feature
deleted file mode 100644
index 95b7d646..00000000
--- a/features/basic_failover.feature
+++ /dev/null
@@ -1,13 +0,0 @@
-Feature: basic failover
-	In order to check that failover works
-	As observers
-	We start the primary and the replica,
-	shut down the primary
-	and check that the replica assumed the primary role.
-
-Scenario: check the basic failover
-	Given basic replication
-	When I shut down postgres0
-	Then postgres1 role is the primary after 10 seconds
-	When I start postgres0
-	Then postgres0 role is the secondary after 10 seconds
\ No newline at end of file
diff --git a/features/basic_failover.py b/features/basic_failover.py
deleted file mode 100644
index 661cfb28..00000000
--- a/features/basic_failover.py
+++ /dev/null
@@ -1,33 +0,0 @@
-from lettuce import world, steps
-
-PATRONI_CONFIG = '{}.yml'
-
-
-@steps
-class BasicFailoverSteps(object):
-
-    def __init__(self, environ):
-        self.env = environ
-
-    def basic_replication(self, step):
-        '''Basic replication'''
-        step.behave_as("""
-            Given I start postgres0
-            And I start postgres1
-            When I add the table foo to postgres0
-            Then table foo is present on postgres1 after 10 seconds
-            """)
-
-    def start_patroni(self, step, pg_name):
-        '''I start (\w+)'''
-        return world.pctl.start_patroni(pg_name)
-
-    def stop_patroni(self, step, pg_name):
-        '''I shut down (\w+)'''
-        return world.pctl.stop_patroni(pg_name)
-
-    def check_role(self, step, pg_name, pg_role, max_promotion_timeout):
-        '''(\w+) role is the (\w+) after (\d+) seconds'''
-        return world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout))
-
-BasicFailoverSteps(world)
diff --git a/features/basic_replication.feature b/features/basic_replication.feature
index 4306bc10..140e0cf2 100644
--- a/features/basic_replication.feature
+++ b/features/basic_replication.feature
@@ -4,9 +4,19 @@ Feature: basic replication
   We start the primary and the replica
   add a table on the primary
   and check that it gets replicated to the replica over time.
+  We stop the primary and check that the replica promotes itself to primary
+  We start the old primary and check that it rejoins as a replica.
 
   Scenario: check replication of a single table
     Given I start postgres0
     And I start postgres1
     When I add the table foo to postgres0
     Then table foo is present on postgres1 after 10 seconds
+
+  Scenario: check the basic failover
+    When I shut down postgres0
+    Then postgres1 role is the primary after 10 seconds
+    When I start postgres0
+    Then postgres0 role is the secondary after 10 seconds
+    When I add the table bar to postgres1
+    Then table bar is present on postgres1 after 10 seconds
\ No newline at end of file
diff --git a/features/basic_replication.py b/features/basic_replication.py
index d856c6fb..4c26da37 100644
--- a/features/basic_replication.py
+++ b/features/basic_replication.py
@@ -34,4 +34,9 @@ class BasicReplicationSteps(object):
             assert False,\
                 "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay)
 
+    def check_role(self, step, pg_name, pg_role, max_promotion_timeout):
+        '''(\w+) role is the (\w+) after (\d+) seconds'''
+        return world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout))
+
+
 BasicReplicationSteps(world)
diff --git a/features/terrain.py b/features/terrain.py
index 3c7a0609..2a9b23cc 100644
--- a/features/terrain.py
+++ b/features/terrain.py
@@ -224,7 +224,7 @@ def etcd_cleanup():
         assert False, "Unable to cleanup etcd: {0}".format(e)
 
 
-@after.each_scenario
+@after.each_feature
 def cleanup(scenario):
     patroni_cleanup_all()
     etcd_cleanup()

From c9b8c2d3a91dc1ac275f367c81ccad42fce6fef8 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Wed, 24 Feb 2016 19:22:42 +0100
Subject: [PATCH 05/33] Bugfixes, add a function to kill patroni daemon, make
 the feature description more concise.

---
 features/basic_replication.feature | 14 ++++----------
 features/basic_replication.py      | 15 ++++++++++++++-
 features/terrain.py                | 12 ++++++++----
 3 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/features/basic_replication.feature b/features/basic_replication.feature
index 140e0cf2..fe766c11 100644
--- a/features/basic_replication.feature
+++ b/features/basic_replication.feature
@@ -1,11 +1,5 @@
 Feature: basic replication
-  In order to check that basic replication works
-  As observers
-  We start the primary and the replica
-  add a table on the primary
-  and check that it gets replicated to the replica over time.
-  We stop the primary and check that the replica promotes itself to primary
-  We start the old primary and check that it rejoins as a replica.
+  We should check that the basic bootstrapping, replication and failover works.
 
   Scenario: check replication of a single table
     Given I start postgres0
@@ -14,9 +8,9 @@ Feature: basic replication
     Then table foo is present on postgres1 after 10 seconds
 
   Scenario: check the basic failover
-    When I shut down postgres0
-    Then postgres1 role is the primary after 10 seconds
+    When I kill postgres0
+    Then postgres1 role is the primary after 30 seconds
     When I start postgres0
     Then postgres0 role is the secondary after 10 seconds
     When I add the table bar to postgres1
-    Then table bar is present on postgres1 after 10 seconds
\ No newline at end of file
+    Then table bar is present on postgres0 after 10 seconds
diff --git a/features/basic_replication.py b/features/basic_replication.py
index 4c26da37..176f1f30 100644
--- a/features/basic_replication.py
+++ b/features/basic_replication.py
@@ -16,6 +16,18 @@ class BasicReplicationSteps(object):
         '''I start (\w+)'''
         return world.pctl.start_patroni(pg_name)
 
+    def stop_patroni(self, step, pg_name):
+        '''I shut down (\w+)'''
+        return world.pctl.stop_patroni(pg_name)
+
+    def kill_patroni(self, step, pg_name):
+        '''I kill (\w+)'''
+        return world.pctl.stop_patroni(pg_name, kill=True)
+
+    def do_sleep(self, step, sleep_seconds):
+        '''I sleep (\w+)'''
+        sleep(int(sleep_seconds))
+
     def add_table(self, step, table_name, pg_name):
         '''I add the table (\w+) to (\w+)'''
         # parse the configuration file and get the port
@@ -36,7 +48,8 @@ class BasicReplicationSteps(object):
 
     def check_role(self, step, pg_name, pg_role, max_promotion_timeout):
         '''(\w+) role is the (\w+) after (\d+) seconds'''
-        return world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout))
+        if not world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)):
+            assert False, "pg_name role didn't change to {0} after {1} seconds".format(pg_role, max_promotion_timeout)
 
 
 BasicReplicationSteps(world)
diff --git a/features/terrain.py b/features/terrain.py
index 2a9b23cc..5f66e780 100644
--- a/features/terrain.py
+++ b/features/terrain.py
@@ -47,9 +47,12 @@ class PatroniController(object):
     def patroni_is_running(self, pg_name):
         return pg_name in self.processes and self.processes[pg_name].pid and (self.processes[pg_name].poll() is None)
 
-    def stop_patroni(self, pg_name):
+    def stop_patroni(self, pg_name, kill=False):
         while self.patroni_is_running(pg_name):
-            self.processes[pg_name].terminate()
+            if not kill:
+                self.processes[pg_name].terminate()
+            else:
+                self.processes[pg_name].kill()
             time.sleep(1)
         self.log.get('pg_name') and self.log[pg_name].close()
         del self.processes[pg_name]
@@ -142,16 +145,17 @@ class PatroniController(object):
 
     def check_role_has_changed_to(self, pg_name, new_role, timeout=10):
         bound_time = time.time() + timeout
-        current_role = 't' if new_role == 'primary' else 'f'
+        recovery_status = False if new_role == 'primary' else True
         role_has_changed = False
         while not role_has_changed:
             cur = self.query(pg_name, "SELECT pg_is_in_recovery()", fail_ok=True)
             if cur:
                 row = cur.fetchone()
-                if row and len(row) > 0 and row[0] != current_role:
+                if row and len(row) > 0 and row[0] == recovery_status:
                     role_has_changed = True
             if time.time() > bound_time:
                 break
+            time.sleep(1)
         return role_has_changed
 
     def stop_all(self):

From 53b5dfe39e6635d6aff29cbd82a1f92c5d082331 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Wed, 24 Feb 2016 19:24:46 +0100
Subject: [PATCH 06/33] Remove an unused function.

---
 features/basic_replication.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/features/basic_replication.py b/features/basic_replication.py
index 176f1f30..02927405 100644
--- a/features/basic_replication.py
+++ b/features/basic_replication.py
@@ -24,10 +24,6 @@ class BasicReplicationSteps(object):
         '''I kill (\w+)'''
         return world.pctl.stop_patroni(pg_name, kill=True)
 
-    def do_sleep(self, step, sleep_seconds):
-        '''I sleep (\w+)'''
-        sleep(int(sleep_seconds))
-
     def add_table(self, step, table_name, pg_name):
         '''I add the table (\w+) to (\w+)'''
         # parse the configuration file and get the port

From 4986db5c6a1f7c28771152018351269c39bff2fc Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Thu, 25 Feb 2016 12:52:45 +0100
Subject: [PATCH 07/33] Code refactoring, no functional changes.

Move etcd code into a separate class.
Reduce the number of global interdependencies.
Clearly define private members of PatroniController and EtcdController.

It would not make the QuantifiedCode entirely happy, since lettuce
passes the step argument to the step definition, that is not used
in the client code, but internally (via the @steps decorator on
the steps class), but that's the issue of the tool used.
---
 features/basic_replication.py |  16 +-
 features/terrain.py           | 319 ++++++++++++++++++----------------
 2 files changed, 176 insertions(+), 159 deletions(-)

diff --git a/features/basic_replication.py b/features/basic_replication.py
index 02927405..77351428 100644
--- a/features/basic_replication.py
+++ b/features/basic_replication.py
@@ -3,8 +3,6 @@ from time import sleep
 
 from lettuce import world, steps
 
-PATRONI_CONFIG = '{}.yml'
-
 
 @steps
 class BasicReplicationSteps(object):
@@ -12,17 +10,17 @@ class BasicReplicationSteps(object):
     def __init__(self, environ):
         self.env = environ
 
-    def start_patroni(self, step, pg_name):
+    def start_patroni(self, step, name):
         '''I start (\w+)'''
-        return world.pctl.start_patroni(pg_name)
+        return world.pctl.start(name)
 
-    def stop_patroni(self, step, pg_name):
+    def stop_patroni(self, step, name):
         '''I shut down (\w+)'''
-        return world.pctl.stop_patroni(pg_name)
+        return world.pctl.stop(name)
 
-    def kill_patroni(self, step, pg_name):
+    def kill_patroni(self, step, name):
         '''I kill (\w+)'''
-        return world.pctl.stop_patroni(pg_name, kill=True)
+        return world.pctl.stop(name, kill=True)
 
     def add_table(self, step, table_name, pg_name):
         '''I add the table (\w+) to (\w+)'''
@@ -45,7 +43,7 @@ class BasicReplicationSteps(object):
     def check_role(self, step, pg_name, pg_role, max_promotion_timeout):
         '''(\w+) role is the (\w+) after (\d+) seconds'''
         if not world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)):
-            assert False, "pg_name role didn't change to {0} after {1} seconds".format(pg_role, max_promotion_timeout)
+            assert False, "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
 
 
 BasicReplicationSteps(world)
diff --git a/features/terrain.py b/features/terrain.py
index 5f66e780..750c1316 100644
--- a/features/terrain.py
+++ b/features/terrain.py
@@ -1,4 +1,4 @@
-from lettuce import *
+from lettuce import world, before, after
 import os.path
 import psycopg2
 import requests
@@ -9,28 +9,19 @@ import time
 import yaml
 
 
-ETCD_VERSION_URL = 'http://127.0.0.1:2379/version'
-ETCD_CLEANUP_URL = 'http://127.0.0.1:2379/v2/keys/service/batman?recursive=true'
-PATRONI_CONFIG = '{}.yml'
-etcd_handle = None
-etcd_dir = None
-pctl = None
-
-
-@world.absorb
 class PatroniController(object):
+    PATRONI_CONFIG = '{}.yml'
     """ starts and stops individual patronis"""
 
     def __init__(self):
-        self.processes = {}
+        self._output_dir = None
         self._patroni_path = None
-        self.connstring = {}
-        self.connections = {}
-        self.cursors = {}
-        self.log = {}
-        self.config = {}
-        self.availability_check_time_limit = 10
-        self.output_dir = None
+        self._connections = {}
+        self._config = {}
+        self._connstring = {}
+        self._cursors = {}
+        self._log = {}
+        self._processes = {}
 
     @property
     def patroni_path(self):
@@ -44,97 +35,43 @@ class PatroniController(object):
             self._patroni_path = cwd
         return self._patroni_path
 
-    def patroni_is_running(self, pg_name):
-        return pg_name in self.processes and self.processes[pg_name].pid and (self.processes[pg_name].poll() is None)
-
-    def stop_patroni(self, pg_name, kill=False):
-        while self.patroni_is_running(pg_name):
-            if not kill:
-                self.processes[pg_name].terminate()
-            else:
-                self.processes[pg_name].kill()
-            time.sleep(1)
-        self.log.get('pg_name') and self.log[pg_name].close()
-        del self.processes[pg_name]
-
-    def make_patroni_test_config(self, pg_name, output_dir):
-        patroni_config_name = PATRONI_CONFIG.format(pg_name)
-        patroni_config_path = os.path.join(output_dir, patroni_config_name)
-
-        with open(patroni_config_name) as f:
-            config = yaml.load(f)
-        postgresql = config['postgresql']['parameters']
-        postgresql['logging_collector'] = 'on'
-        postgresql['log_destination'] = 'csvlog'
-        postgresql['log_directory'] = output_dir
-        postgresql['log_filename'] = '{0}.log'.format(pg_name)
-        postgresql['log_statement'] = 'all'
-        postgresql['log_min_messages'] = 'debug1'
-
-        with open(patroni_config_path, 'w') as f:
-            yaml.dump(config, f, default_flow_style=False)
-
-        return patroni_config_path
-
-    def start_patroni(self, pg_name):
-        if not self.patroni_is_running(pg_name):
-            if pg_name in self.processes:
-                del self.processes[pg_name]
+    def start(self, pg_name, max_wait_limit=15):
+        if not self._is_running(pg_name):
+            if pg_name in self._processes:
+                del self._processes[pg_name]
             cwd = self.patroni_path
-            self.log[pg_name] = open(os.path.join(self.output_dir, 'patroni_{0}.log'.format(pg_name)), 'a')
+            self._log[pg_name] = open(os.path.join(self._output_dir, 'patroni_{0}.log'.format(pg_name)), 'a')
 
-            self.config[pg_name] = self.make_patroni_test_config(pg_name, self.output_dir)
+            self._config[pg_name] = self._make_patroni_test_config(pg_name, self._output_dir)
 
-            p = subprocess.Popen(['python', 'patroni.py', self.config[pg_name]],
-                                 stdout=self.log[pg_name], stderr=subprocess.STDOUT, cwd=cwd)
+            p = subprocess.Popen(['python', 'patroni.py', self._config[pg_name]],
+                                 stdout=self._log[pg_name], stderr=subprocess.STDOUT, cwd=cwd)
             if not (p and p.pid and p.poll() is None):
                 assert False, "PostgreSQL {0} is not running after being started".format(pg_name)
-            self.processes[pg_name] = p
+            self._processes[pg_name] = p
         # wait while patroni is available for queries, but not more than 10 seconds.
-        for tick in range(self.availability_check_time_limit):
+        for tick in range(max_wait_limit):
             if self.query(pg_name, "SELECT 1", fail_ok=True) is not None:
                 break
             time.sleep(1)
         else:
             assert False,\
-                "Patroni instance is not available for queries after {0} seconds".format(self.availability_check_time_limit)
+                "Patroni instance is not available for queries after {0} seconds".format(max_wait_limit)
 
-    def make_connstring(self, pg_name):
-        if pg_name in self.connstring:
-            return self.connstring[pg_name]
-        try:
-            patroni_path = self.patroni_path
-            with open(os.path.join(patroni_path, world.PATRONI_CONFIG.format(pg_name)), 'r') as f:
-                config = yaml.load(f)
-        except OSError:
-            return None
-        connstring = config['postgresql']['connect_address']
-        if ':' in connstring:
-            address, port = connstring.split(':')
-        else:
-            address = connstring
-            port = '5432'
-        user = "postgres"
-        dbname = "postgres"
-        self.connstring[pg_name] = "host={0} port={1} dbname={2} user={3}".format(address, port, dbname, user)
-        return self.connstring[pg_name]
-
-    def connection(self, pg_name):
-        if pg_name not in self.connections or self.connections[pg_name].closed:
-            conn = psycopg2.connect(self.make_connstring(pg_name))
-            conn.autocommit = True
-            self.connections[pg_name] = conn
-        return self.connections[pg_name]
-
-    def cursor(self, pg_name):
-        if pg_name not in self.cursors or self.cursors[pg_name].closed:
-            cursor = self.connection(pg_name).cursor()
-            self.cursors[pg_name] = cursor
-        return self.cursors[pg_name]
+    def stop(self, pg_name, kill=False):
+        while self._is_running(pg_name):
+            if not kill:
+                self._processes[pg_name].terminate()
+            else:
+                self._processes[pg_name].kill()
+            time.sleep(1)
+        self._log.get('pg_name') and self._log[pg_name].close()
+        if pg_name in self._processes:
+            del self._processes[pg_name]
 
     def query(self, pg_name, query, fail_ok=False):
         try:
-            cursor = self.cursor(pg_name)
+            cursor = self._cursor(pg_name)
             cursor.execute(query)
             return cursor
         except psycopg2.Error:
@@ -159,76 +96,158 @@ class PatroniController(object):
         return role_has_changed
 
     def stop_all(self):
-        for patroni in self.processes.copy():
-            self.stop_patroni(patroni)
+        for patroni in self._processes.copy():
+            self.stop(patroni)
+
+    def create_and_set_output_directory(self, feature_name):
+        feature_dir = os.path.join(pctl.patroni_path, "features", "output", feature_name.encode('utf-8').replace(' ', '_'))
+        if os.path.exists(feature_dir):
+            shutil.rmtree(feature_dir)
+        os.makedirs(feature_dir)
+        self._output_dir = feature_dir
+
+    def _is_running(self, pg_name):
+        return pg_name in self._processes and self._processes[pg_name].pid and (self._processes[pg_name].poll() is None)
+
+    def _make_patroni_test_config(self, pg_name, output_dir):
+        patroni_config_name = PatroniController.PATRONI_CONFIG.format(pg_name)
+        patroni_config_path = os.path.join(output_dir, patroni_config_name)
+
+        with open(patroni_config_name) as f:
+            config = yaml.load(f)
+        postgresql = config['postgresql']['parameters']
+        postgresql['logging_collector'] = 'on'
+        postgresql['log_destination'] = 'csvlog'
+        postgresql['log_directory'] = output_dir
+        postgresql['log_filename'] = '{0}.log'.format(pg_name)
+        postgresql['log_statement'] = 'all'
+        postgresql['log_min_messages'] = 'debug1'
+
+        with open(patroni_config_path, 'w') as f:
+            yaml.dump(config, f, default_flow_style=False)
+
+        return patroni_config_path
+
+    def _make_connstring(self, pg_name):
+        if pg_name in self._connstring:
+            return self._connstring[pg_name]
+        try:
+            patroni_path = self.patroni_path
+            with open(os.path.join(patroni_path, PatroniController.PATRONI_CONFIG.format(pg_name)), 'r') as f:
+                config = yaml.load(f)
+        except OSError:
+            return None
+        connstring = config['postgresql']['connect_address']
+        if ':' in connstring:
+            address, port = connstring.split(':')
+        else:
+            address = connstring
+            port = '5432'
+        user = "postgres"
+        dbname = "postgres"
+        self._connstring[pg_name] = "host={0} port={1} dbname={2} user={3}".format(address, port, dbname, user)
+        return self._connstring[pg_name]
+
+    def _connection(self, pg_name):
+        if pg_name not in self._connections or self._connections[pg_name].closed:
+            conn = psycopg2.connect(self._make_connstring(pg_name))
+            conn.autocommit = True
+            self._connections[pg_name] = conn
+        return self._connections[pg_name]
+
+    def _cursor(self, pg_name):
+        if pg_name not in self._cursors or self._cursors[pg_name].closed:
+            cursor = self._connection(pg_name).cursor()
+            self._cursors[pg_name] = cursor
+        return self._cursors[pg_name]
+
+
+class EtcdController(object):
+    """ handles all etcd related tasks, used for the tests setup and cleanup """
+    ETCD_VERSION_URL = 'http://127.0.0.1:2379/version'
+    ETCD_CLEANUP_URL = 'http://127.0.0.1:2379/v2/keys/service/batman?recursive=true'
+
+    def __init__(self):
+        self.handle = None
+        self.work_directory = None
+        self.pid = None
+        self.start_timeot = 5
+
+    def start(self):
+        """ start etcd if it's not already running """
+        if self._is_running():
+            return True
+        self.work_directory = tempfile.mkdtemp()
+        self.handle =\
+            subprocess.Popen(["etcd", "--data-dir", self.work_directory], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        start_time = time.time()
+        while (not self._is_running()):
+            if time.time() - start_time > self.start_timeout:
+                assert False, "Failed to start etcd"
+            time.sleep(1)
+        return True
+
+    def stop_and_remove_work_directory(self):
+        """ terminate etcd and wipe out the temp work directory, but only if we actually started it"""
+        if self._is_running() and self.handle:
+            self.handle.terminate()
+            self.handle = None
+        if self.work_directory:
+            shutil.rmtree(self.work_directory)
+            self.work_directory = None
+
+    @staticmethod
+    def cleanup_service_tree():
+        """ clean all contents stored in the tree used for the tests """
+        r = None
+        try:
+            r = requests.delete(EtcdController.ETCD_CLEANUP_URL)
+            if r and not r.ok:
+                assert False,\
+                    "request to cleanup the etcd contents was not successfull: status code {0}".format(r.status_code)
+        except requests.exceptions.RequestException as e:
+            assert False, "exception when cleanin up etcd contents: {0}".format(e)
+
+    def _is_running(self):
+        # if we have already started etcd
+        if self.handle and self.handle.pid and (self.handle.poll() is None):
+            return True
+        # if etcd is running, but we didn't start it
+        try:
+            r = requests.get(EtcdController.ETCD_VERSION_URL)
+            if r and r.ok and 'etcdserver' in r.content:
+                return True
+        except requests.ConnectionError:
+            pass
+        return False
+
 
 pctl = PatroniController()
+etcd_ctl = EtcdController()
+# export pctl to manage patroni from scenario files
 world.pctl = pctl
-world.patroni_path = pctl.patroni_path
-world.PATRONI_CONFIG = PATRONI_CONFIG
-
-
-def etcd_is_running():
-    # if we have already started etcd
-    if etcd_handle and etcd_handle.pid and (etcd_handle.poll() is None):
-        return True
-    # if etcd is running, but we didn't start it
-    try:
-        r = requests.get(ETCD_VERSION_URL)
-        if r and r.ok and 'etcdserver' in r.content:
-            return True
-    except requests.ConnectionError:
-        pass
-    return False
 
 
+# actions to execute on start/stop of the tests and before running invidual features
 @before.all
 def start_etcd():
-    if not etcd_is_running():
-        global etcd_handle
-        global etcd_dir
-        etcd_dir = tempfile.mkdtemp()
-        etcd_handle = subprocess.Popen(["etcd", "--data-dir", etcd_dir], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        if not etcd_is_running():
-            assert False, "Failed to start etcd"
+    etcd_ctl.start()
 
 
 @after.all
-def stop_etcd(total):
-    global etcd_handle
-    global etcd_dir
-    if etcd_is_running() and etcd_handle:
-        etcd_handle.terminate()
-        etcd_handle = None
-        shutil.rmtree(etcd_dir)
-        etcd_dir = None
+def stop_etcd(*args, **kwargs):
+    etcd_ctl.stop_and_remove_work_directory()
 
 
 @before.each_feature
 def make_test_output_dir(feature):
-    feature_dir = os.path.join(pctl.patroni_path, "features", "output", feature.name.encode('utf-8').replace(' ', '_'))
-    if os.path.exists(feature_dir):
-        shutil.rmtree(feature_dir)
-    os.makedirs(feature_dir)
-    pctl.output_dir = feature_dir
-
-
-def patroni_cleanup_all():
-    pctl.stop_all()
-    # remove the data directory
-    shutil.rmtree(os.path.join(pctl.patroni_path, 'data'))
-
-
-def etcd_cleanup():
-    try:
-        r = requests.delete(ETCD_CLEANUP_URL)
-        if not r.ok:
-            raise Exception('{}'.format(r.reason))
-    except Exception as e:
-        assert False, "Unable to cleanup etcd: {0}".format(e)
+    """ create per-feature output directory to collect Patroni and PostgreSQL logs """
+    pctl.create_and_set_output_directory(feature.name)
 
 
 @after.each_feature
-def cleanup(scenario):
-    patroni_cleanup_all()
-    etcd_cleanup()
+def cleanup(*args, **kwargs):
+    """ stop all Patronis, remove their data directory and cleanup the keys in etcd """
+    pctl.stop_all()
+    shutil.rmtree(os.path.join(pctl.patroni_path, 'data'))
+    etcd_ctl.cleanup_service_tree()

From 481a80a3cecb44d6d109a8b52d135e7e9f05cae8 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Thu, 25 Feb 2016 14:39:22 +0100
Subject: [PATCH 08/33] Fix another couple of warnings from the QuantifiedCode
 and Co.

---
 features/terrain.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/features/terrain.py b/features/terrain.py
index 750c1316..03489c5e 100644
--- a/features/terrain.py
+++ b/features/terrain.py
@@ -42,7 +42,7 @@ class PatroniController(object):
             cwd = self.patroni_path
             self._log[pg_name] = open(os.path.join(self._output_dir, 'patroni_{0}.log'.format(pg_name)), 'a')
 
-            self._config[pg_name] = self._make_patroni_test_config(pg_name, self._output_dir)
+            self._config[pg_name] = self._make_patroni_test_config(pg_name)
 
             p = subprocess.Popen(['python', 'patroni.py', self._config[pg_name]],
                                  stdout=self._log[pg_name], stderr=subprocess.STDOUT, cwd=cwd)
@@ -109,16 +109,16 @@ class PatroniController(object):
     def _is_running(self, pg_name):
         return pg_name in self._processes and self._processes[pg_name].pid and (self._processes[pg_name].poll() is None)
 
-    def _make_patroni_test_config(self, pg_name, output_dir):
+    def _make_patroni_test_config(self, pg_name):
         patroni_config_name = PatroniController.PATRONI_CONFIG.format(pg_name)
-        patroni_config_path = os.path.join(output_dir, patroni_config_name)
+        patroni_config_path = os.path.join(self._output_dir, patroni_config_name)
 
         with open(patroni_config_name) as f:
             config = yaml.load(f)
         postgresql = config['postgresql']['parameters']
         postgresql['logging_collector'] = 'on'
         postgresql['log_destination'] = 'csvlog'
-        postgresql['log_directory'] = output_dir
+        postgresql['log_directory'] = self._output_dir
         postgresql['log_filename'] = '{0}.log'.format(pg_name)
         postgresql['log_statement'] = 'all'
         postgresql['log_min_messages'] = 'debug1'
@@ -215,11 +215,10 @@ class EtcdController(object):
         # if etcd is running, but we didn't start it
         try:
             r = requests.get(EtcdController.ETCD_VERSION_URL)
-            if r and r.ok and 'etcdserver' in r.content:
-                return True
+            running = (r and r.ok and 'etcdserver' in r.content)
         except requests.ConnectionError:
-            pass
-        return False
+            running = False
+        return running
 
 
 pctl = PatroniController()

From 4a8edf44e649a167961c885f22700b7d3250ece9 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Thu, 25 Feb 2016 14:55:29 +0100
Subject: [PATCH 09/33] Convert normal methods to static methods when possible.

---
 features/basic_replication.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/features/basic_replication.py b/features/basic_replication.py
index 77351428..3a39b8b7 100644
--- a/features/basic_replication.py
+++ b/features/basic_replication.py
@@ -10,19 +10,23 @@ class BasicReplicationSteps(object):
     def __init__(self, environ):
         self.env = environ
 
-    def start_patroni(self, step, name):
+    @staticmethod
+    def start_patroni(step, name):
         '''I start (\w+)'''
         return world.pctl.start(name)
 
-    def stop_patroni(self, step, name):
+    @staticmethod
+    def stop_patroni(step, name):
         '''I shut down (\w+)'''
         return world.pctl.stop(name)
 
-    def kill_patroni(self, step, name):
+    @staticmethod
+    def kill_patroni(step, name):
         '''I kill (\w+)'''
         return world.pctl.stop(name, kill=True)
 
-    def add_table(self, step, table_name, pg_name):
+    @staticmethod
+    def add_table(step, table_name, pg_name):
         '''I add the table (\w+) to (\w+)'''
         # parse the configuration file and get the port
         try:
@@ -30,7 +34,8 @@ class BasicReplicationSteps(object):
         except pg.Error as e:
             assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e)
 
-    def table_is_present_on(self, step, table_name, pg_name, max_replication_delay):
+    @staticmethod
+    def table_is_present_on(step, table_name, pg_name, max_replication_delay):
         '''Table (\w+) is present on (\w+) after (\d+) seconds'''
         for i in range(int(max_replication_delay)):
             if world.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
@@ -40,7 +45,8 @@ class BasicReplicationSteps(object):
             assert False,\
                 "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay)
 
-    def check_role(self, step, pg_name, pg_role, max_promotion_timeout):
+    @staticmethod
+    def check_role(step, pg_name, pg_role, max_promotion_timeout):
         '''(\w+) role is the (\w+) after (\d+) seconds'''
         if not world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)):
             assert False, "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)

From 67f55b460668d4868e1b60f74689b16759b65b04 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Thu, 25 Feb 2016 15:11:20 +0100
Subject: [PATCH 10/33] Stylistic issues: clearly mark unused variables.

---
 features/basic_replication.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/features/basic_replication.py b/features/basic_replication.py
index 3a39b8b7..0d2b7172 100644
--- a/features/basic_replication.py
+++ b/features/basic_replication.py
@@ -37,7 +37,7 @@ class BasicReplicationSteps(object):
     @staticmethod
     def table_is_present_on(step, table_name, pg_name, max_replication_delay):
         '''Table (\w+) is present on (\w+) after (\d+) seconds'''
-        for i in range(int(max_replication_delay)):
+        for _ in range(int(max_replication_delay)):
             if world.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
                 break
             sleep(1)

From a84a3fc5e1a3dde792d6e9dc05634caf2ac26545 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Thu, 25 Feb 2016 15:16:35 +0100
Subject: [PATCH 11/33] Changeset missing in the previous commit.

---
 features/terrain.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/features/terrain.py b/features/terrain.py
index 03489c5e..8a8396bb 100644
--- a/features/terrain.py
+++ b/features/terrain.py
@@ -50,7 +50,7 @@ class PatroniController(object):
                 assert False, "PostgreSQL {0} is not running after being started".format(pg_name)
             self._processes[pg_name] = p
         # wait while patroni is available for queries, but not more than 10 seconds.
-        for tick in range(max_wait_limit):
+        for _ in range(max_wait_limit):
             if self.query(pg_name, "SELECT 1", fail_ok=True) is not None:
                 break
             time.sleep(1)

From 83b7c34b003f47181d68f526b531f55be2ae0256 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Thu, 25 Feb 2016 15:35:15 +0100
Subject: [PATCH 12/33] Do not try to close an already closed file.

---
 features/terrain.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/features/terrain.py b/features/terrain.py
index 8a8396bb..76fd345e 100644
--- a/features/terrain.py
+++ b/features/terrain.py
@@ -65,7 +65,8 @@ class PatroniController(object):
             else:
                 self._processes[pg_name].kill()
             time.sleep(1)
-        self._log.get('pg_name') and self._log[pg_name].close()
+        if self._log.get('pg_name') and not self._log['pg_name'].closed:
+            self._log[pg_name].close()
         if pg_name in self._processes:
             del self._processes[pg_name]
 

From 4e9ebf48a80b862e27ebdb879d871b481a3d97ce Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Fri, 26 Feb 2016 17:37:37 +0100
Subject: [PATCH 13/33] Add API tests for a stand-alone node. Bugfixes.

Add tests for patroni API.
Fix test failures when an already running etcd is used.
---
 features/patroni_api.feature | 21 +++++++++
 features/patroni_api.py      | 83 ++++++++++++++++++++++++++++++++++++
 features/terrain.py          | 27 +++++++++---
 3 files changed, 124 insertions(+), 7 deletions(-)
 create mode 100644 features/patroni_api.feature
 create mode 100644 features/patroni_api.py

diff --git a/features/patroni_api.feature b/features/patroni_api.feature
new file mode 100644
index 00000000..3ff28584
--- /dev/null
+++ b/features/patroni_api.feature
@@ -0,0 +1,21 @@
+Feature: patroni api
+	We should check that patroni correctly responds to valid and not-valid API requests.
+
+Scenario: check API requests on a stand-alone server
+	Given I start postgres0
+	And postgres0 is a leader after 10 seconds
+	When I issue a GET request to http://127.0.0.1:8008/
+	Then I receive a response code 200
+	And I receive a response state running
+	And I receive a response role master
+	When I issue a GET request to http://127.0.0.1:8008/replica
+	Then I receive a response code 503
+	When I issue an empty POST request to http://127.0.0.1:8008/reinitialize
+	Then I receive a response code 503
+	And I receive a response text "I am the leader, can not reinitialize"
+	When I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0
+	Then I receive a response code 503
+	And I receive a response text "failover is not possible: cluster does not have members except leader"
+	When I issue an empty POST request to http://127.0.0.1:8008/failover
+	Then I receive a response code 503
+	And I receive a response text "No values given for required parameters leader and member"
\ No newline at end of file
diff --git a/features/patroni_api.py b/features/patroni_api.py
new file mode 100644
index 00000000..96d6e04a
--- /dev/null
+++ b/features/patroni_api.py
@@ -0,0 +1,83 @@
+from lettuce import world, steps
+import time
+import requests
+
+
+@steps
+class PatroniAPISteps(object):
+
+    def __init__(self, environ):
+        self.env = environ
+        self.response = None
+        self.status_code = None
+
+    # there is no way we can find out if the node has already
+    # started as a leader without checking the DCS. We cannot
+    # just rely on the database availability, since there is
+    # a short gap between the time PostgreSQL becomes available
+    # and Patroni assuming the leader role.
+    @staticmethod
+    def is_a_leader(step, name, time_limit):
+        '''(\w+) is a leader after (\d+) seconds'''
+        max_time = time.time() + int(time_limit)
+        while (world.etcd_ctl.query("leader") != name):
+            time.sleep(1)
+            if time.time() > max_time:
+                assert False, "{0} is not a leader in etcd after {1} seconds".format(name, time_limit)
+
+    @staticmethod
+    def sleep_for_n_seconds(step, value):
+        '''I sleep for (\d+) seconds'''
+        time.sleep(int(value))
+
+    def do_get(self, step, url):
+        '''I issue a GET request to (https?://(?:\w|\.|:|/)+)'''
+        try:
+            r = requests.get(url)
+        except requests.exceptions.RequestException:
+            self.code = None
+            self.response = None
+        else:
+            self.status_code = r.status_code
+            try:
+                self.response = r.json()
+            except ValueError:
+                self.response = r.content
+
+    def do_post_empty(self, step, url):
+        '''I issue an empty POST request to (https?://(?:\w|\.|:|/)+)'''
+        self.do_post(step, url, None)
+
+    def do_post(self, step, url, data):
+        '''I issue a POST request to (https?://(?:\w|\.|:|/)+) with (\s*\w+\s*=\s*\w+\s*,?)+'''
+        post_data = {}
+        if data:
+            post_components = data.split(',')
+            for pc in post_components:
+                if '=' in pc:
+                    k, v = pc.split('=', 2)
+                    post_data[k.strip()] = v.strip()
+        try:
+            r = requests.post(url, json=post_data)
+        except requests.exceptions.RequestException:
+            self.code = None
+            self.response = None
+        else:
+            self.status_code = r.status_code
+            try:
+                self.response = r.json()
+            except ValueError:
+                self.response = r.content
+
+    def check_response(self, step, component, data):
+        '''I receive a response (\w+) (.*)'''
+        if component == 'code':
+            assert self.status_code == int(data), "status code {0} != {1}".format(self.status_code, int(data))
+        elif component == 'text':
+            assert self.response == data.strip('"'), "response {0} does not contain {1}".format(self.response, data)
+        else:
+            assert component in self.response, "{0} is not part of the response".format(component)
+            assert self.response[component] == data, "{0} does not contain {1}".format(component, data)
+
+
+PatroniAPISteps(world)
diff --git a/features/terrain.py b/features/terrain.py
index 76fd345e..dd1aeade 100644
--- a/features/terrain.py
+++ b/features/terrain.py
@@ -116,13 +116,15 @@ class PatroniController(object):
 
         with open(patroni_config_name) as f:
             config = yaml.load(f)
-        postgresql = config['postgresql']['parameters']
-        postgresql['logging_collector'] = 'on'
-        postgresql['log_destination'] = 'csvlog'
-        postgresql['log_directory'] = self._output_dir
-        postgresql['log_filename'] = '{0}.log'.format(pg_name)
-        postgresql['log_statement'] = 'all'
-        postgresql['log_min_messages'] = 'debug1'
+        postgresql = config['postgresql']
+        postgresql['name'] = pg_name.encode('utf-8')
+        postgresql_params = postgresql['parameters']
+        postgresql_params['logging_collector'] = 'on'
+        postgresql_params['log_destination'] = 'csvlog'
+        postgresql_params['log_directory'] = self._output_dir
+        postgresql_params['log_filename'] = '{0}.log'.format(pg_name)
+        postgresql_params['log_statement'] = 'all'
+        postgresql_params['log_min_messages'] = 'debug1'
 
         with open(patroni_config_path, 'w') as f:
             yaml.dump(config, f, default_flow_style=False)
@@ -188,6 +190,15 @@ class EtcdController(object):
             time.sleep(1)
         return True
 
+    def query(self, key):
+        """ query etcd for a value of a given key """
+        r = requests.get("http://127.0.0.1:2379/v2/keys/service/batman/{0}".format(key))
+        if r.ok:
+            content = r.json()
+            if content:
+                return content.get('node', {}).get('value', None)
+        return None
+
     def stop_and_remove_work_directory(self):
         """ terminate etcd and wipe out the temp work directory, but only if we actually started it"""
         if self._is_running() and self.handle:
@@ -226,12 +237,14 @@ pctl = PatroniController()
 etcd_ctl = EtcdController()
 # export pctl to manage patroni from scenario files
 world.pctl = pctl
+world.etcd_ctl = etcd_ctl
 
 
 # actions to execute on start/stop of the tests and before running invidual features
 @before.all
 def start_etcd():
     etcd_ctl.start()
+    etcd_ctl.cleanup_service_tree()
 
 
 @after.all

From 0d44e3eb7cc2e71f8449903640343dcfbb9ac165 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Fri, 26 Feb 2016 18:00:11 +0100
Subject: [PATCH 14/33] Add simple API tests for 2 nodes, to be extended.

---
 features/patroni_api.feature | 12 +++++++++++-
 features/patroni_api.py      |  7 +++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/features/patroni_api.feature b/features/patroni_api.feature
index 3ff28584..3649337f 100644
--- a/features/patroni_api.feature
+++ b/features/patroni_api.feature
@@ -18,4 +18,14 @@ Scenario: check API requests on a stand-alone server
 	And I receive a response text "failover is not possible: cluster does not have members except leader"
 	When I issue an empty POST request to http://127.0.0.1:8008/failover
 	Then I receive a response code 503
-	And I receive a response text "No values given for required parameters leader and member"
\ No newline at end of file
+	And I receive a response text "No values given for required parameters leader and member"
+
+Scenario: check API requests for the primary-replica pair
+	Given I start postgres1
+	And replication works after 10 seconds
+	When I issue a GET request to http://127.0.0.1:8009/replica
+	Then I receive a response code 200
+	And I receive a response state running
+	And I receive a response role replica
+	When I issue an empty POST request to http://127.0.0.1:8009/reinitialize
+	Then I receive a response code 200
diff --git a/features/patroni_api.py b/features/patroni_api.py
index 96d6e04a..188f93cb 100644
--- a/features/patroni_api.py
+++ b/features/patroni_api.py
@@ -79,5 +79,12 @@ class PatroniAPISteps(object):
             assert component in self.response, "{0} is not part of the response".format(component)
             assert self.response[component] == data, "{0} does not contain {1}".format(component, data)
 
+    def replication_works(self, step, time_limit):
+        '''And replication works after (\d+) seconds'''
+        step.behave_as("""
+           When I add the table foo to postgres0
+            Then table foo is present on postgres1 after {0} seconds
+            """.format(time_limit))
+
 
 PatroniAPISteps(world)

From fa1a7687e5f1c5bf1ea450221501ddd9ff56d5b1 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 1 Mar 2016 22:00:30 +0100
Subject: [PATCH 15/33] Correct the step definition, randomize the table.

Make sure the step definition does not include "command" worlds.
Use the table name that includes current timestamp in the tests.
---
 features/patroni_api.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/features/patroni_api.py b/features/patroni_api.py
index 188f93cb..44de0f56 100644
--- a/features/patroni_api.py
+++ b/features/patroni_api.py
@@ -49,7 +49,7 @@ class PatroniAPISteps(object):
         self.do_post(step, url, None)
 
     def do_post(self, step, url, data):
-        '''I issue a POST request to (https?://(?:\w|\.|:|/)+) with (\s*\w+\s*=\s*\w+\s*,?)+'''
+        '''I issue a POST request to (https?://(?:\w|\.|:|/)+) with ((?:\s*\w+\s*=\s*\w+\s*,?)+)'''
         post_data = {}
         if data:
             post_components = data.split(',')
@@ -80,11 +80,11 @@ class PatroniAPISteps(object):
             assert self.response[component] == data, "{0} does not contain {1}".format(component, data)
 
     def replication_works(self, step, time_limit):
-        '''And replication works after (\d+) seconds'''
+        '''replication works after (\d+) seconds'''
         step.behave_as("""
-           When I add the table foo to postgres0
-            Then table foo is present on postgres1 after {0} seconds
-            """.format(time_limit))
+           When I add the table test_{0} to postgres0
+            Then table test_{0} is present on postgres1 after {1} seconds
+            """.format(int(time.time()), time_limit))
 
 
 PatroniAPISteps(world)

From ed15f7cd730e60c4784ae6ad2048454a054340cb Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 1 Mar 2016 22:03:38 +0100
Subject: [PATCH 16/33] Improve tests start/stop, add etcd logging.

Toggle the etcd debug logging and write the log to the test dir.

Make sure etcd and patroni are terminated when the tests finish
by sending SIGKILL in case SIGTERM does not work.

Make sure before.all code does the proper cleanup when the exception
is thrown.
---
 features/terrain.py | 47 +++++++++++++++++++++++++++++++--------------
 1 file changed, 33 insertions(+), 14 deletions(-)

diff --git a/features/terrain.py b/features/terrain.py
index dd1aeade..19841a19 100644
--- a/features/terrain.py
+++ b/features/terrain.py
@@ -58,13 +58,16 @@ class PatroniController(object):
             assert False,\
                 "Patroni instance is not available for queries after {0} seconds".format(max_wait_limit)
 
-    def stop(self, pg_name, kill=False):
+    def stop(self, pg_name, kill=False, timeout=15):
+        start_time = time.time()
         while self._is_running(pg_name):
             if not kill:
                 self._processes[pg_name].terminate()
             else:
                 self._processes[pg_name].kill()
             time.sleep(1)
+            if not kill and time.time() - start_time > timeout:
+                kill = True
         if self._log.get('pg_name') and not self._log['pg_name'].closed:
             self._log[pg_name].close()
         if pg_name in self._processes:
@@ -170,19 +173,24 @@ class EtcdController(object):
     ETCD_VERSION_URL = 'http://127.0.0.1:2379/version'
     ETCD_CLEANUP_URL = 'http://127.0.0.1:2379/v2/keys/service/batman?recursive=true'
 
-    def __init__(self):
+    def __init__(self, log_directory):
         self.handle = None
         self.work_directory = None
+        self.log_directory = log_directory
+        self.log_file = None
         self.pid = None
-        self.start_timeot = 5
+        self.start_timeout = 5
 
     def start(self):
         """ start etcd if it's not already running """
         if self._is_running():
             return True
         self.work_directory = tempfile.mkdtemp()
+        # etcd is running throughout the tests, no need to append to the log
+        self.log_file = open(os.path.join(self.log_directory, "features", "output", 'etcd.log'), 'w')
         self.handle =\
-            subprocess.Popen(["etcd", "--data-dir", self.work_directory], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            subprocess.Popen(["etcd", "--debug", "--data-dir", self.work_directory],
+                             stdout=self.log_file, stderr=subprocess.STDOUT)
         start_time = time.time()
         while (not self._is_running()):
             if time.time() - start_time > self.start_timeout:
@@ -199,11 +207,21 @@ class EtcdController(object):
                 return content.get('node', {}).get('value', None)
         return None
 
-    def stop_and_remove_work_directory(self):
+    def stop_and_remove_work_directory(self, timeout=15):
         """ terminate etcd and wipe out the temp work directory, but only if we actually started it"""
-        if self._is_running() and self.handle:
-            self.handle.terminate()
-            self.handle = None
+        kill = False
+        start_time = time.time()
+        while self._is_running() and self.handle:
+            if not kill:
+                self.handle.terminate()
+            else:
+                self.handle.kill()
+            time.sleep(1)
+            if not kill and time.time() - start_time > timeout:
+                kill = True
+        self.handle = None
+        if self.log_file and not self.log_file.closed:
+            self.log_file.close()
         if self.work_directory:
             shutil.rmtree(self.work_directory)
             self.work_directory = None
@@ -218,12 +236,9 @@ class EtcdController(object):
                 assert False,\
                     "request to cleanup the etcd contents was not successfull: status code {0}".format(r.status_code)
         except requests.exceptions.RequestException as e:
-            assert False, "exception when cleanin up etcd contents: {0}".format(e)
+            assert False, "exception when cleaning up etcd contents: {0}".format(e)
 
     def _is_running(self):
-        # if we have already started etcd
-        if self.handle and self.handle.pid and (self.handle.poll() is None):
-            return True
         # if etcd is running, but we didn't start it
         try:
             r = requests.get(EtcdController.ETCD_VERSION_URL)
@@ -234,7 +249,7 @@ class EtcdController(object):
 
 
 pctl = PatroniController()
-etcd_ctl = EtcdController()
+etcd_ctl = EtcdController(pctl.patroni_path)
 # export pctl to manage patroni from scenario files
 world.pctl = pctl
 world.etcd_ctl = etcd_ctl
@@ -244,7 +259,11 @@ world.etcd_ctl = etcd_ctl
 @before.all
 def start_etcd():
     etcd_ctl.start()
-    etcd_ctl.cleanup_service_tree()
+    try:
+        etcd_ctl.cleanup_service_tree()
+    except AssertionError:  # after.all handlers won't be executed in before.all
+        etcd_ctl.stop_and_remove_work_directory()
+        raise
 
 
 @after.all

From 24ebcc72f6a25bb8f5cf53d6efe00bdd4e27c890 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Tue, 1 Mar 2016 22:07:18 +0100
Subject: [PATCH 17/33] Add more tests for the restart and promotion.

---
 features/patroni_api.feature | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/features/patroni_api.feature b/features/patroni_api.feature
index 3649337f..c7779af1 100644
--- a/features/patroni_api.feature
+++ b/features/patroni_api.feature
@@ -29,3 +29,12 @@ Scenario: check API requests for the primary-replica pair
 	And I receive a response role replica
 	When I issue an empty POST request to http://127.0.0.1:8009/reinitialize
 	Then I receive a response code 200
+	Given replication works after 10 seconds
+	When I issue an empty POST request to http://127.0.0.1:8008/restart
+	Then I receive a response code 200
+	And postgres0 is a leader after 5 seconds
+
+Scenario: check promotion via the API
+	Given I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0,candidate=postgres1
+	Then I receive a response code 200
+	And postgres1 is a leader after 10 seconds

From 069440be15c4834316c13806b38879cd99e65086 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Wed, 2 Mar 2016 15:43:44 +0100
Subject: [PATCH 18/33] Improve the "replication work" sentence definition.

Add an ability to specify the origin and the destination for
the replication works clause. Use this ability in the API
promotion test to ensure the replication from the former
replica to the former master.
---
 features/patroni_api.feature |  7 ++++---
 features/patroni_api.py      | 10 +++++-----
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/features/patroni_api.feature b/features/patroni_api.feature
index c7779af1..78f014e2 100644
--- a/features/patroni_api.feature
+++ b/features/patroni_api.feature
@@ -22,14 +22,14 @@ Scenario: check API requests on a stand-alone server
 
 Scenario: check API requests for the primary-replica pair
 	Given I start postgres1
-	And replication works after 10 seconds
+	And replication works from postgres0 to postgres1 after 15 seconds
 	When I issue a GET request to http://127.0.0.1:8009/replica
 	Then I receive a response code 200
 	And I receive a response state running
 	And I receive a response role replica
 	When I issue an empty POST request to http://127.0.0.1:8009/reinitialize
 	Then I receive a response code 200
-	Given replication works after 10 seconds
+	Given replication works from postgres0 to postgres1 after 10 seconds
 	When I issue an empty POST request to http://127.0.0.1:8008/restart
 	Then I receive a response code 200
 	And postgres0 is a leader after 5 seconds
@@ -37,4 +37,5 @@ Scenario: check API requests for the primary-replica pair
 Scenario: check promotion via the API
 	Given I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0,candidate=postgres1
 	Then I receive a response code 200
-	And postgres1 is a leader after 10 seconds
+	And postgres1 is a leader after 5 seconds
+	And replication works from postgres1 to postgres0 after 15 seconds
diff --git a/features/patroni_api.py b/features/patroni_api.py
index 44de0f56..37650d99 100644
--- a/features/patroni_api.py
+++ b/features/patroni_api.py
@@ -79,12 +79,12 @@ class PatroniAPISteps(object):
             assert component in self.response, "{0} is not part of the response".format(component)
             assert self.response[component] == data, "{0} does not contain {1}".format(component, data)
 
-    def replication_works(self, step, time_limit):
-        '''replication works after (\d+) seconds'''
+    def replication_works(self, step, master, replica, time_limit):
+        '''replication works from (\w+) to (\w+) after (\d+) seconds'''
         step.behave_as("""
-           When I add the table test_{0} to postgres0
-            Then table test_{0} is present on postgres1 after {1} seconds
-            """.format(int(time.time()), time_limit))
+           When I add the table test_{0} to {1}
+            Then table test_{0} is present on {2} after {3} seconds
+            """.format(int(time.time()), master, replica, time_limit))
 
 
 PatroniAPISteps(world)

From 3f1c34f5570d1e0f4165e8d765f3def73c2fd4d9 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Wed, 2 Mar 2016 19:39:12 +0100
Subject: [PATCH 19/33] Add tests for the scheduled failover.

The actual amount of time to establish the master and the replication
after the scheduled failover seems sufficient (15 seconds with the
failover in 10 seconds), but occasionally leads to test failures.
This is unlikely the test issue and should be investigated inside
the patroni.
---
 features/patroni_api.feature | 13 ++++++++++---
 features/patroni_api.py      | 13 +++++++++++--
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/features/patroni_api.feature b/features/patroni_api.feature
index 78f014e2..6eedc26c 100644
--- a/features/patroni_api.feature
+++ b/features/patroni_api.feature
@@ -14,10 +14,10 @@ Scenario: check API requests on a stand-alone server
 	Then I receive a response code 503
 	And I receive a response text "I am the leader, can not reinitialize"
 	When I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0
-	Then I receive a response code 503
+	Then I receive a response code 500
 	And I receive a response text "failover is not possible: cluster does not have members except leader"
 	When I issue an empty POST request to http://127.0.0.1:8008/failover
-	Then I receive a response code 503
+	Then I receive a response code 500
 	And I receive a response text "No values given for required parameters leader and member"
 
 Scenario: check API requests for the primary-replica pair
@@ -34,8 +34,15 @@ Scenario: check API requests for the primary-replica pair
 	Then I receive a response code 200
 	And postgres0 is a leader after 5 seconds
 
-Scenario: check promotion via the API
+Scenario: check the failover via the API
 	Given I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0,candidate=postgres1
 	Then I receive a response code 200
 	And postgres1 is a leader after 5 seconds
 	And replication works from postgres1 to postgres0 after 15 seconds
+
+Scenario: check the scheduled failover
+	Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgresq0 in 10 seconds
+	Then I receive a response code 200
+	And postgres0 is a leader after 15 seconds
+	And replication works from postgres0 to postgres1 after 25 seconds
+
diff --git a/features/patroni_api.py b/features/patroni_api.py
index 37650d99..0659bc2c 100644
--- a/features/patroni_api.py
+++ b/features/patroni_api.py
@@ -1,5 +1,7 @@
+from datetime import datetime, timedelta
 from lettuce import world, steps
 import time
+import pytz
 import requests
 
 
@@ -49,7 +51,7 @@ class PatroniAPISteps(object):
         self.do_post(step, url, None)
 
     def do_post(self, step, url, data):
-        '''I issue a POST request to (https?://(?:\w|\.|:|/)+) with ((?:\s*\w+\s*=\s*\w+\s*,?)+)'''
+        '''I issue a POST request to (https?://(?:\w|\.|:|/)+) with ((?:\w+=(?:\w|\.|:|-|\+|\s)+,?)+)'''
         post_data = {}
         if data:
             post_components = data.split(',')
@@ -72,7 +74,8 @@ class PatroniAPISteps(object):
     def check_response(self, step, component, data):
         '''I receive a response (\w+) (.*)'''
         if component == 'code':
-            assert self.status_code == int(data), "status code {0} != {1}".format(self.status_code, int(data))
+            assert self.status_code == int(data),\
+                    "status code {0} != {1}, response: {2}".format(self.status_code, int(data), self.response)
         elif component == 'text':
             assert self.response == data.strip('"'), "response {0} does not contain {1}".format(self.response, data)
         else:
@@ -86,5 +89,11 @@ class PatroniAPISteps(object):
             Then table test_{0} is present on {2} after {3} seconds
             """.format(int(time.time()), master, replica, time_limit))
 
+    def scheduld_failover(self, step, at_url, from_host, to_host, in_seconds):
+        '''I issue a scheduled failover at (https?://(?:\w|\.|:|/)+) from (\w+) to (\w+) in (\d+) seconds'''
+        step.behave_as("""
+            Given I issue a POST request to {0}/failover with leader={1},candidate={2},scheduled_at={3}
+            """.format(at_url, from_host, to_host, datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds))))
+
 
 PatroniAPISteps(world)

From 998f0da3d8c71f9ddd770549747a7e179f5e5985 Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Thu, 10 Mar 2016 16:05:06 +0100
Subject: [PATCH 20/33] Add cascading replication (backup from the replica)
 tests.

---
 features/cascading_replication.feature | 12 +++++++++++
 features/cascading_replication.py      | 26 ++++++++++++++++++++++
 features/terrain.py                    | 30 ++++++++++++++++++++++----
 3 files changed, 64 insertions(+), 4 deletions(-)
 create mode 100644 features/cascading_replication.feature
 create mode 100644 features/cascading_replication.py

diff --git a/features/cascading_replication.feature b/features/cascading_replication.feature
new file mode 100644
index 00000000..31500ae9
--- /dev/null
+++ b/features/cascading_replication.feature
@@ -0,0 +1,12 @@
+Feature: cascading replication
+	We should check that patroni can do base backup and streaming from the replica
+
+Scenario: check a base backup from the replica
+	Given I start postgres0
+	And I start postgres1
+	And replication works from postgres0 to postgres1 after 15 seconds
+	And I create label with "postgres0" in postgres0 data directory
+	And I create label with "postgres1" in postgres1 data directory
+	And I configure and start postgres2 with a tag clonefrom postgres1
+	Then replication works from postgres0 to postgres2 after 15 seconds
+	And there is a label with "postgres1" in postgres2 data directory
\ No newline at end of file
diff --git a/features/cascading_replication.py b/features/cascading_replication.py
new file mode 100644
index 00000000..a5733aa7
--- /dev/null
+++ b/features/cascading_replication.py
@@ -0,0 +1,26 @@
+from lettuce import world, steps
+
+
+@steps
+class CascadingReplicationSteps(object):
+
+    def __init__(self, environ):
+        self.env = environ
+
+    @staticmethod
+    def start_patroni_with_a_name_value_tag(step, name, tag_name, tag_value):
+        '''I configure and start (\w+) with a tag (\w+) (\w+)'''
+        return world.pctl.start(name, tags={tag_name: tag_value})
+
+    @staticmethod
+    def check_label(step, content, name):
+        '''There is a label with "(\w+)" in (\w+) data directory'''
+        label = world.pctl.read_label(name)
+        assert label == content, "{0} is not equal to {1}".format(label, content)
+
+    @staticmethod
+    def write_label(step, content, name):
+        '''I create label with "(\w+)" in (\w+) data directory'''
+        world.pctl.write_label(name, content)
+
+CascadingReplicationSteps(world)
diff --git a/features/terrain.py b/features/terrain.py
index 19841a19..01450070 100644
--- a/features/terrain.py
+++ b/features/terrain.py
@@ -35,14 +35,30 @@ class PatroniController(object):
             self._patroni_path = cwd
         return self._patroni_path
 
-    def start(self, pg_name, max_wait_limit=15):
+    def data_dir(self, pg_name):
+        return os.path.join(self.patroni_path, 'data', pg_name)
+
+    def write_label(self, pg_name, content):
+        with open(os.path.join(self.data_dir(pg_name.encode('utf-8')), 'label'), 'w') as f:
+            f.write(content.encode('utf-8'))
+
+    def read_label(self, pg_name):
+        content = None
+        try:
+            with open(os.path.join(self.data_dir(pg_name.encode('utf-8')), 'label'), 'r') as f:
+                content = f.read()
+        except IOError:
+            return None
+        return content.strip()
+
+    def start(self, pg_name, max_wait_limit=15, tags=None):
         if not self._is_running(pg_name):
             if pg_name in self._processes:
                 del self._processes[pg_name]
             cwd = self.patroni_path
             self._log[pg_name] = open(os.path.join(self._output_dir, 'patroni_{0}.log'.format(pg_name)), 'a')
 
-            self._config[pg_name] = self._make_patroni_test_config(pg_name)
+            self._config[pg_name] = self._make_patroni_test_config(pg_name, tags=tags)
 
             p = subprocess.Popen(['python', 'patroni.py', self._config[pg_name]],
                                  stdout=self._log[pg_name], stderr=subprocess.STDOUT, cwd=cwd)
@@ -113,7 +129,7 @@ class PatroniController(object):
     def _is_running(self, pg_name):
         return pg_name in self._processes and self._processes[pg_name].pid and (self._processes[pg_name].poll() is None)
 
-    def _make_patroni_test_config(self, pg_name):
+    def _make_patroni_test_config(self, pg_name, tags=None):
         patroni_config_name = PatroniController.PATRONI_CONFIG.format(pg_name)
         patroni_config_path = os.path.join(self._output_dir, patroni_config_name)
 
@@ -121,6 +137,7 @@ class PatroniController(object):
             config = yaml.load(f)
         postgresql = config['postgresql']
         postgresql['name'] = pg_name.encode('utf-8')
+        postgresql['data_dir'] = 'data/{0}'.format(pg_name.encode('utf-8'))
         postgresql_params = postgresql['parameters']
         postgresql_params['logging_collector'] = 'on'
         postgresql_params['log_destination'] = 'csvlog'
@@ -129,6 +146,11 @@ class PatroniController(object):
         postgresql_params['log_statement'] = 'all'
         postgresql_params['log_min_messages'] = 'debug1'
 
+        if tags:
+            config['tags'] = {}
+            for tag_name in tags:
+                config['tags'][tag_name.encode('utf-8')] = tags[tag_name].encode('utf-8')
+
         with open(patroni_config_path, 'w') as f:
             yaml.dump(config, f, default_flow_style=False)
 
@@ -141,7 +163,7 @@ class PatroniController(object):
             patroni_path = self.patroni_path
             with open(os.path.join(patroni_path, PatroniController.PATRONI_CONFIG.format(pg_name)), 'r') as f:
                 config = yaml.load(f)
-        except OSError:
+        except IOError:
             return None
         connstring = config['postgresql']['connect_address']
         if ':' in connstring:

From 42d798a3de965b8f9ead9e7a3b64b3b7a252a2bc Mon Sep 17 00:00:00 2001
From: Alexander Kukushkin <alexander.kukushkin@zalando.de>
Date: Thu, 10 Mar 2016 17:19:10 +0100
Subject: [PATCH 21/33] acceptance tests on travis

---
 .travis.yml                        | 13 +++++++++++++
 acceptance_tests.sh                | 16 ++++++++++++++++
 features/basic_replication.feature |  2 +-
 features/basic_replication.py      |  3 ++-
 features/patroni_api.feature       |  2 +-
 features/patroni_api.py            |  2 +-
 features/terrain.py                |  9 +++++++--
 patroni/ha.py                      |  4 ++--
 patroni/postgresql.py              |  4 ++--
 9 files changed, 45 insertions(+), 10 deletions(-)
 create mode 100644 acceptance_tests.sh

diff --git a/.travis.yml b/.travis.yml
index 23c69358..c1ccd74e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,15 +1,28 @@
+sudo: required
 language: python
+addons:
+  postgresql: "9.5"
 python:
   - "2.7"
   - "3.4"
   - "3.5"
 install:
+  - sudo /etc/init.d/postgresql stop
+  - sudo apt-get -y remove --purge postgresql-9.1 postgresql-9.2 postgresql-9.3 postgresql-9.4
+  - sudo apt-get -y autoremove
+  - sudo apt-key adv --keyserver keys.gnupg.net --recv-keys 7FCC7D46ACCC4CF8
+  - sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main 9.5" >> /etc/apt/sources.list.d/postgresql.list'
+  - sudo apt-get update
+  - sudo apt-get -y install postgresql-9.5
+  - sudo /etc/init.d/postgresql stop
+  - pip uninstall boto
   - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py2.txt --use-mirrors; fi
   - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r requirements-py3.txt; fi
   - pip install coveralls codacy-coverage
 script:
   - python setup.py test
   - python setup.py flake8
+  - bash -x acceptance_tests.sh
 after_success:
   - coveralls
   - python-codacy-coverage -r coverage.xml
diff --git a/acceptance_tests.sh b/acceptance_tests.sh
new file mode 100644
index 00000000..2ccc233d
--- /dev/null
+++ b/acceptance_tests.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+ETCDVERSION=2.2.5
+
+BINDIR=bin
+[ -d $BINDIR ] || mkdir $BINDIR
+
+export PATH=$BINDIR:$PATH
+
+# Add etcd
+curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C $BINDIR --strip=1 --wildcards --no-anchored etcd etcdctl
+
+sudo pip2.7 install lettuce python-Levenshtein
+sudo pip2.7 install -r requirements-py2.txt
+
+lettuce
diff --git a/features/basic_replication.feature b/features/basic_replication.feature
index fe766c11..aa07a250 100644
--- a/features/basic_replication.feature
+++ b/features/basic_replication.feature
@@ -11,6 +11,6 @@ Feature: basic replication
     When I kill postgres0
     Then postgres1 role is the primary after 30 seconds
     When I start postgres0
-    Then postgres0 role is the secondary after 10 seconds
+    Then postgres0 role is the secondary after 15 seconds
     When I add the table bar to postgres1
     Then table bar is present on postgres0 after 10 seconds
diff --git a/features/basic_replication.py b/features/basic_replication.py
index 0d2b7172..97913fc6 100644
--- a/features/basic_replication.py
+++ b/features/basic_replication.py
@@ -49,7 +49,8 @@ class BasicReplicationSteps(object):
     def check_role(step, pg_name, pg_role, max_promotion_timeout):
         '''(\w+) role is the (\w+) after (\d+) seconds'''
         if not world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)):
-            assert False, "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
+            assert False,\
+                "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
 
 
 BasicReplicationSteps(world)
diff --git a/features/patroni_api.feature b/features/patroni_api.feature
index 6eedc26c..f97e059f 100644
--- a/features/patroni_api.feature
+++ b/features/patroni_api.feature
@@ -17,7 +17,7 @@ Scenario: check API requests on a stand-alone server
 	Then I receive a response code 500
 	And I receive a response text "failover is not possible: cluster does not have members except leader"
 	When I issue an empty POST request to http://127.0.0.1:8008/failover
-	Then I receive a response code 500
+	Then I receive a response code 400
 	And I receive a response text "No values given for required parameters leader and member"
 
 Scenario: check API requests for the primary-replica pair
diff --git a/features/patroni_api.py b/features/patroni_api.py
index 0659bc2c..632bafa3 100644
--- a/features/patroni_api.py
+++ b/features/patroni_api.py
@@ -75,7 +75,7 @@ class PatroniAPISteps(object):
         '''I receive a response (\w+) (.*)'''
         if component == 'code':
             assert self.status_code == int(data),\
-                    "status code {0} != {1}, response: {2}".format(self.status_code, int(data), self.response)
+                "status code {0} != {1}, response: {2}".format(self.status_code, int(data), self.response)
         elif component == 'text':
             assert self.response == data.strip('"'), "response {0} does not contain {1}".format(self.response, data)
         else:
diff --git a/features/terrain.py b/features/terrain.py
index 01450070..3a845ec5 100644
--- a/features/terrain.py
+++ b/features/terrain.py
@@ -120,7 +120,8 @@ class PatroniController(object):
             self.stop(patroni)
 
     def create_and_set_output_directory(self, feature_name):
-        feature_dir = os.path.join(pctl.patroni_path, "features", "output", feature_name.encode('utf-8').replace(' ', '_'))
+        feature_dir = os.path.join(pctl.patroni_path, "features", "output",
+                                   feature_name.encode('utf-8').replace(' ', '_'))
         if os.path.exists(feature_dir):
             shutil.rmtree(feature_dir)
         os.makedirs(feature_dir)
@@ -145,6 +146,7 @@ class PatroniController(object):
         postgresql_params['log_filename'] = '{0}.log'.format(pg_name)
         postgresql_params['log_statement'] = 'all'
         postgresql_params['log_min_messages'] = 'debug1'
+        postgresql_params['unix_socket_directories'] = '.'
 
         if tags:
             config['tags'] = {}
@@ -209,7 +211,10 @@ class EtcdController(object):
             return True
         self.work_directory = tempfile.mkdtemp()
         # etcd is running throughout the tests, no need to append to the log
-        self.log_file = open(os.path.join(self.log_directory, "features", "output", 'etcd.log'), 'w')
+        output_dir = os.path.join(self.log_directory, "features", "output")
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+        self.log_file = open(os.path.join(output_dir, 'etcd.log'), 'w')
         self.handle =\
             subprocess.Popen(["etcd", "--debug", "--data-dir", self.work_directory],
                              stdout=self.log_file, stderr=subprocess.STDOUT)
diff --git a/patroni/ha.py b/patroni/ha.py
index 98ba4398..f9550e81 100644
--- a/patroni/ha.py
+++ b/patroni/ha.py
@@ -457,8 +457,8 @@ class Ha(object):
             else:
                 # check if we are allowed to join
                 if self.sysid_valid(self.cluster.initialize) and self.cluster.initialize != self.state_handler.sysid:
-                    logger.fatal("system ID mismatch, node {0} belongs to a different cluster".
-                                 format(self.state_handler.name))
+                    logger.fatal("system ID mismatch, node %s belongs to a different cluster: %s != %s",
+                                 self.state_handler.name, self.cluster.initialize, self.state_handler.sysid)
                     sys.exit(1)
 
             # try to start dead postgres
diff --git a/patroni/postgresql.py b/patroni/postgresql.py
index a1cedb58..d875057b 100644
--- a/patroni/postgresql.py
+++ b/patroni/postgresql.py
@@ -207,7 +207,7 @@ class Postgresql(object):
                 options.append('--username={0}'.format(self.superuser['username']))
             if 'password' in self.superuser:
                 (fd, pwfile) = tempfile.mkstemp()
-                os.write(fd, self.superuser['password'].encode())
+                os.write(fd, self.superuser['password'].encode('utf-8'))
                 os.close(fd)
                 options.append('--pwfile={0}'.format(pwfile))
 
@@ -506,7 +506,7 @@ recovery_target_timeline = 'latest'
         try:
             data = subprocess.check_output(['pg_controldata', self.data_dir])
             if data:
-                data = data.decode().splitlines()
+                data = data.decode('utf-8').splitlines()
                 result = {l.split(':')[0].replace('Current ', '', 1): l.split(':')[1].strip() for l in data if l}
         except subprocess.CalledProcessError:
             logger.exception("Error when calling pg_controldata")

From c2d1eea7d05d3c009239761aefbd0dc44f383a19 Mon Sep 17 00:00:00 2001
From: Alexander Kukushkin <alexander.kukushkin@zalando.de>
Date: Thu, 10 Mar 2016 17:19:43 +0100
Subject: [PATCH 22/33] disable clonefrom test

---
 features/cascading_replication.feature | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/features/cascading_replication.feature b/features/cascading_replication.feature
index 31500ae9..37693311 100644
--- a/features/cascading_replication.feature
+++ b/features/cascading_replication.feature
@@ -8,5 +8,5 @@ Scenario: check a base backup from the replica
 	And I create label with "postgres0" in postgres0 data directory
 	And I create label with "postgres1" in postgres1 data directory
 	And I configure and start postgres2 with a tag clonefrom postgres1
-	Then replication works from postgres0 to postgres2 after 15 seconds
-	And there is a label with "postgres1" in postgres2 data directory
\ No newline at end of file
+	Then replication works from postgres0 to postgres2 after 30 seconds
+	And there is a label with "postgres0" in postgres2 data directory

From 33a1de7828cf97298f972329c78ec7d302dc5a95 Mon Sep 17 00:00:00 2001
From: Alexander Kukushkin <alexander.kukushkin@zalando.de>
Date: Thu, 10 Mar 2016 17:23:44 +0100
Subject: [PATCH 23/33] Fix .travis.yml

---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index c1ccd74e..58d2b988 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,7 +15,6 @@ install:
   - sudo apt-get update
   - sudo apt-get -y install postgresql-9.5
   - sudo /etc/init.d/postgresql stop
-  - pip uninstall boto
   - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py2.txt --use-mirrors; fi
   - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r requirements-py3.txt; fi
   - pip install coveralls codacy-coverage

From c955e298057d87a8fee4d5dd9a42104a1b567d0f Mon Sep 17 00:00:00 2001
From: Alexander Kukushkin <wmd@mail.ru>
Date: Thu, 10 Mar 2016 20:02:40 +0100
Subject: [PATCH 24/33] Disable gce boto plugins by overriding BOTO_CONFIG

These plugins are not compatible with python 3 and breaking unit tests
---
 .travis.yml         | 2 ++
 acceptance_tests.sh | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 58d2b988..3bf864f3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,6 +2,8 @@ sudo: required
 language: python
 addons:
   postgresql: "9.5"
+env:
+  - BOTO_CONFIG=''
 python:
   - "2.7"
   - "3.4"
diff --git a/acceptance_tests.sh b/acceptance_tests.sh
index 2ccc233d..d1fbf8d9 100644
--- a/acceptance_tests.sh
+++ b/acceptance_tests.sh
@@ -13,4 +13,4 @@ curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v$
 sudo pip2.7 install lettuce python-Levenshtein
 sudo pip2.7 install -r requirements-py2.txt
 
-lettuce
+exec lettuce

From 30d3982d25d63946ea1c375360f620e2db062eaa Mon Sep 17 00:00:00 2001
From: Alexander Kukushkin <alexander.kukushkin@zalando.de>
Date: Fri, 11 Mar 2016 12:56:29 +0100
Subject: [PATCH 25/33] Acceptance tests with behave

---
 .travis.yml                             |  7 +-
 acceptance_tests.sh                     | 16 ----
 features/basic_replication.py           | 56 --------------
 features/cascading_replication.py       | 26 -------
 features/{terrain.py => environment.py} | 58 ++++++---------
 features/patroni_api.py                 | 99 -------------------------
 features/steps/basic_replication.py     | 55 ++++++++++++++
 features/steps/cascading_replication.py | 17 +++++
 features/steps/patroni_api.py           | 87 ++++++++++++++++++++++
 9 files changed, 186 insertions(+), 235 deletions(-)
 delete mode 100644 acceptance_tests.sh
 delete mode 100644 features/basic_replication.py
 delete mode 100644 features/cascading_replication.py
 rename features/{terrain.py => environment.py} (87%)
 delete mode 100644 features/patroni_api.py
 create mode 100644 features/steps/basic_replication.py
 create mode 100644 features/steps/cascading_replication.py
 create mode 100644 features/steps/patroni_api.py

diff --git a/.travis.yml b/.travis.yml
index 3bf864f3..073b6572 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,7 +3,7 @@ language: python
 addons:
   postgresql: "9.5"
 env:
-  - BOTO_CONFIG=''
+  - BOTO_CONFIG='' ETCDVERSION=2.2.5
 python:
   - "2.7"
   - "3.4"
@@ -19,11 +19,12 @@ install:
   - sudo /etc/init.d/postgresql stop
   - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py2.txt --use-mirrors; fi
   - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r requirements-py3.txt; fi
-  - pip install coveralls codacy-coverage
+  - curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd
+  - pip install behave coveralls codacy-coverage
 script:
   - python setup.py test
   - python setup.py flake8
-  - bash -x acceptance_tests.sh
+  - PATH=.:$PATH behave
 after_success:
   - coveralls
   - python-codacy-coverage -r coverage.xml
diff --git a/acceptance_tests.sh b/acceptance_tests.sh
deleted file mode 100644
index d1fbf8d9..00000000
--- a/acceptance_tests.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash
-
-ETCDVERSION=2.2.5
-
-BINDIR=bin
-[ -d $BINDIR ] || mkdir $BINDIR
-
-export PATH=$BINDIR:$PATH
-
-# Add etcd
-curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C $BINDIR --strip=1 --wildcards --no-anchored etcd etcdctl
-
-sudo pip2.7 install lettuce python-Levenshtein
-sudo pip2.7 install -r requirements-py2.txt
-
-exec lettuce
diff --git a/features/basic_replication.py b/features/basic_replication.py
deleted file mode 100644
index 97913fc6..00000000
--- a/features/basic_replication.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import psycopg2 as pg
-from time import sleep
-
-from lettuce import world, steps
-
-
-@steps
-class BasicReplicationSteps(object):
-
-    def __init__(self, environ):
-        self.env = environ
-
-    @staticmethod
-    def start_patroni(step, name):
-        '''I start (\w+)'''
-        return world.pctl.start(name)
-
-    @staticmethod
-    def stop_patroni(step, name):
-        '''I shut down (\w+)'''
-        return world.pctl.stop(name)
-
-    @staticmethod
-    def kill_patroni(step, name):
-        '''I kill (\w+)'''
-        return world.pctl.stop(name, kill=True)
-
-    @staticmethod
-    def add_table(step, table_name, pg_name):
-        '''I add the table (\w+) to (\w+)'''
-        # parse the configuration file and get the port
-        try:
-            world.pctl.query(pg_name, "CREATE TABLE {0}()".format(table_name))
-        except pg.Error as e:
-            assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e)
-
-    @staticmethod
-    def table_is_present_on(step, table_name, pg_name, max_replication_delay):
-        '''Table (\w+) is present on (\w+) after (\d+) seconds'''
-        for _ in range(int(max_replication_delay)):
-            if world.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
-                break
-            sleep(1)
-        else:
-            assert False,\
-                "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay)
-
-    @staticmethod
-    def check_role(step, pg_name, pg_role, max_promotion_timeout):
-        '''(\w+) role is the (\w+) after (\d+) seconds'''
-        if not world.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)):
-            assert False,\
-                "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
-
-
-BasicReplicationSteps(world)
diff --git a/features/cascading_replication.py b/features/cascading_replication.py
deleted file mode 100644
index a5733aa7..00000000
--- a/features/cascading_replication.py
+++ /dev/null
@@ -1,26 +0,0 @@
-from lettuce import world, steps
-
-
-@steps
-class CascadingReplicationSteps(object):
-
-    def __init__(self, environ):
-        self.env = environ
-
-    @staticmethod
-    def start_patroni_with_a_name_value_tag(step, name, tag_name, tag_value):
-        '''I configure and start (\w+) with a tag (\w+) (\w+)'''
-        return world.pctl.start(name, tags={tag_name: tag_value})
-
-    @staticmethod
-    def check_label(step, content, name):
-        '''There is a label with "(\w+)" in (\w+) data directory'''
-        label = world.pctl.read_label(name)
-        assert label == content, "{0} is not equal to {1}".format(label, content)
-
-    @staticmethod
-    def write_label(step, content, name):
-        '''I create label with "(\w+)" in (\w+) data directory'''
-        world.pctl.write_label(name, content)
-
-CascadingReplicationSteps(world)
diff --git a/features/terrain.py b/features/environment.py
similarity index 87%
rename from features/terrain.py
rename to features/environment.py
index 3a845ec5..90207ab0 100644
--- a/features/terrain.py
+++ b/features/environment.py
@@ -1,4 +1,3 @@
-from lettuce import world, before, after
 import os.path
 import psycopg2
 import requests
@@ -39,13 +38,13 @@ class PatroniController(object):
         return os.path.join(self.patroni_path, 'data', pg_name)
 
     def write_label(self, pg_name, content):
-        with open(os.path.join(self.data_dir(pg_name.encode('utf-8')), 'label'), 'w') as f:
-            f.write(content.encode('utf-8'))
+        with open(os.path.join(self.data_dir(pg_name), 'label'), 'w') as f:
+            f.write(content)
 
     def read_label(self, pg_name):
         content = None
         try:
-            with open(os.path.join(self.data_dir(pg_name.encode('utf-8')), 'label'), 'r') as f:
+            with open(os.path.join(self.data_dir(pg_name), 'label'), 'r') as f:
                 content = f.read()
         except IOError:
             return None
@@ -120,8 +119,8 @@ class PatroniController(object):
             self.stop(patroni)
 
     def create_and_set_output_directory(self, feature_name):
-        feature_dir = os.path.join(pctl.patroni_path, "features", "output",
-                                   feature_name.encode('utf-8').replace(' ', '_'))
+        feature_dir = os.path.join(self.patroni_path, "features", "output",
+                                   feature_name.replace(' ', '_'))
         if os.path.exists(feature_dir):
             shutil.rmtree(feature_dir)
         os.makedirs(feature_dir)
@@ -137,8 +136,8 @@ class PatroniController(object):
         with open(patroni_config_name) as f:
             config = yaml.load(f)
         postgresql = config['postgresql']
-        postgresql['name'] = pg_name.encode('utf-8')
-        postgresql['data_dir'] = 'data/{0}'.format(pg_name.encode('utf-8'))
+        postgresql['name'] = pg_name
+        postgresql['data_dir'] = 'data/{0}'.format(pg_name)
         postgresql_params = postgresql['parameters']
         postgresql_params['logging_collector'] = 'on'
         postgresql_params['log_destination'] = 'csvlog'
@@ -149,9 +148,7 @@ class PatroniController(object):
         postgresql_params['unix_socket_directories'] = '.'
 
         if tags:
-            config['tags'] = {}
-            for tag_name in tags:
-                config['tags'][tag_name.encode('utf-8')] = tags[tag_name].encode('utf-8')
+            config['tags'] = tags
 
         with open(patroni_config_path, 'w') as f:
             yaml.dump(config, f, default_flow_style=False)
@@ -269,44 +266,35 @@ class EtcdController(object):
         # if etcd is running, but we didn't start it
         try:
             r = requests.get(EtcdController.ETCD_VERSION_URL)
-            running = (r and r.ok and 'etcdserver' in r.content)
+            running = (r and r.ok and b'etcdserver' in r.content)
         except requests.ConnectionError:
             running = False
         return running
 
 
-pctl = PatroniController()
-etcd_ctl = EtcdController(pctl.patroni_path)
-# export pctl to manage patroni from scenario files
-world.pctl = pctl
-world.etcd_ctl = etcd_ctl
-
-
 # actions to execute on start/stop of the tests and before running invidual features
-@before.all
-def start_etcd():
-    etcd_ctl.start()
+def before_all(context):
+    context.pctl = PatroniController()
+    context.etcd_ctl = EtcdController(context.pctl.patroni_path)
+    context.etcd_ctl.start()
     try:
-        etcd_ctl.cleanup_service_tree()
+        context.etcd_ctl.cleanup_service_tree()
     except AssertionError:  # after.all handlers won't be executed in before.all
-        etcd_ctl.stop_and_remove_work_directory()
+        context.etcd_ctl.stop_and_remove_work_directory()
         raise
 
 
-@after.all
-def stop_etcd(*args, **kwargs):
-    etcd_ctl.stop_and_remove_work_directory()
+def after_all(context):
+    context.etcd_ctl.stop_and_remove_work_directory()
 
 
-@before.each_feature
-def make_test_output_dir(feature):
+def before_feature(context, feature):
     """ create per-feature output directory to collect Patroni and PostgreSQL logs """
-    pctl.create_and_set_output_directory(feature.name)
+    context.pctl.create_and_set_output_directory(feature.name)
 
 
-@after.each_feature
-def cleanup(*args, **kwargs):
+def after_feature(context, feature):
     """ stop all Patronis, remove their data directory and cleanup the keys in etcd """
-    pctl.stop_all()
-    shutil.rmtree(os.path.join(pctl.patroni_path, 'data'))
-    etcd_ctl.cleanup_service_tree()
+    context.pctl.stop_all()
+    shutil.rmtree(os.path.join(context.pctl.patroni_path, 'data'))
+    context.etcd_ctl.cleanup_service_tree()
diff --git a/features/patroni_api.py b/features/patroni_api.py
deleted file mode 100644
index 632bafa3..00000000
--- a/features/patroni_api.py
+++ /dev/null
@@ -1,99 +0,0 @@
-from datetime import datetime, timedelta
-from lettuce import world, steps
-import time
-import pytz
-import requests
-
-
-@steps
-class PatroniAPISteps(object):
-
-    def __init__(self, environ):
-        self.env = environ
-        self.response = None
-        self.status_code = None
-
-    # there is no way we can find out if the node has already
-    # started as a leader without checking the DCS. We cannot
-    # just rely on the database availability, since there is
-    # a short gap between the time PostgreSQL becomes available
-    # and Patroni assuming the leader role.
-    @staticmethod
-    def is_a_leader(step, name, time_limit):
-        '''(\w+) is a leader after (\d+) seconds'''
-        max_time = time.time() + int(time_limit)
-        while (world.etcd_ctl.query("leader") != name):
-            time.sleep(1)
-            if time.time() > max_time:
-                assert False, "{0} is not a leader in etcd after {1} seconds".format(name, time_limit)
-
-    @staticmethod
-    def sleep_for_n_seconds(step, value):
-        '''I sleep for (\d+) seconds'''
-        time.sleep(int(value))
-
-    def do_get(self, step, url):
-        '''I issue a GET request to (https?://(?:\w|\.|:|/)+)'''
-        try:
-            r = requests.get(url)
-        except requests.exceptions.RequestException:
-            self.code = None
-            self.response = None
-        else:
-            self.status_code = r.status_code
-            try:
-                self.response = r.json()
-            except ValueError:
-                self.response = r.content
-
-    def do_post_empty(self, step, url):
-        '''I issue an empty POST request to (https?://(?:\w|\.|:|/)+)'''
-        self.do_post(step, url, None)
-
-    def do_post(self, step, url, data):
-        '''I issue a POST request to (https?://(?:\w|\.|:|/)+) with ((?:\w+=(?:\w|\.|:|-|\+|\s)+,?)+)'''
-        post_data = {}
-        if data:
-            post_components = data.split(',')
-            for pc in post_components:
-                if '=' in pc:
-                    k, v = pc.split('=', 2)
-                    post_data[k.strip()] = v.strip()
-        try:
-            r = requests.post(url, json=post_data)
-        except requests.exceptions.RequestException:
-            self.code = None
-            self.response = None
-        else:
-            self.status_code = r.status_code
-            try:
-                self.response = r.json()
-            except ValueError:
-                self.response = r.content
-
-    def check_response(self, step, component, data):
-        '''I receive a response (\w+) (.*)'''
-        if component == 'code':
-            assert self.status_code == int(data),\
-                "status code {0} != {1}, response: {2}".format(self.status_code, int(data), self.response)
-        elif component == 'text':
-            assert self.response == data.strip('"'), "response {0} does not contain {1}".format(self.response, data)
-        else:
-            assert component in self.response, "{0} is not part of the response".format(component)
-            assert self.response[component] == data, "{0} does not contain {1}".format(component, data)
-
-    def replication_works(self, step, master, replica, time_limit):
-        '''replication works from (\w+) to (\w+) after (\d+) seconds'''
-        step.behave_as("""
-           When I add the table test_{0} to {1}
-            Then table test_{0} is present on {2} after {3} seconds
-            """.format(int(time.time()), master, replica, time_limit))
-
-    def scheduld_failover(self, step, at_url, from_host, to_host, in_seconds):
-        '''I issue a scheduled failover at (https?://(?:\w|\.|:|/)+) from (\w+) to (\w+) in (\d+) seconds'''
-        step.behave_as("""
-            Given I issue a POST request to {0}/failover with leader={1},candidate={2},scheduled_at={3}
-            """.format(at_url, from_host, to_host, datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds))))
-
-
-PatroniAPISteps(world)
diff --git a/features/steps/basic_replication.py b/features/steps/basic_replication.py
new file mode 100644
index 00000000..98f5879f
--- /dev/null
+++ b/features/steps/basic_replication.py
@@ -0,0 +1,55 @@
+import psycopg2 as pg
+
+from behave import step, then
+from time import sleep, time
+
+
+@step('I start {name}')
+def start_patroni(context, name):
+    return context.pctl.start(name)
+
+
+@step('I shut down {name}')
+def stop_patroni(context, name):
+    return context.pctl.stop(name)
+
+
+@step('I kill {name}')
+def kill_patroni(context, name):
+    return context.pctl.stop(name, kill=True)
+
+
+@step('I add the table {table_name} to {pg_name}')
+def add_table(context, table_name, pg_name):
+    # parse the configuration file and get the port
+    try:
+        context.pctl.query(pg_name, "CREATE TABLE {0}()".format(table_name))
+    except pg.Error as e:
+        assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e)
+
+
+@then('Table {table_name} is present on {pg_name} after {max_replication_delay} seconds')
+def table_is_present_on(context, table_name, pg_name, max_replication_delay):
+    for _ in range(int(max_replication_delay)):
+        if context.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
+            break
+        sleep(1)
+    else:
+        assert False,\
+            "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay)
+
+
+@then('{pg_name} role is the {pg_role} after {max_promotion_timeout} seconds')
+def check_role(context, pg_name, pg_role, max_promotion_timeout):
+    if not context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)):
+        assert False,\
+            "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
+
+
+@step('replication works from {master} to {replica} after {time_limit} seconds')
+@then('replication works from {master} to {replica} after {time_limit} seconds')
+def replication_works(context, master, replica, time_limit):
+    context.execute_steps("""
+        When I add the table test_{0} to {1}
+        Then table test_{0} is present on {2} after {3} seconds
+    """.format(int(time()), master, replica, time_limit))
diff --git a/features/steps/cascading_replication.py b/features/steps/cascading_replication.py
new file mode 100644
index 00000000..07d8fbd2
--- /dev/null
+++ b/features/steps/cascading_replication.py
@@ -0,0 +1,17 @@
+from behave import step, then
+
+
+@step('I configure and start {name} with a tag {tag_name} {tag_value}')
+def start_patroni_with_a_name_value_tag(context, name, tag_name, tag_value):
+    return context.pctl.start(name, tags={tag_name: tag_value})
+
+
+@then('There is a label with "{content}" in {name} data directory')
+def check_label(context, content, name):
+    label = context.pctl.read_label(name)
+    assert label == content, "{0} is not equal to {1}".format(label, content)
+
+
+@step('I create label with "{content}" in {name} data directory')
+def write_label(context, content, name):
+    context.pctl.write_label(name, content)
diff --git a/features/steps/patroni_api.py b/features/steps/patroni_api.py
new file mode 100644
index 00000000..309a20af
--- /dev/null
+++ b/features/steps/patroni_api.py
@@ -0,0 +1,87 @@
+import time
+import pytz
+import requests
+
+from datetime import datetime, timedelta
+from behave import step, then
+
+
+# there is no way we can find out if the node has already
+# started as a leader without checking the DCS. We cannot
+# just rely on the database availability, since there is
+# a short gap between the time PostgreSQL becomes available
+# and Patroni assuming the leader role.
+@step('{name} is a leader after {time_limit} seconds')
+@then('{name} is a leader after {time_limit} seconds')
+def is_a_leader(context, name, time_limit):
+    max_time = time.time() + int(time_limit)
+    while (context.etcd_ctl.query("leader") != name):
+        time.sleep(1)
+        if time.time() > max_time:
+            assert False, "{0} is not a leader in etcd after {1} seconds".format(name, time_limit)
+
+
+@step('I sleep for {value} seconds')
+def sleep_for_n_seconds(context, value):
+    time.sleep(int(value))
+
+
+@step('I issue a GET request to {url}')
+def do_get(context, url):
+    try:
+        r = requests.get(url)
+    except requests.exceptions.RequestException:
+        context.status_code = None
+        context.response = None
+    else:
+        context.status_code = r.status_code
+        try:
+            context.response = r.json()
+        except ValueError:
+            context.response = r.content.decode('utf-8')
+
+
+@step('I issue an empty POST request to {url}')
+def do_post_empty(context, url):
+    do_post(context, url, None)
+
+
+@step('I issue a POST request to {url} with {data}')
+def do_post(context, url, data):
+    post_data = {}
+    if data:
+        post_components = data.split(',')
+        for pc in post_components:
+            if '=' in pc:
+                k, v = pc.split('=', 2)
+                post_data[k.strip()] = v.strip()
+    try:
+        r = requests.post(url, json=post_data)
+    except requests.exceptions.RequestException:
+        context.status_code = None
+        context.response = None
+    else:
+        context.status_code = r.status_code
+        try:
+            context.response = r.json()
+        except ValueError:
+            context.response = r.content.decode('utf-8')
+
+
+@then('I receive a response {component} {data}')
+def check_response(context, component, data):
+    if component == 'code':
+        assert context.status_code == int(data),\
+            "status code {0} != {1}, response: {2}".format(context.status_code, int(data), context.response)
+    elif component == 'text':
+        assert context.response == data.strip('"'), "response {0} does not contain {1}".format(context.response, data)
+    else:
+        assert component in context.response, "{0} is not part of the response".format(component)
+        assert context.response[component] == data, "{0} does not contain {1}".format(component, data)
+
+
+@step('I issue a scheduled failover at {at_url} from {from_host} to {to_host} in {in_seconds} seconds')
+def scheduld_failover(context, at_url, from_host, to_host, in_seconds):
+    context.execute_steps("""
+        Given I issue a POST request to {0}/failover with leader={1},candidate={2},scheduled_at={3}
+    """.format(at_url, from_host, to_host, datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds))))

From 8b81d270bcbdc4acfec09716cc021002f41f330b Mon Sep 17 00:00:00 2001
From: Alexander Kukushkin <alexander.kukushkin@zalando.de>
Date: Fri, 11 Mar 2016 13:47:57 +0100
Subject: [PATCH 26/33] BUGFIX: Assertion Failed: Steps must be unicode

---
 features/steps/basic_replication.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/features/steps/basic_replication.py b/features/steps/basic_replication.py
index 98f5879f..26af60e9 100644
--- a/features/steps/basic_replication.py
+++ b/features/steps/basic_replication.py
@@ -49,7 +49,7 @@ def check_role(context, pg_name, pg_role, max_promotion_timeout):
 @step('replication works from {master} to {replica} after {time_limit} seconds')
 @then('replication works from {master} to {replica} after {time_limit} seconds')
 def replication_works(context, master, replica, time_limit):
-    context.execute_steps("""
+    context.execute_steps(u"""
         When I add the table test_{0} to {1}
         Then table test_{0} is present on {2} after {3} seconds
     """.format(int(time()), master, replica, time_limit))

From 5f6beae22f6fe4383b3ae79c8a0fd409c6083073 Mon Sep 17 00:00:00 2001
From: Alexander Kukushkin <alexander.kukushkin@zalando.de>
Date: Fri, 11 Mar 2016 14:46:14 +0100
Subject: [PATCH 27/33] Enforce data-type checks for step matcher

and increase default timeout for patroni start
---
 features/environment.py                 |  6 ++--
 features/steps/basic_replication.py     | 16 +++++------
 features/steps/cascading_replication.py |  6 ++--
 features/steps/patroni_api.py           | 38 +++++++++++++++++--------
 4 files changed, 40 insertions(+), 26 deletions(-)

diff --git a/features/environment.py b/features/environment.py
index 90207ab0..f031f19b 100644
--- a/features/environment.py
+++ b/features/environment.py
@@ -1,8 +1,8 @@
-import os.path
+import os
 import psycopg2
 import requests
-import subprocess
 import shutil
+import subprocess
 import tempfile
 import time
 import yaml
@@ -50,7 +50,7 @@ class PatroniController(object):
             return None
         return content.strip()
 
-    def start(self, pg_name, max_wait_limit=15, tags=None):
+    def start(self, pg_name, max_wait_limit=20, tags=None):
         if not self._is_running(pg_name):
             if pg_name in self._processes:
                 del self._processes[pg_name]
diff --git a/features/steps/basic_replication.py b/features/steps/basic_replication.py
index 26af60e9..364de61f 100644
--- a/features/steps/basic_replication.py
+++ b/features/steps/basic_replication.py
@@ -4,22 +4,22 @@ from behave import step, then
 from time import sleep, time
 
 
-@step('I start {name}')
+@step('I start {name:w}')
 def start_patroni(context, name):
     return context.pctl.start(name)
 
 
-@step('I shut down {name}')
+@step('I shut down {name:w}')
 def stop_patroni(context, name):
     return context.pctl.stop(name)
 
 
-@step('I kill {name}')
+@step('I kill {name:w}')
 def kill_patroni(context, name):
     return context.pctl.stop(name, kill=True)
 
 
-@step('I add the table {table_name} to {pg_name}')
+@step('I add the table {table_name:w} to {pg_name:w}')
 def add_table(context, table_name, pg_name):
     # parse the configuration file and get the port
     try:
@@ -28,7 +28,7 @@ def add_table(context, table_name, pg_name):
         assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e)
 
 
-@then('Table {table_name} is present on {pg_name} after {max_replication_delay} seconds')
+@then('Table {table_name:w} is present on {pg_name:w} after {max_replication_delay:d} seconds')
 def table_is_present_on(context, table_name, pg_name, max_replication_delay):
     for _ in range(int(max_replication_delay)):
         if context.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
@@ -39,15 +39,15 @@ def table_is_present_on(context, table_name, pg_name, max_replication_delay):
             "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay)
 
 
-@then('{pg_name} role is the {pg_role} after {max_promotion_timeout} seconds')
+@then('{pg_name:w} role is the {pg_role:w} after {max_promotion_timeout:d} seconds')
 def check_role(context, pg_name, pg_role, max_promotion_timeout):
     if not context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)):
         assert False,\
             "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
 
 
-@step('replication works from {master} to {replica} after {time_limit} seconds')
-@then('replication works from {master} to {replica} after {time_limit} seconds')
+@step('replication works from {master:w} to {replica:w} after {time_limit:d} seconds')
+@then('replication works from {master:w} to {replica:w} after {time_limit:d} seconds')
 def replication_works(context, master, replica, time_limit):
     context.execute_steps(u"""
         When I add the table test_{0} to {1}
diff --git a/features/steps/cascading_replication.py b/features/steps/cascading_replication.py
index 07d8fbd2..59399c97 100644
--- a/features/steps/cascading_replication.py
+++ b/features/steps/cascading_replication.py
@@ -1,17 +1,17 @@
 from behave import step, then
 
 
-@step('I configure and start {name} with a tag {tag_name} {tag_value}')
+@step('I configure and start {name:w} with a tag {tag_name:w} {tag_value:w}')
 def start_patroni_with_a_name_value_tag(context, name, tag_name, tag_value):
     return context.pctl.start(name, tags={tag_name: tag_value})
 
 
-@then('There is a label with "{content}" in {name} data directory')
+@then('There is a label with "{content:w}" in {name:w} data directory')
 def check_label(context, content, name):
     label = context.pctl.read_label(name)
     assert label == content, "{0} is not equal to {1}".format(label, content)
 
 
-@step('I create label with "{content}" in {name} data directory')
+@step('I create label with "{content:w}" in {name:w} data directory')
 def write_label(context, content, name):
     context.pctl.write_label(name, content)
diff --git a/features/steps/patroni_api.py b/features/steps/patroni_api.py
index 309a20af..0f802dee 100644
--- a/features/steps/patroni_api.py
+++ b/features/steps/patroni_api.py
@@ -1,9 +1,23 @@
-import time
+import parse
 import pytz
 import requests
+import time
 
+from behave import register_type, step, then
 from datetime import datetime, timedelta
-from behave import step, then
+
+
+@parse.with_pattern(r'https?://(?:\w|\.|:|/)+')
+def parse_url(text):
+    return text
+
+
+@parse.with_pattern(r'(?:\w+=(?:\w|\.|:|-|\+|\s)+,?)+')
+def parse_data(text):
+    return text
+
+
+register_type(url=parse_url, data=parse_data)
 
 
 # there is no way we can find out if the node has already
@@ -11,8 +25,8 @@ from behave import step, then
 # just rely on the database availability, since there is
 # a short gap between the time PostgreSQL becomes available
 # and Patroni assuming the leader role.
-@step('{name} is a leader after {time_limit} seconds')
-@then('{name} is a leader after {time_limit} seconds')
+@step('{name:w} is a leader after {time_limit:d} seconds')
+@then('{name:w} is a leader after {time_limit:d} seconds')
 def is_a_leader(context, name, time_limit):
     max_time = time.time() + int(time_limit)
     while (context.etcd_ctl.query("leader") != name):
@@ -21,12 +35,12 @@ def is_a_leader(context, name, time_limit):
             assert False, "{0} is not a leader in etcd after {1} seconds".format(name, time_limit)
 
 
-@step('I sleep for {value} seconds')
+@step('I sleep for {value:d} seconds')
 def sleep_for_n_seconds(context, value):
     time.sleep(int(value))
 
 
-@step('I issue a GET request to {url}')
+@step('I issue a GET request to {url:url}')
 def do_get(context, url):
     try:
         r = requests.get(url)
@@ -41,12 +55,12 @@ def do_get(context, url):
             context.response = r.content.decode('utf-8')
 
 
-@step('I issue an empty POST request to {url}')
+@step('I issue an empty POST request to {url:url}')
 def do_post_empty(context, url):
     do_post(context, url, None)
 
 
-@step('I issue a POST request to {url} with {data}')
+@step('I issue a POST request to {url:url} with {data:data}')
 def do_post(context, url, data):
     post_data = {}
     if data:
@@ -68,7 +82,7 @@ def do_post(context, url, data):
             context.response = r.content.decode('utf-8')
 
 
-@then('I receive a response {component} {data}')
+@then('I receive a response {component:w} {data}')
 def check_response(context, component, data):
     if component == 'code':
         assert context.status_code == int(data),\
@@ -80,8 +94,8 @@ def check_response(context, component, data):
         assert context.response[component] == data, "{0} does not contain {1}".format(component, data)
 
 
-@step('I issue a scheduled failover at {at_url} from {from_host} to {to_host} in {in_seconds} seconds')
-def scheduld_failover(context, at_url, from_host, to_host, in_seconds):
-    context.execute_steps("""
+@step('I issue a scheduled failover at {at_url:url} from {from_host:w} to {to_host:w} in {in_seconds:d} seconds')
+def scheduled_failover(context, at_url, from_host, to_host, in_seconds):
+    context.execute_steps(u"""
         Given I issue a POST request to {0}/failover with leader={1},candidate={2},scheduled_at={3}
     """.format(at_url, from_host, to_host, datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds))))

From ba444adb67674438fdf04e089396df68ae07a137 Mon Sep 17 00:00:00 2001
From: Alexander Kukushkin <alexander.kukushkin@zalando.de>
Date: Fri, 11 Mar 2016 15:32:16 +0100
Subject: [PATCH 28/33] make codacy and quantifiedcode happier

---
 features/environment.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/features/environment.py b/features/environment.py
index f031f19b..f822007b 100644
--- a/features/environment.py
+++ b/features/environment.py
@@ -222,13 +222,14 @@ class EtcdController(object):
             time.sleep(1)
         return True
 
-    def query(self, key):
+    @staticmethod
+    def query(key):
         """ query etcd for a value of a given key """
         r = requests.get("http://127.0.0.1:2379/v2/keys/service/batman/{0}".format(key))
         if r.ok:
             content = r.json()
             if content:
-                return content.get('node', {}).get('value', None)
+                return content.get('node', {}).get('value')
         return None
 
     def stop_and_remove_work_directory(self, timeout=15):
@@ -262,7 +263,8 @@ class EtcdController(object):
         except requests.exceptions.RequestException as e:
             assert False, "exception when cleaning up etcd contents: {0}".format(e)
 
-    def _is_running(self):
+    @staticmethod
+    def _is_running():
         # if etcd is running, but we didn't start it
         try:
             r = requests.get(EtcdController.ETCD_VERSION_URL)

From 6985df3aca1f20a306aebfb17c77c973432f2b8b Mon Sep 17 00:00:00 2001
From: Oleksii Kliukin <oleksii.kliukin@zalando.de>
Date: Fri, 11 Mar 2016 16:59:35 +0100
Subject: [PATCH 29/33] Restore the test for the clone from the replica.

---
 features/cascading_replication.feature | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/features/cascading_replication.feature b/features/cascading_replication.feature
index 37693311..7d297e41 100644
--- a/features/cascading_replication.feature
+++ b/features/cascading_replication.feature
@@ -9,4 +9,4 @@ Scenario: check a base backup from the replica
 	And I create label with "postgres1" in postgres1 data directory
 	And I configure and start postgres2 with a tag clonefrom postgres1
 	Then replication works from postgres0 to postgres2 after 30 seconds
-	And there is a label with "postgres0" in postgres2 data directory
+	And there is a label with "postgres1" in postgres2 data directory

From 62f11ab747dbef7af59ff4ff96c8a7c028c9936b Mon Sep 17 00:00:00 2001
From: Alexander Kukushkin <wmd@mail.ru>
Date: Sun, 13 Mar 2016 09:09:31 +0100
Subject: [PATCH 30/33] Attempt to export acceptance tests coverage results to
 coveralls

---
 .travis.yml                            | 11 ++++++-----
 features/basic_replication.feature     |  3 ++-
 features/cascading_replication.feature |  1 +
 features/environment.py                |  5 ++++-
 4 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 073b6572..f20910d1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,7 +3,8 @@ language: python
 addons:
   postgresql: "9.5"
 env:
-  - BOTO_CONFIG='' ETCDVERSION=2.2.5
+  - TEST_SUITE="python setup.py test"
+  - TEST_SUITE="behave"
 python:
   - "2.7"
   - "3.4"
@@ -19,12 +20,12 @@ install:
   - sudo /etc/init.d/postgresql stop
   - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py2.txt --use-mirrors; fi
   - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r requirements-py3.txt; fi
-  - curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd
-  - pip install behave coveralls codacy-coverage
+  - ETCDVERSION=2.2.5 curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd
+  - pip install behave codacy-coverage coverage coveralls 
 script:
   - python setup.py test
+  - BOTO_CONFIG='' PATH=.:$PATH $TEST_SUITE
   - python setup.py flake8
-  - PATH=.:$PATH behave
 after_success:
   - coveralls
-  - python-codacy-coverage -r coverage.xml
+  - if [[ -f coverage.xml ]]; then python-codacy-coverage -r coverage.xml; fi
diff --git a/features/basic_replication.feature b/features/basic_replication.feature
index aa07a250..b3307e40 100644
--- a/features/basic_replication.feature
+++ b/features/basic_replication.feature
@@ -3,9 +3,10 @@ Feature: basic replication
 
   Scenario: check replication of a single table
     Given I start postgres0
+    And postgres0 is a leader after 10 seconds
     And I start postgres1
     When I add the table foo to postgres0
-    Then table foo is present on postgres1 after 10 seconds
+    Then table foo is present on postgres1 after 15 seconds
 
   Scenario: check the basic failover
     When I kill postgres0
diff --git a/features/cascading_replication.feature b/features/cascading_replication.feature
index 7d297e41..8a3d2f80 100644
--- a/features/cascading_replication.feature
+++ b/features/cascading_replication.feature
@@ -3,6 +3,7 @@ Feature: cascading replication
 
 Scenario: check a base backup from the replica
 	Given I start postgres0
+	And postgres0 is a leader after 10 seconds
 	And I start postgres1
 	And replication works from postgres0 to postgres1 after 15 seconds
 	And I create label with "postgres0" in postgres0 data directory
diff --git a/features/environment.py b/features/environment.py
index f822007b..bd6d5054 100644
--- a/features/environment.py
+++ b/features/environment.py
@@ -59,7 +59,7 @@ class PatroniController(object):
 
             self._config[pg_name] = self._make_patroni_test_config(pg_name, tags=tags)
 
-            p = subprocess.Popen(['python', 'patroni.py', self._config[pg_name]],
+            p = subprocess.Popen(['coverage', 'run', '--branch', '--source=patroni', '-p', 'patroni.py', self._config[pg_name]],
                                  stdout=self._log[pg_name], stderr=subprocess.STDOUT, cwd=cwd)
             if not (p and p.pid and p.poll() is None):
                 assert False, "PostgreSQL {0} is not running after being started".format(pg_name)
@@ -190,6 +190,7 @@ class PatroniController(object):
 
 
 class EtcdController(object):
+
     """ handles all etcd related tasks, used for the tests setup and cleanup """
     ETCD_VERSION_URL = 'http://127.0.0.1:2379/version'
     ETCD_CLEANUP_URL = 'http://127.0.0.1:2379/v2/keys/service/batman?recursive=true'
@@ -300,3 +301,5 @@ def after_feature(context, feature):
     context.pctl.stop_all()
     shutil.rmtree(os.path.join(context.pctl.patroni_path, 'data'))
     context.etcd_ctl.cleanup_service_tree()
+    subprocess.call(['coverage', 'combine'])
+    subprocess.call(['coverage', 'report'])

From 7e0723a7fc5f94e38a9d75c9a9b4acaaa0087614 Mon Sep 17 00:00:00 2001
From: Alexander Kukushkin <wmd@mail.ru>
Date: Sun, 13 Mar 2016 09:17:52 +0100
Subject: [PATCH 31/33] Attempt to export acceptance tests coverage results to
 coveralls

---
 .travis.yml | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index f20910d1..6f92756a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,8 +3,11 @@ language: python
 addons:
   postgresql: "9.5"
 env:
-  - TEST_SUITE="python setup.py test"
-  - TEST_SUITE="behave"
+  global:
+    - BOTO_CONFIG='' ETCDVERSION=2.2.5
+  matrix:
+    - TEST_SUITE="python setup.py test"
+    - TEST_SUITE="behave"
 python:
   - "2.7"
   - "3.4"
@@ -20,11 +23,11 @@ install:
   - sudo /etc/init.d/postgresql stop
   - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py2.txt --use-mirrors; fi
   - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r requirements-py3.txt; fi
-  - ETCDVERSION=2.2.5 curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd
+  - curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd
   - pip install behave codacy-coverage coverage coveralls 
 script:
   - python setup.py test
-  - BOTO_CONFIG='' PATH=.:$PATH $TEST_SUITE
+  - PATH=.:$PATH $TEST_SUITE
   - python setup.py flake8
 after_success:
   - coveralls

From f3a238ccbcbca7da7919ffc26b8ba39f552f401f Mon Sep 17 00:00:00 2001
From: Alexander Kukushkin <wmd@mail.ru>
Date: Sun, 13 Mar 2016 09:24:01 +0100
Subject: [PATCH 32/33] Attempt to export acceptance tests coverage results to
 coveralls

---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 6f92756a..38dddc77 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -26,7 +26,6 @@ install:
   - curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd
   - pip install behave codacy-coverage coverage coveralls 
 script:
-  - python setup.py test
   - PATH=.:$PATH $TEST_SUITE
   - python setup.py flake8
 after_success:

From 79f4d9a13b63cd872f447efcbbda194aacf94f9e Mon Sep 17 00:00:00 2001
From: Alexander Kukushkin <wmd@mail.ru>
Date: Sun, 13 Mar 2016 09:42:02 +0100
Subject: [PATCH 33/33] Attempt to export acceptance tests coverage results to
 coveralls

---
 features/environment.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/features/environment.py b/features/environment.py
index bd6d5054..73f77011 100644
--- a/features/environment.py
+++ b/features/environment.py
@@ -289,6 +289,8 @@ def before_all(context):
 
 def after_all(context):
     context.etcd_ctl.stop_and_remove_work_directory()
+    subprocess.call(['coverage', 'combine'])
+    subprocess.call(['coverage', 'report'])
 
 
 def before_feature(context, feature):
@@ -301,5 +303,3 @@ def after_feature(context, feature):
     context.pctl.stop_all()
     shutil.rmtree(os.path.join(context.pctl.patroni_path, 'data'))
     context.etcd_ctl.cleanup_service_tree()
-    subprocess.call(['coverage', 'combine'])
-    subprocess.call(['coverage', 'report'])