Bumped version to 0.90

urllib3.exceptions.HTTPError fixes for python 3.5.1

.travis.yml

@@ -1,32 +1,82 @@
-sudo: required
+sudo: false
 language: python
+python:
+- "3.5"
 addons:
+  apt:
+    packages:
+    - postgresql-contrib-9.5
   postgresql: "9.5"
 env:
   global:
-    - BOTO_CONFIG='' ETCDVERSION=2.2.5
+  - ETCDVERSION=2.3.2 ZKVERSION=3.4.6 CONSULVERSION=0.6.4
   matrix:
-    - TEST_SUITE="python setup.py test"
-    - TEST_SUITE="behave"
-python:
-  - "2.7"
-  - "3.4"
-  - "3.5"
+  - TEST_SUITE="python setup.py"
+  - DCS="etcd" TEST_SUITE="behave"
+  - DCS="exhibitor" TEST_SUITE="behave"
+  - DCS="consul" TEST_SUITE="behave"
+cache:
+  directories:
+  - $HOME/virtualenv/python2.7.9
+  - $HOME/virtualenv/python3.4.2
+  - $HOME/virtualenv/python3.5.0
 install:
-  - sudo /etc/init.d/postgresql stop
-  - sudo apt-get -y remove --purge postgresql-9.1 postgresql-9.2 postgresql-9.3 postgresql-9.4
-  - sudo apt-get -y autoremove
-  - sudo apt-key adv --keyserver keys.gnupg.net --recv-keys 7FCC7D46ACCC4CF8
-  - sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main 9.5" >> /etc/apt/sources.list.d/postgresql.list'
-  - sudo apt-get update
-  - sudo apt-get -y install postgresql-9.5
-  - sudo /etc/init.d/postgresql stop
-  - pip install -r requirements.txt
-  - curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd
-  - pip install behave codacy-coverage coverage coveralls 
+  - |
+    set -e
+
+    if [[ $TEST_SUITE == "behave" ]]; then
+        if [[ $DCS == "consul" ]]; then
+            curl -L https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip \
+                | gunzip > consul
+            chmod +x consul
+        fi
+
+        if [[ $DCS == "etcd" ]]; then
+            curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
+                | tar xz -C . --strip=1 --wildcards --no-anchored etcd
+        fi
+
+        if [[ $DCS == "exhibitor" ]]; then
+            curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz
+            mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg
+            zookeeper-${ZKVERSION}/bin/zkServer.sh start
+            # following lines are 'emulating' exhibitor REST API
+            while true; do
+                echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \
+                    | nc -l 8181 &> /dev/null
+            done&
+        fi
+    fi
+
+    for pv in "2.7" "3.4" "3.5"; do
+        source ~/virtualenv/python${pv}/bin/activate
+        # explicitly install all needed python modules to cache them
+        for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8 mock pytest-cov pytest'; do
+            pip install $p
+        done
+    done
 script:
-  - PATH=.:$PATH $TEST_SUITE
-  - python setup.py flake8
+  - |
+    for pv in "2.7" "3.4" "3.5"; do
+        source ~/virtualenv/python${pv}/bin/activate
+
+        if [[ $TEST_SUITE == "behave" ]]; then
+            if [[ $pv != "3.4" ]]; then
+                echo Running acceptance tests using python${pv}
+                if ! PATH=.:$PATH $TEST_SUITE; then
+                    # output all log files when tests are failing
+                    grep . features/output/*/*postgres?.*
+                    exit 1
+                fi
+            fi
+        else
+            echo Running unit tests using python${pv}
+            $TEST_SUITE test
+            $TEST_SUITE flake8
+        fi
+    done
+
+    set +e
 after_success:
   - coveralls
-  - if [[ -f coverage.xml ]]; then python-codacy-coverage -r coverage.xml; fi
+  - if [[ $TEST_SUITE != "behave" ]]; then python-codacy-coverage -r coverage.xml; fi

README.rst

@@ -1,6 +1,6 @@
 |Build Status| |Coverage Status|
 
-Patroni: A Template for PostgreSQL HA with ZooKeeper or etcd
+Patroni: A Template for PostgreSQL HA with ZooKeeper, etcd or Consul
 ------------------------------------------------------------
 
 Patroni was previously known as Governor.
@@ -8,7 +8,7 @@ Patroni was previously known as Governor.
 *There are many ways to run high availability with PostgreSQL. Here, we
 present a template for you to create your own customized, high-availability 
 solution using Python and — for maximum accessibility — a distributed 
-configuration store like ZooKeeper or etcd.*
+configuration store like ZooKeeper, etcd or Consul.*
 
 Getting Started
 ---------------
@@ -61,12 +61,17 @@ For an example file, see ``postgres0.yml``. Regarding settings:
     -  *keyfile*: (optional) Specifies a file with the secret key in the PEM format.
 
 -  *etcd*:
-    -  *scope*: the relative path used on etcd's HTTP API for this deployment; makes it possible to run multiple HA deployments from a single etcd.
+    -  *scope*: the relative path used on etcd's HTTP API for this deployment; makes it possible to run multiple HA deployments from a single etcd cluster.
     -  *ttl*: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process.
     -  *host*: the host:port for the etcd endpoint.
 
+-  *consul*:
+    -  *scope*: the relative path used on Consul's HTTP API for this deployment; makes it possible to run multiple HA deployments from a single Consul cluster.
+    -  *ttl*: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process.
+    -  *host*: the host:port for the Consul endpoint.
+
 -  *zookeeper*:
-    -  *scope*: the relative path used on etcd's HTTP API for this deployment; makes it possible to run multiple HA deployments from a single etcd.
+    -  *scope*: the relative path used on ZooKeeper for this deployment; makes it possible to run multiple HA deployments from a single ZooKeeper cluster.
     -  *session\_timeout*: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process.
     -  *reconnect\_timeout*: how long we should try to reconnect to ZooKeeper after a connection loss. After this timeout, assume that you no longer have a lock and restart in read-only mode.
     -  *hosts*: list of ZooKeeper cluster members in format: ['host1:port1', 'host2:port2', 'etc...']
@@ -91,11 +96,11 @@ For an example file, see ``postgres0.yml``. Regarding settings:
 
     -  *pg\_hba*: list of lines which should be added to pg\_hba.conf.
         -  *- host all all 0.0.0.0/0 md5*.
+        -  *- host replication replicator 127.0.0.1/32 md5* # A line like this is required for replication
 
     -  *replication*:
         -  *username*: replication username; user will be created during initialization.
         -  *password*: replication password; user will be created during initialization.
-        -  *network*: network setting for replication in pg\_hba.conf.
 
     -  *callbacks* callback scripts to run on certain actions. Patroni will pass the action, role and cluster name. See scripts/aws.py as an example on how to write them.
         -  *on\_start*: a script to run when the cluster starts.

extras/startup-scripts/patroni.service

@@ -17,6 +17,9 @@ Group=postgres
 
 ExecStart=/bin/patroni /etc/patroni.yml
 
+# only kill the patroni process, not it's children, so it will gracefully stop postgres
+KillMode=process
+
 # Give a reasonable amount of time for the server to start up/shut down
 TimeoutSec=10
 

features/basic_replication.feature

@@ -6,12 +6,12 @@ Feature: basic replication
     And postgres0 is a leader after 10 seconds
     And I start postgres1
     When I add the table foo to postgres0
-    Then table foo is present on postgres1 after 15 seconds
+    Then table foo is present on postgres1 after 20 seconds
 
   Scenario: check the basic failover
     When I kill postgres0
-    Then postgres1 role is the primary after 30 seconds
+    Then postgres1 role is the primary after 32 seconds
     When I start postgres0
-    Then postgres0 role is the secondary after 15 seconds
+    Then postgres0 role is the secondary after 20 seconds
     When I add the table bar to postgres1
-    Then table bar is present on postgres0 after 10 seconds
+    Then table bar is present on postgres0 after 20 seconds

features/cascading_replication.feature

@@ -1,13 +1,13 @@
 Feature: cascading replication
 	We should check that patroni can do base backup and streaming from the replica
 
-Scenario: check a base backup from the replica
+Scenario: check a base backup and streaming replication from a replica
 	Given I start postgres0
 	And postgres0 is a leader after 10 seconds
-	And I start postgres1
-	And replication works from postgres0 to postgres1 after 15 seconds
+	And I configure and start postgres1 with a tag clonefrom true
+	And replication works from postgres0 to postgres1 after 20 seconds
 	And I create label with "postgres0" in postgres0 data directory
 	And I create label with "postgres1" in postgres1 data directory
-	And I configure and start postgres2 with a tag clonefrom postgres1
+	And I configure and start postgres2 with a tag replicatefrom postgres1
 	Then replication works from postgres0 to postgres2 after 30 seconds
 	And there is a label with "postgres1" in postgres2 data directory

features/environment.py

@@ -1,294 +1,374 @@
+import abc
+import consul
+import etcd
+import kazoo.client
+import kazoo.exceptions
 import os
 import psycopg2
-import requests
 import shutil
+import six
 import subprocess
 import tempfile
 import time
 import yaml
 
 
-class PatroniController(object):
-    PATRONI_CONFIG = '{}.yml'
-    """ starts and stops individual patronis"""
+@six.add_metaclass(abc.ABCMeta)
+class AbstractController(object):
 
-    def __init__(self):
-        self._output_dir = None
-        self._patroni_path = None
-        self._connections = {}
-        self._config = {}
-        self._connstring = {}
-        self._cursors = {}
-        self._log = {}
-        self._processes = {}
+    def __init__(self, name, work_directory, output_dir):
+        self._name = name
+        self._work_directory = work_directory
+        self._output_dir = output_dir
+        self._handle = None
+        self._log = None
 
-    @property
-    def patroni_path(self):
-        if self._patroni_path is None:
-            cwd = os.path.realpath(__file__)
-            while True:
-                path, entry = os.path.split(cwd)
-                cwd = path
-                if entry == 'features' or cwd == '/':
-                    break
-            self._patroni_path = cwd
-        return self._patroni_path
+    def _has_started(self):
+        return self._handle and self._handle.pid and self._handle.poll() is None
 
-    def data_dir(self, pg_name):
-        return os.path.join(self.patroni_path, 'data', pg_name)
+    def _is_running(self):
+        return self._has_started()
 
-    def write_label(self, pg_name, content):
-        with open(os.path.join(self.data_dir(pg_name), 'label'), 'w') as f:
-            f.write(content)
+    @abc.abstractmethod
+    def _is_accessible(self):
+        """process is accessible for queries"""
 
-    def read_label(self, pg_name):
-        content = None
-        try:
-            with open(os.path.join(self.data_dir(pg_name), 'label'), 'r') as f:
-                content = f.read()
-        except IOError:
-            return None
-        return content.strip()
+    @abc.abstractmethod
+    def _start(self):
+        """start process"""
 
-    def start(self, pg_name, max_wait_limit=20, tags=None):
-        if not self._is_running(pg_name):
-            if pg_name in self._processes:
-                del self._processes[pg_name]
-            cwd = self.patroni_path
-            self._log[pg_name] = open(os.path.join(self._output_dir, 'patroni_{0}.log'.format(pg_name)), 'a')
+    def start(self, max_wait_limit=5):
+        if self._is_running():
+            return True
 
-            self._config[pg_name] = self._make_patroni_test_config(pg_name, tags=tags)
+        self._log = open(os.path.join(self._output_dir, self._name + '.log'), 'a')
+        self._handle = self._start()
+
+        assert self._has_started(), "Process {0} is not running after being started".format(self._name)
 
-            p = subprocess.Popen(['coverage', 'run', '--branch', '--source=patroni', '-p', 'patroni.py', self._config[pg_name]],
-                                 stdout=self._log[pg_name], stderr=subprocess.STDOUT, cwd=cwd)
-            if not (p and p.pid and p.poll() is None):
-                assert False, "PostgreSQL {0} is not running after being started".format(pg_name)
-            self._processes[pg_name] = p
-        # wait while patroni is available for queries, but not more than 10 seconds.
         for _ in range(max_wait_limit):
-            if self.query(pg_name, "SELECT 1", fail_ok=True) is not None:
+            if self._is_accessible():
                 break
             time.sleep(1)
         else:
             assert False,\
-                "Patroni instance is not available for queries after {0} seconds".format(max_wait_limit)
+                "{0} instance is not available for queries after {1} seconds".format(self._name, max_wait_limit)
 
-    def stop(self, pg_name, kill=False, timeout=15):
+    def stop(self, kill=False, timeout=15):
+        term = False
         start_time = time.time()
-        while self._is_running(pg_name):
-            if not kill:
-                self._processes[pg_name].terminate()
-            else:
-                self._processes[pg_name].kill()
+
+        while self._handle and self._is_running():
+            if kill:
+                self._handle.kill()
+            elif not term:
+                self._handle.terminate()
+                term = True
             time.sleep(1)
             if not kill and time.time() - start_time > timeout:
                 kill = True
-        if self._log.get('pg_name') and not self._log['pg_name'].closed:
-            self._log[pg_name].close()
-        if pg_name in self._processes:
-            del self._processes[pg_name]
 
-    def query(self, pg_name, query, fail_ok=False):
+        if self._log:
+            self._log.close()
+
+
+class PatroniController(AbstractController):
+    __PORT = 5440
+    PATRONI_CONFIG = '{}.yml'
+    """ starts and stops individual patronis"""
+
+    def __init__(self, dcs, name, work_directory, output_dir, tags=None):
+        super(PatroniController, self).__init__('patroni_' + name, work_directory, output_dir)
+        PatroniController.__PORT += 1
+        self._data_dir = os.path.join(work_directory, 'data', name)
+        self._connstring = None
+        self._config = self._make_patroni_test_config(name, dcs, tags)
+
+        self._conn = None
+        self._curs = None
+
+    def write_label(self, content):
+        with open(os.path.join(self._data_dir, 'label'), 'w') as f:
+            f.write(content)
+
+    def read_label(self):
         try:
-            cursor = self._cursor(pg_name)
-            cursor.execute(query)
-            return cursor
-        except psycopg2.Error:
-            if fail_ok:
-                return None
-            else:
-                raise
+            with open(os.path.join(self._data_dir, 'label'), 'r') as f:
+                return f.read().strip()
+        except IOError:
+            return None
 
-    def check_role_has_changed_to(self, pg_name, new_role, timeout=10):
-        bound_time = time.time() + timeout
-        recovery_status = False if new_role == 'primary' else True
-        role_has_changed = False
-        while not role_has_changed:
-            cur = self.query(pg_name, "SELECT pg_is_in_recovery()", fail_ok=True)
-            if cur:
-                row = cur.fetchone()
-                if row and len(row) > 0 and row[0] == recovery_status:
-                    role_has_changed = True
-            if time.time() > bound_time:
-                break
-            time.sleep(1)
-        return role_has_changed
+    def _start(self):
+        return subprocess.Popen(['coverage', 'run', '--source=patroni', '-p', 'patroni.py', self._config],
+                                stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory)
 
-    def stop_all(self):
-        for patroni in self._processes.copy():
-            self.stop(patroni)
+    def _is_accessible(self):
+        return self.query("SELECT 1", fail_ok=True) is not None
 
-    def create_and_set_output_directory(self, feature_name):
-        feature_dir = os.path.join(self.patroni_path, "features", "output",
-                                   feature_name.replace(' ', '_'))
-        if os.path.exists(feature_dir):
-            shutil.rmtree(feature_dir)
-        os.makedirs(feature_dir)
-        self._output_dir = feature_dir
-
-    def _is_running(self, pg_name):
-        return pg_name in self._processes and self._processes[pg_name].pid and (self._processes[pg_name].poll() is None)
-
-    def _make_patroni_test_config(self, pg_name, tags=None):
-        patroni_config_name = PatroniController.PATRONI_CONFIG.format(pg_name)
+    def _make_patroni_test_config(self, name, dcs, tags):
+        patroni_config_name = self.PATRONI_CONFIG.format(name)
         patroni_config_path = os.path.join(self._output_dir, patroni_config_name)
 
         with open(patroni_config_name) as f:
             config = yaml.load(f)
-        postgresql = config['postgresql']
-        postgresql['name'] = pg_name
-        postgresql['data_dir'] = 'data/{0}'.format(pg_name)
-        postgresql_params = postgresql['parameters']
-        postgresql_params['logging_collector'] = 'on'
-        postgresql_params['log_destination'] = 'csvlog'
-        postgresql_params['log_directory'] = self._output_dir
-        postgresql_params['log_filename'] = '{0}.log'.format(pg_name)
-        postgresql_params['log_statement'] = 'all'
-        postgresql_params['log_min_messages'] = 'debug1'
-        postgresql_params['unix_socket_directories'] = '.'
+
+        host = config['postgresql']['listen'].split(':')[0]
+
+        config['postgresql']['listen'] = config['postgresql']['connect_address'] = '{0}:{1}'.format(host, self.__PORT)
+
+        user = config['postgresql'].get('superuser', {})
+        self._connkwargs = {k: user[n] for n, k in [('username', 'user'), ('password', 'password')] if n in user}
+        self._connkwargs.update({'host': host, 'port': self.__PORT, 'database': 'postgres'})
+
+        config['postgresql'].update({'name': name, 'data_dir': self._data_dir})
+        config['postgresql']['parameters'].update({
+            'logging_collector': 'on', 'log_destination': 'csvlog', 'log_directory': self._output_dir,
+            'log_filename': name + '.log', 'log_statement': 'all', 'log_min_messages': 'debug1'})
 
         if tags:
             config['tags'] = tags
 
+        if dcs != 'etcd':
+            dcs_config = config.pop('etcd')
+            dcs_config.pop('host')
+
+            if dcs == 'consul':
+                config[dcs] = dcs_config
+            else:
+                dcs_config.update({'session_timeout': dcs_config.pop('ttl'), 'reconnect_timeout': config['loop_wait']})
+                if dcs == 'exhibitor':
+                    dcs_config['exhibitor'] = {'hosts': ['127.0.0.1'], 'port': 8181}
+                else:
+                    dcs_config['hosts'] = ['127.0.0.1:2181']
+                config['zookeeper'] = dcs_config
+
         with open(patroni_config_path, 'w') as f:
             yaml.dump(config, f, default_flow_style=False)
 
         return patroni_config_path
 
-    def _make_connstring(self, pg_name):
-        if pg_name in self._connstring:
-            return self._connstring[pg_name]
+    def _connection(self):
+        if not self._conn or self._conn.closed != 0:
+            self._conn = psycopg2.connect(**self._connkwargs)
+            self._conn.autocommit = True
+        return self._conn
+
+    def _cursor(self):
+        if not self._curs or self._curs.closed or self._curs.connection.closed != 0:
+            self._curs = self._connection().cursor()
+        return self._curs
+
+    def query(self, query, fail_ok=False):
         try:
-            patroni_path = self.patroni_path
-            with open(os.path.join(patroni_path, PatroniController.PATRONI_CONFIG.format(pg_name)), 'r') as f:
-                config = yaml.load(f)
-        except IOError:
-            return None
-        connstring = config['postgresql']['connect_address']
-        if ':' in connstring:
-            address, port = connstring.split(':')
-        else:
-            address = connstring
-            port = '5432'
-        user = "postgres"
-        dbname = "postgres"
-        self._connstring[pg_name] = "host={0} port={1} dbname={2} user={3}".format(address, port, dbname, user)
-        return self._connstring[pg_name]
+            cursor = self._cursor()
+            cursor.execute(query)
+            return cursor
+        except psycopg2.Error:
+            if not fail_ok:
+                raise
 
-    def _connection(self, pg_name):
-        if pg_name not in self._connections or self._connections[pg_name].closed:
-            conn = psycopg2.connect(self._make_connstring(pg_name))
-            conn.autocommit = True
-            self._connections[pg_name] = conn
-        return self._connections[pg_name]
-
-    def _cursor(self, pg_name):
-        if pg_name not in self._cursors or self._cursors[pg_name].closed:
-            cursor = self._connection(pg_name).cursor()
-            self._cursors[pg_name] = cursor
-        return self._cursors[pg_name]
-
-
-class EtcdController(object):
-
-    """ handles all etcd related tasks, used for the tests setup and cleanup """
-    ETCD_VERSION_URL = 'http://127.0.0.1:2379/version'
-    ETCD_CLEANUP_URL = 'http://127.0.0.1:2379/v2/keys/service/batman?recursive=true'
-
-    def __init__(self, log_directory):
-        self.handle = None
-        self.work_directory = None
-        self.log_directory = log_directory
-        self.log_file = None
-        self.pid = None
-        self.start_timeout = 5
-
-    def start(self):
-        """ start etcd if it's not already running """
-        if self._is_running():
-            return True
-        self.work_directory = tempfile.mkdtemp()
-        # etcd is running throughout the tests, no need to append to the log
-        output_dir = os.path.join(self.log_directory, "features", "output")
-        if not os.path.exists(output_dir):
-            os.makedirs(output_dir)
-        self.log_file = open(os.path.join(output_dir, 'etcd.log'), 'w')
-        self.handle =\
-            subprocess.Popen(["etcd", "--debug", "--data-dir", self.work_directory],
-                             stdout=self.log_file, stderr=subprocess.STDOUT)
-        start_time = time.time()
-        while (not self._is_running()):
-            if time.time() - start_time > self.start_timeout:
-                assert False, "Failed to start etcd"
+    def check_role_has_changed_to(self, new_role, timeout=10):
+        bound_time = time.time() + timeout
+        recovery_status = new_role != 'primary'
+        while time.time() < bound_time:
+            cur = self.query("SELECT pg_is_in_recovery()", fail_ok=True)
+            if cur:
+                row = cur.fetchone()
+                if row and row[0] == recovery_status:
+                    return True
             time.sleep(1)
-        return True
+        return False
 
-    @staticmethod
-    def query(key):
-        """ query etcd for a value of a given key """
-        r = requests.get("http://127.0.0.1:2379/v2/keys/service/batman/{0}".format(key))
-        if r.ok:
-            content = r.json()
-            if content:
-                return content.get('node', {}).get('value')
-        return None
+
+class AbstractDcsController(AbstractController):
+
+    _CLUSTER_NODE = 'service/batman'
+
+    def _is_accessible(self):
+        return self._is_running()
 
     def stop_and_remove_work_directory(self, timeout=15):
-        """ terminate etcd and wipe out the temp work directory, but only if we actually started it"""
-        kill = False
-        start_time = time.time()
-        while self._is_running() and self.handle:
-            if not kill:
-                self.handle.terminate()
-            else:
-                self.handle.kill()
-            time.sleep(1)
-            if not kill and time.time() - start_time > timeout:
-                kill = True
-        self.handle = None
-        if self.log_file and not self.log_file.closed:
-            self.log_file.close()
-        if self.work_directory:
-            shutil.rmtree(self.work_directory)
-            self.work_directory = None
+        """ terminate process and wipe out the temp work directory, but only if we actually started it"""
+        self.stop(timeout=timeout)
+        if self._work_directory:
+            shutil.rmtree(self._work_directory)
 
-    @staticmethod
-    def cleanup_service_tree():
+    @abc.abstractmethod
+    def query(self, key):
+        """ query for a value of a given key """
+
+    @abc.abstractmethod
+    def cleanup_service_tree(self):
         """ clean all contents stored in the tree used for the tests """
-        r = None
+
+
+class ConsulController(AbstractDcsController):
+
+    def __init__(self, output_dir):
+        super(ConsulController, self).__init__('consul', tempfile.mkdtemp(), output_dir)
+        self._client = consul.Consul()
+
+    def _start(self):
+        return subprocess.Popen(['consul', 'agent', '-server', '-bootstrap', '-advertise=127.0.0.1',
+                                 '-data-dir', self._work_directory], stdout=self._log, stderr=subprocess.STDOUT)
+
+    def _is_running(self):
         try:
-            r = requests.delete(EtcdController.ETCD_CLEANUP_URL)
-            if r and not r.ok:
-                assert False,\
-                    "request to cleanup the etcd contents was not successfull: status code {0}".format(r.status_code)
-        except requests.exceptions.RequestException as e:
+            return bool(self._client.status.leader())
+        except Exception:
+            return False
+
+    def query(self, key):
+        _, value = self._client.kv.get('{0}/{1}'.format(self._CLUSTER_NODE, key))
+        return value and value['Value'].decode('utf-8')
+
+    def cleanup_service_tree(self):
+        self._client.kv.delete(self._CLUSTER_NODE, recurse=True)
+
+
+class EtcdController(AbstractDcsController):
+
+    """ handles all etcd related tasks, used for the tests setup and cleanup """
+
+    def __init__(self, output_dir):
+        super(EtcdController, self).__init__('etcd', tempfile.mkdtemp(), output_dir)
+        self._client = etcd.Client()
+
+    def _start(self):
+        return subprocess.Popen(["etcd", "--debug", "--data-dir", self._work_directory],
+                                stdout=self._log, stderr=subprocess.STDOUT)
+
+    def query(self, key):
+        try:
+            return self._client.get('/{0}/{1}'.format(self._CLUSTER_NODE, key)).value
+        except etcd.EtcdKeyNotFound:
+            return None
+
+    def cleanup_service_tree(self):
+        try:
+            self._client.delete('/' + self._CLUSTER_NODE, recursive=True)
+        except (etcd.EtcdKeyNotFound, etcd.EtcdConnectionFailed):
+            return
+        except Exception as e:
             assert False, "exception when cleaning up etcd contents: {0}".format(e)
 
-    @staticmethod
-    def _is_running():
+    def _is_running(self):
         # if etcd is running, but we didn't start it
         try:
-            r = requests.get(EtcdController.ETCD_VERSION_URL)
-            running = (r and r.ok and b'etcdserver' in r.content)
-        except requests.ConnectionError:
-            running = False
-        return running
+            return bool(self._client.machines)
+        except Exception:
+            return False
+
+
+class ZooKeeperController(AbstractDcsController):
+
+    """ handles all zookeeper related tasks, used for the tests setup and cleanup """
+
+    def __init__(self, output_dir):
+        super(ZooKeeperController, self).__init__('zookeeper', None, output_dir)
+        self._client = kazoo.client.KazooClient()
+
+    def _start(self):
+        pass  # TODO: implement later
+
+    def query(self, key):
+        try:
+            return self._client.get('/{0}/{1}'.format(self._CLUSTER_NODE, key))[0].decode('utf-8')
+        except kazoo.exceptions.NoNodeError:
+            return None
+
+    def cleanup_service_tree(self):
+        try:
+            self._client.delete('/' + self._CLUSTER_NODE, recursive=True)
+        except (kazoo.exceptions.NoNodeError):
+            return
+        except Exception as e:
+            assert False, "exception when cleaning up zookeeper contents: {0}".format(e)
+
+    def _is_running(self):
+        # if zookeeper is running, but we didn't start it
+        if self._client.connected:
+            return True
+        try:
+            return self._client.start(1) or True
+        except Exception:
+            return False
+
+
+class PatroniPoolController(object):
+
+    KNOWN_DCS = {'consul': ConsulController, 'etcd': EtcdController,
+                 'zookeeper': ZooKeeperController, 'exhibitor': ZooKeeperController}
+
+    def __init__(self):
+        self._dcs = None
+        self._output_dir = None
+        self._patroni_path = None
+        self._processes = {}
+        self.create_and_set_output_directory('')
+
+    @property
+    def patroni_path(self):
+        if self._patroni_path is None:
+            cwd = os.path.realpath(__file__)
+            while True:
+                cwd, entry = os.path.split(cwd)
+                if entry == 'features' or cwd == '/':
+                    break
+            self._patroni_path = cwd
+        return self._patroni_path
+
+    @property
+    def output_dir(self):
+        return self._output_dir
+
+    def start(self, pg_name, max_wait_limit=20, tags=None):
+        if pg_name not in self._processes:
+            self._processes[pg_name] = PatroniController(self.dcs, pg_name, self.patroni_path, self._output_dir, tags)
+        self._processes[pg_name].start(max_wait_limit)
+
+    def __getattr__(self, func):
+        if func not in ['stop', 'query', 'write_label', 'read_label', 'check_role_has_changed_to']:
+            raise AttributeError("PatroniPoolController instance has no attribute '{0}'".format(func))
+
+        def wrapper(pg_name, *args, **kwargs):
+            return getattr(self._processes[pg_name], func)(*args, **kwargs)
+        return wrapper
+
+    def stop_all(self):
+        for ctl in self._processes.values():
+            ctl.stop()
+        self._processes.clear()
+
+    def create_and_set_output_directory(self, feature_name):
+        feature_dir = os.path.join(self.patroni_path, 'features/output', feature_name.replace(' ', '_'))
+        if os.path.exists(feature_dir):
+            shutil.rmtree(feature_dir)
+        os.makedirs(feature_dir)
+        self._output_dir = feature_dir
+
+    @property
+    def dcs(self):
+        if self._dcs is None:
+            self._dcs = os.environ.get('DCS', 'etcd')
+            assert self._dcs in self.KNOWN_DCS, 'Unsupported dcs: ' + self.dcs
+        return self._dcs
 
 
 # actions to execute on start/stop of the tests and before running invidual features
 def before_all(context):
-    context.pctl = PatroniController()
-    context.etcd_ctl = EtcdController(context.pctl.patroni_path)
-    context.etcd_ctl.start()
+    context.pctl = PatroniPoolController()
+    context.dcs_ctl = context.pctl.KNOWN_DCS[context.pctl.dcs](context.pctl.output_dir)
+    context.dcs_ctl.start()
     try:
-        context.etcd_ctl.cleanup_service_tree()
-    except AssertionError:  # after.all handlers won't be executed in before.all
-        context.etcd_ctl.stop_and_remove_work_directory()
+        context.dcs_ctl.cleanup_service_tree()
+    except AssertionError:  # after_all handlers won't be executed in before_all
+        context.dcs_ctl.stop_and_remove_work_directory()
         raise
 
 
 def after_all(context):
-    context.etcd_ctl.stop_and_remove_work_directory()
+    context.dcs_ctl.stop_and_remove_work_directory()
     subprocess.call(['coverage', 'combine'])
     subprocess.call(['coverage', 'report'])
 
@@ -302,4 +382,4 @@ def after_feature(context, feature):
     """ stop all Patronis, remove their data directory and cleanup the keys in etcd """
     context.pctl.stop_all()
     shutil.rmtree(os.path.join(context.pctl.patroni_path, 'data'))
-    context.etcd_ctl.cleanup_service_tree()
+    context.dcs_ctl.cleanup_service_tree()

features/patroni_api.feature

@@ -18,31 +18,36 @@ Scenario: check API requests on a stand-alone server
 	And I receive a response text "failover is not possible: cluster does not have members except leader"
 	When I issue an empty POST request to http://127.0.0.1:8008/failover
 	Then I receive a response code 400
-	And I receive a response text "No values given for required parameters leader and member"
+	And I receive a response text "No values given for required parameters leader and candidate"
 
 Scenario: check API requests for the primary-replica pair
 	Given I start postgres1
-	And replication works from postgres0 to postgres1 after 15 seconds
+	And replication works from postgres0 to postgres1 after 20 seconds
 	When I issue a GET request to http://127.0.0.1:8009/replica
 	Then I receive a response code 200
 	And I receive a response state running
 	And I receive a response role replica
 	When I issue an empty POST request to http://127.0.0.1:8009/reinitialize
 	Then I receive a response code 200
-	Given replication works from postgres0 to postgres1 after 10 seconds
 	When I issue an empty POST request to http://127.0.0.1:8008/restart
 	Then I receive a response code 200
-	And postgres0 is a leader after 5 seconds
+        And postgres0 role is the primary after 5 seconds
+        When I sleep for 10 seconds
+        Then postgres1 role is the secondary after 15 seconds
 
 Scenario: check the failover via the API
 	Given I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0,candidate=postgres1
 	Then I receive a response code 200
 	And postgres1 is a leader after 5 seconds
-	And replication works from postgres1 to postgres0 after 15 seconds
+        And postgres1 role is the primary after 5 seconds
+        And postgres0 role is the secondary after 10 seconds
+	And replication works from postgres1 to postgres0 after 20 seconds
 
 Scenario: check the scheduled failover
-	Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgresq0 in 10 seconds
+	Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgres0 in 1 seconds
 	Then I receive a response code 200
-	And postgres0 is a leader after 15 seconds
+	And postgres0 is a leader after 20 seconds
+        And postgres0 role is the primary after 5 seconds
+        And postgres1 role is the secondary after 10 seconds
 	And replication works from postgres0 to postgres1 after 25 seconds
 

features/steps/basic_replication.py

@@ -41,9 +41,8 @@ def table_is_present_on(context, table_name, pg_name, max_replication_delay):
 
 @then('{pg_name:w} role is the {pg_role:w} after {max_promotion_timeout:d} seconds')
 def check_role(context, pg_name, pg_role, max_promotion_timeout):
-    if not context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)):
-        assert False,\
-            "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
+    assert context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)),\
+        "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
 
 
 @step('replication works from {master:w} to {replica:w} after {time_limit:d} seconds')

features/steps/patroni_api.py

@@ -29,10 +29,9 @@ register_type(url=parse_url, data=parse_data)
 @then('{name:w} is a leader after {time_limit:d} seconds')
 def is_a_leader(context, name, time_limit):
     max_time = time.time() + int(time_limit)
-    while (context.etcd_ctl.query("leader") != name):
+    while (context.dcs_ctl.query("leader") != name):
         time.sleep(1)
-        if time.time() > max_time:
-            assert False, "{0} is not a leader in etcd after {1} seconds".format(name, time_limit)
+        assert time.time() < max_time, "{0} is not a leader in dcs after {1} seconds".format(name, time_limit)
 
 
 @step('I sleep for {value:d} seconds')

patroni/__init__.py

@@ -5,21 +5,22 @@ import time
 import yaml
 
 from patroni.api import RestApiServer
-from patroni.etcd import Etcd
+from patroni.exceptions import PatroniException
 from patroni.ha import Ha
 from patroni.postgresql import Postgresql
-from patroni.utils import setup_signal_handlers, reap_children
-from patroni.zookeeper import ZooKeeper
-from .version import __version__
+from patroni.utils import reap_children, set_ignore_sigterm, setup_signal_handlers
+from patroni.version import __version__
 
 logger = logging.getLogger(__name__)
 
 
 class Patroni(object):
+    PATRONI_CONFIG_VARIABLE = 'PATRONI_CONFIGURATION'
 
     def __init__(self, config):
         self.nap_time = config['loop_wait']
-        self.tags = config.get('tags', dict())
+        self.tags = {tag: value for tag, value in config.get('tags', {}).items()
+                     if tag not in ('clonefrom', 'nofailover', 'noloadbalance') or value}
         self.postgresql = Postgresql(config['postgresql'])
         self.dcs = self.get_dcs(self.postgresql.name, config)
         self.version = __version__
@@ -27,6 +28,10 @@ class Patroni(object):
         self.ha = Ha(self)
         self.next_run = time.time()
 
+    @property
+    def noloadbalance(self):
+        return self.tags.get('noloadbalance', False)
+
     @property
     def nofailover(self):
         return self.tags.get('nofailover', False)
@@ -35,17 +40,18 @@ class Patroni(object):
     def replicatefrom(self):
         return self.tags.get('replicatefrom')
 
-    @property
-    def clonefrom(self):
-        return self.tags.get('clonefrom')
-
     @staticmethod
     def get_dcs(name, config):
         if 'etcd' in config:
+            from patroni.etcd import Etcd
             return Etcd(name, config['etcd'])
         if 'zookeeper' in config:
+            from patroni.zookeeper import ZooKeeper
             return ZooKeeper(name, config['zookeeper'])
-        raise Exception('Can not find suitable configuration of distributed configuration store')
+        if 'consul' in config:
+            from patroni.consul import Consul
+            return Consul(name, config['consul'])
+        raise PatroniException('Can not find suitable configuration of distributed configuration store')
 
     def schedule_next_run(self):
         self.next_run += self.nap_time
@@ -71,19 +77,30 @@ def main():
     logging.getLogger('requests').setLevel(logging.WARNING)
     setup_signal_handlers()
 
-    if len(sys.argv) < 2 or not os.path.isfile(sys.argv[1]):
-        print('Usage: {0} config.yml'.format(sys.argv[0]))
-        return
+    # Patroni reads the configuration from the command-line argument if it exists, and from the environment otherwise.
+    use_env = False
+    use_file = (len(sys.argv) >= 2 and os.path.isfile(sys.argv[1]))
+    if not use_file:
+        config_env = os.environ.get(Patroni.PATRONI_CONFIG_VARIABLE)
+        use_env = config_env is not None
+        if not use_env:
+            print('Usage: {0} config.yml'.format(sys.argv[0]))
+            print('\tPatroni may also read the configuration from the {} environment variable'.
+                  format(Patroni.PATRONI_CONFIG_VARIABLE))
+            return
 
-    with open(sys.argv[1], 'r') as f:
-        config = yaml.load(f)
+    if use_file:
+        with open(sys.argv[1], 'r') as f:
+            config = yaml.load(f)
+    elif use_env:
+        config = yaml.load(config_env)
 
     patroni = Patroni(config)
     try:
         patroni.run()
     except KeyboardInterrupt:
-        pass
+        set_ignore_sigterm()
     finally:
         patroni.api.shutdown()
-        patroni.postgresql.stop()
+        patroni.postgresql.stop(checkpoint=False)
         patroni.dcs.delete_leader()

patroni/api.py

@@ -34,12 +34,19 @@ def check_auth(func):
 
 class RestApiHandler(BaseHTTPRequestHandler):
 
+    def _write_response(self, status_code, body, headers=None):
+        self.send_response(status_code)
+        if body is not None:
+            headers = headers or {}
+            if 'Content-Type' not in headers:
+                headers['Content-Type'] = 'text/html'
+            for name, value in (headers or {}).items():
+                self.send_header(name, value)
+            self.end_headers()
+            self.wfile.write(body.encode('utf-8'))
+
     def send_auth_request(self, body):
-        self.send_response(401)
-        self.send_header('WWW-Authenticate', 'Basic realm=\"Patroni\"')
-        self.send_header('Content-type', 'text/html')
-        self.end_headers()
-        self.wfile.write(body.encode('utf-8'))
+        self._write_response(401, body, {'WWW-Authenticate': 'Basic realm=\"Patroni\"'})
 
     def finish(self, *args, **kwargs):
         try:
@@ -55,15 +62,23 @@ class RestApiHandler(BaseHTTPRequestHandler):
         status = self.server.check_auth_header(auth_header)
         return not status or self.send_auth_request(status)
 
-    def do_OPTIONS(self):
-        self.do_GET(options=True)
+    def _write_status_response(self, status_code, response, options=False):
+        if options:
+            body = None
+        else:
+            patroni = self.server.patroni
+            response.update({'tags': patroni.tags} if patroni.tags else {})
+            if patroni.postgresql.sysid:
+                response['database_system_identifier'] = patroni.postgresql.sysid
+            response['patroni'] = {'version': patroni.version, 'scope': patroni.postgresql.scope}
+            body = json.dumps(response)
+        self._write_response(status_code, body, {'Content-Type': 'application/json'})
 
     def do_GET(self, options=False):
         """Default method for processing all GET requests which can not be routed to other methods"""
 
         path = '/master' if self.path == '/' else self.path
         response = self.get_postgresql_status()
-        response.update(self.get_tags())
 
         patroni = self.server.patroni
         cluster = patroni.dcs.cluster
@@ -74,49 +89,36 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 status_code = 503
             elif response['role'] == 'master':  # running as master but without leader lock!!!!
                 status_code = 503
-            elif response['role'] in path:
-                status_code = 200
+            elif response['role'] in path:  # response['role'] != 'master'
+                status_code = 503 if patroni.noloadbalance else 200
             else:
                 status_code = 503
         elif 'role' in response and response['role'] in path:
-            status_code = 200
+            status_code = 503 if response['role'] != 'master' and patroni.noloadbalance else 200
         elif patroni.ha.restart_scheduled() and patroni.postgresql.role == 'master' and 'master' in path:
             # exceptional case for master node when the postgres is being restarted via API
             status_code = 200
         else:
             status_code = 503
+        self._write_status_response(status_code, response, options)
 
-        self.send_response(status_code)
-        if not options:
-            self.send_header('Content-Type', 'application/json')
-            self.end_headers()
-            self.wfile.write(json.dumps(response).encode('utf-8'))
+    def do_OPTIONS(self):
+        self.do_GET(options=True)
 
     def do_GET_patroni(self):
         response = self.get_postgresql_status(True)
-        response.update(self.get_tags())
-        response['patroni'] = {'version': self.server.patroni.version, 'scope': self.server.patroni.postgresql.scope}
-
-        self.send_response(200)
-        self.send_header('Content-Type', 'application/json')
-        self.end_headers()
-        self.wfile.write(json.dumps(response).encode('utf-8'))
+        self._write_status_response(200, response)
 
     @check_auth
     def do_POST_restart(self):
         status_code = 500
-        data = b'restart failed'
+        data = 'restart failed'
         try:
-            status, msg = self.server.patroni.ha.restart()
+            status, data = self.server.patroni.ha.restart()
             status_code = 200 if status else 503
-            data = msg.encode('utf-8')
-        except:
+        except Exception:
             logger.exception('Exception during restart')
-
-        self.send_response(status_code)
-        self.send_header('Content-Type', 'text/html')
-        self.end_headers()
-        self.wfile.write(data)
+        self._write_response(status_code, data)
 
     @check_auth
     def do_POST_reinitialize(self):
@@ -124,52 +126,51 @@ class RestApiHandler(BaseHTTPRequestHandler):
         cluster = ha.dcs.get_cluster()
         if cluster.is_unlocked():
             status_code = 503
-            data = b'Cluster has no leader, can not reinitialize'
+            data = 'Cluster has no leader, can not reinitialize'
         elif cluster.leader.name == ha.state_handler.name:
             status_code = 503
-            data = b'I am the leader, can not reinitialize'
+            data = 'I am the leader, can not reinitialize'
         else:
             action = ha.schedule_reinitialize()
             if action is not None:
                 status_code = 503
-                data = (action + ' already in progress').encode('utf-8')
+                data = action + ' already in progress'
             else:
                 status_code = 200
-                data = b'reinitialize scheduled'
+                data = 'reinitialize scheduled'
+        self._write_response(status_code, data)
 
-        self.send_response(status_code)
-        self.send_header('Content-Type', 'text/html')
-        self.end_headers()
-        self.wfile.write(data)
-
-    def poll_failover_result(self, leader, member):
+    def poll_failover_result(self, leader, candidate):
         for _ in range(0, 15):
             time.sleep(1)
             try:
                 cluster = self.server.patroni.dcs.get_cluster()
                 if cluster.leader and cluster.leader.name != leader:
-                    return 200, ('Successfully failed over to ' + cluster.leader.name).encode('utf-8')
+                    if not candidate or candidate == cluster.leader.name:
+                        return 200, 'Successfully failed over to "{0}"'.format(cluster.leader.name)
+                    else:
+                        return 200, 'Failed over to "{0}" instead of "{1}"'.format(cluster.leader.name, candidate)
                 if not cluster.failover:
-                    return 503, b'Failover failed'
-            except:
-                pass
-        return 503, b'Failover status unknown'
+                    return 503, 'Failover failed'
+            except Exception as e:
+                logger.debug('Exception occured during polling failover result: %s', e)
+        return 503, 'Failover status unknown'
 
-    def is_failover_possible(self, cluster, leader, member):
+    def is_failover_possible(self, cluster, leader, candidate):
         if leader and not cluster.leader or cluster.leader.name != leader:
-            return b'leader name does not match'
-        if member:
-            members = [m for m in cluster.members if m.name == member]
+            return 'leader name does not match'
+        if candidate:
+            members = [m for m in cluster.members if m.name == candidate]
             if not members:
-                return b'member does not exists'
+                return 'candidate does not exists'
         else:
             members = [m for m in cluster.members if m.name != cluster.leader.name and m.api_url]
             if not members:
-                return b'failover is not possible: cluster does not have members except leader'
-        for member, reachable, _, xlog_location, tags in self.server.patroni.ha.fetch_nodes_statuses(members):
+                return 'failover is not possible: cluster does not have members except leader'
+        for _, reachable, _, _, tags in self.server.patroni.ha.fetch_nodes_statuses(members):
             if reachable and not tags.get('nofailover', False):
                 return None
-        return b'failover is not possible: no good candidates have been found'
+        return 'failover is not possible: no good candidates have been found'
 
     @check_auth
     def do_POST_failover(self):
@@ -179,48 +180,49 @@ class RestApiHandler(BaseHTTPRequestHandler):
         except ValueError:
             request = {}
         leader = request.get('leader')
-        member = request.get('member')
+        candidate = request.get('candidate') or request.get('member')
+        scheduled_at = request.get('scheduled_at')
         cluster = self.server.patroni.ha.dcs.get_cluster()
         status_code = 500
 
-        logger.info("received failover request with leader {0} member {1} scheduled_at {2}".
-                    format(leader, member, request.get("scheduled_at")))
+        logger.info("received failover request with leader=%s candidate=%s scheduled_at=%s",
+                    leader, candidate, scheduled_at)
 
-        data = b''
-        if leader or member:
-            if request.get('scheduled_at'):
+        data = ''
+        if leader or candidate:
+            if scheduled_at:
                 try:
-                    scheduled_at = dateutil.parser.parse(request['scheduled_at'])
+                    scheduled_at = dateutil.parser.parse(scheduled_at)
                     if scheduled_at.tzinfo is None:
-                        data = b'Timezone information is mandatory for scheduled_at'
+                        data = 'Timezone information is mandatory for scheduled_at'
                         status_code = 400
                     elif scheduled_at < datetime.datetime.now(pytz.utc):
-                        data = b'Cannot schedule failover in the past'
+                        data = 'Cannot schedule failover in the past'
                         status_code = 422
-                    elif self.server.patroni.dcs.manual_failover(leader, member, scheduled_at):
-                        data = b'Failover scheduled'
+                    elif self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
+                        self.server.patroni.dcs.event.set()
+                        data = 'Failover scheduled'
                         status_code = 200
+                    else:
+                        data = 'failed to write failover key into DCS'
+                        status_code = 503
                 except (ValueError, TypeError):
-                    logger.exception('Invalid scheduled failover time: {}'.format(request['scheduled_at']))
-                    data = b'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601'
+                    logger.exception('Invalid scheduled failover time: %s', request['scheduled_at'])
+                    data = 'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601'
                     status_code = 422
             else:
-                data = self.is_failover_possible(cluster, leader, member)
+                data = self.is_failover_possible(cluster, leader, candidate)
                 if not data:
-                    if not self.server.patroni.dcs.manual_failover(leader, member):
-                        data = b'failed to write failover key into DCS'
-                        status_code = 503
-                    else:
+                    if self.server.patroni.dcs.manual_failover(leader, candidate):
                         self.server.patroni.dcs.event.set()
-                        status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, member)
+                        status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, candidate)
+                    else:
+                        data = 'failed to write failover key into DCS'
+                        status_code = 503
         else:
             status_code = 400
-            data = b'No values given for required parameters leader and member'
-
-        self.send_response(status_code)
-        self.send_header('Content-Type', 'text/html')
-        self.end_headers()
-        self.wfile.write(data)
+            data = 'No values given for required parameters leader and candidate'
+        self._write_response(status_code, data)
 
     def parse_request(self):
         """Override parse_request method to enrich basic functionality of `BaseHTTPRequestHandler` class
@@ -284,9 +286,6 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 state = 'unknown'
             return {'state': state}
 
-    def get_tags(self):
-        return {'tags': self.server.patroni.tags}
-
     def log_message(self, fmt, *args):
         logger.debug("API thread: %s - - [%s] %s", self.client_address[0], self.log_date_time_string(), fmt % args)
 

patroni/consul.py

@@ -0,0 +1,244 @@
+from __future__ import absolute_import
+import logging
+import os
+import time
+import six
+
+from consul import ConsulException, NotFound, base, std
+from patroni.dcs import AbstractDCS, Cluster, Failover, Leader, Member
+from patroni.exceptions import DCSError
+from patroni.utils import sleep
+from requests.exceptions import RequestException
+
+logger = logging.getLogger(__name__)
+
+
+class ConsulError(DCSError):
+    pass
+
+
+class HTTPClient(std.HTTPClient):
+
+    def __init__(self, *args, **kwargs):
+        super(HTTPClient, self).__init__(*args, **kwargs)
+        self._patch_default_timeout()
+
+    def _patch_default_timeout(self):
+        # Set a default timeout for the `request.session.request` method, that is used
+        # internally  by the methods request.session.get, request.session.post and
+        # others. We monkey-patch here to avoid reimplementing each individual method from
+        # `std.HTTPClient`. By default, the timeout is not set. It means that a new
+        # session may hang almost indefinitely waiting for the server to respond,
+        # which is not what we want in Patroni.
+
+        request_func = getattr(self.session.request, '__func__' if six.PY3 else 'im_func')
+        defaults_attr_name = '__defaults__' if six.PY3 else 'func_defaults'
+        defaults = list(getattr(request_func, defaults_attr_name))
+        code = request_func.__code__ if six.PY3 else request_func.func_code
+        defaults[code.co_varnames[code.co_argcount - len(defaults):code.co_argcount].index('timeout')] = 5
+        setattr(request_func, defaults_attr_name, tuple(defaults))  # monkeypatching
+
+    def get(self, callback, path, params=None):
+        # The get function is overridden to handle a special case of it being called
+        # with an index and wait parameters. That form indicates that a user needs to
+        # wait for the given key to change its value, with a wait timeout supplied. We
+        # don't want our monkey-patched timeout to be less than the value of the wait
+        # parameter, therefore, we set it to either the value of wait or a default of 5 minutes.
+
+        if isinstance(params, dict) and 'index' in params:
+            timeout = (float(params['wait'][:-1]) if 'wait' in params else 300) + 1
+        else:
+            timeout = None
+        return callback(self.response(self.session.get(self.uri(path, params), verify=self.verify, timeout=timeout)))
+
+
+class ConsulClient(base.Consul):
+
+    @staticmethod
+    def connect(host, port, scheme, verify=True):
+        return HTTPClient(host, port, scheme, verify)
+
+
+def catch_consul_errors(func):
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except (ConsulException, RequestException):
+            return False
+    return wrapper
+
+
+class Consul(AbstractDCS):
+
+    def __init__(self, name, config):
+        super(Consul, self).__init__(name, config)
+        self.ttl = int((config.get('ttl') or 30)/2)  # My experiments have shown that session expires after 2*ttl time
+        host, port = config.get('host', '127.0.0.1:8500').split(':')
+        self._client = ConsulClient(host=host, port=port)
+        self._scope = config['scope']
+        self._session = None
+        self._my_member_data = None
+        self.create_or_restore_session()
+
+    def create_or_restore_session(self):
+        while not self._session:
+            try:
+                _, member = self._client.kv.get(self.member_path)
+                self._session = (member or {}).get('Session')
+                if self.refresh_session():
+                    self._client.kv.delete(self.member_path)
+            except (ConsulException, RequestException):
+                logger.info('waiting on consul')
+                sleep(5)
+
+    def refresh_session(self):
+        """:returns: `!True` if it had to create new session"""
+        if self._session:
+            try:
+                return self._client.session.renew(self._session) is None
+            except NotFound:
+                self._session = None
+        if not self._session:
+            name = self._scope + '-' + self._name
+            try:
+                self._session = self._client.session.create(name=name, lock_delay=0, behavior='delete', ttl=self.ttl)
+            except (ConsulException, RequestException):
+                logger.exception('session.create')
+        if not self._session:
+            raise ConsulError('Failed to renew/create session')
+        return True
+
+    def client_path(self, path):
+        return super(Consul, self).client_path(path)[1:]
+
+    @staticmethod
+    def member(node):
+        return Member.from_node(node['ModifyIndex'], os.path.basename(node['Key']), node.get('Session'), node['Value'])
+
+    def _load_cluster(self):
+        try:
+            path = self.client_path('/')
+            _, results = self._client.kv.get(path, recurse=True)
+
+            if results is None:
+                raise NotFound
+
+            nodes = {}
+            for node in results:
+                node['Value'] = (node['Value'] or b'').decode('utf-8')
+                nodes[os.path.relpath(node['Key'], path)] = node
+
+            # get initialize flag
+            initialize = nodes.get(self._INITIALIZE)
+            initialize = initialize and initialize['Value']
+
+            # get last leader operation
+            last_leader_operation = nodes.get(self._LEADER_OPTIME)
+            last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation['Value'])
+
+            # get list of members
+            members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
+
+            # get leader
+            leader = nodes.get(self._LEADER)
+            if leader and leader['Value'] == self._name and self._session != leader.get('Session', 'x'):
+                logger.info('I am leader but not owner of the session. Removing leader node')
+                self._client.kv.delete(self.leader_path, cas=leader['ModifyIndex'])
+                leader = None
+
+            if leader:
+                member = Member(-1, leader['Value'], None, {})
+                member = ([m for m in members if m.name == leader['Value']] or [member])[0]
+                leader = Leader(leader['ModifyIndex'], leader.get('Session'), member)
+
+            # failover key
+            failover = nodes.get(self._FAILOVER)
+            if failover:
+                failover = Failover.from_node(failover['ModifyIndex'], failover['Value'])
+
+            self._cluster = Cluster(initialize, leader, last_leader_operation, members, failover)
+        except NotFound:
+            self._cluster = Cluster(False, None, None, [], None)
+        except:
+            logger.exception('get_cluster')
+            raise ConsulError('Consul is not responding properly')
+
+    def touch_member(self, data, **kwargs):
+        create_member = self.refresh_session()
+        cluster = self.cluster
+        member = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
+        if create_member and member:
+            try:
+                self._client.kv.delete(self.member_path)
+            except Exception:
+                return False
+
+        if not create_member and member and data == self._my_member_data:
+            return True
+
+        try:
+            self._client.kv.put(self.member_path, data, acquire=self._session)
+            self._my_member_data = data
+            return True
+        except Exception:
+            logger.exception('touch_member')
+        return False
+
+    @catch_consul_errors
+    def attempt_to_acquire_leader(self):
+        ret = self._client.kv.put(self.leader_path, self._name, acquire=self._session)
+        if not ret:
+            logger.info('Could not take out TTL lock')
+        return ret
+
+    def take_leader(self):
+        return self.attempt_to_acquire_leader()
+
+    @catch_consul_errors
+    def set_failover_value(self, value, index=None):
+        return self._client.kv.put(self.failover_path, value, cas=index)
+
+    @catch_consul_errors
+    def write_leader_optime(self, last_operation):
+        return self._client.kv.put(self.leader_optime_path, last_operation)
+
+    @staticmethod
+    def update_leader():
+        return True
+
+    @catch_consul_errors
+    def initialize(self, create_new=True, sysid=''):
+        kwargs = {'cas': 0} if create_new else {}
+        return self._client.kv.put(self.initialize_path, sysid, **kwargs)
+
+    @catch_consul_errors
+    def cancel_initialization(self):
+        return self._client.kv.delete(self.initialize_path)
+
+    @catch_consul_errors
+    def delete_cluster(self):
+        return self._client.kv.delete(self.client_path(''), recurse=True)
+
+    @catch_consul_errors
+    def delete_leader(self):
+        cluster = self.cluster
+        if cluster and isinstance(cluster.leader, Leader) and cluster.leader.name == self._name:
+            return self._client.kv.delete(self.leader_path, cas=cluster.leader.index)
+
+    def watch(self, timeout):
+        cluster = self.cluster
+        if cluster and cluster.leader and cluster.leader.name != self._name and cluster.leader.index:
+            end_time = time.time() + timeout
+            while timeout >= 1:
+                try:
+                    idx, _ = self._client.kv.get(self.leader_path, index=cluster.leader.index, wait=str(timeout) + 's')
+                    return str(idx) != str(cluster.leader.index)
+                except (ConsulException, RequestException):
+                    logging.exception('watch')
+
+                timeout = end_time - time.time()
+
+        try:
+            return super(Consul, self).watch(timeout)
+        finally:
+            self.event.clear()

patroni/ctl.py

@@ -3,59 +3,54 @@ Patroni Control
 '''
 
 import click
-import os
-import yaml
+import datetime
+import dateutil
 import json
-import time
+import logging
+import os
 import psycopg2
 import random
 import requests
-import datetime
+import time
+import tzlocal
+import yaml
+
+from click import ClickException
+from patroni import Patroni, PatroniException
+from patroni.postgresql import parseurl
 from prettytable import PrettyTable
 from six.moves.urllib_parse import urlparse
-import logging
-import dateutil
-import tzlocal
-
-from .etcd import Etcd
-from .zookeeper import ZooKeeper
-from .exceptions import PatroniCtlException
-from .postgresql import parseurl
 
 CONFIG_DIR_PATH = click.get_app_dir('patroni')
 CONFIG_FILE_PATH = os.path.join(CONFIG_DIR_PATH, 'patronictl.yaml')
 LOGLEVEL = 'WARNING'
+DCS_DEFAULTS = {'zookeeper': {'port': 2181, 'template': "zookeeper:\n hosts: ['{host}:{port}']"},
+                'exhibitor': {'port': 8181, 'template': "zookeeper:\n exhibitor:\n  hosts: [{host}]\n  port: {port}"},
+                'consul': {'port': 8500, 'template': "consul:\n host: '{host}:{port}'"},
+                'etcd': {'port': 4001, 'template': "etcd:\n host: '{host}:{port}'"}}
+
+
+class PatroniCtlException(ClickException):
+    pass
 
 
 def parse_dcs(dcs):
-    """
-    Break up the provided dcs string
-    >>> parse_dcs('localhost') == {'scheme': 'etcd', 'hostname': 'localhost', 'port': 4001}
-    True
-    >>> parse_dcs('localhost:8500') == {'scheme': 'consul', 'hostname': 'localhost', 'port': 8500}
-    True
-    >>> parse_dcs('zookeeper://localhost') == {'scheme': 'zookeeper', 'hostname': 'localhost', 'port': 2181}
-    True
-    """
-
-    if not dcs:
-        return {}
+    if dcs is None:
+        return None
 
     parsed = urlparse(dcs)
     scheme = parsed.scheme
     if scheme == '' and parsed.netloc == '':
         parsed = urlparse('//' + dcs)
+    port = int(parsed.port) if parsed.port else None
 
     if scheme == '':
-        default_schemes = {'2181': 'zookeeper', '8181': 'exhibitor', '8500': 'consul'}
-        scheme = default_schemes.get(str(parsed.port), 'etcd')
+        scheme = ([k for k, v in DCS_DEFAULTS.items() if v['port'] == port] or ['etcd'])[0]
+    elif scheme not in DCS_DEFAULTS:
+        raise PatroniCtlException('Unknown dcs scheme: {}'.format(scheme))
 
-    port = parsed.port
-    if port is None:
-        default_ports = {'consul': 8500, 'zookeeper': 2181, 'exhibitor': 8181}
-        port = default_ports.get(str(scheme), 4001)
-
-    return {'scheme': str(scheme), 'hostname': str(parsed.hostname), 'port': int(port)}
+    dcs_info = DCS_DEFAULTS[scheme]
+    return yaml.load(dcs_info['template'].format(host=parsed.hostname or 'localhost', port=port or dcs_info['port']))
 
 
 def load_config(path, dcs):
@@ -67,10 +62,7 @@ def load_config(path, dcs):
     except (IOError, yaml.YAMLError):
         logging.exception('Could not load configuration file')
 
-    if dcs:
-        config['dcs'] = parse_dcs(dcs)
-    else:
-        config['dcs'] = parse_dcs(config.get('dcs_api'))
+    config.update(parse_dcs(dcs) or parse_dcs(config.get('dcs_api')) or {})
 
     return config
 
@@ -101,18 +93,12 @@ def ctl(ctx):
 
 
 def get_dcs(config, scope):
-    scheme, hostname, port = map(config.get('dcs', {}).get, ('scheme', 'hostname', 'port'))
-
-    if scheme == 'etcd':
-        return Etcd(name=scope, config={'scope': scope, 'host': '{0}:{1}'.format(hostname, port)})
-
-    if scheme == 'zookeeper':
-        return ZooKeeper(name=scope, config={'scope': scope, 'hosts': [hostname], 'port': port})
-
-    if scheme == 'exhibitor':
-        return ZooKeeper(name=scope, config={'scope': scope, 'exhibitor': {'hosts': [hostname], 'port': port}})
-
-    raise PatroniCtlException('Can not find suitable configuration of distributed configuration store')
+    for k in set(DCS_DEFAULTS.keys()) & set(config.keys()):
+        config[k].setdefault('scope', scope)
+    try:
+        return Patroni.get_dcs(scope, config)
+    except PatroniException as e:
+        raise PatroniCtlException(str(e))
 
 
 def post_patroni(member, endpoint, content, headers=None):
@@ -202,16 +188,14 @@ def get_all_members(cluster, role='master'):
 
 
 def get_any_member(cluster, role='master', member=None):
-    members = get_all_members(cluster=cluster, role=role)
+    members = get_all_members(cluster, role)
     for m in members:
         if member is None or m.name == member:
             return m
 
-    return None
-
 
 def get_cursor(cluster, role='master', member=None, connect_parameters=None):
-    member = get_any_member(cluster=cluster, role=role, member=member)
+    member = get_any_member(cluster, role=role, member=member)
     if member is None:
         return None
 
@@ -248,7 +232,7 @@ def dsn(cluster_name, config_file, dcs, role, member):
         role = 'master'
 
     _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
-    m = get_any_member(cluster=cluster, role=role, member=member)
+    m = get_any_member(cluster, role=role, member=member)
     if m is None:
         raise PatroniCtlException('Can not find a suitable member')
 
@@ -311,13 +295,12 @@ def query(
     if p_file is not None:
         command = p_file.read()
 
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
 
     cursor = None
     for _ in watching(w, watch, clear=False):
 
-        output, cursor = query_member(cluster=cluster, cursor=cursor, member=member, role=role, command=command,
-                                      connect_parameters=connect_parameters)
+        output, cursor = query_member(cluster, cursor, member, role, command, connect_parameters)
         print_output(None, output, fmt=fmt, delimiter=delimiter)
 
         if cursor is None:
@@ -361,12 +344,9 @@ def query_member(cluster, cursor, member, role, command, connect_parameters=None
 @option_format
 @option_dcs
 def remove(config_file, cluster_name, fmt, dcs):
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
 
-    if not isinstance(dcs, Etcd):
-        raise PatroniCtlException('We have not implemented this for DCS of type {0}'.format(type(dcs)))
-
-    output_members(cluster, fmt=fmt)
+    output_members(cluster, cluster_name, fmt)
 
     confirm = click.prompt('Please confirm the cluster name to remove', type=str)
     if confirm != cluster_name:
@@ -384,7 +364,7 @@ def remove(config_file, cluster_name, fmt, dcs):
         if confirm != cluster.leader.name:
             raise PatroniCtlException('You did not specify the current master of the cluster')
 
-    dcs.client.delete(dcs.client_path(''), recursive=True)
+    dcs.delete_cluster()
 
 
 def wait_for_leader(dcs, timeout=30):
@@ -445,9 +425,9 @@ def ctl_load_config(cluster_name, config_file, dcs):
 @option_force
 @option_dcs
 def restart(cluster_name, member_names, config_file, dcs, force, role, p_any):
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
 
-    role_names = [m.name for m in get_all_members(cluster=cluster, role=role)]
+    role_names = [m.name for m in get_all_members(cluster, role)]
 
     if member_names:
         member_names = list(set(member_names) & set(role_names))
@@ -458,7 +438,7 @@ def restart(cluster_name, member_names, config_file, dcs, force, role, p_any):
         random.shuffle(member_names)
         member_names = member_names[:1]
 
-    output_members(cluster)
+    output_members(cluster, cluster_name)
     empty_post_to_members(cluster, member_names, force, 'restart')
 
 
@@ -469,7 +449,7 @@ def restart(cluster_name, member_names, config_file, dcs, force, role, p_any):
 @option_force
 @option_dcs
 def reinit(cluster_name, member_names, config_file, dcs, force):
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
     empty_post_to_members(cluster, member_names, force, 'reinitialize')
 
 
@@ -490,7 +470,7 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
         If so, we trigger a failover and keep the client up to date.
     """
 
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
 
     if cluster.leader is None:
         raise PatroniCtlException('This cluster has no master')
@@ -532,16 +512,16 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
             if scheduled_at.tzinfo is None:
                 scheduled_at = tzlocal.get_localzone().localize(scheduled_at)
         except (ValueError, TypeError):
-            message = 'Unable to parse scheduled timestamp ({}). It should be in an unambiguous format (e.g. ISO 8601)'
+            message = 'Unable to parse scheduled timestamp ({0}). It should be in an unambiguous format (e.g. ISO 8601)'
             raise PatroniCtlException(message.format(scheduled))
         scheduled_at = scheduled_at.isoformat()
 
-    failover_value = {'leader': master, 'member': candidate, 'scheduled_at': scheduled_at}
+    failover_value = {'leader': master, 'candidate': candidate, 'scheduled_at': scheduled_at}
     logging.debug(failover_value)
 
     # By now we have established that the leader exists and the candidate exists
     click.echo('Current cluster topology')
-    output_members(dcs.get_cluster(), name=cluster_name)
+    output_members(dcs.get_cluster(), cluster_name)
 
     if not force:
         a = \
@@ -561,17 +541,17 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
         else:
             click.echo('Failover failed, details: {0}, {1}'.format(r.status_code, r.text))
             return
-    except:
+    except Exception:
         logging.exception(r)
         logging.warning('Failing over to DCS')
         click.echo(timestamp() + ' Could not failover using Patroni api, falling back to DCS')
         click.echo(timestamp() + ' Initializing failover from master {0}'.format(master))
-        dcs.manual_failover(leader=master, member=candidate, scheduled_at=failover_value)
+        dcs.manual_failover(master, candidate, scheduled_at=failover_value)
 
-    output_members(cluster, name=cluster_name)
+    output_members(cluster, cluster_name)
 
 
-def output_members(cluster, name=None, fmt='pretty'):
+def output_members(cluster, name, fmt='pretty'):
     rows = []
     logging.debug(cluster)
     leader_name = None
@@ -631,11 +611,11 @@ def members(config_file, cluster_names, fmt, watch, w, dcs):
         return
 
     config = load_config(config_file, dcs)
-    for cn in cluster_names:
-        dcs = get_dcs(config, cn)
+    for cluster_name in cluster_names:
+        dcs = get_dcs(config, cluster_name)
 
         for _ in watching(w, watch):
-            output_members(dcs.get_cluster(), name=cn, fmt=fmt)
+            output_members(dcs.get_cluster(), cluster_name, fmt)
 
 
 def timestamp(precision=6):

patroni/dcs.py

@@ -1,8 +1,10 @@
 import abc
-import json
 import dateutil
+import json
+import six
 
 from collections import namedtuple
+from random import randint
 from six.moves.urllib_parse import urlparse, urlunparse, parse_qsl
 from threading import Event, Lock
 
@@ -63,13 +65,21 @@ class Member(namedtuple('Member', 'index,name,session,data')):
     def api_url(self):
         return self.data.get('api_url')
 
+    @property
+    def tags(self):
+        return self.data.get('tags', {})
+
     @property
     def nofailover(self):
-        return self.data.get('tags', {}).get('nofailover', False)
+        return self.tags.get('nofailover', False)
 
     @property
     def replicatefrom(self):
-        return self.data.get('tags', {}).get('replicatefrom')
+        return self.tags.get('replicatefrom')
+
+    @property
+    def clonefrom(self):
+        return self.tags.get('clonefrom', False)
 
 
 class Leader(namedtuple('Leader', 'index,session,member')):
@@ -89,16 +99,16 @@ class Leader(namedtuple('Leader', 'index,session,member')):
         return self.member.conn_url
 
 
-class Failover(namedtuple('Failover', 'index,leader,member,scheduled_at')):
+class Failover(namedtuple('Failover', 'index,leader,candidate,scheduled_at')):
 
     """
     >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader"}'))
     True
-    >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader", "member": "cluster:member"}'))
+    >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader", "member": "cluster_candidate"}'))
     True
     >>> Failover.from_node(1, 'null') is None
     True
-    >>> n = '{"leader": "cluster_leader", "member": "cluster:member", "scheduled_at": "2016-01-14T10:09:57.1394Z"}'
+    >>> n = '{"leader": "cluster_leader", "member": "cluster_candidate", "scheduled_at": "2016-01-14T10:09:57.1394Z"}'
     >>> 'tzinfo=' in str(Failover.from_node(1, n))
     True
     >>> Failover.from_node(1, None) is None
@@ -146,14 +156,17 @@ class Cluster(namedtuple('Cluster', 'initialize,leader,last_leader_operation,mem
     def has_member(self, member_name):
         return any(m for m in self.members if m.name == member_name)
 
-    def get_member(self, member_name):
-        return ([m for m in self.members if m.name == member_name] or [None])[0]
+    def get_member(self, member_name, fallback_to_leader=True):
+        return ([m for m in self.members if m.name == member_name] or [self.leader if fallback_to_leader else None])[0]
+
+    def get_clone_member(self):
+        candidates = [m for m in self.members if m.clonefrom and (not self.leader or m.name != self.leader.name)]
+        return candidates[randint(0, len(candidates) - 1)] if candidates else self.leader
 
 
+@six.add_metaclass(abc.ABCMeta)
 class AbstractDCS(object):
 
-    __metaclass__ = abc.ABCMeta
-
     _INITIALIZE = 'initialize'
     _LEADER = 'leader'
     _FAILOVER = 'failover'
@@ -258,13 +271,13 @@ class AbstractDCS(object):
     def set_failover_value(self, value, index=None):
         """Create or update `/failover` key"""
 
-    def manual_failover(self, leader, member, scheduled_at=None, index=None):
-        failover_value = dict()
+    def manual_failover(self, leader, candidate, scheduled_at=None, index=None):
+        failover_value = {}
         if leader:
             failover_value['leader'] = leader
 
-        if member:
-            failover_value['member'] = member
+        if candidate:
+            failover_value['member'] = candidate
 
         if scheduled_at:
             failover_value['scheduled_at'] = scheduled_at.isoformat()
@@ -308,6 +321,10 @@ class AbstractDCS(object):
     def cancel_initialization(self):
         """ Removes the initialize key for a cluster """
 
+    @abc.abstractmethod
+    def delete_cluster(self):
+        """Delete cluster from DCS"""
+
     def watch(self, timeout):
         """If the current node is a master it should just sleep.
         Any other node should watch for changes of leader key with a given timeout

patroni/etcd.py

@@ -6,13 +6,13 @@ import random
 import requests
 import socket
 import time
-import urllib3
 
 from dns.exception import DNSException
 from dns import resolver
 from patroni.dcs import AbstractDCS, Cluster, Failover, Leader, Member
 from patroni.exceptions import DCSError
 from patroni.utils import Retry, RetryFailedError, sleep
+from urllib3.exceptions import HTTPError, ReadTimeoutError
 from requests.exceptions import RequestException
 from six.moves.http_client import HTTPException
 
@@ -55,9 +55,9 @@ class Client(etcd.Client):
             response = request_executor(method, url, fields=fields, **kwargs)
             response.data.decode('utf-8')
             self._check_cluster_id(response)
-        except (urllib3.exceptions.HTTPError, HTTPException, socket.error) as e:
+        except (HTTPError, HTTPException, socket.error, socket.timeout) as e:
             if (isinstance(fields, dict) and fields.get("wait") == "true" and
-                    isinstance(e, urllib3.exceptions.ReadTimeoutError)):
+                    isinstance(e, ReadTimeoutError)):
                 logger.debug("Watch timed out.")
                 raise etcd.EtcdWatchTimedOut("Watch timed out: {0}".format(e), cause=e)
             logger.error("Request to server %s failed: %r", self._base_uri, e)
@@ -199,7 +199,7 @@ class Etcd(AbstractDCS):
                                               etcd.EtcdLeaderElectionInProgress,
                                               etcd.EtcdWatcherCleared,
                                               etcd.EtcdEventIndexCleared))
-        self.client = self.get_etcd_client(config)
+        self._client = self.get_etcd_client(config)
 
     def retry(self, *args, **kwargs):
         return self._retry.copy()(*args, **kwargs)
@@ -221,7 +221,7 @@ class Etcd(AbstractDCS):
 
     def _load_cluster(self):
         try:
-            result = self.retry(self.client.read, self.client_path(''), recursive=True)
+            result = self.retry(self._client.read, self.client_path(''), recursive=True)
             nodes = {os.path.relpath(node.key, result.key): node for node in result.leaves}
 
             # get initialize flag
@@ -256,15 +256,15 @@ class Etcd(AbstractDCS):
 
     @catch_etcd_errors
     def touch_member(self, connection_string, ttl=None):
-        return self.retry(self.client.set, self.member_path, connection_string, ttl or self.ttl)
+        return self.retry(self._client.set, self.member_path, connection_string, ttl or self.ttl)
 
     @catch_etcd_errors
     def take_leader(self):
-        return self.retry(self.client.set, self.leader_path, self._name, self.ttl)
+        return self.retry(self._client.set, self.leader_path, self._name, self.ttl)
 
     def attempt_to_acquire_leader(self):
         try:
-            return bool(self.retry(self.client.write, self.leader_path, self._name, ttl=self.ttl, prevExist=False))
+            return bool(self.retry(self._client.write, self.leader_path, self._name, ttl=self.ttl, prevExist=False))
         except etcd.EtcdAlreadyExist:
             logger.info('Could not take out TTL lock')
         except (RetryFailedError, etcd.EtcdException):
@@ -273,43 +273,46 @@ class Etcd(AbstractDCS):
 
     @catch_etcd_errors
     def set_failover_value(self, value, index=None):
-        return self.client.write(self.failover_path, value, prevIndex=index or 0)
+        return self._client.write(self.failover_path, value, prevIndex=index or 0)
 
     @catch_etcd_errors
     def write_leader_optime(self, last_operation):
-        return self.client.set(self.leader_optime_path, last_operation)
+        return self._client.set(self.leader_optime_path, last_operation)
 
     @catch_etcd_errors
     def update_leader(self):
-        return self.retry(self.client.test_and_set, self.leader_path, self._name, self._name, self.ttl)
+        return self.retry(self._client.test_and_set, self.leader_path, self._name, self._name, self.ttl)
 
     @catch_etcd_errors
     def initialize(self, create_new=True, sysid=""):
-        return self.retry(self.client.write, self.initialize_path, sysid, prevExist=(not create_new))
+        return self.retry(self._client.write, self.initialize_path, sysid, prevExist=(not create_new))
 
     @catch_etcd_errors
     def delete_leader(self):
-        return self.client.delete(self.leader_path, prevValue=self._name)
+        return self._client.delete(self.leader_path, prevValue=self._name)
 
     @catch_etcd_errors
     def cancel_initialization(self):
-        return self.retry(self.client.delete, self.initialize_path)
+        return self.retry(self._client.delete, self.initialize_path)
+
+    @catch_etcd_errors
+    def delete_cluster(self):
+        return self.retry(self._client.delete, self.client_path(''), recursive=True)
 
     def watch(self, timeout):
         cluster = self.cluster
         # watch on leader key changes if it is defined and current node is not lock owner
-        if cluster and cluster.leader and cluster.leader.name != self._name:
+        if cluster and cluster.leader and cluster.leader.name != self._name and cluster.leader.index:
             end_time = time.time() + timeout
-            index = cluster.leader.index
 
-            while index and timeout >= 1:  # when timeout is too small urllib3 doesn't have enough time to connect
+            while timeout >= 1:  # when timeout is too small urllib3 doesn't have enough time to connect
                 try:
-                    self.client.watch(self.leader_path, index=index + 1, timeout=timeout + 0.5)
+                    self._client.watch(self.leader_path, index=cluster.leader.index + 1, timeout=timeout + 0.5)
                     # Synchronous work of all cluster members with etcd is less expensive
                     # than reestablishing http connection every time from every replica.
                     return True
                 except etcd.EtcdWatchTimedOut:
-                    self.client.http.clear()
+                    self._client.http.clear()
                     return False
                 except etcd.EtcdException:
                     logging.exception('watch')

patroni/exceptions.py

@@ -1,6 +1,3 @@
-from click import ClickException
-
-
 class PatroniException(Exception):
 
     """Parent class for all kind of exceptions related to selected distributed configuration store"""
@@ -16,10 +13,6 @@ class PatroniException(Exception):
         return repr(self.value)
 
 
-class PatroniCtlException(ClickException):
-    pass
-
-
 class PostgresException(PatroniException):
     pass
 

patroni/ha.py

@@ -38,7 +38,7 @@ class Ha(object):
 
     def update_lock(self):
         ret = self.dcs.update_leader()
-        if ret:
+        if ret and not self._async_executor.busy:
             try:
                 self.dcs.write_leader_optime(self.state_handler.last_operation())
             except:
@@ -55,35 +55,33 @@ class Ha(object):
             'conn_url': self.state_handler.connection_string,
             'api_url': self.patroni.api.connection_string,
             'state': self.state_handler.state,
-            'role': self.state_handler.role,
-            'tags': self.patroni.tags
+            'role': self.state_handler.role
         }
-        if data['state'] in ['running', 'restarting', 'starting']:
+        if self.patroni.tags:
+            data['tags'] = self.patroni.tags
+        if not self._async_executor.busy and data['state'] in ['running', 'restarting', 'starting']:
             try:
                 data['xlog_location'] = self.state_handler.xlog_position()
             except:
                 pass
         self.dcs.touch_member(json.dumps(data, separators=(',', ':')))
 
-    def clone(self, clone_member, clone_member_name="leader"):
+    def clone(self, clone_member=None, msg='(without leader)'):
         if self.state_handler.bootstrap(cluster_initialized=True, clone_member=clone_member):
-            logger.info('bootstrapped from {0}'.format(clone_member_name)
-                        if clone_member else 'bootstrapped without leader')
+            logger.info('bootstrapped %s', msg)
         else:
+            logger.error('failed to bootstrap %s', msg)
             self.state_handler.stop('immediate')
             self.state_handler.remove_data_directory()
-            logger.error('failed to bootstrap from {0}'.format(clone_member_name)
-                         if clone_member else 'failed to bootstrap (without leader)')
 
     def bootstrap(self):
         if not self.cluster.is_unlocked():  # cluster already has leader
-            clonefrom = self.patroni.clonefrom
-            clone_member = self.cluster.get_member(clonefrom)\
-                if self.cluster.has_member(clonefrom) else self.cluster.leader
-            clone_member_name = 'leader' if clone_member == self.cluster.leader else 'replica \'{0}\''.format(clonefrom)
-            self._async_executor.schedule('bootstrap from {0}'.format(clone_member_name))
-            self._async_executor.run_async(self.clone, args=(clone_member, clone_member_name))
-            return 'trying to bootstrap from {0}'.format(clone_member_name)
+            clone_member = self.cluster.get_clone_member()
+            member_role = 'leader' if clone_member == self.cluster.leader else 'replica'
+            msg = "from {0} '{1}'".format(member_role, clone_member.name)
+            self._async_executor.schedule('bootstrap {0}'.format(msg))
+            self._async_executor.run_async(self.clone, args=(clone_member, msg))
+            return 'trying to bootstrap {0}'.format(msg)
         elif not self.cluster.initialize and not self.patroni.nofailover:  # no initialize key
             if self.dcs.initialize(create_new=True):  # race for initialization
                 try:
@@ -103,39 +101,30 @@ class Ha(object):
                 return 'failed to acquire initialize lock'
         else:
             if self.state_handler.can_create_replica_without_replication_connection():
-                self._async_executor.run_async(self.clone, args=(None, ))
-                return "trying to bootstrap without leader"
+                self._async_executor.run_async(self.clone)
+                return "trying to bootstrap (without leader)"
             return 'waiting for leader to bootstrap'
 
     def recover(self):
-        # try to see if we are the former master that crashed. If so - we likely need to run pg_rewind
-        # in order to join the former standby being promoted.
-        pg_controldata = self.state_handler.controldata()
-        if (self.state_handler.role == 'master') and pg_controldata and\
-                pg_controldata.get('Database cluster state', '') == 'in production':  # crashed master
-            self.state_handler.require_rewind()
         self.recovering = True
-        return self.follow("started as readonly because i had the session lock",
-                           "started as a secondary",
-                           refresh=True, recovery=True)
+        return self.follow("starting as readonly because i had the session lock", "starting as a secondary", True, True)
 
     def follow(self, demote_reason, follow_reason, refresh=True, recovery=False):
         if refresh:
             self.load_cluster_from_dcs()
 
-        if not recovery and self.state_handler.is_leader() or recovery and self.state_handler.role == 'master':
-            ret = demote_reason
-        else:
-            ret = follow_reason
+        ret = demote_reason if not recovery and self.state_handler.is_leader() else follow_reason
 
         # determine the node to follow. If replicatefrom tag is set,
         # try to follow the node mentioned there, otherwise, follow the leader.
+
         if self.patroni.replicatefrom:
-            node_to_follow = [m for m in self.cluster.members if m.name == self.patroni.replicatefrom]
-            node_to_follow = node_to_follow[0] if node_to_follow else self.cluster.leader
+            node_to_follow = self.cluster.get_member(self.patroni.replicatefrom, fallback_to_leader=True)
         else:
             node_to_follow = self.cluster.leader
-        node_to_follow = None if node_to_follow and node_to_follow.name == self.state_handler.name else node_to_follow
+        if node_to_follow and node_to_follow.name == self.state_handler.name:
+            ret = demote_reason
+            node_to_follow = None
         if not self.state_handler.check_recovery_conf(node_to_follow) or recovery:
             self._async_executor.schedule('changing primary_conninfo and restarting')
             self._async_executor.run_async(self.state_handler.follow, (node_to_follow, recovery))
@@ -222,14 +211,14 @@ class Ha(object):
 
     def manual_failover_process_no_leader(self):
         failover = self.cluster.failover
-        if failover.member:  # manual failover to specific member
-            if failover.member == self.state_handler.name:  # manual failover to me
+        if failover.candidate:  # manual failover to specific member
+            if failover.candidate == self.state_handler.name:  # manual failover to me
                 return True
 
             # find specific node and check that it is healthy
-            members = [m for m in self.cluster.members if m.name == failover.member]
-            if members:
-                member, reachable, _, _, tags = self.fetch_node_status(members[0])
+            member = self.cluster.get_member(failover.candidate, fallback_to_leader=False)
+            if member:
+                member, reachable, _, _, tags = self.fetch_node_status(member)
                 if reachable and not tags.get('nofailover', False):  # node is healthy
                     logger.info('manual failover: to %s, i am %s', member.name, self.state_handler.name)
                     return False
@@ -240,13 +229,13 @@ class Ha(object):
                     logger.warning('manual failover: member %s is not allowed to promote', member.name)
 
             # at this point we should consider all members as a candidates for failover
-            # i.e. we assume that failover.member is None
+            # i.e. we assume that failover.candidate is None
 
         # try to pick some other members to failover and check that they are healthy
         if failover.leader:
             if self.state_handler.name == failover.leader:  # I was the leader
-                # exclude me and desired member which is unhealthy (failover.member can be None)
-                members = [m for m in self.cluster.members if m.name not in (failover.member, failover.leader)]
+                # exclude me and desired member which is unhealthy (failover.candidate can be None)
+                members = [m for m in self.cluster.members if m.name not in (failover.candidate, failover.leader)]
                 if self.is_failover_possible(members):  # check that there are healthy members
                     return False
                 else:  # I was the leader and it looks like currently I am the only healthy member
@@ -279,7 +268,10 @@ class Ha(object):
             self.dcs.delete_leader()
             self.touch_member()
             self.dcs.reset_cluster()
-        self.state_handler.follow(None)
+            sleep(2)  # Give a time to somebody to promote
+            self.recover()
+        else:
+            self.state_handler.follow(None)
 
     def process_manual_failover_from_leader(self):
         failover = self.cluster.failover
@@ -298,7 +290,7 @@ class Ha(object):
                     return
                 elif delta < - int(self.patroni.nap_time * 1.5):
                     logger.warning('Found a stale failover value, cleaning up: %s', failover.scheduled_at)
-                    self.dcs.manual_failover('', '', self.cluster.failover.index)
+                    self.dcs.manual_failover('', '', index=self.cluster.failover.index)
                     return
 
                 # The value is very close to now
@@ -308,8 +300,8 @@ class Ha(object):
                 logger.warning('Incorrect value in of scheduled_at: %s', failover.scheduled_at)
 
         if not failover.leader or failover.leader == self.state_handler.name:
-            if not failover.member or failover.member != self.state_handler.name:
-                members = [m for m in self.cluster.members if not failover.member or m.name == failover.member]
+            if not failover.candidate or failover.candidate != self.state_handler.name:
+                members = [m for m in self.cluster.members if not failover.candidate or m.name == failover.candidate]
                 if self.is_failover_possible(members):  # check that there are healthy members
                     self._async_executor.schedule('manual failover: demote')
                     self._async_executor.run_async(self.demote)
@@ -323,7 +315,7 @@ class Ha(object):
                            self.cluster.failover.leader, self.state_handler.name)
 
         logger.info('Trying to clean up failover key')
-        self.dcs.manual_failover('', '', self.cluster.failover.index)
+        self.dcs.manual_failover('', '', index=self.cluster.failover.index)
 
     def process_unhealthy_cluster(self):
         if self.is_healthiest_node():
@@ -331,7 +323,7 @@ class Ha(object):
                 if self.cluster.failover:
                     logger.info('Cleaning up failover key after acquiring leader lock...')
                     self.dcs.manual_failover('', '')
-                self.dcs.get_cluster()
+                self.load_cluster_from_dcs()
                 return self.enforce_master_role('acquired session lock as a leader',
                                                 'promoted self to leader by acquiring session lock')
             else:
@@ -389,7 +381,10 @@ class Ha(object):
     def reinitialize(self, cluster):
         self.state_handler.stop('immediate')
         self.state_handler.remove_data_directory()
-        self.clone(cluster.leader)
+
+        clone_member = cluster.get_clone_member()
+        member_role = 'leader' if clone_member == cluster.leader else 'replica'
+        self.clone(clone_member, "from {0} '{1}'".format(member_role, clone_member.name))
 
     def process_scheduled_action(self):
         if self.reinitialize_scheduled():

patroni/postgresql.py

@@ -44,17 +44,18 @@ class Postgresql(object):
     def __init__(self, config):
         self.config = config
         self.name = config['name']
-        self.server_parameters = config.get('parameters', {})
+        self.server_parameters = config.get('parameters') or {}
         self.scope = config['scope']
         self.listen_addresses, self.port = config['listen'].split(':')
         self.data_dir = config['data_dir']
         self.replication = config['replication']
-        self.superuser = config['superuser']
-        self.admin = config['admin']
-        self.initdb_options = config.get('initdb', [])
+        self.superuser = config.get('superuser') or {}
+        self.admin = config.get('admin') or {}
+
+        self.initdb_options = config.get('initdb') or []
         self.pgpass = config.get('pgpass') or os.path.join(os.path.expanduser('~'), 'pgpass')
-        self.pg_rewind = config.get('pg_rewind', {})
-        self.callback = config.get('callbacks', {})
+        self.pg_rewind = config.get('pg_rewind') or {}
+        self.callback = config.get('callbacks') or {}
         self.use_slots = config.get('use_slots', True)
         self.schedule_load_slots = self.use_slots
         self.recovery_conf = os.path.join(self.data_dir, 'recovery.conf')
@@ -73,14 +74,13 @@ class Postgresql(object):
 
         self._connection = None
         self._cursor_holder = None
-        self._need_rewind = False
         self._sysid = None
         self.replication_slots = []  # list of already existing replication slots
         self.retry = Retry(max_tries=-1, deadline=5, max_delay=1, retry_exceptions=PostgresConnectionException)
 
         self._state = 'stopped'
         self._state_lock = Lock()
-        self._role = 'replica'
+        self._role = self.get_postgres_role_from_data_directory()
         self._role_lock = Lock()
 
         if self.is_running():
@@ -115,9 +115,6 @@ class Postgresql(object):
             self._sysid = data.get('Database system identifier', "")
         return self._sysid
 
-    def require_rewind(self):
-        self._need_rewind = True
-
     def get_local_address(self):
         listen_addresses = self.listen_addresses.split(',')
         local_address = listen_addresses[0].strip()  # take first address from listen_addresses
@@ -128,6 +125,9 @@ class Postgresql(object):
                 break
         return local_address + ':' + self.port
 
+    def get_postgres_role_from_data_directory(self):
+        return 'replica' if os.path.exists(self.recovery_conf) else 'master'
+
     @property
     def _connect_kwargs(self):
         r = parseurl('postgres://{0}/postgres'.format(self.local_address))
@@ -269,9 +269,8 @@ class Postgresql(object):
         # If there is no configuration key, or no value is specified, use basebackup
         replica_methods = self.config.get('create_replica_method') or ['basebackup']
         # if we don't have any source, leave only replica methods that work without it
-        replica_methods = \
-            [r for r in replica_methods if self.replica_method_can_work_without_replication_connection(r)]\
-            if not clone_member else replica_methods
+        replica_methods = replica_methods if clone_member else \
+            [r for r in replica_methods if self.replica_method_can_work_without_replication_connection(r)]
         # go through them in priority order
         ret = 1
         for replica_method in replica_methods:
@@ -352,7 +351,7 @@ class Postgresql(object):
             logger.error('Cannot start PostgreSQL because one is already running.')
             return True
 
-        self.set_role('replica' if os.path.exists(self.recovery_conf) else 'master')
+        self.set_role(self.get_postgres_role_from_data_directory())
         if os.path.exists(self.postmaster_pid):
             os.remove(self.postmaster_pid)
             logger.info('Removed %s', self.postmaster_pid)
@@ -360,8 +359,9 @@ class Postgresql(object):
         if not block_callbacks:
             self.set_state('starting')
 
-        env = os.environ.copy()
-        if 'username' in self.superuser:
+        env = {'PATH': os.environ.get('PATH')}
+        # pg_ctl will write a FATAL if the username is incorrect. exporting PGUSER if necessary
+        if 'username' in self.superuser and self.superuser['username'] != os.environ.get('USER'):
             env['PGUSER'] = self.superuser['username']
         ret = subprocess.call(self._pg_ctl + ['start', '-o', self.server_options()], env=env, preexec_fn=os.setsid) == 0
 
@@ -388,7 +388,7 @@ class Postgresql(object):
         except psycopg2.Error:
             logging.exception('Exception during CHECKPOINT')
 
-    def stop(self, mode='fast', block_callbacks=False):
+    def stop(self, mode='fast', block_callbacks=False, checkpoint=True):
         # make sure we close all connections established against
         # the former node, otherwise, we might get a stalled one
         # after kill -9, which would report incorrect data to
@@ -400,9 +400,10 @@ class Postgresql(object):
                 self.set_state('stopped')
             return True
 
-        if block_callbacks:
+        if checkpoint:
             self.checkpoint()
-        else:
+
+        if not block_callbacks:
             self.set_state('stopping')
 
         ret = subprocess.call(self._pg_ctl + ['stop', '-m', mode]) == 0
@@ -447,11 +448,7 @@ class Postgresql(object):
 
     def write_pg_hba(self):
         with open(os.path.join(self.data_dir, 'pg_hba.conf'), 'a') as f:
-            f.write('\nhost replication {username} {network} md5\n'.format(**self.replication))
-            for line in self.config.get('pg_hba', []):
-                if line.split()[0].strip() == 'hostssl' and self.server_parameters.get('ssl', 'off').lower() != 'on':
-                    continue
-                f.write(line + '\n')
+            f.write('\n{}\n'.format('\n'.join(self.config.get('pg_hba', []))))
 
     @staticmethod
     def primary_conninfo(leader_url):
@@ -470,17 +467,15 @@ class Postgresql(object):
                     return pattern and (pattern in line)
         return not pattern
 
-    def write_recovery_conf(self, leader, bootstrap=False):
+    def write_recovery_conf(self, leader):
         with open(self.recovery_conf, 'w') as f:
-            f.write("""standby_mode = 'on'
-recovery_target_timeline = 'latest'
-""")
+            f.write("standby_mode = 'on'\nrecovery_target_timeline = 'latest'\n")
             if leader and leader.conn_url:
-                f.write("""primary_conninfo = '{0}'\n""".format(self.primary_conninfo(leader.conn_url)))
+                f.write("primary_conninfo = '{0}'\n".format(self.primary_conninfo(leader.conn_url)))
                 if self.use_slots:
-                    f.write("""primary_slot_name = '{0}'\n""".format(self.name))
-            if (leader and leader.conn_url) or bootstrap:
-                for name, value in self.config.get('recovery_conf', {}).items():
+                    f.write("primary_slot_name = '{0}'\n".format(self.name))
+            for name, value in self.config.get('recovery_conf', {}).items():
+                if name not in ('standby_mode', 'recovery_target_timeline', 'primary_conninfo', 'primary_slot_name'):
                     f.write("{0} = '{1}'\n".format(name, value))
 
     def rewind(self, leader):
@@ -564,13 +559,12 @@ recovery_target_timeline = 'latest'
     def follow(self, leader, recovery=False):
         if self.check_recovery_conf(leader) and not recovery:
             return True
-
         change_role = self.role == 'master'
-        self._need_rewind = (self._need_rewind or change_role) and self.can_rewind
-        if self._need_rewind:
+        need_rewind = change_role and self.can_rewind
+        if need_rewind:
             logger.info("set the rewind flag after demote")
         self.write_recovery_conf(leader)
-        if leader and self._need_rewind:  # we have a leader and need to rewind
+        if leader and need_rewind:  # we have a leader and need to rewind
             if self.is_running():
                 self.stop()
             # at present, pg_rewind only runs when the cluster is shut down cleanly
@@ -594,7 +588,6 @@ recovery_target_timeline = 'latest'
                 logger.error("unable to rewind the former master")
                 self.remove_data_directory()
                 ret = True
-            self._need_rewind = False
         else:  # do not rewind until the leader becomes available
             ret = self.restart()
         if change_role and ret:
@@ -630,7 +623,6 @@ recovery_target_timeline = 'latest'
         if ret:
             self.set_role('master')
             logger.info("cleared rewind flag after becoming the leader")
-            self._need_rewind = False
             self.call_nowait(ACTION_ON_ROLE_CHANGE)
         return ret
 
@@ -733,7 +725,7 @@ $$""".format(name, options), name, password, password)
         else:
             if self.sync_replica(clone_member):
                 self.restore_configuration_files()
-                self.write_recovery_conf(clone_member, True)
+                self.write_recovery_conf(clone_member)
                 ret = self.start()
         return ret
 

patroni/utils.py

@@ -33,10 +33,14 @@ def calculate_ttl(expiration):
     return int((expiration - now).total_seconds())
 
 
-def sigterm_handler(signo, stack_frame):
+def set_ignore_sigterm(value=True):
     global __ignore_sigterm
+    __ignore_sigterm = value
+
+
+def sigterm_handler(signo, stack_frame):
     if not __ignore_sigterm:
-        __ignore_sigterm = True
+        set_ignore_sigterm()
         sys.exit()
 
 

patroni/version.py

@@ -1 +1 @@
-__version__ = '0.80'
+__version__ = '0.90'

patroni/zookeeper.py

@@ -83,32 +83,29 @@ class ZooKeeper(AbstractDCS):
             self.exhibitor = ExhibitorEnsembleProvider(exhibitor['hosts'], exhibitor['port'], poll_interval=interval)
             hosts = self.exhibitor.zookeeper_hosts
 
-        self.client = KazooClient(hosts=hosts,
-                                  timeout=(config.get('session_timeout') or 30),
-                                  command_retry={
-                                      'deadline': (config.get('reconnect_timeout') or 10),
-                                      'max_delay': 1,
-                                      'max_tries': -1},
-                                  connection_retry={'max_delay': 1, 'max_tries': -1})
-        self.client.add_listener(self.session_listener)
+        self._client = KazooClient(hosts=hosts, timeout=(config.get('session_timeout') or 30),
+                                   command_retry={'deadline': (config.get('reconnect_timeout') or 10),
+                                                  'max_delay': 1, 'max_tries': -1},
+                                   connection_retry={'max_delay': 1, 'max_tries': -1})
+        self._client.add_listener(self.session_listener)
 
         self._my_member_data = None
-        self.fetch_cluster = True
-        self.last_leader_operation = 0
+        self._fetch_cluster = True
+        self._last_leader_operation = 0
 
-        self.client.start(None)
+        self._client.start()
 
     def session_listener(self, state):
         if state in [KazooState.SUSPENDED, KazooState.LOST]:
             self.cluster_watcher(None)
 
     def cluster_watcher(self, event):
-        self.fetch_cluster = True
+        self._fetch_cluster = True
         self.event.set()
 
     def get_node(self, key, watch=None):
         try:
-            ret = self.client.get(key, watch)
+            ret = self._client.get(key, watch)
             return (ret[0].decode('utf-8'), ret[1])
         except NoNodeError:
             return None
@@ -119,7 +116,7 @@ class ZooKeeper(AbstractDCS):
 
     def get_children(self, key, watch=None):
         try:
-            return self.client.get_children(key, watch)
+            return self._client.get_children(key, watch)
         except NoNodeError:
             return []
 
@@ -132,11 +129,11 @@ class ZooKeeper(AbstractDCS):
         return members
 
     def _inner_load_cluster(self):
-        self.fetch_cluster = False
+        self._fetch_cluster = False
         self.event.clear()
         nodes = set(self.get_children(self.client_path(''), self.cluster_watcher))
         if not nodes:
-            self.fetch_cluster = True
+            self._fetch_cluster = True
 
         # get initialize flag
         initialize = (self.get_node(self.initialize_path) or [None])[0] if self._INITIALIZE in nodes else None
@@ -147,17 +144,17 @@ class ZooKeeper(AbstractDCS):
         # get leader
         leader = self.get_node(self.leader_path) if self._LEADER in nodes else None
         if leader:
-            client_id = self.client.client_id
+            client_id = self._client.client_id
             if leader[0] == self._name and client_id is not None and client_id[0] != leader[1].ephemeralOwner:
                 logger.info('I am leader but not owner of the session. Removing leader node')
-                self.client.delete(self.leader_path)
+                self._client.delete(self.leader_path)
                 leader = None
 
             if leader:
                 member = Member(-1, leader[0], None, {})
                 member = ([m for m in members if m.name == leader[0]] or [member])[0]
                 leader = Leader(leader[1].version, leader[1].ephemeralOwner, member)
-                self.fetch_cluster = member.index == -1
+                self._fetch_cluster = member.index == -1
 
         # failover key
         failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
@@ -165,17 +162,17 @@ class ZooKeeper(AbstractDCS):
             failover = Failover.from_node(failover[1].version, failover[0])
 
         # get last leader operation
-        optime = self.get_node(self.leader_optime_path) if self._OPTIME in nodes and self.fetch_cluster else None
-        self.last_leader_operation = 0 if optime is None else int(optime[0])
-        self._cluster = Cluster(initialize, leader, self.last_leader_operation, members, failover)
+        optime = self.get_node(self.leader_optime_path) if self._OPTIME in nodes and self._fetch_cluster else None
+        self._last_leader_operation = 0 if optime is None else int(optime[0])
+        self._cluster = Cluster(initialize, leader, self._last_leader_operation, members, failover)
 
     def _load_cluster(self):
         if self.exhibitor and self.exhibitor.poll():
-            self.client.set_hosts(self.exhibitor.zookeeper_hosts)
+            self._client.set_hosts(self.exhibitor.zookeeper_hosts)
 
-        if self.fetch_cluster:
+        if self._fetch_cluster or self._cluster is None:
             try:
-                self.client.retry(self._inner_load_cluster)
+                self._client.retry(self._inner_load_cluster)
             except:
                 logger.exception('get_cluster')
                 self.session_listener(KazooState.LOST)
@@ -183,20 +180,20 @@ class ZooKeeper(AbstractDCS):
 
     def _create(self, path, value, **kwargs):
         try:
-            self.client.retry(self.client.create, path, value.encode('utf-8'), **kwargs)
+            self._client.retry(self._client.create, path, value.encode('utf-8'), **kwargs)
             return True
         except:
             return False
 
     def attempt_to_acquire_leader(self):
         ret = self._create(self.leader_path, self._name, makepath=True, ephemeral=True)
-        if ret:
+        if not ret:
             logger.info('Could not take out TTL lock')
         return ret
 
     def set_failover_value(self, value, index=None):
         try:
-            self.client.retry(self.client.set, self.failover_path, value.encode('utf-8'), version=index or -1)
+            self._client.retry(self._client.set, self.failover_path, value.encode('utf-8'), version=index or -1)
             return True
         except NoNodeError:
             return value == '' or (not index and self._create(self.failover_path, value))
@@ -206,36 +203,35 @@ class ZooKeeper(AbstractDCS):
 
     def initialize(self, create_new=True, sysid=""):
         return self._create(self.initialize_path, sysid, makepath=True) if create_new \
-            else self.client.retry(self.client.set, self.initialize_path,  sysid.encode("utf-8"))
+            else self._client.retry(self._client.set, self.initialize_path,  sysid.encode("utf-8"))
 
     def touch_member(self, data, ttl=None):
         cluster = self.cluster
-        me = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
+        member = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
         path = self.member_path
         data = data.encode('utf-8')
-        create = not me
-        if me and self.client.client_id is not None and me.session != self.client.client_id[0]:
+        if member and self._client.client_id is not None and member.session != self._client.client_id[0]:
             try:
-                self.client.retry(self.client.delete, path)
+                self._client.retry(self._client.delete, path)
             except NoNodeError:
                 pass
             except:
                 return False
-            create = True
+            member = None
 
-        if not create and data == self._my_member_data:
+        if member and data == self._my_member_data:
             return True
 
         try:
-            if create:
-                self.client.retry(self.client.create, path, data, makepath=True, ephemeral=True)
+            if member:
+                self._client.retry(self._client.set, path, data)
             else:
-                self.client.retry(self.client.set, path, data)
+                self._client.retry(self._client.create, path, data, makepath=True, ephemeral=True)
             self._my_member_data = data
             return True
         except NodeExistsError:
             try:
-                self.client.retry(self.client.set, path, data)
+                self._client.retry(self._client.set, path, data)
                 self._my_member_data = data
                 return True
             except:
@@ -249,14 +245,14 @@ class ZooKeeper(AbstractDCS):
 
     def write_leader_optime(self, last_operation):
         last_operation = last_operation.encode('utf-8')
-        if last_operation != self.last_leader_operation:
-            self.last_leader_operation = last_operation
+        if last_operation != self._last_leader_operation:
+            self._last_leader_operation = last_operation
             path = self.leader_optime_path
             try:
-                self.client.retry(self.client.set, path, last_operation)
+                self._client.retry(self._client.set, path, last_operation)
             except NoNodeError:
                 try:
-                    self.client.retry(self.client.create, path, last_operation, makepath=True)
+                    self._client.retry(self._client.create, path, last_operation, makepath=True)
                 except:
                     logger.exception('Failed to create %s', path)
             except:
@@ -266,22 +262,28 @@ class ZooKeeper(AbstractDCS):
         return True
 
     def delete_leader(self):
-        self.client.restart()
+        self._client.restart()
         self._my_member_data = None
         return True
 
     def _cancel_initialization(self):
         node = self.get_node(self.initialize_path)
         if node:
-            self.client.delete(self.initialize_path, version=node[1].version)
+            self._client.delete(self.initialize_path, version=node[1].version)
 
     def cancel_initialization(self):
         try:
-            self.client.retry(self._cancel_initialization)
+            self._client.retry(self._cancel_initialization)
         except:
             logger.exception("Unable to delete initialize key")
 
+    def delete_cluster(self):
+        try:
+            return self._client.retry(self._client.delete, self.client_path(''), recursive=True)
+        except NoNodeError:
+            return True
+
     def watch(self, timeout):
         if super(ZooKeeper, self).watch(timeout):
-            self.fetch_cluster = True
-        return self.fetch_cluster
+            self._fetch_cluster = True
+        return self._fetch_cluster

postgres0.yml

@@ -12,6 +12,10 @@ etcd:
   ttl: *ttl
   host: 127.0.0.1:4001
   #discovery_srv: my-etcd.domain
+#consul:
+#  scope: *scope
+#  ttl: *ttl
+#  host: 127.0.0.1:8500
 #zookeeper:
 #  scope: *scope
 #  session_timeout: *ttl
@@ -56,12 +60,12 @@ postgresql:
     username: postgres
     password: zalando
   pg_hba:
+  - host replication replicator 127.0.0.1/32 md5
   - host all all 0.0.0.0/0 md5
-  - hostssl all all 0.0.0.0/0 md5
+  # - hostssl all all 0.0.0.0/0 md5
   replication:
     username: replicator
     password: rep-pass
-    network:  127.0.0.1/32
   superuser:
     username: postgres
     password: zalando
@@ -93,6 +97,7 @@ postgresql:
     max_replication_slots: 10
     hot_standby: "on"
     wal_log_hints: "on"
+    unix_socket_directories: '.'
 tags:
     nofailover: False
     noloadbalance: False

postgres1.yml

@@ -12,6 +12,10 @@ etcd:
   ttl: *ttl
   host: 127.0.0.1:4001
   #discovery_srv: my-etcd.domain
+#consul:
+#  scope: *scope
+#  ttl: *ttl
+#  host: 127.0.0.1:8500
 #zookeeper:
 #  scope: *scope
 #  session_timeout: *ttl
@@ -56,12 +60,12 @@ postgresql:
     username: postgres
     password: zalando
   pg_hba:
+  - host replication replicator 127.0.0.1/32 md5
   - host all all 0.0.0.0/0 md5
-  - hostssl all all 0.0.0.0/0 md5
+  # - hostssl all all 0.0.0.0/0 md5
   replication:
     username: replicator
     password: rep-pass
-    network: 127.0.0.1/32
   superuser:
     username: postgres
     password: zalando
@@ -94,6 +98,7 @@ postgresql:
     max_replication_slots: 10
     hot_standby: "on"
     wal_log_hints: "on"
+    unix_socket_directories: '.'
 tags:
     nofailover: False
     noloadbalance: False

postgres2.yml

@@ -12,6 +12,10 @@ etcd:
   ttl: *ttl
   host: 127.0.0.1:4001
   #discovery_srv: my-etcd.domain
+#consul:
+#  scope: *scope
+#  ttl: *ttl
+#  host: 127.0.0.1:8500
 #zookeeper:
 #  scope: *scope
 #  session_timeout: *ttl
@@ -56,12 +60,12 @@ postgresql:
     username: postgres
     password: zalando
   pg_hba:
+  - host replication replicator 127.0.0.1/32 md5
   - host all all 0.0.0.0/0 md5
-  - hostssl all all 0.0.0.0/0 md5
+  # - hostssl all all 0.0.0.0/0 md5
   replication:
     username: replicator
     password: rep-pass
-    network: 127.0.0.1/32
   superuser:
     username: postgres
     password: zalando
@@ -94,6 +98,7 @@ postgresql:
     max_replication_slots: 10
     hot_standby: "on"
     wal_log_hints: "on"
+    unix_socket_directories: '.'
 tags:
     nofailover: False
     noloadbalance: False

requirements.txt

@@ -5,6 +5,7 @@ requests
 six >= 1.7
 kazoo>=2.2.1
 python-etcd==0.4.3
+python-consul==0.6.0
 click>=4.1
 prettytable>=0.7
 tzlocal

setup.py

@@ -4,13 +4,12 @@
     Setup file for patroni
 """
 
-import sys
-import os
 import inspect
+import os
+import sys
 
-import setuptools
 from setuptools.command.test import test as TestCommand
-from setuptools import setup
+from setuptools import find_packages, setup
 
 if sys.version_info < (2, 7, 0):
     sys.stderr.write('FATAL: patroni needs to be run with Python 2.7+\n')
@@ -94,7 +93,12 @@ class PyTest(TestCommand):
             params['args'] += self.cov
         if self.junitxml:
             params['args'] += self.junitxml
-        params['args'] += ['--doctest-modules', MAIN_PACKAGE, '-s', '-vv']
+        params['args'] += ['--doctest-modules', MAIN_PACKAGE, '-vv']
+
+        import logging
+        silence = logging.WARNING
+        logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=os.getenv('LOGLEVEL', silence))
+        params['args'] += ['-s' if logging.getLogger().getEffectiveLevel() < silence else '--capture=fd']
         errno = pytest.main(**params)
         sys.exit(errno)
 
@@ -138,7 +142,7 @@ def setup_package():
         long_description=read('README.rst'),
         classifiers=CLASSIFIERS,
         test_suite='tests',
-        packages=setuptools.find_packages(exclude=['tests', 'tests.*']),
+        packages=find_packages(exclude=['tests', 'tests.*']),
         package_data={MAIN_PACKAGE: ["*.json"]},
         install_requires=install_reqs,
         setup_requires=['flake8'],

tests/test_api.py

@@ -11,32 +11,25 @@ import socket
 from test_postgresql import psycopg2_connect, MockCursor
 
 
-class MockPostgresql(Mock):
+class MockPostgresql(object):
 
     name = 'test'
     state = 'running'
     role = 'master'
     server_version = '999999'
+    sysid = 'dummysysid'
     scope = 'dummy'
 
     @staticmethod
     def connection():
         return psycopg2_connect()
 
-    @staticmethod
-    def is_running():
-        return True
 
-
-class MockHa(Mock):
+class MockHa(object):
 
     dcs = Mock()
     state_handler = MockPostgresql()
 
-    @staticmethod
-    def schedule_restart():
-        return 'restart'
-
     @staticmethod
     def schedule_reinitialize():
         return 'reinitialize'
@@ -54,13 +47,14 @@ class MockHa(Mock):
         return [[None, True, None, None, {}]]
 
 
-class MockPatroni(Mock):
+class MockPatroni(object):
 
     postgresql = MockPostgresql()
     ha = MockHa()
     dcs = Mock()
     tags = {}
     version = '0.00'
+    noloadbalance = Mock(return_value=False)
 
 
 class MockRequest(object):
@@ -113,8 +107,8 @@ class TestRestApiHandler(unittest.TestCase):
         # make sure socket.error gets propagated via wfile object in finalize()
         with patch.object(MockRequest, 'makefile') as makefile:
             makefile.return_value.closed = False
-            makefile.return_value.readline.side_effect = lambda x: b"foo"
-            makefile.return_value.flush = Mock(side_effect=socket.error("foo"))
+            makefile.return_value.readline = Mock(return_value=b'foo')
+            makefile.return_value.flush = Mock(side_effect=socket.error('foo'))
             MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0')
 
     def test_do_GET_patroni(self):
@@ -157,34 +151,48 @@ class TestRestApiHandler(unittest.TestCase):
         request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\
                   b'Content-Length: 0\n\n'
         MockRestApiServer(RestApiHandler, request)
+
+        cluster.leader.name = 'postgresql1'
+        MockRestApiServer(RestApiHandler, request)
+
         request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\
                   b'Content-Length: 25\n\n{"leader": "postgresql1"}'
         MockRestApiServer(RestApiHandler, request)
+
+        cluster.leader.name = 'postgresql2'
+        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\
+                  b'Content-Length: 53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
+        MockRestApiServer(RestApiHandler, request)
+
         cluster.leader.name = 'postgresql1'
         MockRestApiServer(RestApiHandler, request)
-        cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'})]
+
+        cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
+                           Member(0, 'postgresql2', 30, {'api_url': 'http'})]
         MockRestApiServer(RestApiHandler, request)
         with patch.object(MockPatroni, 'dcs') as d:
             cluster = d.get_cluster.return_value
             cluster.leader.name = 'postgresql0'
             MockRestApiServer(RestApiHandler, request)
+            cluster.leader.name = 'postgresql2'
+            MockRestApiServer(RestApiHandler, request)
             cluster.leader.name = 'postgresql1'
             cluster.failover = None
             MockRestApiServer(RestApiHandler, request)
-            d.get_cluster = Mock(side_effect=Exception())
+            d.get_cluster = Mock(side_effect=Exception)
             MockRestApiServer(RestApiHandler, request)
             d.manual_failover.return_value = False
             MockRestApiServer(RestApiHandler, request)
         with patch.object(MockHa, 'fetch_nodes_statuses', Mock(return_value=[])):
             MockRestApiServer(RestApiHandler, request)
-        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\
-                  b'Content-Length: 50\n\n{"leader": "postgresql1", "member": "postgresql2"}'
-        MockRestApiServer(RestApiHandler, request)
 
         # Valid future date
         request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\
                   b'"postgresql1", "member": "postgresql2", "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}'
         MockRestApiServer(RestApiHandler, request)
+        with patch.object(MockPatroni, 'dcs') as d:
+            d.manual_failover.return_value = False
+            MockRestApiServer(RestApiHandler, request)
 
         # Exception: No timezone specified
         request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 97\n\n{"leader": ' +\

tests/test_consul.py

@@ -0,0 +1,128 @@
+import consul
+import unittest
+
+from patroni.dcs import AbstractDCS
+from mock import Mock, patch
+from patroni.consul import Cluster, Consul, ConsulError, ConsulException, HTTPClient, NotFound
+from test_etcd import SleepException
+
+
+def kv_get(self, key, **kwargs):
+    if key == 'service/test/members/postgresql1':
+        return '1', {'Session': 'fd4f44fe-2cac-bba5-a60b-304b51ff39b7'}
+    if key == 'service/test/':
+        return None, None
+    if key == 'service/good/leader':
+        return '1', None
+    if key == 'service/good/':
+        return ('6429',
+                [{'CreateIndex': 1334, 'Flags': 0, 'Key': key + 'failover', 'LockIndex': 0,
+                  'ModifyIndex': 1334, 'Value': b''},
+                 {'CreateIndex': 1334, 'Flags': 0, 'Key': key + 'initialize', 'LockIndex': 0,
+                  'ModifyIndex': 1334, 'Value': b'postgresql0'},
+                 {'CreateIndex': 2621, 'Flags': 0, 'Key': key + 'leader', 'LockIndex': 1,
+                  'ModifyIndex': 2621, 'Session': 'fd4f44fe-2cac-bba5-a60b-304b51ff39b7', 'Value': b'postgresql1'},
+                 {'CreateIndex': 6156, 'Flags': 0, 'Key': key + 'members/postgresql0', 'LockIndex': 1,
+                  'ModifyIndex': 6156, 'Session': '782e6da4-ed02-3aef-7963-99a90ed94b53',
+                  'Value': ('postgres://replicator:[email protected]:5432/postgres' +
+                            '?application_name=http://127.0.0.1:8008/patroni').encode('utf-8')},
+                 {'CreateIndex': 2630, 'Flags': 0, 'Key': key + 'members/postgresql1', 'LockIndex': 1,
+                  'ModifyIndex': 2630, 'Session': 'fd4f44fe-2cac-bba5-a60b-304b51ff39b7',
+                  'Value': ('postgres://replicator:[email protected]:5433/postgres' +
+                            '?application_name=http://127.0.0.1:8009/patroni').encode('utf-8')},
+                 {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'optime/leader', 'LockIndex': 0,
+                  'ModifyIndex': 6429, 'Value': b'4496294792'}])
+    raise ConsulException
+
+
+class TestHTTPClient(unittest.TestCase):
+
+    def test_get(self):
+        self.client = HTTPClient('127.0.0.1', '8500', 'http', False)
+        self.client.session.get = Mock()
+        self.client.get(Mock(), '')
+        self.client.get(Mock(), '', {'wait': '1s', 'index': 1})
+
+
+@patch.object(consul.Consul.KV, 'get', kv_get)
+class TestConsul(unittest.TestCase):
+
+    @patch.object(consul.Consul.Session, 'create', Mock(return_value='fd4f44fe-2cac-bba5-a60b-304b51ff39b7'))
+    @patch.object(consul.Consul.Session, 'renew', Mock(side_effect=NotFound))
+    @patch.object(consul.Consul.KV, 'get', kv_get)
+    @patch.object(consul.Consul.KV, 'delete', Mock())
+    def setUp(self):
+        self.c = Consul('postgresql1', {'ttl': 30, 'scope': 'test', 'host': 'localhost:1'})
+        self.c._base_path = '/service/good'
+        self.c._load_cluster()
+
+    @patch('time.sleep', Mock(side_effect=SleepException))
+    def test_create_or_restore_session(self):
+        self.c._session = None
+        self.assertRaises(SleepException, self.c.create_or_restore_session)
+
+    @patch.object(consul.Consul.Session, 'renew', Mock(side_effect=NotFound))
+    @patch.object(consul.Consul.Session, 'create', Mock(side_effect=ConsulException))
+    def test_referesh_session(self):
+        self.c._session = '1'
+        self.c._name = ''
+        self.assertRaises(ConsulError, self.c.refresh_session)
+
+    @patch.object(consul.Consul.KV, 'delete', Mock())
+    def test_get_cluster(self):
+        self.c._base_path = '/service/test'
+        self.assertIsInstance(self.c.get_cluster(), Cluster)
+        self.assertIsInstance(self.c.get_cluster(), Cluster)
+        self.c._base_path = '/service/fail'
+        self.assertRaises(ConsulError, self.c.get_cluster)
+        self.c._base_path = '/service/good'
+        self.c._session = 'fd4f44fe-2cac-bba5-a60b-304b51ff39b8'
+        self.assertIsInstance(self.c.get_cluster(), Cluster)
+
+    @patch.object(consul.Consul.KV, 'delete', Mock(side_effect=[ConsulException, True, True]))
+    @patch.object(consul.Consul.KV, 'put', Mock(side_effect=[True, ConsulException]))
+    def test_touch_member(self):
+        self.c.refresh_session = Mock(return_value=True)
+        self.c.touch_member('balbla')
+        self.c.touch_member('balbla')
+        self.c.touch_member('balbla')
+        self.c.refresh_session = Mock(return_value=False)
+        self.c.touch_member('balbla')
+
+    @patch.object(consul.Consul.KV, 'put', Mock(return_value=False))
+    def test_take_leader(self):
+        self.c.take_leader()
+
+    @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
+    def test_set_failover_value(self):
+        self.c.set_failover_value('')
+
+    @patch.object(consul.Consul.KV, 'put', Mock(side_effect=ConsulException))
+    def test_write_leader_optime(self):
+        self.c.write_leader_optime('')
+
+    def test_update_leader(self):
+        self.c.update_leader()
+
+    @patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
+    def test_delete_leader(self):
+        self.c.delete_leader()
+
+    @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
+    def test_initialize(self):
+        self.c.initialize()
+
+    @patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
+    def test_cancel_initialization(self):
+        self.c.cancel_initialization()
+
+    @patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
+    def test_delete_cluster(self):
+        self.c.delete_cluster()
+
+    @patch.object(AbstractDCS, 'watch', Mock())
+    def test_watch(self):
+        self.c._name = ''
+        self.c.watch(1)
+        with patch.object(consul.Consul.KV, 'get', Mock(side_effect=ConsulException)):
+            self.c.watch(1)

tests/test_ctl.py

@@ -1,18 +1,16 @@
+import etcd
 import os
 import pytest
 import requests.exceptions
 import unittest
 
 from click.testing import CliRunner
-from etcd import EtcdException
-from mock import patch, Mock, MagicMock
-from patroni.ctl import ctl, members, store_config, load_config, output_members, post_patroni, get_dcs, \
-    wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure
+from mock import patch, Mock
+from patroni.ctl import ctl, members, store_config, load_config, output_members, post_patroni, get_dcs, parse_dcs, \
+    wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException
 from patroni.etcd import Etcd, Client
-from patroni.exceptions import PatroniCtlException
 from psycopg2 import OperationalError
-from test_etcd import etcd_read, etcd_write, requests_get, socket_getaddrinfo, MockResponse
-from test_zookeeper import MockKazooClient
+from test_etcd import etcd_read, requests_get, socket_getaddrinfo, MockResponse
 from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \
     get_cluster_initialized_with_only_leader
 from test_postgresql import MockConnect, psycopg2_connect
@@ -42,8 +40,11 @@ def test_rw_config():
     load_config(CONFIG_FILE_PATH, None)
     load_config(CONFIG_FILE_PATH, '0.0.0.0')
 
+    store_config({'dcs_api': None}, CONFIG_FILE_PATH)
+    load_config(CONFIG_FILE_PATH, None)
 
-@patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'etcd', 'hostname': 'localhost', 'port': 4001}}))
+
+@patch('patroni.ctl.load_config', Mock(return_value={'etcd': {'host': 'localhost:4001'}}))
 class TestCtl(unittest.TestCase):
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
@@ -52,9 +53,6 @@ class TestCtl(unittest.TestCase):
         with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
             self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'})
-            self.e.client.read = etcd_read
-            self.e.client.write = etcd_write
-            self.e.client.delete = Mock(side_effect=EtcdException)
 
     @patch('psycopg2.connect', psycopg2_connect)
     def test_get_cursor(self):
@@ -67,6 +65,16 @@ class TestCtl(unittest.TestCase):
 
         self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), role='any'))
 
+    def test_parse_dcs(self):
+        assert parse_dcs(None) is None
+        assert parse_dcs('localhost') == {'etcd': {'host': 'localhost:4001'}}
+        assert parse_dcs('') == {'etcd': {'host': 'localhost:4001'}}
+        assert parse_dcs('localhost:8500') == {'consul': {'host': 'localhost:8500'}}
+        assert parse_dcs('zookeeper://localhost') == {'zookeeper': {'hosts': ['localhost:2181']}}
+        assert parse_dcs('exhibitor://dummy') == {'zookeeper': {'exhibitor': {'hosts': ['dummy'], 'port': 8181}}}
+        assert parse_dcs('consul://localhost') == {'consul': {'host': 'localhost:8500'}}
+        self.assertRaises(PatroniCtlException, parse_dcs, 'invalid://test')
+
     def test_output_members(self):
         cluster = get_cluster_initialized_with_leader()
         self.assertIsNone(output_members(cluster, name='abc', fmt='pretty'))
@@ -75,125 +83,79 @@ class TestCtl(unittest.TestCase):
 
     @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
     @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
-    @patch('patroni.etcd.Etcd.set_failover_value', Mock(return_value=None))
-    @patch('patroni.ctl.wait_for_leader', Mock(return_value=get_cluster_initialized_with_leader()))
-    @patch('requests.get', requests_get)
-    @patch('requests.post', requests_get)
     @patch('patroni.ctl.post_patroni', Mock(return_value=MockResponse()))
     def test_failover(self):
-        with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())):
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
-other
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n\ny''')
+        assert 'leader' in result.output
 
-y''')
-            assert 'leader' in result.output
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n2100-01-01T12:23:00\ny''')
+        assert result.exit_code == 0
 
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
-other
-2100-01-01T12:23:00
-y''')
-            assert result.exit_code == 0
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n2030-01-01T12:23:00\ny''')
+        assert result.exit_code == 0
 
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
-other
-2030-01-01T12:23:00
-y''')
-            assert result.exit_code == 0
+        # Aborting failover,as we anser NO to the confirmation
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n\nN''')
+        assert result.exit_code == 1
 
-            # Aborting failover,as we anser NO to the confirmation
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
-other
+        # Target and source are equal
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nleader\n\ny''')
+        assert result.exit_code == 1
 
-N''')
-            assert result.exit_code == 1
+        # Reality is not part of this cluster
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nReality\n\ny''')
+        assert result.exit_code == 1
 
-            # Target and source are equal
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
-leader
+        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force'])
+        assert 'Member' in result.output
 
-y''')
-            assert result.exit_code == 1
+        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
+        assert result.exit_code == 0
 
-            # Reality is not part of this cluster
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
-Reality
+        # Invalid timestamp
+        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', 'invalid'])
+        assert result.exit_code != 0
 
-y''')
-            assert result.exit_code == 1
+        # Invalid timestamp
+        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2115-02-30T12:00:00+01:00'])
+        assert result.exit_code != 0
 
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--force'])
-            assert 'Member' in result.output
-
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--force',
-                                              '--scheduled', '2015-01-01T12:00:00+01:00'])
-            assert result.exit_code == 0
-
-            # Invalid timestamp
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', 'invalid'])
-            assert result.exit_code != 0
-
-            # Invalid timestamp
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--force',
-                                              '--scheduled', '2115-02-30T12:00:00+01:00'])
-            assert result.exit_code != 0
-
-            # Specifying wrong leader
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='dummy')
-            assert result.exit_code == 1
+        # Specifying wrong leader
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='dummy')
+        assert result.exit_code == 1
 
         with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_only_leader())):
             # No members available
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
-other
-
-y''')
+            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n\ny''')
             assert result.exit_code == 1
 
         with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_without_leader())):
             # No master available
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
-other
-
-y''')
+            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n\ny''')
             assert result.exit_code == 1
 
-        with patch('patroni.ctl.post_patroni', Mock(side_effect=Exception())):
+        with patch('patroni.ctl.post_patroni', Mock(side_effect=Exception)):
             # Non-responding patroni
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
-other
-
-y''')
+            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n\ny''')
             assert 'falling back to DCS' in result.output
 
-        mocked = Mock()
-        mocked.return_value.status_code = 500
-        with patch('patroni.ctl.post_patroni', Mock(return_value=mocked)):
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
-other
-
-y''')
+        with patch('patroni.ctl.post_patroni') as mocked:
+            mocked.return_value.status_code = 500
+            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n\ny''')
             assert 'Failover failed' in result.output
 
-    @patch('patroni.zookeeper.KazooClient', MockKazooClient)
-    @patch('requests.get', requests_get)
     def test_get_dcs(self):
-        self.assertIsNotNone(get_dcs({'dcs': {'scheme': 'zookeeper', 'hostname': 'foo', 'port': 2181}}, 'dummy'))
-        self.assertIsNotNone(get_dcs({'dcs': {'scheme': 'exhibitor', 'hostname': 'exhibitor', 'port': 8181}}, 'dummy'))
-        self.assertRaises(PatroniCtlException, get_dcs, {'scheme': 'dummy'}, 'dummy')
+        self.assertRaises(PatroniCtlException, get_dcs, {'dummy': {}}, 'dummy')
+        with patch('patroni.Patroni.get_dcs', Mock(return_value=self.e)):
+            assert get_dcs({'etcd': {'host': 'none'}}, 'dummy').client_path('') == '/service/test/'
 
     @patch('psycopg2.connect', psycopg2_connect)
     @patch('patroni.ctl.query_member', Mock(return_value=([['mock column']], None)))
+    @patch.object(etcd.Client, 'read', etcd_read)
     def test_query(self):
         with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)):
             # Mutually exclusive
-            result = self.runner.invoke(ctl, [
-                'query',
-                'alpha',
-                '--member',
-                'abc',
-                '--role',
-                'master',
-            ])
+            result = self.runner.invoke(ctl, ['query', 'alpha', '--member', 'abc', '--role', 'master'])
             assert result.exit_code == 1
 
             with self.runner.isolated_filesystem():
@@ -201,17 +163,11 @@ y''')
                     dummy_file.write('SELECT 1')
 
                 # Mutually exclusive
-                result = self.runner.invoke(ctl, [
-                    'query',
-                    'alpha',
-                    '--file',
-                    'dummy',
-                    '--command',
-                    'dummy',
-                ])
+                result = self.runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy', '--command', 'dummy'])
                 assert result.exit_code == 1
 
                 result = self.runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy'])
+                assert result.exit_code == 0
 
                 os.remove('dummy')
 
@@ -226,13 +182,16 @@ y''')
                                               '--password', '--dbname', 'postgres'], input='ab\nab')
             assert 'mock column' in result.output
 
-    @patch('patroni.ctl.get_cursor', Mock(return_value=MockConnect().cursor()))
     def test_query_member(self):
-        rows = query_member(None, None, None, 'master', 'SELECT pg_is_in_recovery()')
-        self.assertTrue('False' in str(rows))
+        with patch('patroni.ctl.get_cursor', Mock(return_value=MockConnect().cursor())):
+            rows = query_member(None, None, None, 'master', 'SELECT pg_is_in_recovery()')
+            self.assertTrue('False' in str(rows))
 
-        rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
-        self.assertEquals(rows, (None, None))
+            rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
+            self.assertEquals(rows, (None, None))
+
+            with patch('test_postgresql.MockCursor.execute', Mock(side_effect=OperationalError('bla'))):
+                rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
 
         with patch('patroni.ctl.get_cursor', Mock(return_value=None)):
             rows = query_member(None, None, None, None, 'SELECT pg_is_in_recovery()')
@@ -244,24 +203,14 @@ y''')
         with patch('patroni.ctl.get_cursor', Mock(side_effect=OperationalError('bla'))):
             rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
 
-        with patch('test_postgresql.MockCursor.execute', Mock(side_effect=OperationalError('bla'))):
-            rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
-
     @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
     def test_dsn(self):
         with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)):
-            result = self.runner.invoke(ctl, ['dsn', 'alpha', '--dcs', '8.8.8.8'])
+            result = self.runner.invoke(ctl, ['dsn', 'alpha'])
             assert 'host=127.0.0.1 port=5435' in result.output
 
             # Mutually exclusive options
-            result = self.runner.invoke(ctl, [
-                'dsn',
-                'alpha',
-                '--role',
-                'master',
-                '--member',
-                'dummy',
-            ])
+            result = self.runner.invoke(ctl, ['dsn', 'alpha', '--role', 'master', '--member', 'dummy'])
             assert result.exit_code == 1
 
             # Non-existing member
@@ -270,79 +219,60 @@ y''')
 
     @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
     @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
-    @patch('requests.get', requests_get)
     @patch('requests.post', requests_get)
     def test_restart_reinit(self):
-        result = self.runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='y')
+        result = self.runner.invoke(ctl, ['restart', 'alpha'], input='y')
+        assert 'restart failed for' in result.output
         assert result.exit_code == 0
 
-        result = self.runner.invoke(ctl, ['reinit', 'alpha', '--dcs', '8.8.8.8'], input='y')
+        result = self.runner.invoke(ctl, ['reinit', 'alpha'], input='y')
         assert result.exit_code == 1
 
         # Aborted restart
-        result = self.runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='N')
+        result = self.runner.invoke(ctl, ['restart', 'alpha'], input='N')
         assert result.exit_code == 1
 
         # Not a member
-        result = self.runner.invoke(ctl, [
-            'restart',
-            'alpha',
-            '--dcs',
-            '8.8.8.8',
-            'dummy',
-            '--any',
-        ], input='y')
+        result = self.runner.invoke(ctl, ['restart', 'alpha', 'dummy', '--any'], input='y')
         assert result.exit_code == 1
 
         with patch('requests.post', Mock(return_value=MockResponse())):
-            result = self.runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='y')
+            result = self.runner.invoke(ctl, ['restart', 'alpha'], input='y')
+            assert result.exit_code == 0
 
     @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
-    @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
+    @patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
     def test_remove(self):
-        result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='alpha\nslave')
-        assert 'Please confirm' in result.output
-        assert 'You are about to remove all' in result.output
-        # Not typing an exact confirmation
-        assert result.exit_code == 1
-
-        # master specified does not match master of cluster
-        result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='''alpha
-Yes I am aware
-slave''')
-        assert result.exit_code == 1
-
-        # cluster specified on cmdline does not match verification prompt
-        result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='beta\nleader')
-        assert result.exit_code == 1
-
-        with patch('patroni.etcd.Etcd.get_cluster', get_cluster_initialized_with_leader):
-            result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'],
-                                        input='''alpha
-Yes I am aware
-leader''')
-            assert 'object has no attribute' in str(result.exception)
-
-        with patch('patroni.ctl.get_dcs', Mock(return_value=Mock())):
-            # Not implemented DCS
-            result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='''alpha
-Yes I am aware
-leader''')
+        with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)):
+            result = self.runner.invoke(ctl, ['remove', 'alpha'], input='alpha\nslave')
+            assert 'Please confirm' in result.output
+            assert 'You are about to remove all' in result.output
+            # Not typing an exact confirmation
             assert result.exit_code == 1
 
+            # master specified does not match master of cluster
+            result = self.runner.invoke(ctl, ['remove', 'alpha'], input='''alpha\nYes I am aware\nslave''')
+            assert result.exit_code == 1
+
+            # cluster specified on cmdline does not match verification prompt
+            result = self.runner.invoke(ctl, ['remove', 'alpha'], input='beta\nleader')
+            assert result.exit_code == 1
+
+            result = self.runner.invoke(ctl, ['remove', 'alpha'], input='''alpha\nYes I am aware\nleader''')
+            assert result.exit_code == 0
+
     @patch('patroni.etcd.Etcd.watch', Mock(return_value=None))
     @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
     def test_wait_for_leader(self):
-        dcs = self.e
-        self.assertRaises(PatroniCtlException, wait_for_leader, dcs, 0)
+        self.assertRaises(PatroniCtlException, wait_for_leader, self.e, 0)
 
-        cluster = wait_for_leader(dcs=dcs, timeout=2)
+        cluster = wait_for_leader(self.e, timeout=2)
         assert cluster.leader.member.name == 'leader'
 
+    @patch('requests.post', Mock(side_effect=requests.exceptions.ConnectionError('foo')))
     def test_post_patroni(self):
-        with patch('requests.post', MagicMock(side_effect=requests.exceptions.ConnectionError('foo'))):
-            member = get_cluster_initialized_with_leader().leader.member
-            self.assertRaises(requests.exceptions.ConnectionError, post_patroni, member, 'dummy', {})
+        member = get_cluster_initialized_with_leader().leader.member
+        self.assertRaises(requests.exceptions.ConnectionError, post_patroni, member, 'dummy', {})
 
     def test_ctl(self):
         self.runner.invoke(ctl, ['list'])
@@ -371,20 +301,11 @@ leader''')
 
     @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
     @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
-    @patch('requests.get', requests_get)
-    @patch('requests.post', requests_get)
     def test_members(self):
         result = self.runner.invoke(members, ['alpha'])
+        assert '127.0.0.1' in result.output
         assert result.exit_code == 0
 
     def test_configure(self):
-        result = self.runner.invoke(configure, [
-            '--dcs',
-            'abc',
-            '-c',
-            'dummy',
-            '-n',
-            'bla',
-        ])
-
+        result = self.runner.invoke(configure, ['--dcs', 'abc', '-c', 'dummy', '-n', 'bla'])
         assert result.exit_code == 0

tests/test_etcd.py

@@ -1,15 +1,15 @@
 import etcd
 import json
 import requests
-import urllib3
 import socket
 import unittest
 
 from dns.exception import DNSException
 from mock import Mock, patch
-from patroni.dcs import Cluster
+from patroni.dcs import Cluster, AbstractDCS
 from patroni.etcd import Client, Etcd, EtcdError
 from patroni.exceptions import DCSError
+from urllib3.exceptions import ReadTimeoutError
 
 
 class MockResponse(object):
@@ -36,16 +36,6 @@ class MockResponse(object):
         return ''
 
 
-class MockPostgresql(Mock):
-
-    server_version = '999999'
-    scope = 'dummy'
-
-    @staticmethod
-    def last_operation():
-        return '0'
-
-
 def requests_get(url, **kwargs):
     members = '[{"id":14855829450254237642,"peerURLs":["http://localhost:2380","http://localhost:7001"],' +\
               '"name":"default","clientURLs":["http://localhost:2379","http://localhost:4001"]}]'
@@ -64,20 +54,16 @@ def requests_get(url, **kwargs):
     return response
 
 
-def etcd_watch(key, index=None, timeout=None, recursive=None):
+def etcd_watch(self, key, index=None, timeout=None, recursive=None):
     if timeout == 2.0:
         raise etcd.EtcdWatchTimedOut
     elif timeout == 5.0:
         return etcd.EtcdResult('delete', {})
     elif timeout == 10.0:
         raise etcd.EtcdException
-    elif index == 20729:
-        return etcd.EtcdResult('set', {'value': 'postgresql1', 'modifiedIndex': index + 1})
-    elif index == 20731:
-        return etcd.EtcdResult('set', {'value': 'postgresql2', 'modifiedIndex': index + 1})
 
 
-def etcd_write(key, value, **kwargs):
+def etcd_write(self, key, value, **kwargs):
     if key == '/service/exists/leader':
         raise etcd.EtcdAlreadyExist
     if key in ['/service/test/leader', '/patroni/test/leader'] and \
@@ -86,7 +72,7 @@ def etcd_write(key, value, **kwargs):
     raise etcd.EtcdException
 
 
-def etcd_read(key, **kwargs):
+def etcd_read(self, key, **kwargs):
     if key == '/service/noleader/':
         raise DCSError('noleader')
     elif key == '/service/nocluster/':
@@ -144,7 +130,7 @@ def socket_getaddrinfo(*args):
 
 def http_request(method, url, **kwargs):
     if url == 'http://localhost:2379/timeout':
-        raise urllib3.exceptions.ReadTimeoutError(None, None, None)
+        raise ReadTimeoutError(None, None, None)
     if url == 'http://localhost:2379/':
         return MockResponse()
     raise socket.error
@@ -198,15 +184,15 @@ class TestClient(unittest.TestCase):
 
 
 @patch('requests.get', requests_get)
+@patch.object(etcd.Client, 'write', etcd_write)
+@patch.object(etcd.Client, 'read', etcd_read)
+@patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
 class TestEtcd(unittest.TestCase):
 
     def setUp(self):
         with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
             self.etcd = Etcd('foo', {'namespace': '/patroni/', 'ttl': 30, 'host': 'localhost:2379', 'scope': 'test'})
-            self.etcd.client.write = etcd_write
-            self.etcd.client.read = etcd_read
-            self.etcd.client.delete = Mock(side_effect=etcd.EtcdException())
 
     def test_base_path(self):
         self.assertEquals(self.etcd._base_path, '/patroni/test')
@@ -254,14 +240,14 @@ class TestEtcd(unittest.TestCase):
     def test_delete_leader(self):
         self.assertFalse(self.etcd.delete_leader())
 
+    @patch.object(etcd.Client, 'watch', etcd_watch)
     def test_watch(self):
-        self.etcd.client.watch = etcd_watch
         self.etcd.watch(0)
         self.etcd.get_cluster()
         self.etcd.watch(1.5)
         self.etcd.watch(4.5)
-        self.etcd.watch(9.5)
-        self.etcd.watch(100)
+        with patch.object(AbstractDCS, 'watch', Mock()):
+            self.etcd.watch(9.5)
 
     @patch('patroni.etcd.Etcd.retry', Mock(side_effect=AttributeError("foo")))
     def test_other_exceptions(self):

tests/test_ha.py

@@ -1,8 +1,8 @@
+import etcd
 import unittest
 import datetime
 import pytz
 
-from etcd import EtcdException
 from mock import Mock, MagicMock, patch
 from patroni.dcs import Cluster, Failover, Leader, Member
 from patroni.etcd import Client, Etcd
@@ -33,7 +33,7 @@ def get_cluster_initialized_without_leader(leader=False, failover=None):
                                   'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location': 4})
     l = Leader(0, 0, m1) if leader else None
     m2 = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
-                                 'api_url': 'http://127.0.0.1:8011/patroni'})
+                                 'api_url': 'http://127.0.0.1:8011/patroni', 'tags': {'clonefrom': True}})
     return get_cluster(True, l, [m1, m2], failover)
 
 
@@ -52,7 +52,7 @@ class MockPatroni(object):
         self.postgresql = p
         self.dcs = d
         self.api = Mock()
-        self.tags = {}
+        self.tags = {'foo': 'bar'}
         self.nofailover = None
         self.nap_time = 10
         self.replicatefrom = None
@@ -76,10 +76,14 @@ def run_async(func, args=()):
 @patch.object(Postgresql, 'write_recovery_conf', Mock())
 @patch.object(Postgresql, 'query', Mock())
 @patch.object(Postgresql, 'checkpoint', Mock())
+@patch.object(etcd.Client, 'write', etcd_write)
+@patch.object(etcd.Client, 'read', etcd_read)
+@patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
 @patch('subprocess.call', Mock(return_value=0))
 class TestHa(unittest.TestCase):
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
+    @patch.object(etcd.Client, 'read', etcd_read)
     def setUp(self):
         with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
@@ -87,12 +91,10 @@ class TestHa(unittest.TestCase):
                                  'data_dir': 'data/postgresql0', 'superuser': {}, 'admin': {},
                                  'replication': {'username': '', 'password': '', 'network': ''}})
             self.p.set_state('running')
+            self.p.set_role('replica')
             self.p.check_replication_lag = true
             self.p.can_create_replica_without_replication_connection = MagicMock(return_value=False)
             self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'})
-            self.e.client.read = etcd_read
-            self.e.client.write = etcd_write
-            self.e.client.delete = Mock(side_effect=EtcdException())
             self.ha = Ha(MockPatroni(self.p, self.e))
             self.ha._async_executor.run_async = run_async
             self.ha.old_cluster = self.e.get_cluster()
@@ -109,25 +111,25 @@ class TestHa(unittest.TestCase):
 
     def test_start_as_replica(self):
         self.p.is_healthy = false
-        self.assertEquals(self.ha.run_cycle(), 'started as a secondary')
+        self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
 
     def test_recover_replica_failed(self):
         self.p.controldata = lambda: {'Database cluster state': 'in production'}
         self.p.is_healthy = false
         self.p.is_running = false
         self.p.follow = false
-        self.assertEquals(self.ha.run_cycle(), 'started as a secondary')
+        self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
         self.assertEquals(self.ha.run_cycle(), 'failed to start postgres')
 
     def test_recover_master_failed(self):
         self.p.follow = false
         self.p.is_healthy = false
         self.p.is_running = false
-        self.ha.has_lock = true
+        self.p.name = 'leader'
         self.p.set_role('master')
         self.p.controldata = lambda: {'Database cluster state': 'in production'}
-        self.assertEquals(self.ha.run_cycle(), 'started as readonly because i had the session lock')
-        self.assertEquals(self.ha.run_cycle(), 'removed leader key after trying and failing to start postgres')
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.assertEquals(self.ha.run_cycle(), 'starting as readonly because i had the session lock')
 
     @patch('sys.exit', return_value=1)
     @patch('patroni.ha.Ha.sysid_valid', MagicMock(return_value=True))
@@ -201,14 +203,8 @@ class TestHa(unittest.TestCase):
         self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
         self.assertEquals(self.ha.run_cycle(), 'demoted self because DCS is not accessible and i was a leader')
 
-    def test_bootstrap_from_leader(self):
-        self.ha.cluster = get_cluster_initialized_with_leader()
-        self.p.bootstrap = false
-        self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap from leader')
-
     def test_bootstrap_from_another_member(self):
         self.ha.cluster = get_cluster_initialized_with_leader()
-        self.ha.patroni.clonefrom = 'other'
         self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap from replica \'other\'')
 
     def test_bootstrap_waiting_for_leader(self):
@@ -218,7 +214,7 @@ class TestHa(unittest.TestCase):
     def test_bootstrap_without_leader(self):
         self.ha.cluster = get_cluster_initialized_without_leader()
         self.p.can_create_replica_without_replication_connection = MagicMock(return_value=True)
-        self.assertEquals(self.ha.bootstrap(), "trying to bootstrap without leader")
+        self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap (without leader)')
 
     def test_bootstrap_initialize_lock_failed(self):
         self.ha.cluster = get_cluster_not_initialized_without_leader()
@@ -273,6 +269,7 @@ class TestHa(unittest.TestCase):
         self.assertEquals(self.ha.run_cycle(), 'failed to update leader lock during restart')
 
     @patch('requests.get', requests_get)
+    @patch('time.sleep', Mock())
     def test_manual_failover_from_leader(self):
         self.ha.has_lock = true
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None))

tests/test_patroni.py

@@ -1,3 +1,5 @@
+import etcd
+import os
 import sys
 import time
 import unittest
@@ -6,8 +8,9 @@ import yaml
 from mock import Mock, patch
 from patroni.api import RestApiServer
 from patroni.async_executor import AsyncExecutor
+from patroni.consul import Consul
 from patroni.etcd import Etcd
-from patroni import Patroni, main as _main
+from patroni import Patroni, PatroniException, main as _main
 from patroni.zookeeper import ZooKeeper
 from six.moves import BaseHTTPServer
 from test_etcd import Client, SleepException, etcd_read, etcd_write
@@ -22,6 +25,8 @@ from test_zookeeper import MockKazooClient
 @patch.object(Postgresql, 'write_recovery_conf', Mock())
 @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
 @patch.object(AsyncExecutor, 'run', Mock())
+@patch.object(etcd.Client, 'write', etcd_write)
+@patch.object(etcd.Client, 'read', etcd_read)
 class TestPatroni(unittest.TestCase):
 
     def setUp(self):
@@ -35,15 +40,15 @@ class TestPatroni(unittest.TestCase):
             with open('postgres0.yml', 'r') as f:
                 config = yaml.load(f)
                 self.p = Patroni(config)
-                self.p.ha.dcs.client.write = etcd_write
-                self.p.ha.dcs.client.read = etcd_read
 
     @patch('patroni.zookeeper.KazooClient', MockKazooClient())
+    @patch.object(Consul, 'create_or_restore_session', Mock())
     def test_get_dcs(self):
         self.assertIsInstance(self.p.get_dcs('', {'zookeeper': {'scope': '', 'hosts': ''}}), ZooKeeper)
-        self.assertRaises(Exception, self.p.get_dcs, '', {})
+        self.assertIsInstance(self.p.get_dcs('', {'consul': {'scope': '', 'hosts': '127.0.0.1:1'}}), Consul)
+        self.assertRaises(PatroniException, self.p.get_dcs, '', {})
 
-    @patch('time.sleep', Mock(side_effect=SleepException()))
+    @patch('time.sleep', Mock(side_effect=SleepException))
     @patch.object(Etcd, 'delete_leader', Mock())
     @patch.object(Client, 'machines')
     def test_patroni_main(self, mock_machines):
@@ -51,17 +56,21 @@ class TestPatroni(unittest.TestCase):
         sys.argv = ['patroni.py', 'postgres0.yml']
 
         mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
-        with patch.object(Patroni, 'run', Mock(side_effect=SleepException())):
+        with patch.object(Patroni, 'run', Mock(side_effect=SleepException)):
             self.assertRaises(SleepException, _main)
         with patch.object(Patroni, 'run', Mock(side_effect=KeyboardInterrupt())):
             _main()
+        sys.argv = ['patroni.py']
+        # read the content of the yaml configuration file into the environment variable
+        # in order to test how does patroni handle the configuration passed from the environment.
+        with open('postgres0.yml', 'r') as f:
+            os.environ[Patroni.PATRONI_CONFIG_VARIABLE] = f.read()
+        with patch.object(Patroni, 'run', Mock(side_effect=SleepException())):
+            self.assertRaises(SleepException, _main)
+        del os.environ[Patroni.PATRONI_CONFIG_VARIABLE]
 
-    @patch('time.sleep', Mock(side_effect=SleepException()))
     def test_run(self):
-        self.p.ha.dcs.watch = Mock(side_effect=SleepException())
-        self.assertRaises(SleepException, self.p.run)
-
-        self.p.ha.state_handler.is_leader = Mock(return_value=False)
+        self.p.ha.dcs.watch = Mock(side_effect=SleepException)
         self.p.api.start = Mock()
         self.assertRaises(SleepException, self.p.run)
 
@@ -71,6 +80,10 @@ class TestPatroni(unittest.TestCase):
         self.p.next_run = time.time() - self.p.nap_time - 1
         self.p.schedule_next_run()
 
+    def test_noloadbalance(self):
+        self.p.tags['noloadbalance'] = True
+        self.assertTrue(self.p.noloadbalance)
+
     def test_nofailover(self):
         self.p.tags['nofailover'] = True
         self.assertTrue(self.p.nofailover)

tests/test_postgresql.py

@@ -54,10 +54,6 @@ class MockCursor(object):
     def fetchall(self):
         return self.results
 
-    @staticmethod
-    def close():
-        pass
-
     def __iter__(self):
         for i in self.results:
             yield i
@@ -69,8 +65,9 @@ class MockCursor(object):
         pass
 
 
-class MockConnect(Mock):
+class MockConnect(object):
 
+    server_version = '99999'
     autocommit = False
     closed = 0
 
@@ -83,6 +80,10 @@ class MockConnect(Mock):
     def __exit__(self, *args):
         pass
 
+    @staticmethod
+    def close():
+        pass
+
 
 def pg_controldata_string(*args, **kwargs):
     return b"""
@@ -163,13 +164,14 @@ class TestPostgresql(unittest.TestCase):
     def setUp(self):
         self.p = Postgresql({'name': 'test0', 'scope': 'batman', 'data_dir': 'data/test0',
                              'listen': '127.0.0.1, *:5432', 'connect_address': '127.0.0.2:5432',
-                             'pg_hba': ['hostssl all all 0.0.0.0/0 md5', 'host all all 0.0.0.0/0 md5'],
+                             'pg_hba': ['host replication replicator 127.0.0.1/32 md5',
+                                        'hostssl all all 0.0.0.0/0 md5',
+                                        'host all all 0.0.0.0/0 md5'],
                              'superuser': {'username': 'test', 'password': 'test'},
                              'admin': {'username': 'admin', 'password': 'admin'},
                              'pg_rewind': {'username': 'admin', 'password': 'admin'},
                              'replication': {'username': 'replicator',
-                                             'password': 'rep-pass',
-                                             'network': '127.0.0.1/32'},
+                                             'password': 'rep-pass'},
                              'parameters': {'foo': 'bar'}, 'recovery_conf': {'foo': 'bar'},
                              'callbacks': {'on_start': 'true', 'on_stop': 'true',
                                            'on_restart': 'true', 'on_role_change': 'true',
@@ -204,6 +206,11 @@ class TestPostgresql(unittest.TestCase):
         self.assertTrue(self.p.initialize())
         self.assertTrue(os.path.exists(os.path.join(self.p.data_dir, 'pg_hba.conf')))
 
+        with open(os.path.join(self.p.data_dir, 'pg_hba.conf')) as f:
+            lines = f.readlines()
+            assert 'host replication replicator 127.0.0.1/32 md5\n' in lines
+            assert 'host all all 0.0.0.0/0 md5\n' in lines
+
     @patch('os.path.exists', Mock(return_value=True))
     @patch('os.unlink', Mock())
     def test_delete_trigger_file(self):
@@ -256,12 +263,10 @@ class TestPostgresql(unittest.TestCase):
         self.p.follow(Leader(-1, 28, self.other))
         self.p.rewind = mock_pg_rewind
         self.p.follow(self.leader)
-        self.p.require_rewind()
         with mock.patch('os.path.islink', MagicMock(return_value=True)):
             with mock.patch('patroni.postgresql.Postgresql.can_rewind', new_callable=PropertyMock(return_value=True)):
                 with mock.patch('os.unlink', MagicMock(return_value=True)):
                     self.p.follow(self.leader, recovery=True)
-        self.p.require_rewind()
         with mock.patch('patroni.postgresql.Postgresql.can_rewind', new_callable=PropertyMock(return_value=True)):
             self.p.rewind.return_value = True
             self.p.follow(self.leader, recovery=True)

tests/test_utils.py

@@ -2,7 +2,8 @@ import unittest
 
 from mock import Mock, patch
 from patroni.exceptions import PatroniException
-from patroni.utils import Retry, RetryFailedError, reap_children, sigchld_handler, sigterm_handler, sleep
+from patroni.utils import reap_children, Retry, RetryFailedError, set_ignore_sigterm,\
+    sigchld_handler, sigterm_handler, sleep
 
 
 def time_sleep(_):
@@ -12,6 +13,7 @@ def time_sleep(_):
 class TestUtils(unittest.TestCase):
 
     def test_sigterm_handler(self):
+        set_ignore_sigterm(False)
         self.assertRaises(SystemExit, sigterm_handler, None, None)
 
     @patch('time.sleep', Mock())

tests/test_wale_restore.py

@@ -6,22 +6,6 @@ from mock import MagicMock, patch, PropertyMock
 from patroni.scripts.wale_restore import WALERestore, main as _main
 
 
-def fake_cursor_fetchone(*args, **kwargs):
-    return ('16777216',)
-
-
-def fake_call_fail_for_wal_e(*args, **kwargs):
-    if len(args) > 0 and 'backup-fetch' in args[0]:
-        return 1
-    return 0
-
-
-def fake_call_fail_for_base_backup(*args, **kwargs):
-    if len(args) > 0 and 'backup-fetch' in args[0]:
-        return 0
-    return 1
-
-
 def fake_backup_data(self, *args, **kwargs):
     """ return the fake result of WAL-E backup-list"""
     return """name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop
@@ -52,7 +36,6 @@ base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z 167772160   000
 @patch('os.makedirs', MagicMock(return_value=True))
 @patch('os.path.exists', MagicMock(return_value=True))
 @patch('os.path.isdir', MagicMock(return_value=True))
-@patch('psycopg2.extensions.cursor.fetchone', MagicMock(side_effect=fake_cursor_fetchone))
 @patch('psycopg2.extensions.cursor', MagicMock(autospec=True))
 @patch('psycopg2.extensions.connection', MagicMock(autospec=True))
 @patch('psycopg2.connect', MagicMock(autospec=True))
@@ -60,11 +43,7 @@ base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z 167772160   000
 class TestWALERestore(unittest.TestCase):
 
     def setUp(self):
-        self.wale_restore = WALERestore("batman", "/data",
-                                        "host=batman port=5432 user=batman", "/etc", 100, 100, 1, 0)
-
-    def tearDown(self):
-        pass
+        self.wale_restore = WALERestore("batman", "/data", "host=batman port=5432 user=batman", "/etc", 100, 100, 1, 0)
 
     def test_should_use_s3_to_create_replica(self):
         with patch('psycopg2.connect', MagicMock(side_effect=psycopg2.Error("foo"))):

tests/test_zookeeper.py

@@ -14,7 +14,6 @@ class MockKazooClient(Mock):
 
     leader = False
     exists = True
-    handler = Mock()
 
     @property
     def client_id(self):
@@ -34,8 +33,6 @@ class MockKazooClient(Mock):
                 b'postgres://repuser:rep-pass@localhost:5434/postgres?application_name=http://127.0.0.1:8009/patroni',
                 ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0 if self.exists else -1, 0, 0, 0)
             )
-        elif path.endswith('/optime/leader'):
-            return (b'1', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))
         elif path.endswith('/leader'):
             if self.leader:
                 return (b'foo', ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0))
@@ -86,13 +83,11 @@ class MockKazooClient(Mock):
             raise TypeError("Invalid type for 'path' (string expected)")
         self.exists = False
         if path == '/service/test/leader':
-            if self.leader:
-                return
             self.leader = True
             raise Exception
         elif path == '/service/test/members/buzz':
             raise Exception
-        elif path.endswith('/initialize') or path == '/service/test/members/bar':
+        elif path.endswith('/') or path.endswith('/initialize') or path == '/service/test/members/bar':
             raise NoNodeError
 
 
@@ -152,16 +147,18 @@ class TestZooKeeper(unittest.TestCase):
         self.zk._name = 'bar'
         self.zk.touch_member('new')
         self.zk._name = 'na'
-        self.zk.client.exists = 1
+        self.zk._client.exists = 1
         self.zk.touch_member('exists')
         self.zk._name = 'bar'
         self.zk.touch_member('retry')
-        self.zk.fetch_cluster = True
+        self.zk._fetch_cluster = True
         self.zk.get_cluster()
         self.zk.touch_member('retry')
 
     def test_take_leader(self):
         self.zk.take_leader()
+        with patch.object(MockKazooClient, 'create', Mock(side_effect=Exception)):
+            self.zk.take_leader()
 
     def test_update_leader(self):
         self.assertTrue(self.zk.update_leader())
@@ -172,6 +169,9 @@ class TestZooKeeper(unittest.TestCase):
         self.zk._base_path = self.zk._base_path.replace('test', 'bla')
         self.zk.write_leader_optime('2')
 
+    def test_delete_cluster(self):
+        self.assertTrue(self.zk.delete_cluster())
+
     def test_watch(self):
         self.zk.watch(0)
         self.zk.event.isSet = lambda: True