Fix the README in order to show BDR-related PostgreSQL options.

Patroni changes to support BDR.
Currently, BDR and physical replication at the same time is not supported. BDR requires additional postgresql configuration and a patched version of PostgreSQL 9.4 + BDR plugin. Included is the Docker image to try it locally.
2026-08-28 08:29:21 +00:00 · 2015-12-18 17:56:01 +01:00 · 2015-12-18 17:15:40 +01:00
50 changed files with 1568 additions and 2871 deletions
@@ -1,82 +1,14 @@
-sudo: false
 language: python
 python:
- "3.5"
-addons:
-  apt:
-    packages:
-    - postgresql-contrib-9.5
-  postgresql: "9.5"
-env:
-  global:
-  - ETCDVERSION=2.3.2 ZKVERSION=3.4.6 CONSULVERSION=0.6.4
-  matrix:
-  - TEST_SUITE="python setup.py"
-  - DCS="etcd" TEST_SUITE="behave"
-  - DCS="exhibitor" TEST_SUITE="behave"
-  - DCS="consul" TEST_SUITE="behave"
-cache:
-  directories:
-  - $HOME/virtualenv/python2.7.9
-  - $HOME/virtualenv/python3.4.2
-  - $HOME/virtualenv/python3.5.0
+  - "2.7"
+  - "3.3"
+  - "3.4"
 install:
-  - |
-    set -e
-
-    if [[ $TEST_SUITE == "behave" ]]; then
-        if [[ $DCS == "consul" ]]; then
-            curl -L https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip \
-                | gunzip > consul
-            chmod +x consul
-        fi
-
-        if [[ $DCS == "etcd" ]]; then
-            curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
-                | tar xz -C . --strip=1 --wildcards --no-anchored etcd
-        fi
-
-        if [[ $DCS == "exhibitor" ]]; then
-            curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz
-            mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg
-            zookeeper-${ZKVERSION}/bin/zkServer.sh start
-            # following lines are 'emulating' exhibitor REST API
-            while true; do
-                echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \
-                    | nc -l 8181 &> /dev/null
-            done&
-        fi
-    fi
-
-    for pv in "2.7" "3.4" "3.5"; do
-        source ~/virtualenv/python${pv}/bin/activate
-        # explicitly install all needed python modules to cache them
-        for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8 mock pytest-cov pytest'; do
-            pip install $p
-        done
-    done
+  - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py2.txt --use-mirrors; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r requirements-py3.txt --use-mirrors; fi
+  - pip install coveralls
 script:
-  - |
-    for pv in "2.7" "3.4" "3.5"; do
-        source ~/virtualenv/python${pv}/bin/activate
-
-        if [[ $TEST_SUITE == "behave" ]]; then
-            if [[ $pv != "3.4" ]]; then
-                echo Running acceptance tests using python${pv}
-                if ! PATH=.:$PATH $TEST_SUITE; then
-                    # output all log files when tests are failing
-                    grep . features/output/*/*postgres?.*
-                    exit 1
-                fi
-            fi
-        else
-            echo Running unit tests using python${pv}
-            $TEST_SUITE test
-            $TEST_SUITE flake8
-        fi
-    done
-
-    set +e
+  - python setup.py test
+  - python setup.py flake8
 after_success:
  - coveralls
-  - if [[ $TEST_SUITE != "behave" ]]; then python-codacy-coverage -r coverage.xml; fi
@@ -1,12 +0,0 @@
-approvals:
-  # PR needs at least 4 approvals
-  minimum: 1
-  # approval = comment that matches this regex
-  pattern: "^:?\\+1:?$"
-  from:
-    # commenter must be either one of:
-    # a public zalando org member
-    orgs:
-      - zalando
-    # a collaborator of the repo
-    collaborators: true
@@ -6,28 +6,30 @@ MAINTAINER Feike Steenbergen <[email protected]>
 # We need curl
 RUN apt-get update -y && apt-get install curl -y

-# Add PGDG repositories
+# Add PGDG and BDR repositories
 RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
+RUN echo "deb http://packages.2ndquadrant.com/bdr/apt/ $(lsb_release -cs)-2ndquadrant main" >> /etc/apt/sources.list.d/pgdg.list
 RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+# import the BDR key
+RUN curl -s -o - http://packages.2ndquadrant.com/bdr/apt/AA7A6805.asc | sudo apt-key add -
+
 RUN apt-get update -y
 RUN apt-get upgrade -y

-ENV PGVERSION 9.5
-RUN apt-get install postgresql-${PGVERSION} postgresql-server-dev-${PGVERSION} -y
-RUN apt-get install python python-dev python-pip -y
-ADD requirements-py2.txt /requirements-py2.txt
-RUN pip install -r /requirements-py2.txt
+ENV PGVERSION 9.4
+ENV PGTYPE postgresql-bdr
+ENV PGCOMPATIBLETYPE postgresql
+
+RUN apt-get install python python-yaml python-requests python-boto ${PGTYPE}-${PGVERSION} ${PGTYPE}-${PGVERSION}-bdr-plugin python-dnspython python-kazoo python-pip -y
+RUN apt-get install python-dev ${PGTYPE}-server-dev-${PGVERSION} -y
+RUN pip install python-etcd psycopg2

 ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:$PATH

 ADD patroni.py /patroni.py
-ADD patronictl.py /patronictl.py
 ADD patroni/ /patroni

-RUN ln -s /patroni.py /usr/local/bin/patroni
-RUN ln -s /patronictl.py /usr/local/bin/patronictl
-
-ENV ETCDVERSION 2.2.5
+ENV ETCDVERSION 2.0.13
 RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C /bin --strip=1 --wildcards --no-anchored etcd etcdctl

 ### Setting up a simple script that will serve as an entrypoint
@@ -1,3 +0,0 @@
-Alexander Kukushkin <[email protected]>
-Feike Steenbergen <[email protected]>
-Oleksii Kliukin <[email protected]>
@@ -1,6 +1,6 @@
 |Build Status| |Coverage Status|

-Patroni: A Template for PostgreSQL HA with ZooKeeper, etcd or Consul
+Patroni: A Template for PostgreSQL HA with ZooKeeper or etcd
 ------------------------------------------------------------

 Patroni was previously known as Governor.
@@ -8,7 +8,7 @@ Patroni was previously known as Governor.
 *There are many ways to run high availability with PostgreSQL. Here, we
 present a template for you to create your own customized, high-availability 
 solution using Python and — for maximum accessibility — a distributed 
-configuration store like ZooKeeper, etcd or Consul.*
+configuration store like ZooKeeper or etcd.*

 Getting Started
 ---------------
@@ -61,17 +61,12 @@ For an example file, see ``postgres0.yml``. Regarding settings:
    -  *keyfile*: (optional) Specifies a file with the secret key in the PEM format.

 -  *etcd*:
-    -  *scope*: the relative path used on etcd's HTTP API for this deployment; makes it possible to run multiple HA deployments from a single etcd cluster.
+    -  *scope*: the relative path used on etcd's HTTP API for this deployment; makes it possible to run multiple HA deployments from a single etcd.
    -  *ttl*: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process.
    -  *host*: the host:port for the etcd endpoint.

-  *consul*:
-    -  *scope*: the relative path used on Consul's HTTP API for this deployment; makes it possible to run multiple HA deployments from a single Consul cluster.
-    -  *ttl*: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process.
-    -  *host*: the host:port for the Consul endpoint.
-
 -  *zookeeper*:
-    -  *scope*: the relative path used on ZooKeeper for this deployment; makes it possible to run multiple HA deployments from a single ZooKeeper cluster.
+    -  *scope*: the relative path used on etcd's HTTP API for this deployment; makes it possible to run multiple HA deployments from a single etcd.
    -  *session\_timeout*: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process.
    -  *reconnect\_timeout*: how long we should try to reconnect to ZooKeeper after a connection loss. After this timeout, assume that you no longer have a lock and restart in read-only mode.
    -  *hosts*: list of ZooKeeper cluster members in format: ['host1:port1', 'host2:port2', 'etc...']
@@ -80,10 +75,13 @@ For an example file, see ``postgres0.yml``. Regarding settings:
        -  *port*: Exhibitor port.
        -  *hosts*: initial list of Exhibitor (ZooKeeper) nodes in format: ['host1', 'host2', 'etc...' ]. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes.

+-  *bdr*:
+    - *enable*: on if you want to enable BDR
+    - *database*: database name to support BDR (only a single database is supported)
+
 -  *postgresql*:
    -  *name*: the name of the Postgres host. Must be unique for the cluster.
-    -  *listen*: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. The first address from this list will be used by Patroni to establish local connections to the PostgreSQL node. 
-    
+    -  *listen*: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication.
    -  *connect\_address*: IP address + port through which Postgres is accessible from other nodes and applications.
    -  *data\_dir*: file path to initialize and store Postgres data files.
    -  *maximum\_lag\_on\_failover*: the maximum bytes a follower may lag.
@@ -96,11 +94,11 @@ For an example file, see ``postgres0.yml``. Regarding settings:

    -  *pg\_hba*: list of lines which should be added to pg\_hba.conf.
        -  *- host all all 0.0.0.0/0 md5*.
-        -  *- host replication replicator 127.0.0.1/32 md5* # A line like this is required for replication

    -  *replication*:
        -  *username*: replication username; user will be created during initialization.
        -  *password*: replication password; user will be created during initialization.
+        -  *network*: network setting for replication in pg\_hba.conf.

    -  *callbacks* callback scripts to run on certain actions. Patroni will pass the action, role and cluster name. See scripts/aws.py as an example on how to write them.
        -  *on\_start*: a script to run when the cluster starts.
@@ -169,6 +167,31 @@ Choosing your replication schema is dependent on your business
 considerations. Investigate both async and sync replication, as well as other
 HA solutions, to determine which solution is best for you.

+You can also use BDR (bi-directional replication) if you have a compatible
+PostgreSQL version with the BDR plugin installed (see http://bdr-project.org/docs/next/installation.html).
+It will require adding a BDR shared_library to your configuration, as well as
+setting the following options for postgresql (see http://bdr-project.org/docs/next/settings-prerequisite.html):
+
+.. code:: YAML
+
+        max_worker_processes: 10
+        max_replication_slots: 10
+        max_wal_senders: 10
+        shared_preload_libraries: 'bdr'
+        track_commit_timestamp: 'on'
+        wal_level: 'logical'
+
+At the moment Patroni BDR is not compatible with a streaming replication,
+if BDR is enabled normal replica node won't be able to join. This is not
+a principal limitation of BDR, and we might resolve this in the future
+(although 'promotion' will only work between nodes running a physical
+replication, i.e. a replica won't be able to attach to a different
+multimaster node).
+
+Another limitation is that only one database is supported at the moment.
+BDR requires the replication user to be also a superuser, so you might
+want to excersie extra caution when choosing the password for this user.
+
 Applications Should Not Use Superusers
 --------------------------------------

@@ -1,16 +1,4 @@
 Failover
 ========
- When determining who should become master, include the minor version of PostgreSQL in the decision.
- Create a way to disable governance of a cluster, something like the existence of a "nogover" or "admin" file in PGDATA will stop patroni from changing the cluster state.
-
-Configuration
-==============
- Provide a way to change postgresql.conf and pg_hba.conf of a running cluster on the Patroni level, without changing individual nodes.
- Provide hooks to store and retrieve cluster-wide passwords without exposing them in a plain-text form to unauthorized users.
- Implement patronictl command to create initial configuration of the cluster with leader and member keys fixed to the user-supplied values in order to simplify migrations.
- Implement support for consul in addtion to etcd and zookeeper
- Complete zookeeper support in patronictl
-
-Documentation
-==============
- Document how to run cascading replication and possibly initialize the cluster without an access to the master node.
+- When determining who should become master, include the minor version of PostgreSQL in the decision
+- Create a way to disable governance of a cluster, something like the existence of a "nogover" or "admin" file in PGDATA will stop governor from changing the cluster state
@@ -79,12 +79,7 @@ then
    ETCD_CLUSTER="127.0.0.1:4001"
 fi

-mkdir -p ~postgres/.config/patroni
-cat > ~postgres/.config/patroni/patronictl.yaml <<__EOF__
-{dcs_api: 'etcd://${ETCD_CLUSTER}', namespace: /service/}
-__EOF__
-
-cat > /patroni/postgres.yaml <<__EOF__
+cat > /patroni/postgres.yml <<__EOF__

 ttl: &ttl 30
 loop_wait: &loop_wait 10
@@ -97,6 +92,9 @@ etcd:
  scope: *scope
  ttl: *ttl
  host: ${ETCD_CLUSTER}
+bdr:
+   enable: 'on'
+   database: 'bdrtest'
 postgresql:
  name: ${HOSTNAME}
  scope: *scope
@@ -124,22 +122,24 @@ postgresql:
    archive_command: 'true'
    max_wal_senders: 20
    listen_addresses: 0.0.0.0
-    max_wal_size: 1GB
-    min_wal_size: 128MB
+    checkpoint_segments: 64
    wal_keep_segments: 64
    archive_timeout: 1800s
    max_replication_slots: 20
    hot_standby: "on"
+    max_worker_processes: 10
+    max_replication_slots: 10
+    max_wal_senders: 10
+    shared_preload_libraries: 'bdr'
+    track_commit_timestamp: true
+    wal_level: 'logical'
 __EOF__

-cat /patroni/postgres.yaml
+cat /patroni/postgres.yml

 if [ ! -z $CHEAT ]
 then
-    while :
-    do
-        sleep 60
-    done
+    exec bash
 else
-    exec python /patroni.py /patroni/postgres.yaml
+    exec python /patroni.py /patroni/postgres.yml
 fi
@@ -8,6 +8,3 @@ Scripts supplied:
 ### patroni.upstart.conf

 Upstart job for Ubuntu 12.04 or 14.04.  Requires Upstart > 1.4. Intended for systems where Patroni has been installed on a base system, rather than in Docker.
-
-### patroni.service
-Systemd service file, to be copied to /etc/systemd/system/patroni.service, tested on Centos 7.1 with Patroni installed from pip. 
@@ -1,31 +0,0 @@
-# This is an example systemd config file for Patroni
-# You can copy it to "/etc/systemd/system/patroni.service",
-
-[Unit]
-Description=Runners to orchestrate a high-availability PostgreSQL
-After=syslog.target network.target
-
-[Service]
-Type=simple
-
-User=postgres
-Group=postgres
-
-# Where to send early-startup messages from the server
-# This is normally controlled by the global default set by systemd
-# StandardOutput=syslog
-
-ExecStart=/bin/patroni /etc/patroni.yml
-
-# only kill the patroni process, not it's children, so it will gracefully stop postgres
-KillMode=process
-
-# Give a reasonable amount of time for the server to start up/shut down
-TimeoutSec=10
-
-# Do not restart the service if it crashes, we want to manually inspect database on failure
-Restart=no
-
-[Install]
-WantedBy=multi-user.target
-
@@ -1,17 +0,0 @@
-Feature: basic replication
-  We should check that the basic bootstrapping, replication and failover works.
-
-  Scenario: check replication of a single table
-    Given I start postgres0
-    And postgres0 is a leader after 10 seconds
-    And I start postgres1
-    When I add the table foo to postgres0
-    Then table foo is present on postgres1 after 20 seconds
-
-  Scenario: check the basic failover
-    When I kill postgres0
-    Then postgres1 role is the primary after 32 seconds
-    When I start postgres0
-    Then postgres0 role is the secondary after 20 seconds
-    When I add the table bar to postgres1
-    Then table bar is present on postgres0 after 20 seconds
@@ -1,13 +0,0 @@
-Feature: cascading replication
-	We should check that patroni can do base backup and streaming from the replica
-
-Scenario: check a base backup and streaming replication from a replica
-	Given I start postgres0
-	And postgres0 is a leader after 10 seconds
-	And I configure and start postgres1 with a tag clonefrom true
-	And replication works from postgres0 to postgres1 after 20 seconds
-	And I create label with "postgres0" in postgres0 data directory
-	And I create label with "postgres1" in postgres1 data directory
-	And I configure and start postgres2 with a tag replicatefrom postgres1
-	Then replication works from postgres0 to postgres2 after 30 seconds
-	And there is a label with "postgres1" in postgres2 data directory
@@ -1,385 +0,0 @@
-import abc
-import consul
-import etcd
-import kazoo.client
-import kazoo.exceptions
-import os
-import psycopg2
-import shutil
-import six
-import subprocess
-import tempfile
-import time
-import yaml
-
-
-@six.add_metaclass(abc.ABCMeta)
-class AbstractController(object):
-
-    def __init__(self, name, work_directory, output_dir):
-        self._name = name
-        self._work_directory = work_directory
-        self._output_dir = output_dir
-        self._handle = None
-        self._log = None
-
-    def _has_started(self):
-        return self._handle and self._handle.pid and self._handle.poll() is None
-
-    def _is_running(self):
-        return self._has_started()
-
-    @abc.abstractmethod
-    def _is_accessible(self):
-        """process is accessible for queries"""
-
-    @abc.abstractmethod
-    def _start(self):
-        """start process"""
-
-    def start(self, max_wait_limit=5):
-        if self._is_running():
-            return True
-
-        self._log = open(os.path.join(self._output_dir, self._name + '.log'), 'a')
-        self._handle = self._start()
-
-        assert self._has_started(), "Process {0} is not running after being started".format(self._name)
-
-        for _ in range(max_wait_limit):
-            if self._is_accessible():
-                break
-            time.sleep(1)
-        else:
-            assert False,\
-                "{0} instance is not available for queries after {1} seconds".format(self._name, max_wait_limit)
-
-    def stop(self, kill=False, timeout=15):
-        term = False
-        start_time = time.time()
-
-        while self._handle and self._is_running():
-            if kill:
-                self._handle.kill()
-            elif not term:
-                self._handle.terminate()
-                term = True
-            time.sleep(1)
-            if not kill and time.time() - start_time > timeout:
-                kill = True
-
-        if self._log:
-            self._log.close()
-
-
-class PatroniController(AbstractController):
-    __PORT = 5440
-    PATRONI_CONFIG = '{}.yml'
-    """ starts and stops individual patronis"""
-
-    def __init__(self, dcs, name, work_directory, output_dir, tags=None):
-        super(PatroniController, self).__init__('patroni_' + name, work_directory, output_dir)
-        PatroniController.__PORT += 1
-        self._data_dir = os.path.join(work_directory, 'data', name)
-        self._connstring = None
-        self._config = self._make_patroni_test_config(name, dcs, tags)
-
-        self._conn = None
-        self._curs = None
-
-    def write_label(self, content):
-        with open(os.path.join(self._data_dir, 'label'), 'w') as f:
-            f.write(content)
-
-    def read_label(self):
-        try:
-            with open(os.path.join(self._data_dir, 'label'), 'r') as f:
-                return f.read().strip()
-        except IOError:
-            return None
-
-    def _start(self):
-        return subprocess.Popen(['coverage', 'run', '--source=patroni', '-p', 'patroni.py', self._config],
-                                stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory)
-
-    def _is_accessible(self):
-        return self.query("SELECT 1", fail_ok=True) is not None
-
-    def _make_patroni_test_config(self, name, dcs, tags):
-        patroni_config_name = self.PATRONI_CONFIG.format(name)
-        patroni_config_path = os.path.join(self._output_dir, patroni_config_name)
-
-        with open(patroni_config_name) as f:
-            config = yaml.load(f)
-
-        host = config['postgresql']['listen'].split(':')[0]
-
-        config['postgresql']['listen'] = config['postgresql']['connect_address'] = '{0}:{1}'.format(host, self.__PORT)
-
-        user = config['postgresql'].get('superuser', {})
-        self._connkwargs = {k: user[n] for n, k in [('username', 'user'), ('password', 'password')] if n in user}
-        self._connkwargs.update({'host': host, 'port': self.__PORT, 'database': 'postgres'})
-
-        config['postgresql'].update({'name': name, 'data_dir': self._data_dir})
-        config['postgresql']['parameters'].update({
-            'logging_collector': 'on', 'log_destination': 'csvlog', 'log_directory': self._output_dir,
-            'log_filename': name + '.log', 'log_statement': 'all', 'log_min_messages': 'debug1'})
-
-        if tags:
-            config['tags'] = tags
-
-        if dcs != 'etcd':
-            dcs_config = config.pop('etcd')
-            dcs_config.pop('host')
-
-            if dcs == 'consul':
-                config[dcs] = dcs_config
-            else:
-                dcs_config.update({'session_timeout': dcs_config.pop('ttl'), 'reconnect_timeout': config['loop_wait']})
-                if dcs == 'exhibitor':
-                    dcs_config['exhibitor'] = {'hosts': ['127.0.0.1'], 'port': 8181}
-                else:
-                    dcs_config['hosts'] = ['127.0.0.1:2181']
-                config['zookeeper'] = dcs_config
-
-        with open(patroni_config_path, 'w') as f:
-            yaml.dump(config, f, default_flow_style=False)
-
-        return patroni_config_path
-
-    def _connection(self):
-        if not self._conn or self._conn.closed != 0:
-            self._conn = psycopg2.connect(**self._connkwargs)
-            self._conn.autocommit = True
-        return self._conn
-
-    def _cursor(self):
-        if not self._curs or self._curs.closed or self._curs.connection.closed != 0:
-            self._curs = self._connection().cursor()
-        return self._curs
-
-    def query(self, query, fail_ok=False):
-        try:
-            cursor = self._cursor()
-            cursor.execute(query)
-            return cursor
-        except psycopg2.Error:
-            if not fail_ok:
-                raise
-
-    def check_role_has_changed_to(self, new_role, timeout=10):
-        bound_time = time.time() + timeout
-        recovery_status = new_role != 'primary'
-        while time.time() < bound_time:
-            cur = self.query("SELECT pg_is_in_recovery()", fail_ok=True)
-            if cur:
-                row = cur.fetchone()
-                if row and row[0] == recovery_status:
-                    return True
-            time.sleep(1)
-        return False
-
-
-class AbstractDcsController(AbstractController):
-
-    _CLUSTER_NODE = 'service/batman'
-
-    def _is_accessible(self):
-        return self._is_running()
-
-    def stop_and_remove_work_directory(self, timeout=15):
-        """ terminate process and wipe out the temp work directory, but only if we actually started it"""
-        self.stop(timeout=timeout)
-        if self._work_directory:
-            shutil.rmtree(self._work_directory)
-
-    @abc.abstractmethod
-    def query(self, key):
-        """ query for a value of a given key """
-
-    @abc.abstractmethod
-    def cleanup_service_tree(self):
-        """ clean all contents stored in the tree used for the tests """
-
-
-class ConsulController(AbstractDcsController):
-
-    def __init__(self, output_dir):
-        super(ConsulController, self).__init__('consul', tempfile.mkdtemp(), output_dir)
-        self._client = consul.Consul()
-
-    def _start(self):
-        return subprocess.Popen(['consul', 'agent', '-server', '-bootstrap', '-advertise=127.0.0.1',
-                                 '-data-dir', self._work_directory], stdout=self._log, stderr=subprocess.STDOUT)
-
-    def _is_running(self):
-        try:
-            return bool(self._client.status.leader())
-        except Exception:
-            return False
-
-    def query(self, key):
-        _, value = self._client.kv.get('{0}/{1}'.format(self._CLUSTER_NODE, key))
-        return value and value['Value'].decode('utf-8')
-
-    def cleanup_service_tree(self):
-        self._client.kv.delete(self._CLUSTER_NODE, recurse=True)
-
-
-class EtcdController(AbstractDcsController):
-
-    """ handles all etcd related tasks, used for the tests setup and cleanup """
-
-    def __init__(self, output_dir):
-        super(EtcdController, self).__init__('etcd', tempfile.mkdtemp(), output_dir)
-        self._client = etcd.Client()
-
-    def _start(self):
-        return subprocess.Popen(["etcd", "--debug", "--data-dir", self._work_directory],
-                                stdout=self._log, stderr=subprocess.STDOUT)
-
-    def query(self, key):
-        try:
-            return self._client.get('/{0}/{1}'.format(self._CLUSTER_NODE, key)).value
-        except etcd.EtcdKeyNotFound:
-            return None
-
-    def cleanup_service_tree(self):
-        try:
-            self._client.delete('/' + self._CLUSTER_NODE, recursive=True)
-        except (etcd.EtcdKeyNotFound, etcd.EtcdConnectionFailed):
-            return
-        except Exception as e:
-            assert False, "exception when cleaning up etcd contents: {0}".format(e)
-
-    def _is_running(self):
-        # if etcd is running, but we didn't start it
-        try:
-            return bool(self._client.machines)
-        except Exception:
-            return False
-
-
-class ZooKeeperController(AbstractDcsController):
-
-    """ handles all zookeeper related tasks, used for the tests setup and cleanup """
-
-    def __init__(self, output_dir):
-        super(ZooKeeperController, self).__init__('zookeeper', None, output_dir)
-        self._client = kazoo.client.KazooClient()
-
-    def _start(self):
-        pass  # TODO: implement later
-
-    def query(self, key):
-        try:
-            return self._client.get('/{0}/{1}'.format(self._CLUSTER_NODE, key))[0].decode('utf-8')
-        except kazoo.exceptions.NoNodeError:
-            return None
-
-    def cleanup_service_tree(self):
-        try:
-            self._client.delete('/' + self._CLUSTER_NODE, recursive=True)
-        except (kazoo.exceptions.NoNodeError):
-            return
-        except Exception as e:
-            assert False, "exception when cleaning up zookeeper contents: {0}".format(e)
-
-    def _is_running(self):
-        # if zookeeper is running, but we didn't start it
-        if self._client.connected:
-            return True
-        try:
-            return self._client.start(1) or True
-        except Exception:
-            return False
-
-
-class PatroniPoolController(object):
-
-    KNOWN_DCS = {'consul': ConsulController, 'etcd': EtcdController,
-                 'zookeeper': ZooKeeperController, 'exhibitor': ZooKeeperController}
-
-    def __init__(self):
-        self._dcs = None
-        self._output_dir = None
-        self._patroni_path = None
-        self._processes = {}
-        self.create_and_set_output_directory('')
-
-    @property
-    def patroni_path(self):
-        if self._patroni_path is None:
-            cwd = os.path.realpath(__file__)
-            while True:
-                cwd, entry = os.path.split(cwd)
-                if entry == 'features' or cwd == '/':
-                    break
-            self._patroni_path = cwd
-        return self._patroni_path
-
-    @property
-    def output_dir(self):
-        return self._output_dir
-
-    def start(self, pg_name, max_wait_limit=20, tags=None):
-        if pg_name not in self._processes:
-            self._processes[pg_name] = PatroniController(self.dcs, pg_name, self.patroni_path, self._output_dir, tags)
-        self._processes[pg_name].start(max_wait_limit)
-
-    def __getattr__(self, func):
-        if func not in ['stop', 'query', 'write_label', 'read_label', 'check_role_has_changed_to']:
-            raise AttributeError("PatroniPoolController instance has no attribute '{0}'".format(func))
-
-        def wrapper(pg_name, *args, **kwargs):
-            return getattr(self._processes[pg_name], func)(*args, **kwargs)
-        return wrapper
-
-    def stop_all(self):
-        for ctl in self._processes.values():
-            ctl.stop()
-        self._processes.clear()
-
-    def create_and_set_output_directory(self, feature_name):
-        feature_dir = os.path.join(self.patroni_path, 'features/output', feature_name.replace(' ', '_'))
-        if os.path.exists(feature_dir):
-            shutil.rmtree(feature_dir)
-        os.makedirs(feature_dir)
-        self._output_dir = feature_dir
-
-    @property
-    def dcs(self):
-        if self._dcs is None:
-            self._dcs = os.environ.get('DCS', 'etcd')
-            assert self._dcs in self.KNOWN_DCS, 'Unsupported dcs: ' + self.dcs
-        return self._dcs
-
-
-# actions to execute on start/stop of the tests and before running invidual features
-def before_all(context):
-    context.pctl = PatroniPoolController()
-    context.dcs_ctl = context.pctl.KNOWN_DCS[context.pctl.dcs](context.pctl.output_dir)
-    context.dcs_ctl.start()
-    try:
-        context.dcs_ctl.cleanup_service_tree()
-    except AssertionError:  # after_all handlers won't be executed in before_all
-        context.dcs_ctl.stop_and_remove_work_directory()
-        raise
-
-
-def after_all(context):
-    context.dcs_ctl.stop_and_remove_work_directory()
-    subprocess.call(['coverage', 'combine'])
-    subprocess.call(['coverage', 'report'])
-
-
-def before_feature(context, feature):
-    """ create per-feature output directory to collect Patroni and PostgreSQL logs """
-    context.pctl.create_and_set_output_directory(feature.name)
-
-
-def after_feature(context, feature):
-    """ stop all Patronis, remove their data directory and cleanup the keys in etcd """
-    context.pctl.stop_all()
-    shutil.rmtree(os.path.join(context.pctl.patroni_path, 'data'))
-    context.dcs_ctl.cleanup_service_tree()
@@ -1,53 +0,0 @@
-Feature: patroni api
-	We should check that patroni correctly responds to valid and not-valid API requests.
-
-Scenario: check API requests on a stand-alone server
-	Given I start postgres0
-	And postgres0 is a leader after 10 seconds
-	When I issue a GET request to http://127.0.0.1:8008/
-	Then I receive a response code 200
-	And I receive a response state running
-	And I receive a response role master
-	When I issue a GET request to http://127.0.0.1:8008/replica
-	Then I receive a response code 503
-	When I issue an empty POST request to http://127.0.0.1:8008/reinitialize
-	Then I receive a response code 503
-	And I receive a response text "I am the leader, can not reinitialize"
-	When I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0
-	Then I receive a response code 500
-	And I receive a response text "failover is not possible: cluster does not have members except leader"
-	When I issue an empty POST request to http://127.0.0.1:8008/failover
-	Then I receive a response code 400
-	And I receive a response text "No values given for required parameters leader and candidate"
-
-Scenario: check API requests for the primary-replica pair
-	Given I start postgres1
-	And replication works from postgres0 to postgres1 after 20 seconds
-	When I issue a GET request to http://127.0.0.1:8009/replica
-	Then I receive a response code 200
-	And I receive a response state running
-	And I receive a response role replica
-	When I issue an empty POST request to http://127.0.0.1:8009/reinitialize
-	Then I receive a response code 200
-	When I issue an empty POST request to http://127.0.0.1:8008/restart
-	Then I receive a response code 200
-        And postgres0 role is the primary after 5 seconds
-        When I sleep for 10 seconds
-        Then postgres1 role is the secondary after 15 seconds
-
-Scenario: check the failover via the API
-	Given I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0,candidate=postgres1
-	Then I receive a response code 200
-	And postgres1 is a leader after 5 seconds
-        And postgres1 role is the primary after 5 seconds
-        And postgres0 role is the secondary after 10 seconds
-	And replication works from postgres1 to postgres0 after 20 seconds
-
-Scenario: check the scheduled failover
-	Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgres0 in 1 seconds
-	Then I receive a response code 200
-	And postgres0 is a leader after 20 seconds
-        And postgres0 role is the primary after 5 seconds
-        And postgres1 role is the secondary after 10 seconds
-	And replication works from postgres0 to postgres1 after 25 seconds
-
@@ -1,54 +0,0 @@
-import psycopg2 as pg
-
-from behave import step, then
-from time import sleep, time
-
-
-@step('I start {name:w}')
-def start_patroni(context, name):
-    return context.pctl.start(name)
-
-
-@step('I shut down {name:w}')
-def stop_patroni(context, name):
-    return context.pctl.stop(name)
-
-
-@step('I kill {name:w}')
-def kill_patroni(context, name):
-    return context.pctl.stop(name, kill=True)
-
-
-@step('I add the table {table_name:w} to {pg_name:w}')
-def add_table(context, table_name, pg_name):
-    # parse the configuration file and get the port
-    try:
-        context.pctl.query(pg_name, "CREATE TABLE {0}()".format(table_name))
-    except pg.Error as e:
-        assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e)
-
-
-@then('Table {table_name:w} is present on {pg_name:w} after {max_replication_delay:d} seconds')
-def table_is_present_on(context, table_name, pg_name, max_replication_delay):
-    for _ in range(int(max_replication_delay)):
-        if context.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
-            break
-        sleep(1)
-    else:
-        assert False,\
-            "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay)
-
-
-@then('{pg_name:w} role is the {pg_role:w} after {max_promotion_timeout:d} seconds')
-def check_role(context, pg_name, pg_role, max_promotion_timeout):
-    assert context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)),\
-        "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
-
-
-@step('replication works from {master:w} to {replica:w} after {time_limit:d} seconds')
-@then('replication works from {master:w} to {replica:w} after {time_limit:d} seconds')
-def replication_works(context, master, replica, time_limit):
-    context.execute_steps(u"""
-        When I add the table test_{0} to {1}
-        Then table test_{0} is present on {2} after {3} seconds
-    """.format(int(time()), master, replica, time_limit))
@@ -1,17 +0,0 @@
-from behave import step, then
-
-
-@step('I configure and start {name:w} with a tag {tag_name:w} {tag_value:w}')
-def start_patroni_with_a_name_value_tag(context, name, tag_name, tag_value):
-    return context.pctl.start(name, tags={tag_name: tag_value})
-
-
-@then('There is a label with "{content:w}" in {name:w} data directory')
-def check_label(context, content, name):
-    label = context.pctl.read_label(name)
-    assert label == content, "{0} is not equal to {1}".format(label, content)
-
-
-@step('I create label with "{content:w}" in {name:w} data directory')
-def write_label(context, content, name):
-    context.pctl.write_label(name, content)
@@ -1,100 +0,0 @@
-import parse
-import pytz
-import requests
-import time
-
-from behave import register_type, step, then
-from datetime import datetime, timedelta
-
-
-@parse.with_pattern(r'https?://(?:\w|\.|:|/)+')
-def parse_url(text):
-    return text
-
-
-@parse.with_pattern(r'(?:\w+=(?:\w|\.|:|-|\+|\s)+,?)+')
-def parse_data(text):
-    return text
-
-
-register_type(url=parse_url, data=parse_data)
-
-
-# there is no way we can find out if the node has already
-# started as a leader without checking the DCS. We cannot
-# just rely on the database availability, since there is
-# a short gap between the time PostgreSQL becomes available
-# and Patroni assuming the leader role.
-@step('{name:w} is a leader after {time_limit:d} seconds')
-@then('{name:w} is a leader after {time_limit:d} seconds')
-def is_a_leader(context, name, time_limit):
-    max_time = time.time() + int(time_limit)
-    while (context.dcs_ctl.query("leader") != name):
-        time.sleep(1)
-        assert time.time() < max_time, "{0} is not a leader in dcs after {1} seconds".format(name, time_limit)
-
-
-@step('I sleep for {value:d} seconds')
-def sleep_for_n_seconds(context, value):
-    time.sleep(int(value))
-
-
-@step('I issue a GET request to {url:url}')
-def do_get(context, url):
-    try:
-        r = requests.get(url)
-    except requests.exceptions.RequestException:
-        context.status_code = None
-        context.response = None
-    else:
-        context.status_code = r.status_code
-        try:
-            context.response = r.json()
-        except ValueError:
-            context.response = r.content.decode('utf-8')
-
-
-@step('I issue an empty POST request to {url:url}')
-def do_post_empty(context, url):
-    do_post(context, url, None)
-
-
-@step('I issue a POST request to {url:url} with {data:data}')
-def do_post(context, url, data):
-    post_data = {}
-    if data:
-        post_components = data.split(',')
-        for pc in post_components:
-            if '=' in pc:
-                k, v = pc.split('=', 2)
-                post_data[k.strip()] = v.strip()
-    try:
-        r = requests.post(url, json=post_data)
-    except requests.exceptions.RequestException:
-        context.status_code = None
-        context.response = None
-    else:
-        context.status_code = r.status_code
-        try:
-            context.response = r.json()
-        except ValueError:
-            context.response = r.content.decode('utf-8')
-
-
-@then('I receive a response {component:w} {data}')
-def check_response(context, component, data):
-    if component == 'code':
-        assert context.status_code == int(data),\
-            "status code {0} != {1}, response: {2}".format(context.status_code, int(data), context.response)
-    elif component == 'text':
-        assert context.response == data.strip('"'), "response {0} does not contain {1}".format(context.response, data)
-    else:
-        assert component in context.response, "{0} is not part of the response".format(component)
-        assert context.response[component] == data, "{0} does not contain {1}".format(component, data)
-
-
-@step('I issue a scheduled failover at {at_url:url} from {from_host:w} to {to_host:w} in {in_seconds:d} seconds')
-def scheduled_failover(context, at_url, from_host, to_host, in_seconds):
-    context.execute_steps(u"""
-        Given I issue a POST request to {0}/failover with leader={1},candidate={2},scheduled_at={3}
-    """.format(at_url, from_host, to_host, datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds))))
@@ -5,53 +5,38 @@ import time
 import yaml

 from patroni.api import RestApiServer
-from patroni.exceptions import PatroniException
+from patroni.etcd import Etcd
 from patroni.ha import Ha
 from patroni.postgresql import Postgresql
-from patroni.utils import reap_children, set_ignore_sigterm, setup_signal_handlers
-from patroni.version import __version__
+from patroni.utils import setup_signal_handlers, reap_children
+from patroni.zookeeper import ZooKeeper

 logger = logging.getLogger(__name__)


-class Patroni(object):
-    PATRONI_CONFIG_VARIABLE = 'PATRONI_CONFIGURATION'
+class Patroni:

    def __init__(self, config):
        self.nap_time = config['loop_wait']
-        self.tags = {tag: value for tag, value in config.get('tags', {}).items()
-                     if tag not in ('clonefrom', 'nofailover', 'noloadbalance') or value}
-        self.postgresql = Postgresql(config['postgresql'])
+        self.tags = config.get('tags', dict())
+        self.bdr = config.get('bdr', dict())
+        self.postgresql = Postgresql(config['postgresql'], self.bdr)
        self.dcs = self.get_dcs(self.postgresql.name, config)
-        self.version = __version__
        self.api = RestApiServer(self, config['restapi'])
        self.ha = Ha(self)
        self.next_run = time.time()

-    @property
-    def noloadbalance(self):
-        return self.tags.get('noloadbalance', False)
-
    @property
    def nofailover(self):
        return self.tags.get('nofailover', False)

-    @property
-    def replicatefrom(self):
-        return self.tags.get('replicatefrom')
-
    @staticmethod
    def get_dcs(name, config):
        if 'etcd' in config:
-            from patroni.etcd import Etcd
            return Etcd(name, config['etcd'])
        if 'zookeeper' in config:
-            from patroni.zookeeper import ZooKeeper
            return ZooKeeper(name, config['zookeeper'])
-        if 'consul' in config:
-            from patroni.consul import Consul
-            return Consul(name, config['consul'])
-        raise PatroniException('Can not find suitable configuration of distributed configuration store')
+        raise Exception('Can not find suitable configuration of distributed configuration store')

    def schedule_next_run(self):
        self.next_run += self.nap_time
@@ -77,30 +62,19 @@ def main():
    logging.getLogger('requests').setLevel(logging.WARNING)
    setup_signal_handlers()

-    # Patroni reads the configuration from the command-line argument if it exists, and from the environment otherwise.
-    use_env = False
-    use_file = (len(sys.argv) >= 2 and os.path.isfile(sys.argv[1]))
-    if not use_file:
-        config_env = os.environ.get(Patroni.PATRONI_CONFIG_VARIABLE)
-        use_env = config_env is not None
-        if not use_env:
-            print('Usage: {0} config.yml'.format(sys.argv[0]))
-            print('\tPatroni may also read the configuration from the {} environment variable'.
-                  format(Patroni.PATRONI_CONFIG_VARIABLE))
-            return
+    if len(sys.argv) < 2 or not os.path.isfile(sys.argv[1]):
+        print('Usage: {} config.yml'.format(sys.argv[0]))
+        return

-    if use_file:
-        with open(sys.argv[1], 'r') as f:
-            config = yaml.load(f)
-    elif use_env:
-        config = yaml.load(config_env)
+    with open(sys.argv[1], 'r') as f:
+        config = yaml.load(f)

    patroni = Patroni(config)
    try:
        patroni.run()
    except KeyboardInterrupt:
-        set_ignore_sigterm()
+        pass
    finally:
        patroni.api.shutdown()
-        patroni.postgresql.stop(checkpoint=False)
+        patroni.postgresql.stop()
        patroni.dcs.delete_leader()
@@ -5,9 +5,6 @@ import logging
 import psycopg2
 import socket
 import time
-import dateutil
-import datetime
-import pytz

 from patroni.exceptions import PostgresConnectionException
 from patroni.utils import Retry, RetryFailedError
@@ -34,19 +31,12 @@ def check_auth(func):

 class RestApiHandler(BaseHTTPRequestHandler):

-    def _write_response(self, status_code, body, headers=None):
-        self.send_response(status_code)
-        if body is not None:
-            headers = headers or {}
-            if 'Content-Type' not in headers:
-                headers['Content-Type'] = 'text/html'
-            for name, value in (headers or {}).items():
-                self.send_header(name, value)
-            self.end_headers()
-            self.wfile.write(body.encode('utf-8'))
-
    def send_auth_request(self, body):
-        self._write_response(401, body, {'WWW-Authenticate': 'Basic realm=\"Patroni\"'})
+        self.send_response(401)
+        self.send_header('WWW-Authenticate', 'Basic realm=\"Patroni\"')
+        self.send_header('Content-type', 'text/html')
+        self.end_headers()
+        self.wfile.write(body.encode('utf-8'))

    def finish(self, *args, **kwargs):
        try:
@@ -62,23 +52,15 @@ class RestApiHandler(BaseHTTPRequestHandler):
        status = self.server.check_auth_header(auth_header)
        return not status or self.send_auth_request(status)

-    def _write_status_response(self, status_code, response, options=False):
-        if options:
-            body = None
-        else:
-            patroni = self.server.patroni
-            response.update({'tags': patroni.tags} if patroni.tags else {})
-            if patroni.postgresql.sysid:
-                response['database_system_identifier'] = patroni.postgresql.sysid
-            response['patroni'] = {'version': patroni.version, 'scope': patroni.postgresql.scope}
-            body = json.dumps(response)
-        self._write_response(status_code, body, {'Content-Type': 'application/json'})
+    def do_OPTIONS(self):
+        self.do_GET(options=True)

    def do_GET(self, options=False):
        """Default method for processing all GET requests which can not be routed to other methods"""

        path = '/master' if self.path == '/' else self.path
        response = self.get_postgresql_status()
+        response.update(self.get_tags())

        patroni = self.server.patroni
        cluster = patroni.dcs.cluster
@@ -89,36 +71,48 @@ class RestApiHandler(BaseHTTPRequestHandler):
                status_code = 503
            elif response['role'] == 'master':  # running as master but without leader lock!!!!
                status_code = 503
-            elif response['role'] in path:  # response['role'] != 'master'
-                status_code = 503 if patroni.noloadbalance else 200
+            elif response['role'] in path:
+                status_code = 200
            else:
                status_code = 503
        elif 'role' in response and response['role'] in path:
-            status_code = 503 if response['role'] != 'master' and patroni.noloadbalance else 200
+            status_code = 200
        elif patroni.ha.restart_scheduled() and patroni.postgresql.role == 'master' and 'master' in path:
            # exceptional case for master node when the postgres is being restarted via API
            status_code = 200
        else:
            status_code = 503
-        self._write_status_response(status_code, response, options)

-    def do_OPTIONS(self):
-        self.do_GET(options=True)
+        self.send_response(status_code)
+        if not options:
+            self.send_header('Content-Type', 'application/json')
+            self.end_headers()
+            self.wfile.write(json.dumps(response).encode('utf-8'))

    def do_GET_patroni(self):
        response = self.get_postgresql_status(True)
-        self._write_status_response(200, response)
+        response.update(self.get_tags())
+
+        self.send_response(200)
+        self.send_header('Content-Type', 'application/json')
+        self.end_headers()
+        self.wfile.write(json.dumps(response).encode('utf-8'))

    @check_auth
    def do_POST_restart(self):
-        status_code = 500
-        data = 'restart failed'
+        status_code = 503
+        data = b'restart failed'
        try:
-            status, data = self.server.patroni.ha.restart()
+            status, msg = self.server.patroni.ha.restart()
            status_code = 200 if status else 503
-        except Exception:
+            data = msg.encode('utf-8')
+        except:
            logger.exception('Exception during restart')
-        self._write_response(status_code, data)
+
+        self.send_response(status_code)
+        self.send_header('Content-Type', 'text/html')
+        self.end_headers()
+        self.wfile.write(data)

    @check_auth
    def do_POST_reinitialize(self):
@@ -126,103 +120,73 @@ class RestApiHandler(BaseHTTPRequestHandler):
        cluster = ha.dcs.get_cluster()
        if cluster.is_unlocked():
            status_code = 503
-            data = 'Cluster has no leader, can not reinitialize'
+            data = b'Cluster has no leader, can not reinitialize'
        elif cluster.leader.name == ha.state_handler.name:
            status_code = 503
-            data = 'I am the leader, can not reinitialize'
+            data = b'I am the leader, can not reinitialize'
        else:
            action = ha.schedule_reinitialize()
            if action is not None:
                status_code = 503
-                data = action + ' already in progress'
+                data = (action + ' already in progress').encode('utf-8')
            else:
                status_code = 200
-                data = 'reinitialize scheduled'
-        self._write_response(status_code, data)
+                data = b'reinitialize scheduled'

-    def poll_failover_result(self, leader, candidate):
-        for _ in range(0, 15):
+        self.send_response(status_code)
+        self.send_header('Content-Type', 'text/html')
+        self.end_headers()
+        self.wfile.write(data)
+
+    def poll_failover_result(self, leader, member):
+        for a in range(0, 15):
            time.sleep(1)
            try:
                cluster = self.server.patroni.dcs.get_cluster()
                if cluster.leader and cluster.leader.name != leader:
-                    if not candidate or candidate == cluster.leader.name:
-                        return 200, 'Successfully failed over to "{0}"'.format(cluster.leader.name)
-                    else:
-                        return 200, 'Failed over to "{0}" instead of "{1}"'.format(cluster.leader.name, candidate)
+                    return 200, ('Successfully failed over to ' + cluster.leader.name).encode('utf-8')
                if not cluster.failover:
-                    return 503, 'Failover failed'
-            except Exception as e:
-                logger.debug('Exception occured during polling failover result: %s', e)
-        return 503, 'Failover status unknown'
+                    return 503, b'Failover failed'
+            except:
+                pass
+        return 503, b'Failover status unknown'

-    def is_failover_possible(self, cluster, leader, candidate):
+    def is_failover_possible(self, cluster, leader, member):
        if leader and not cluster.leader or cluster.leader.name != leader:
-            return 'leader name does not match'
-        if candidate:
-            members = [m for m in cluster.members if m.name == candidate]
+            return b'leader name does not match'
+        if member:
+            members = [m for m in cluster.members if m.name == member]
            if not members:
-                return 'candidate does not exists'
+                return b'member does not exists'
        else:
            members = [m for m in cluster.members if m.name != cluster.leader.name and m.api_url]
            if not members:
-                return 'failover is not possible: cluster does not have members except leader'
-        for _, reachable, _, _, tags in self.server.patroni.ha.fetch_nodes_statuses(members):
+                return b'failover is not possible: cluster does not have members except leader'
+        for member, reachable, in_recovery, xlog_location, tags in self.server.patroni.ha.fetch_nodes_statuses(members):
            if reachable and not tags.get('nofailover', False):
                return None
-        return 'failover is not possible: no good candidates have been found'
+        return b'failover is not possible: no good candidates have been found'

    @check_auth
    def do_POST_failover(self):
        content_length = int(self.headers.get('content-length', 0))
-        try:
-            request = json.loads(self.rfile.read(content_length).decode('utf-8'))
-        except ValueError:
-            request = {}
-        leader = request.get('leader')
-        candidate = request.get('candidate') or request.get('member')
-        scheduled_at = request.get('scheduled_at')
+        request = json.loads(self.rfile.read(content_length).decode('utf-8'))
+        leader = request.get('leader', None)
+        member = request.get('member', None)
        cluster = self.server.patroni.ha.dcs.get_cluster()
-        status_code = 500
-
-        logger.info("received failover request with leader=%s candidate=%s scheduled_at=%s",
-                    leader, candidate, scheduled_at)
-
-        data = ''
-        if leader or candidate:
-            if scheduled_at:
-                try:
-                    scheduled_at = dateutil.parser.parse(scheduled_at)
-                    if scheduled_at.tzinfo is None:
-                        data = 'Timezone information is mandatory for scheduled_at'
-                        status_code = 400
-                    elif scheduled_at < datetime.datetime.now(pytz.utc):
-                        data = 'Cannot schedule failover in the past'
-                        status_code = 422
-                    elif self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
-                        self.server.patroni.dcs.event.set()
-                        data = 'Failover scheduled'
-                        status_code = 200
-                    else:
-                        data = 'failed to write failover key into DCS'
-                        status_code = 503
-                except (ValueError, TypeError):
-                    logger.exception('Invalid scheduled failover time: %s', request['scheduled_at'])
-                    data = 'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601'
-                    status_code = 422
+        status_code = 503
+        data = self.is_failover_possible(cluster, leader, member)
+        if not data:
+            if not self.server.patroni.dcs.manual_failover(leader, member):
+                data = b'failed to write failover key into DCS'
            else:
-                data = self.is_failover_possible(cluster, leader, candidate)
-                if not data:
-                    if self.server.patroni.dcs.manual_failover(leader, candidate):
-                        self.server.patroni.dcs.event.set()
-                        status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, candidate)
-                    else:
-                        data = 'failed to write failover key into DCS'
-                        status_code = 503
-        else:
-            status_code = 400
-            data = 'No values given for required parameters leader and candidate'
-        self._write_response(status_code, data)
+                self.server.patroni.dcs.event.set()
+                status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, member)
+
+        self.send_response(status_code)
+        self.send_header('Content-Type', 'text/html')
+        self.end_headers()
+        self.wfile.write(data)

    def parse_request(self):
        """Override parse_request method to enrich basic functionality of `BaseHTTPRequestHandler` class
@@ -264,18 +228,15 @@ class RestApiHandler(BaseHTTPRequestHandler):
                                       END,
                                       pg_xlog_location_diff(pg_last_xlog_receive_location(), '0/0')::bigint,
                                       pg_xlog_location_diff(pg_last_xlog_replay_location(), '0/0')::bigint,
-                                       to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
                                       pg_is_in_recovery() AND pg_is_xlog_replay_paused()""", retry=retry)[0]
            return {
                'state': self.server.patroni.postgresql.state,
                'postmaster_start_time': row[0],
                'role': 'replica' if row[1] else 'master',
-                'server_version': self.server.patroni.postgresql.server_version,
                'xlog': ({
                    'received_location': row[3],
                    'replayed_location': row[4],
-                    'replayed_timestamp': row[5],
-                    'paused': row[6]} if row[1] else {
+                    'paused': row[5]} if row[1] else {
                    'location': row[2]
                })
            }
@@ -286,8 +247,11 @@ class RestApiHandler(BaseHTTPRequestHandler):
                state = 'unknown'
            return {'state': state}

-    def log_message(self, fmt, *args):
-        logger.debug("API thread: %s - - [%s] %s", self.client_address[0], self.log_date_time_string(), fmt % args)
+    def get_tags(self):
+        return {'tags': self.server.patroni.tags}
+
+    def log_message(self, format, *args):
+        logger.debug("API thread: " + format % args)


 class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
@@ -304,12 +268,12 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
        # wrap socket with ssl if 'certfile' is defined in a config.yaml
        # Sometime it's also needed to pass reference to a 'keyfile'.
        options = {option: config[option] for option in ['certfile', 'keyfile'] if option in config}
-        if options.get('certfile'):
+        if options.get('certfile', None):
            import ssl
            self.socket = ssl.wrap_socket(self.socket, server_side=True, **options)
            protocol = 'https'

-        self.connection_string = '{0}://{1}/patroni'.format(protocol, config.get('connect_address', config['listen']))
+        self.connection_string = '{}://{}/patroni'.format(protocol, config.get('connect_address', config['listen']))

        self.patroni = patroni
        self.daemon = True
@@ -317,7 +281,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
    def query(self, sql, *params):
        cursor = None
        try:
-            with self.patroni.postgresql.connection().cursor() as cursor:
+            with self.patroni.postgresql.connection('postgres').cursor() as cursor:
                cursor.execute(sql, params)
                return [r for r in cursor]
        except psycopg2.Error as e:
@@ -4,9 +4,10 @@ from threading import Lock, Thread
 logger = logging.getLogger(__name__)


-class AsyncExecutor(object):
+class AsyncExecutor:

    def __init__(self):
+        Lock.__init__(self)
        self._busy = False
        self._thread_lock = Lock()
        self._scheduled_action = None
@@ -50,5 +51,5 @@ class AsyncExecutor(object):
    def __enter__(self):
        self._thread_lock.acquire()

-    def __exit__(self, *args):
+    def __exit__(self, type, value, traceback):
        self._thread_lock.release()
@@ -1,244 +0,0 @@
-from __future__ import absolute_import
-import logging
-import os
-import time
-import six
-
-from consul import ConsulException, NotFound, base, std
-from patroni.dcs import AbstractDCS, Cluster, Failover, Leader, Member
-from patroni.exceptions import DCSError
-from patroni.utils import sleep
-from requests.exceptions import RequestException
-
-logger = logging.getLogger(__name__)
-
-
-class ConsulError(DCSError):
-    pass
-
-
-class HTTPClient(std.HTTPClient):
-
-    def __init__(self, *args, **kwargs):
-        super(HTTPClient, self).__init__(*args, **kwargs)
-        self._patch_default_timeout()
-
-    def _patch_default_timeout(self):
-        # Set a default timeout for the `request.session.request` method, that is used
-        # internally  by the methods request.session.get, request.session.post and
-        # others. We monkey-patch here to avoid reimplementing each individual method from
-        # `std.HTTPClient`. By default, the timeout is not set. It means that a new
-        # session may hang almost indefinitely waiting for the server to respond,
-        # which is not what we want in Patroni.
-
-        request_func = getattr(self.session.request, '__func__' if six.PY3 else 'im_func')
-        defaults_attr_name = '__defaults__' if six.PY3 else 'func_defaults'
-        defaults = list(getattr(request_func, defaults_attr_name))
-        code = request_func.__code__ if six.PY3 else request_func.func_code
-        defaults[code.co_varnames[code.co_argcount - len(defaults):code.co_argcount].index('timeout')] = 5
-        setattr(request_func, defaults_attr_name, tuple(defaults))  # monkeypatching
-
-    def get(self, callback, path, params=None):
-        # The get function is overridden to handle a special case of it being called
-        # with an index and wait parameters. That form indicates that a user needs to
-        # wait for the given key to change its value, with a wait timeout supplied. We
-        # don't want our monkey-patched timeout to be less than the value of the wait
-        # parameter, therefore, we set it to either the value of wait or a default of 5 minutes.
-
-        if isinstance(params, dict) and 'index' in params:
-            timeout = (float(params['wait'][:-1]) if 'wait' in params else 300) + 1
-        else:
-            timeout = None
-        return callback(self.response(self.session.get(self.uri(path, params), verify=self.verify, timeout=timeout)))
-
-
-class ConsulClient(base.Consul):
-
-    @staticmethod
-    def connect(host, port, scheme, verify=True):
-        return HTTPClient(host, port, scheme, verify)
-
-
-def catch_consul_errors(func):
-    def wrapper(*args, **kwargs):
-        try:
-            return func(*args, **kwargs)
-        except (ConsulException, RequestException):
-            return False
-    return wrapper
-
-
-class Consul(AbstractDCS):
-
-    def __init__(self, name, config):
-        super(Consul, self).__init__(name, config)
-        self.ttl = int((config.get('ttl') or 30)/2)  # My experiments have shown that session expires after 2*ttl time
-        host, port = config.get('host', '127.0.0.1:8500').split(':')
-        self._client = ConsulClient(host=host, port=port)
-        self._scope = config['scope']
-        self._session = None
-        self._my_member_data = None
-        self.create_or_restore_session()
-
-    def create_or_restore_session(self):
-        while not self._session:
-            try:
-                _, member = self._client.kv.get(self.member_path)
-                self._session = (member or {}).get('Session')
-                if self.refresh_session():
-                    self._client.kv.delete(self.member_path)
-            except (ConsulException, RequestException):
-                logger.info('waiting on consul')
-                sleep(5)
-
-    def refresh_session(self):
-        """:returns: `!True` if it had to create new session"""
-        if self._session:
-            try:
-                return self._client.session.renew(self._session) is None
-            except NotFound:
-                self._session = None
-        if not self._session:
-            name = self._scope + '-' + self._name
-            try:
-                self._session = self._client.session.create(name=name, lock_delay=0, behavior='delete', ttl=self.ttl)
-            except (ConsulException, RequestException):
-                logger.exception('session.create')
-        if not self._session:
-            raise ConsulError('Failed to renew/create session')
-        return True
-
-    def client_path(self, path):
-        return super(Consul, self).client_path(path)[1:]
-
-    @staticmethod
-    def member(node):
-        return Member.from_node(node['ModifyIndex'], os.path.basename(node['Key']), node.get('Session'), node['Value'])
-
-    def _load_cluster(self):
-        try:
-            path = self.client_path('/')
-            _, results = self._client.kv.get(path, recurse=True)
-
-            if results is None:
-                raise NotFound
-
-            nodes = {}
-            for node in results:
-                node['Value'] = (node['Value'] or b'').decode('utf-8')
-                nodes[os.path.relpath(node['Key'], path)] = node
-
-            # get initialize flag
-            initialize = nodes.get(self._INITIALIZE)
-            initialize = initialize and initialize['Value']
-
-            # get last leader operation
-            last_leader_operation = nodes.get(self._LEADER_OPTIME)
-            last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation['Value'])
-
-            # get list of members
-            members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
-
-            # get leader
-            leader = nodes.get(self._LEADER)
-            if leader and leader['Value'] == self._name and self._session != leader.get('Session', 'x'):
-                logger.info('I am leader but not owner of the session. Removing leader node')
-                self._client.kv.delete(self.leader_path, cas=leader['ModifyIndex'])
-                leader = None
-
-            if leader:
-                member = Member(-1, leader['Value'], None, {})
-                member = ([m for m in members if m.name == leader['Value']] or [member])[0]
-                leader = Leader(leader['ModifyIndex'], leader.get('Session'), member)
-
-            # failover key
-            failover = nodes.get(self._FAILOVER)
-            if failover:
-                failover = Failover.from_node(failover['ModifyIndex'], failover['Value'])
-
-            self._cluster = Cluster(initialize, leader, last_leader_operation, members, failover)
-        except NotFound:
-            self._cluster = Cluster(False, None, None, [], None)
-        except:
-            logger.exception('get_cluster')
-            raise ConsulError('Consul is not responding properly')
-
-    def touch_member(self, data, **kwargs):
-        create_member = self.refresh_session()
-        cluster = self.cluster
-        member = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
-        if create_member and member:
-            try:
-                self._client.kv.delete(self.member_path)
-            except Exception:
-                return False
-
-        if not create_member and member and data == self._my_member_data:
-            return True
-
-        try:
-            self._client.kv.put(self.member_path, data, acquire=self._session)
-            self._my_member_data = data
-            return True
-        except Exception:
-            logger.exception('touch_member')
-        return False
-
-    @catch_consul_errors
-    def attempt_to_acquire_leader(self):
-        ret = self._client.kv.put(self.leader_path, self._name, acquire=self._session)
-        if not ret:
-            logger.info('Could not take out TTL lock')
-        return ret
-
-    def take_leader(self):
-        return self.attempt_to_acquire_leader()
-
-    @catch_consul_errors
-    def set_failover_value(self, value, index=None):
-        return self._client.kv.put(self.failover_path, value, cas=index)
-
-    @catch_consul_errors
-    def write_leader_optime(self, last_operation):
-        return self._client.kv.put(self.leader_optime_path, last_operation)
-
-    @staticmethod
-    def update_leader():
-        return True
-
-    @catch_consul_errors
-    def initialize(self, create_new=True, sysid=''):
-        kwargs = {'cas': 0} if create_new else {}
-        return self._client.kv.put(self.initialize_path, sysid, **kwargs)
-
-    @catch_consul_errors
-    def cancel_initialization(self):
-        return self._client.kv.delete(self.initialize_path)
-
-    @catch_consul_errors
-    def delete_cluster(self):
-        return self._client.kv.delete(self.client_path(''), recurse=True)
-
-    @catch_consul_errors
-    def delete_leader(self):
-        cluster = self.cluster
-        if cluster and isinstance(cluster.leader, Leader) and cluster.leader.name == self._name:
-            return self._client.kv.delete(self.leader_path, cas=cluster.leader.index)
-
-    def watch(self, timeout):
-        cluster = self.cluster
-        if cluster and cluster.leader and cluster.leader.name != self._name and cluster.leader.index:
-            end_time = time.time() + timeout
-            while timeout >= 1:
-                try:
-                    idx, _ = self._client.kv.get(self.leader_path, index=cluster.leader.index, wait=str(timeout) + 's')
-                    return str(idx) != str(cluster.leader.index)
-                except (ConsulException, RequestException):
-                    logging.exception('watch')
-
-                timeout = end_time - time.time()
-
-        try:
-            return super(Consul, self).watch(timeout)
-        finally:
-            self.event.clear()
@@ -3,80 +3,86 @@ Patroni Control
 '''

 import click
-import datetime
-import dateutil
-import json
-import logging
 import os
+import yaml
+import json
+import time
 import psycopg2
 import random
 import requests
-import time
-import tzlocal
-import yaml
-
-from click import ClickException
-from patroni import Patroni, PatroniException
-from patroni.postgresql import parseurl
+import datetime
 from prettytable import PrettyTable
 from six.moves.urllib_parse import urlparse
+import logging
+
+from .etcd import Etcd
+from .exceptions import PatroniCtlException
+from .postgresql import parseurl

 CONFIG_DIR_PATH = click.get_app_dir('patroni')
 CONFIG_FILE_PATH = os.path.join(CONFIG_DIR_PATH, 'patronictl.yaml')
 LOGLEVEL = 'WARNING'
-DCS_DEFAULTS = {'zookeeper': {'port': 2181, 'template': "zookeeper:\n hosts: ['{host}:{port}']"},
-                'exhibitor': {'port': 8181, 'template': "zookeeper:\n exhibitor:\n  hosts: [{host}]\n  port: {port}"},
-                'consul': {'port': 8500, 'template': "consul:\n host: '{host}:{port}'"},
-                'etcd': {'port': 4001, 'template': "etcd:\n host: '{host}:{port}'"}}
-
-
-class PatroniCtlException(ClickException):
-    pass


 def parse_dcs(dcs):
-    if dcs is None:
-        return None
+    """
+    Break up the provided dcs string
+    >>> parse_dcs('localhost') == {'scheme': 'etcd', 'hostname': 'localhost', 'port': 4001}
+    True
+    >>> parse_dcs('localhost:8500') == {'scheme': 'consul', 'hostname': 'localhost', 'port': 8500}
+    True
+    >>> parse_dcs('zookeeper://localhost') == {'scheme': 'zookeeper', 'hostname': 'localhost', 'port': 2181}
+    True
+    """
+
+    if not dcs:
+        return {}

    parsed = urlparse(dcs)
    scheme = parsed.scheme
    if scheme == '' and parsed.netloc == '':
        parsed = urlparse('//' + dcs)
-    port = int(parsed.port) if parsed.port else None

    if scheme == '':
-        scheme = ([k for k, v in DCS_DEFAULTS.items() if v['port'] == port] or ['etcd'])[0]
-    elif scheme not in DCS_DEFAULTS:
-        raise PatroniCtlException('Unknown dcs scheme: {}'.format(scheme))
+        default_schemes = {'2181': 'zookeeper', '8500': 'consul'}
+        scheme = default_schemes.get(str(parsed.port), 'etcd')

-    dcs_info = DCS_DEFAULTS[scheme]
-    return yaml.load(dcs_info['template'].format(host=parsed.hostname or 'localhost', port=port or dcs_info['port']))
+    port = parsed.port
+    if port is None:
+        default_ports = {'consul': 8500, 'zookeeper': 2181}
+        port = default_ports.get(str(scheme), 4001)
+
+    return {'scheme': str(scheme), 'hostname': str(parsed.hostname), 'port': int(port)}


 def load_config(path, dcs):
-    logging.debug('Loading configuration from file %s', path)
+    logging.debug('Loading configuration from file {}'.format(path))
    config = dict()
    try:
        with open(path, 'rb') as fd:
            config = yaml.safe_load(fd)
-    except (IOError, yaml.YAMLError):
+    except:
        logging.exception('Could not load configuration file')

-    config.update(parse_dcs(dcs) or parse_dcs(config.get('dcs_api')) or {})
+    if dcs:
+        config['dcs'] = parse_dcs(dcs)
+    else:
+        config['dcs'] = parse_dcs(config.get('dcs_api'))

    return config


 def store_config(config, path):
    dir_path = os.path.dirname(path)
-    if dir_path and not os.path.isdir(dir_path):
-        os.makedirs(dir_path)
+    if dir_path:
+        if not os.path.isdir(dir_path):
+            os.makedirs(dir_path)
    with open(path, 'w') as fd:
        yaml.dump(config, fd)


 option_config_file = click.option('--config-file', '-c', help='Configuration file', default=CONFIG_FILE_PATH)
-option_format = click.option('--format', '-f', 'fmt', help='Output format (pretty, json)', default='pretty')
+option_format = click.option('--format', '-f', help='Output format (pretty, json)', default='pretty')
 option_dcs = click.option('--dcs', '-d', help='Use this DCS', envvar='DCS')
 option_watchrefresh = click.option('-w', '--watch', type=float, help='Auto update the screen every X seconds')
 option_watch = click.option('-W', is_flag=True, help='Auto update the screen every 2 seconds')
@@ -93,25 +99,23 @@ def ctl(ctx):


 def get_dcs(config, scope):
-    for k in set(DCS_DEFAULTS.keys()) & set(config.keys()):
-        config[k].setdefault('scope', scope)
-    try:
-        return Patroni.get_dcs(scope, config)
-    except PatroniException as e:
-        raise PatroniCtlException(str(e))
+    scheme, hostname, port = map(config.get('dcs', {}).get, ('scheme', 'hostname', 'port'))
+
+    if scheme == 'etcd':
+        return Etcd(name=scope, config={'scope': scope, 'host': '{}:{}'.format(hostname, port)})
+
+    raise PatroniCtlException('Can not find suitable configuration of distributed configuration store')


-def post_patroni(member, endpoint, content, headers=None):
+def post_patroni(member, endpoint, content, headers={'Content-Type': 'application/json'}):
    url = urlparse(member.api_url)
    logging.debug(url)
-    return requests.post('{0}://{1}/{2}'.format(url.scheme, url.netloc, endpoint),
-                         headers=headers or {'Content-Type': 'application/json'},
+    return requests.post('{}://{}/{}'.format(url.scheme, url.netloc, endpoint), headers=headers,
                         data=json.dumps(content), timeout=60)


-def print_output(columns, rows=None, alignment=None, fmt='pretty', header=True, delimiter='\t'):
-    rows = rows or []
-    if fmt == 'pretty':
+def print_output(columns, rows=[], alignment=None, format='pretty', header=True, delimiter='\t'):
+    if format == 'pretty':
        t = PrettyTable(columns)
        for k, v in (alignment or {}).items():
            t.align[k] = v
@@ -120,18 +124,18 @@ def print_output(columns, rows=None, alignment=None, fmt='pretty', header=True,
        click.echo(t)
        return

-    if fmt == 'json':
+    if format == 'json':
        elements = list()
        for r in rows:
            elements.append(dict(zip(columns, r)))

        click.echo(json.dumps(elements))

-    if fmt == 'tsv':
+    if format == 'tsv':
        if columns is not None and header:
            click.echo(delimiter.join(columns) + '\n')

-        for r in rows:
+        for r in rows or []:
            c = [str(c) for c in r]
            click.echo(delimiter.join(c))

@@ -164,8 +168,8 @@ def watching(w, watch, max_count=None, clear=True):
        yield 0


-def build_connect_parameters(conn_url, connect_parameters=None):
-    params = (connect_parameters or {}).copy()
+def build_connect_parameters(conn_url, connect_parameters={}):
+    params = connect_parameters.copy()
    parsed = parseurl(conn_url)
    params['host'] = parsed['host']
    params['port'] = parsed['port']
@@ -188,18 +192,20 @@ def get_all_members(cluster, role='master'):


 def get_any_member(cluster, role='master', member=None):
-    members = get_all_members(cluster, role)
+    members = get_all_members(cluster=cluster, role=role)
    for m in members:
        if member is None or m.name == member:
            return m

+    return None

-def get_cursor(cluster, role='master', member=None, connect_parameters=None):
-    member = get_any_member(cluster, role=role, member=member)
+
+def get_cursor(cluster, role='master', member=None, connect_parameters={}):
+    member = get_any_member(cluster=cluster, role=role, member=member)
    if member is None:
        return None

-    params = build_connect_parameters(member.conn_url, connect_parameters)
+    params = build_connect_parameters(member.conn_url, connect_parameters=connect_parameters)

    conn = psycopg2.connect(**params)
    conn.autocommit = True
@@ -231,23 +237,21 @@ def dsn(cluster_name, config_file, dcs, role, member):
    if member is None and role is None:
        role = 'master'

-    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
-    m = get_any_member(cluster, role=role, member=member)
+    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    m = get_any_member(cluster=cluster, role=role, member=member)
    if m is None:
        raise PatroniCtlException('Can not find a suitable member')

    params = build_connect_parameters(m.conn_url)
-    click.echo('host={host} port={port}'.format(**params))
+    click.echo('host={} port={}'.format(params['host'], params['port']))


@ctl.command('query', help='Query a Patroni PostgreSQL member')
@click.argument('cluster_name')
@option_config_file
@option_format
-@click.option('--format', 'fmt', help='Output format (pretty, json)', default='tsv')
-@click.option('--file', '-f', 'p_file', help='Execute the SQL commands from this file', type=click.File('rb'))
-@click.option('--password', help='force password prompt', is_flag=True)
-@click.option('-U', '--username', help='database user name', type=str)
+@click.option('--format', help='Output format (pretty, json)', default='tsv')
+@click.option('--file', '-f', help='Execute the SQL commands from this file', type=click.File('rb'))
@option_dcs
@option_watch
@option_watchrefresh
@@ -256,7 +260,6 @@ def dsn(cluster_name, config_file, dcs, role, member):
@click.option('--member', '-m', help='Query a specific member', type=str)
@click.option('--delimiter', help='The column delimiter', default='\t')
@click.option('--command', '-c', help='The SQL commands to execute')
-@click.option('-d', '--dbname', help='database name to connect to', type=str)
 def query(
    cluster_name,
    config_file,
@@ -267,56 +270,42 @@ def query(
    watch,
    delimiter,
    command,
-    p_file,
-    password,
-    username,
-    dbname,
-    fmt='tsv',
+    file,
+    format='tsv',
 ):
    if role is not None and member is not None:
        raise PatroniCtlException('--role and --member are mutually exclusive options')
    if member is None and role is None:
        role = 'master'

-    if p_file is not None and command is not None:
+    if file is not None and command is not None:
        raise PatroniCtlException('--file and --command are mutually exclusive options')

-    if p_file is None and command is None:
-        raise PatroniCtlException('You need to specify either --command or --file')
+    if file is not None:
+        command = file.read()

-    connect_parameters = dict()
-    if username:
-        connect_parameters['user'] = username
-    if password:
-        connect_parameters['password'] = click.prompt('Password', hide_input=True, type=str)
-    if dbname:
-        connect_parameters['database'] = dbname
-
-    if p_file is not None:
-        command = p_file.read()
-
-    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)

    cursor = None
    for _ in watching(w, watch, clear=False):

-        output, cursor = query_member(cluster, cursor, member, role, command, connect_parameters)
-        print_output(None, output, fmt=fmt, delimiter=delimiter)
+        output, cursor = query_member(cluster=cluster, cursor=cursor, member=member, role=role, command=command)
+        print_output(None, output, format=format, delimiter=delimiter)

        if cursor is None:
            cluster = dcs.get_cluster()


-def query_member(cluster, cursor, member, role, command, connect_parameters=None):
+def query_member(cluster, cursor, member, role, command):
    try:
        if cursor is None:
-            cursor = get_cursor(cluster, role=role, member=member, connect_parameters=connect_parameters)
+            cursor = get_cursor(cluster, role=role, member=member)

        if cursor is None:
            if role is None:
-                message = 'No connection to member {0} is available'.format(member)
+                message = 'No connection to member {} is available'.format(member)
            else:
-                message = 'No connection to role={0} is available'.format(role)
+                message = 'No connection to role={} is available'.format(role)
            logging.debug(message)
            return [[timestamp(0), message]], None

@@ -335,7 +324,7 @@ def query_member(cluster, cursor, member, role, command, connect_parameters=None
            cursor.connection.close()
        message = oe.pgcode or oe.pgerror or str(oe)
        message = message.replace('\n', ' ')
-        return [[timestamp(0), 'ERROR, SQLSTATE: {0}'.format(message)]], None
+        return [[timestamp(0), 'ERROR, SQLSTATE: {}'.format(message)]], None


@ctl.command('remove', help='Remove cluster from DCS')
@@ -343,10 +332,13 @@ def query_member(cluster, cursor, member, role, command, connect_parameters=None
@option_config_file
@option_format
@option_dcs
-def remove(config_file, cluster_name, fmt, dcs):
-    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+def remove(config_file, cluster_name, format, dcs):
+    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)

-    output_members(cluster, cluster_name, fmt)
+    if not isinstance(dcs, Etcd):
+        raise PatroniCtlException('We have not implemented this for DCS of type {}'.format(type(dcs)))
+
+    output_members(cluster, format=format)

    confirm = click.prompt('Please confirm the cluster name to remove', type=str)
    if confirm != cluster_name:
@@ -354,17 +346,17 @@ def remove(config_file, cluster_name, fmt, dcs):

    message = 'Yes I am aware'
    confirm = \
-        click.prompt('You are about to remove all information in DCS for {0}, please type: "{1}"'.format(cluster_name,
+        click.prompt('You are about to remove all information in DCS for {}, please type: "{}"'.format(cluster_name,
                     message), type=str)
    if message != confirm:
-        raise PatroniCtlException('You did not exactly type "{0}"'.format(message))
+        raise PatroniCtlException('You did not exactly type "{}"'.format(message))

    if cluster.leader:
        confirm = click.prompt('This cluster currently is healthy. Please specify the master name to continue')
        if confirm != cluster.leader.name:
            raise PatroniCtlException('You did not specify the current master of the cluster')

-    dcs.delete_cluster()
+    dcs.client.delete(dcs._base_path, recursive=True)


 def wait_for_leader(dcs, timeout=30):
@@ -386,25 +378,25 @@ def empty_post_to_members(cluster, member_names, force, endpoint):
    for m in cluster.members:
        candidates[m.name] = m

-    if not member_names:
-        member_names = [click.prompt('Which member do you want to {0} [{1}]?'.format(endpoint,
+    if len(member_names) == 0:
+        member_names = [click.prompt('Which member do you want to {} [{}]?'.format(endpoint,
                        ', '.join(candidates.keys())), type=str, default='')]

    for mn in member_names:
        if mn not in candidates.keys():
-            raise PatroniCtlException('{0} is not a member of cluster'.format(mn))
+            raise PatroniCtlException('{} is not a member of cluster'.format(mn))

    if not force:
-        confirm = click.confirm('Are you sure you want to {0} members {1}?'.format(endpoint, ', '.join(member_names)))
+        confirm = click.confirm('Are you sure you want to {} members {}?'.format(endpoint, ', '.join(member_names)))
        if not confirm:
-            raise PatroniCtlException('Aborted {0}'.format(endpoint))
+            raise PatroniCtlException('Aborted {}'.format(endpoint))

    for mn in member_names:
        r = post_patroni(candidates[mn], endpoint, '')
        if r.status_code != 200:
-            click.echo('{0} failed for member {1}, status code={2}, ({3})'.format(endpoint, mn, r.status_code, r.text))
+            click.echo('{} failed for member {}, status code={}, ({})'.format(endpoint, mn, r.status_code, r.text))
        else:
-            click.echo('Succesful {0} on member {1}'.format(endpoint, mn))
+            click.echo('Succesful {} on member {}'.format(endpoint, mn))


 def ctl_load_config(cluster_name, config_file, dcs):
@@ -420,25 +412,25 @@ def ctl_load_config(cluster_name, config_file, dcs):
@click.argument('member_names', nargs=-1)
@click.option('--role', '-r', help='Restart only members with this role', default='any',
              type=click.Choice(['master', 'replica', 'any']))
-@click.option('--any', 'p_any', help='Restart a single member only', is_flag=True)
+@click.option('--any', help='Restart a single member only', is_flag=True)
@option_config_file
@option_force
@option_dcs
-def restart(cluster_name, member_names, config_file, dcs, force, role, p_any):
-    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+def restart(cluster_name, member_names, config_file, dcs, force, role, any):
+    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)

-    role_names = [m.name for m in get_all_members(cluster, role)]
+    role_names = [m.name for m in get_all_members(cluster=cluster, role=role)]

-    if member_names:
+    if len(member_names) > 0:
        member_names = list(set(member_names) & set(role_names))
    else:
        member_names = role_names

-    if p_any:
+    if any:
        random.shuffle(member_names)
        member_names = member_names[:1]

-    output_members(cluster, cluster_name)
+    output_members(cluster)
    empty_post_to_members(cluster, member_names, force, 'restart')


@@ -449,7 +441,7 @@ def restart(cluster_name, member_names, config_file, dcs, force, role, p_any):
@option_force
@option_dcs
 def reinit(cluster_name, member_names, config_file, dcs, force):
-    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
    empty_post_to_members(cluster, member_names, force, 'reinitialize')


@@ -457,12 +449,10 @@ def reinit(cluster_name, member_names, config_file, dcs, force):
@click.argument('cluster_name')
@click.option('--master', help='The name of the current master', default=None)
@click.option('--candidate', help='The name of the candidate', default=None)
-@click.option('--scheduled', help='Timestamp of a scheduled failover in unambiguous format (e.g. ISO 8601)',
-              default=None)
@click.option('--force', is_flag=True)
@option_config_file
@option_dcs
-def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled):
+def failover(config_file, cluster_name, master, candidate, force, dcs):
    """
        We want to trigger a failover for the specified cluster name.

@@ -470,7 +460,7 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
        If so, we trigger a failover and keep the client up to date.
    """

-    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)

    if cluster.leader is None:
        raise PatroniCtlException('This cluster has no master')
@@ -482,13 +472,13 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
            master = click.prompt('Master', type=str, default=cluster.leader.member.name)

    if cluster.leader.member.name != master:
-        raise PatroniCtlException('Member {0} is not the leader of cluster {1}'.format(master, cluster_name))
+        raise PatroniCtlException('Member {} is not the leader of cluster {}'.format(master, cluster_name))

    candidate_names = [str(m.name) for m in cluster.members if m.name != master]
    # We sort the names for consistent output to the client
    candidate_names.sort()

-    if not candidate_names:
+    if len(candidate_names) == 0:
        raise PatroniCtlException('No candidates found to failover to')

    if candidate is None and not force:
@@ -498,60 +488,52 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
        raise PatroniCtlException('Failover target and source are the same.')

    if candidate and candidate not in candidate_names:
-        raise PatroniCtlException('Member {0} does not exist in cluster {1}'.format(candidate, cluster_name))
-
-    if scheduled is None and not force:
-        scheduled = click.prompt('When should the failover take place (e.g. 2015-10-01T14:30) ', type=str,
-                                 default='now')
-
-    if (scheduled or 'now') == 'now':
-        scheduled_at = None
-    else:
-        try:
-            scheduled_at = dateutil.parser.parse(scheduled)
-            if scheduled_at.tzinfo is None:
-                scheduled_at = tzlocal.get_localzone().localize(scheduled_at)
-        except (ValueError, TypeError):
-            message = 'Unable to parse scheduled timestamp ({0}). It should be in an unambiguous format (e.g. ISO 8601)'
-            raise PatroniCtlException(message.format(scheduled))
-        scheduled_at = scheduled_at.isoformat()
-
-    failover_value = {'leader': master, 'candidate': candidate, 'scheduled_at': scheduled_at}
-    logging.debug(failover_value)
+        raise PatroniCtlException('Member {} does not exist in cluster {}'.format(candidate, cluster_name))

    # By now we have established that the leader exists and the candidate exists
    click.echo('Current cluster topology')
-    output_members(dcs.get_cluster(), cluster_name)
+    output_members(dcs.get_cluster(), name=cluster_name)

    if not force:
        a = \
-            click.confirm('Are you sure you want to failover cluster {0}, demoting current master {1}?'.format(
+            click.confirm('Are you sure you want to failover cluster {}, demoting current master {}?'.format(
                cluster_name, master))
        if not a:
            raise PatroniCtlException('Aborting failover')

+    failover_value = '{}:{}'.format(master, candidate or '')
+
+    t_started = time.time()
    r = None
    try:
-        r = post_patroni(cluster.leader.member, 'failover', failover_value)
+        r = post_patroni(cluster.leader.member, 'failover', {'leader': master, 'member': candidate or ''})
        if r.status_code == 200:
            logging.debug(r)
+            logging.debug(r.text)
            cluster = dcs.get_cluster()
-            logging.debug(cluster)
-            click.echo('{0} {1}'.format(timestamp(), r.text))
+            click.echo(timestamp() + ' Failing over to new leader: {}'.format(cluster.leader.member.name))
        else:
-            click.echo('Failover failed, details: {0}, {1}'.format(r.status_code, r.text))
+            click.echo('Failover failed, details: {}, {}'.format(r.status_code, r.text))
            return
-    except Exception:
+    except:
        logging.exception(r)
        logging.warning('Failing over to DCS')
        click.echo(timestamp() + ' Could not failover using Patroni api, falling back to DCS')
-        click.echo(timestamp() + ' Initializing failover from master {0}'.format(master))
-        dcs.manual_failover(master, candidate, scheduled_at=failover_value)
+        dcs.set_failover_value(failover_value)
+        click.echo(timestamp() + ' Initialized failover from master {}'.format(master))
+        # The failover process should within a minute update the failover key, we will keep watching it until it changes
+        # or we timeout
+        cluster = wait_for_leader(dcs, timeout=60)
+        if cluster.leader.member.name == master:
+            click.echo('Failover failed, master did not change after {:0.1f} seconds'.format(time.time() - t_started))
+            return

-    output_members(cluster, cluster_name)
+    click.echo(timestamp() + ' Failover completed in {:0.1f} seconds, new leader is {}'.format(time.time() - t_started,
+               str(cluster.leader.member.name)))
+    output_members(cluster, name=cluster_name)


-def output_members(cluster, name, fmt='pretty'):
+def output_members(cluster, name=None, format='pretty'):
    rows = []
    logging.debug(cluster)
    leader_name = None
@@ -571,9 +553,10 @@ def output_members(cluster, name, fmt='pretty'):

        host = build_connect_parameters(m.conn_url)['host']

-        xlog_location = m.data.get('xlog_location') or 0
-        lag = ''
-        if (xlog_location_cluster >= xlog_location):
+        xlog_location = m.data.get('xlog_location')
+        if xlog_location is None or (xlog_location_cluster < xlog_location):
+            lag = ''
+        else:
            lag = round((xlog_location_cluster - xlog_location)/1024/1024)

        rows.append([
@@ -595,7 +578,7 @@ def output_members(cluster, name, fmt='pretty'):
    ]
    alignment = {'Cluster': 'l', 'Member': 'l', 'Host': 'l', 'Lag in MB': 'r'}

-    print_output(columns, rows, alignment, fmt)
+    print_output(columns, rows, alignment, format)


@ctl.command('list', help='List the Patroni members for a given Patroni')
@@ -605,17 +588,17 @@ def output_members(cluster, name, fmt='pretty'):
@option_watch
@option_watchrefresh
@option_dcs
-def members(config_file, cluster_names, fmt, watch, w, dcs):
-    if not cluster_names:
+def members(config_file, cluster_names, format, watch, w, dcs):
+    if len(cluster_names) == 0:
        logging.warning('Listing members: No cluster names were provided')
        return

    config = load_config(config_file, dcs)
-    for cluster_name in cluster_names:
-        dcs = get_dcs(config, cluster_name)
+    for cn in cluster_names:
+        dcs = get_dcs(config, cn)

        for _ in watching(w, watch):
-            output_members(dcs.get_cluster(), cluster_name, fmt)
+            output_members(dcs.get_cluster(), name=cn, format=format)


 def timestamp(precision=6):
@@ -1,10 +1,8 @@
 import abc
-import dateutil
 import json
-import six

 from collections import namedtuple
-from random import randint
+from patroni.exceptions import DCSError
 from six.moves.urllib_parse import urlparse, urlunparse, parse_qsl
 from threading import Event, Lock

@@ -53,33 +51,21 @@ class Member(namedtuple('Member', 'index,name,session,data')):
        else:
            try:
                data = json.loads(data)
-            except (TypeError, ValueError):
+            except:
                data = {}
        return Member(index, name, session, data)

    @property
    def conn_url(self):
-        return self.data.get('conn_url')
+        return self.data.get('conn_url', None)

    @property
    def api_url(self):
-        return self.data.get('api_url')
-
-    @property
-    def tags(self):
-        return self.data.get('tags', {})
+        return self.data.get('api_url', None)

    @property
    def nofailover(self):
-        return self.tags.get('nofailover', False)
-
-    @property
-    def replicatefrom(self):
-        return self.tags.get('replicatefrom')
-
-    @property
-    def clonefrom(self):
-        return self.tags.get('clonefrom', False)
+        return self.data.get('tags', {}).get('nofailover', False)


 class Leader(namedtuple('Leader', 'index,session,member')):
@@ -99,44 +85,12 @@ class Leader(namedtuple('Leader', 'index,session,member')):
        return self.member.conn_url


-class Failover(namedtuple('Failover', 'index,leader,candidate,scheduled_at')):
+class Failover(namedtuple('Failover', 'index,leader,member')):

-    """
-    >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader"}'))
-    True
-    >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader", "member": "cluster_candidate"}'))
-    True
-    >>> Failover.from_node(1, 'null') is None
-    True
-    >>> n = '{"leader": "cluster_leader", "member": "cluster_candidate", "scheduled_at": "2016-01-14T10:09:57.1394Z"}'
-    >>> 'tzinfo=' in str(Failover.from_node(1, n))
-    True
-    >>> Failover.from_node(1, None) is None
-    True
-    >>> Failover.from_node(1, '{}') is None
-    True
-    >>> 'abc' in Failover.from_node(1, 'abc:def')
-    True
-    """
    @staticmethod
    def from_node(index, value):
-        if not value:
-            return None
-
-        try:
-            data = json.loads(value)
-            if not data:
-                return None
-        except ValueError:
-            t = [a.strip() for a in value.split(':')]
-            leader = t[0]
-            candidate = t[1] if len(t) > 1 else None
-            return Failover(index, leader, candidate, None) if leader or candidate else None
-
-        if data.get('scheduled_at'):
-            data['scheduled_at'] = dateutil.parser.parse(data['scheduled_at'])
-
-        return Failover(index, data.get('leader'), data.get('member'), data.get('scheduled_at'))
+        t = [a.strip() for a in value.split(':')] + ['']
+        return Failover(index, t[0], t[1]) if t[0] or t[1] else None


 class Cluster(namedtuple('Cluster', 'initialize,leader,last_leader_operation,members,failover')):
@@ -153,19 +107,10 @@ class Cluster(namedtuple('Cluster', 'initialize,leader,last_leader_operation,mem
    def is_unlocked(self):
        return not (self.leader and self.leader.name)

-    def has_member(self, member_name):
-        return any(m for m in self.members if m.name == member_name)

-    def get_member(self, member_name, fallback_to_leader=True):
-        return ([m for m in self.members if m.name == member_name] or [self.leader if fallback_to_leader else None])[0]
+class AbstractDCS:

-    def get_clone_member(self):
-        candidates = [m for m in self.members if m.clonefrom and (not self.leader or m.name != self.leader.name)]
-        return candidates[randint(0, len(candidates) - 1)] if candidates else self.leader
-
-
-@six.add_metaclass(abc.ABCMeta)
-class AbstractDCS(object):
+    __metaclass__ = abc.ABCMeta

    _INITIALIZE = 'initialize'
    _LEADER = 'leader'
@@ -181,7 +126,7 @@ class AbstractDCS(object):
            i.e.: `zookeeper` for zookeeper, `etcd` for etcd, etc...
        """
        self._name = name
-        self._namespace = '/{0}'.format(config.get('namespace', '/service/').strip('/'))
+        self._namespace = '/{}'.format(config.get('namespace', '/service/').strip('/'))
        self._base_path = '/'.join([self._namespace, config['scope']])

        self._cluster = None
@@ -271,18 +216,15 @@ class AbstractDCS(object):
    def set_failover_value(self, value, index=None):
        """Create or update `/failover` key"""

-    def manual_failover(self, leader, candidate, scheduled_at=None, index=None):
-        failover_value = {}
-        if leader:
-            failover_value['leader'] = leader
+    def manual_failover(self, leader, member, index=None):
+        return self.set_failover_value(leader + (':' + member if member else ''), index)

-        if candidate:
-            failover_value['member'] = candidate
-
-        if scheduled_at:
-            failover_value['scheduled_at'] = scheduled_at.isoformat()
-
-        return self.set_failover_value(json.dumps(failover_value), index)
+    def current_leader(self):
+        try:
+            cluster = self.get_cluster()
+            return None if cluster.is_unlocked() else cluster.leader
+        except DCSError:
+            return None

    @abc.abstractmethod
    def touch_member(self, connection_string, ttl=None):
@@ -321,10 +263,6 @@ class AbstractDCS(object):
    def cancel_initialization(self):
        """ Removes the initialize key for a cluster """

-    @abc.abstractmethod
-    def delete_cluster(self):
-        """Delete cluster from DCS"""
-
    def watch(self, timeout):
        """If the current node is a master it should just sleep.
        Any other node should watch for changes of leader key with a given timeout
@@ -6,15 +6,14 @@ import random
 import requests
 import socket
 import time
+import urllib3

 from dns.exception import DNSException
 from dns import resolver
 from patroni.dcs import AbstractDCS, Cluster, Failover, Leader, Member
 from patroni.exceptions import DCSError
 from patroni.utils import Retry, RetryFailedError, sleep
-from urllib3.exceptions import HTTPError, ReadTimeoutError
 from requests.exceptions import RequestException
-from six.moves.http_client import HTTPException

 logger = logging.getLogger(__name__)

@@ -50,57 +49,11 @@ class Client(etcd.Client):
            self._update_machines_cache = True
            return [self._base_uri]

-    def _do_http_request(self, request_executor, method, url, fields=None, **kwargs):
-        try:
-            response = request_executor(method, url, fields=fields, **kwargs)
-            response.data.decode('utf-8')
-            self._check_cluster_id(response)
-        except (HTTPError, HTTPException, socket.error, socket.timeout) as e:
-            if (isinstance(fields, dict) and fields.get("wait") == "true" and
-                    isinstance(e, ReadTimeoutError)):
-                logger.debug("Watch timed out.")
-                raise etcd.EtcdWatchTimedOut("Watch timed out: {0}".format(e), cause=e)
-            logger.error("Request to server %s failed: %r", self._base_uri, e)
-            logger.info("Reconnection allowed, looking for another server.")
-            self._base_uri = self._next_server(cause=e)
-            response = False
-        return response
-
-    def api_execute(self, path, method, params=None, timeout=None):
-        if not path.startswith('/'):
-            raise ValueError('Path does not start with /')
-
-        if timeout is None:
-            timeout = self.read_timeout
-
-        if timeout == 0:
-            timeout = None
-
-        kwargs = {'timeout': timeout, 'fields': params, 'redirect': self.allow_redirect,
-                  'headers': self._get_headers(), 'preload_content': False}
-
-        if method in [self._MGET, self._MDELETE]:
-            request_executor = self.http.request
-        elif method in [self._MPUT, self._MPOST]:
-            request_executor = self.http.request_encode_body
-            kwargs['encode_multipart'] = False
-        else:
-            raise etcd.EtcdException('HTTP method {0} not supported'.format(method))
-
+    def api_execute(self, path, method, **kwargs):
        # Update machines_cache if previous attempt of update has failed
-        if self._update_machines_cache:
-            self._load_machines_cache()
-
-        response = False
-
+        self._update_machines_cache and self._load_machines_cache()
        try:
-            while not response:
-                response = self._do_http_request(request_executor, method, self._base_uri + path, **kwargs)
-
-                if response is False and not self._use_proxies:
-                    self._machines_cache = self.machines
-                    self._machines_cache.remove(self._base_uri)
-            return self._handle_server_response(response)
+            return super(Client, self).api_execute(path, method, **kwargs)
        except etcd.EtcdConnectionFailed:
            self._update_machines_cache = True
            raise
@@ -113,6 +66,16 @@ class Client(etcd.Client):
            logger.exception('Can not resolve SRV for %s', host)
        return []

+    # try to workarond bug in python-etcd: https://github.com/jplana/python-etcd/issues/81
+    def _result_from_response(self, response):
+        try:
+            response.data.decode('utf-8')
+        except urllib3.exceptions.TimeoutError:
+            raise
+        except Exception as e:
+            raise etcd.EtcdException('Unable to decode server response: %s' % e)
+        return super(Client, self)._result_from_response(response)
+
    def _get_machines_cache_from_srv(self, discovery_srv):
        """Fetch list of etcd-cluster member by resolving _etcd-server._tcp. SRV record.
        This record should contain list of host and peer ports which could be used to run
@@ -120,7 +83,7 @@ class Client(etcd.Client):

        ret = []
        for host, port in self.get_srv_record(discovery_srv):
-            url = '{0}://{1}:{2}/members'.format(self._protocol, host, port)
+            url = '{}://{}:{}/members'.format(self._protocol, host, port)
            try:
                response = requests.get(url, timeout=5)
                if response.ok:
@@ -138,10 +101,10 @@ class Client(etcd.Client):
        host, port = addr.split(':')
        try:
            for r in set(socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)):
-                ret.append('{0}://{1}:{2}'.format(self._protocol, r[4][0], r[4][1]))
+                ret.append('{}://{}:{}'.format(self._protocol, r[4][0], r[4][1]))
        except socket.error:
            logger.exception('Can not resolve %s', host)
-        return list(set(ret)) if ret else ['{0}://{1}:{2}'.format(self._protocol, host, port)]
+        return list(set(ret)) if ret else ['{}://{}:{}'.format(self._protocol, host, port)]

    def _load_machines_cache(self):
        """This method should fill up `_machines_cache` from scratch.
@@ -169,9 +132,7 @@ class Client(etcd.Client):
        # After filling up initial list of machines_cache we should ask etcd-cluster about actual list
        self._base_uri = self._machines_cache.pop(0)
        self._machines_cache = self.machines
-
-        if self._base_uri in self._machines_cache:
-            self._machines_cache.remove(self._base_uri)
+        self._base_uri in self._machines_cache and self._machines_cache.remove(self._base_uri)

        self._update_machines_cache = False

@@ -179,7 +140,7 @@ class Client(etcd.Client):
 def catch_etcd_errors(func):
    def wrapper(*args, **kwargs):
        try:
-            return func(*args, **kwargs) is not None
+            return not func(*args, **kwargs) is None
        except (RetryFailedError, etcd.EtcdException):
            return False
        except:
@@ -199,13 +160,12 @@ class Etcd(AbstractDCS):
                                              etcd.EtcdLeaderElectionInProgress,
                                              etcd.EtcdWatcherCleared,
                                              etcd.EtcdEventIndexCleared))
-        self._client = self.get_etcd_client(config)
+        self.client = self.get_etcd_client(config)

    def retry(self, *args, **kwargs):
        return self._retry.copy()(*args, **kwargs)

-    @staticmethod
-    def get_etcd_client(config):
+    def get_etcd_client(self, config):
        client = None
        while not client:
            try:
@@ -221,29 +181,29 @@ class Etcd(AbstractDCS):

    def _load_cluster(self):
        try:
-            result = self.retry(self._client.read, self.client_path(''), recursive=True)
+            result = self.retry(self.client.read, self.client_path(''), recursive=True)
            nodes = {os.path.relpath(node.key, result.key): node for node in result.leaves}

            # get initialize flag
-            initialize = nodes.get(self._INITIALIZE)
+            initialize = nodes.get(self._INITIALIZE, None)
            initialize = initialize and initialize.value

            # get last leader operation
-            last_leader_operation = nodes.get(self._LEADER_OPTIME)
+            last_leader_operation = nodes.get(self._LEADER_OPTIME, None)
            last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation.value)

            # get list of members
            members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]

            # get leader
-            leader = nodes.get(self._LEADER)
+            leader = nodes.get(self._LEADER, None)
            if leader:
                member = Member(-1, leader.value, None, {})
                member = ([m for m in members if m.name == leader.value] or [member])[0]
                leader = Leader(leader.modifiedIndex, leader.ttl, member)

            # failover key
-            failover = nodes.get(self._FAILOVER)
+            failover = nodes.get(self._FAILOVER, None)
            if failover:
                failover = Failover.from_node(failover.modifiedIndex, failover.value)

@@ -256,15 +216,15 @@ class Etcd(AbstractDCS):

    @catch_etcd_errors
    def touch_member(self, connection_string, ttl=None):
-        return self.retry(self._client.set, self.member_path, connection_string, ttl or self.ttl)
+        return self.retry(self.client.set, self.member_path, connection_string, ttl or self.ttl)

    @catch_etcd_errors
    def take_leader(self):
-        return self.retry(self._client.set, self.leader_path, self._name, self.ttl)
+        return self.retry(self.client.set, self.leader_path, self._name, self.ttl)

    def attempt_to_acquire_leader(self):
        try:
-            return bool(self.retry(self._client.write, self.leader_path, self._name, ttl=self.ttl, prevExist=False))
+            return bool(self.retry(self.client.write, self.leader_path, self._name, ttl=self.ttl, prevExist=False))
        except etcd.EtcdAlreadyExist:
            logger.info('Could not take out TTL lock')
        except (RetryFailedError, etcd.EtcdException):
@@ -273,46 +233,43 @@ class Etcd(AbstractDCS):

    @catch_etcd_errors
    def set_failover_value(self, value, index=None):
-        return self._client.write(self.failover_path, value, prevIndex=index or 0)
+        return self.client.write(self.failover_path, value, prevIndex=index or 0)

    @catch_etcd_errors
    def write_leader_optime(self, last_operation):
-        return self._client.set(self.leader_optime_path, last_operation)
+        return self.client.set(self.leader_optime_path, last_operation)

    @catch_etcd_errors
    def update_leader(self):
-        return self.retry(self._client.test_and_set, self.leader_path, self._name, self._name, self.ttl)
+        return self.retry(self.client.test_and_set, self.leader_path, self._name, self._name, self.ttl)

    @catch_etcd_errors
    def initialize(self, create_new=True, sysid=""):
-        return self.retry(self._client.write, self.initialize_path, sysid, prevExist=(not create_new))
+        return self.retry(self.client.write, self.initialize_path, sysid, prevExist=(not create_new))

    @catch_etcd_errors
    def delete_leader(self):
-        return self._client.delete(self.leader_path, prevValue=self._name)
+        return self.client.delete(self.leader_path, prevValue=self._name)

    @catch_etcd_errors
    def cancel_initialization(self):
-        return self.retry(self._client.delete, self.initialize_path)
-
-    @catch_etcd_errors
-    def delete_cluster(self):
-        return self.retry(self._client.delete, self.client_path(''), recursive=True)
+        return self.retry(self.client.delete, self.initialize_path)

    def watch(self, timeout):
        cluster = self.cluster
        # watch on leader key changes if it is defined and current node is not lock owner
-        if cluster and cluster.leader and cluster.leader.name != self._name and cluster.leader.index:
+        if cluster and cluster.leader and cluster.leader.name != self._name:
            end_time = time.time() + timeout
+            index = cluster.leader.index

-            while timeout >= 1:  # when timeout is too small urllib3 doesn't have enough time to connect
+            while index and timeout >= 1:  # when timeout is too small urllib3 doesn't have enough time to connect
                try:
-                    self._client.watch(self.leader_path, index=cluster.leader.index + 1, timeout=timeout + 0.5)
+                    self.client.watch(self.leader_path, index=index + 1, timeout=timeout + 0.5)
                    # Synchronous work of all cluster members with etcd is less expensive
                    # than reestablishing http connection every time from every replica.
                    return True
-                except etcd.EtcdWatchTimedOut:
-                    self._client.http.clear()
+                except urllib3.exceptions.TimeoutError:
+                    self.client.http.clear()
                    return False
                except etcd.EtcdException:
                    logging.exception('watch')
@@ -13,6 +13,10 @@ class PatroniException(Exception):
        return repr(self.value)


+class PatroniCtlException(Exception):
+    pass
+
+
 class PostgresException(PatroniException):
    pass

@@ -3,18 +3,15 @@ import logging
 import psycopg2
 import requests
 import sys
-import datetime
-import pytz

-from multiprocessing.pool import ThreadPool
 from patroni.async_executor import AsyncExecutor
 from patroni.exceptions import DCSError, PostgresConnectionException
-from patroni.utils import sleep
+from multiprocessing.pool import ThreadPool

 logger = logging.getLogger(__name__)


-class Ha(object):
+class Ha:

    def __init__(self, patroni):
        self.patroni = patroni
@@ -22,13 +19,17 @@ class Ha(object):
        self.dcs = patroni.dcs
        self.cluster = None
        self.old_cluster = None
-        self.recovering = False
        self._async_executor = AsyncExecutor()
+        self.bdr = patroni.bdr
+        self.use_bdr = self.bdr and\
+            self.bdr.get('enable') and\
+            (self.bdr.get('database') is not None)
+        self.bdr_may_need_reinitialize = False

    def load_cluster_from_dcs(self):
        cluster = self.dcs.get_cluster()

-        # We want to keep the state of cluster when it was healthy
+        # We want to keep the state of cluster when it was healhy
        if not cluster.is_unlocked() or not self.old_cluster:
            self.old_cluster = cluster
        self.cluster = cluster
@@ -38,7 +39,7 @@ class Ha(object):

    def update_lock(self):
        ret = self.dcs.update_leader()
-        if ret and not self._async_executor.busy:
+        if ret:
            try:
                self.dcs.write_leader_optime(self.state_handler.last_operation())
            except:
@@ -50,42 +51,52 @@ class Ha(object):
        logger.info('Lock owner: %s; I am %s', lock_owner, self.state_handler.name)
        return lock_owner == self.state_handler.name

-    def touch_member(self):
+    def touch_member(self, ttl=None):
        data = {
            'conn_url': self.state_handler.connection_string,
            'api_url': self.patroni.api.connection_string,
            'state': self.state_handler.state,
-            'role': self.state_handler.role
+            'role': self.state_handler.role,
+            'tags': self.patroni.tags
        }
-        if self.patroni.tags:
-            data['tags'] = self.patroni.tags
-        if not self._async_executor.busy and data['state'] in ['running', 'restarting', 'starting']:
+        if data['state'] in ['running', 'restarting', 'starting']:
            try:
                data['xlog_location'] = self.state_handler.xlog_position()
            except:
                pass
-        self.dcs.touch_member(json.dumps(data, separators=(',', ':')))
+        self.dcs.touch_member(json.dumps(data, separators=(',', ':')), ttl=ttl)

-    def clone(self, clone_member=None, msg='(without leader)'):
-        if self.state_handler.bootstrap(cluster_initialized=True, clone_member=clone_member):
-            logger.info('bootstrapped %s', msg)
+    def copy_backup_from_leader(self, leader):
+        if self.state_handler.bootstrap(leader):
+            logger.info('bootstrapped from leader')
        else:
-            logger.error('failed to bootstrap %s', msg)
            self.state_handler.stop('immediate')
            self.state_handler.remove_data_directory()
+            logger.error('failed to bootstrap from leader')

-    def bootstrap(self):
+    def initialize_bdr(self, leader):
+        if self.state_handler.initialize_bdr_database(leader):
+            logger.info("bootstrapped from leader")
+        else:
+            self.state_handler.stop('immediate')
+            self.state_handler.remove_data_directory()
+            logger.error('failed to bootstrap from leader')
+
+    def bootstrap(self, bdr=False):
        if not self.cluster.is_unlocked():  # cluster already has leader
-            clone_member = self.cluster.get_clone_member()
-            member_role = 'leader' if clone_member == self.cluster.leader else 'replica'
-            msg = "from {0} '{1}'".format(member_role, clone_member.name)
-            self._async_executor.schedule('bootstrap {0}'.format(msg))
-            self._async_executor.run_async(self.clone, args=(clone_member, msg))
-            return 'trying to bootstrap {0}'.format(msg)
+            self._async_executor.schedule('bootstrap from leader')
+            if not bdr:
+                self._async_executor.run_async(self.copy_backup_from_leader, args=(self.cluster.leader, ))
+            else:
+                self._async_executor.run_async(self.initialize_bdr, args=(self.cluster.leader, ))
+            return 'trying to bootstrap from leader'
        elif not self.cluster.initialize and not self.patroni.nofailover:  # no initialize key
            if self.dcs.initialize(create_new=True):  # race for initialization
                try:
-                    self.state_handler.bootstrap()
+                    if not bdr:
+                        self.state_handler.bootstrap()
+                    else:
+                        self.state_handler.initialize_bdr_database()
                    self.dcs.initialize(create_new=False, sysid=self.state_handler.sysid)
                except:  # initdb or start failed
                    # remove initialization key and give a chance to other members
@@ -95,39 +106,64 @@ class Ha(object):
                    self.state_handler.move_data_directory()
                    raise
                self.dcs.take_leader()
-                self.load_cluster_from_dcs()
                return 'initialized a new cluster'
            else:
                return 'failed to acquire initialize lock'
        else:
-            if self.state_handler.can_create_replica_without_replication_connection():
-                self._async_executor.run_async(self.clone)
-                return "trying to bootstrap (without leader)"
            return 'waiting for leader to bootstrap'

+    def bootstrap_bdr(self):
+        return self.bootstrap(bdr=True)
+
    def recover(self):
-        self.recovering = True
-        return self.follow("starting as readonly because i had the session lock", "starting as a secondary", True, True)
+        has_lock = self.has_lock()

-    def follow(self, demote_reason, follow_reason, refresh=True, recovery=False):
-        if refresh:
-            self.load_cluster_from_dcs()
+        # try to see if we are the former master that crashed. If so - we likely need to run pg_rewind
+        # in order to join the former standby being promoted.
+        pg_controldata = self.state_handler.controldata()
+        if not has_lock and pg_controldata and\
+                pg_controldata.get('Database cluster state', '') == 'in production':  # crashed master
+            self.state_handler.require_rewind()

-        ret = demote_reason if not recovery and self.state_handler.is_leader() else follow_reason
+        # XXX: follow the leader calls stop, which might take quite some time.
+        # perhaps we should run sync asynchronously
+        # (we still need the exit code from follow_the_leader)
+        ret = self.state_handler.follow_the_leader(None if has_lock else self.cluster.leader, recovery=True)
+        if not ret:
+            if not has_lock:
+                return 'failed to start postgres'
+            self.dcs.delete_leader()
+            self.dcs.reset_cluster()
+            return 'removed leader key after trying and failing to start postgres'
+        if not has_lock:
+            return 'started as a secondary'
+        logger.info('started as readonly because i had the session lock')
+        self.load_cluster_from_dcs()

-        # determine the node to follow. If replicatefrom tag is set,
-        # try to follow the node mentioned there, otherwise, follow the leader.
+    def recover_bdr(self):
+        if not self.bdr_may_need_reinitialize:
+            self.state_handler.start_bdr()
+            return "started as a member of BDR group"
+        has_lock = self.has_lock()
+        if self.cluster.is_unlocked():  # no leader yet, and we need to reinitialize from the leader
+            return "waiting for another member of the BDR group"
+        if has_lock:
+            self.dcs.delete_leader()
+            return "released the lock to another member of the BDR group"
+        if self.state_handler.initialize_bdr_database(self.cluster.leader, empty=False):
+            # we reinitialized this node, make sure the cluster is aware of it.
+            self.touch_member()
+            self.bdr_may_need_reinitialize = False
+            return "reinitialized and started as a member of the BDR group"

-        if self.patroni.replicatefrom:
-            node_to_follow = self.cluster.get_member(self.patroni.replicatefrom, fallback_to_leader=True)
-        else:
-            node_to_follow = self.cluster.leader
-        if node_to_follow and node_to_follow.name == self.state_handler.name:
-            ret = demote_reason
-            node_to_follow = None
-        if not self.state_handler.check_recovery_conf(node_to_follow) or recovery:
+    def follow_the_leader(self, demote_reason, follow_reason, refresh=True):
+        refresh and self.load_cluster_from_dcs()
+        ret = demote_reason if self.state_handler.is_leader() else follow_reason
+        leader = self.cluster.leader
+        leader = None if (leader and leader.name) == self.state_handler.name else leader
+        if not self.state_handler.check_recovery_conf(leader):
            self._async_executor.schedule('changing primary_conninfo and restarting')
-            self._async_executor.run_async(self.state_handler.follow, (node_to_follow, recovery))
+            self._async_executor.run_async(self.state_handler.follow_the_leader, (leader, ))
        return ret

    def enforce_master_role(self, message, promote_message):
@@ -198,7 +234,7 @@ class Ha(object):
        ret = False
        members = [m for m in members if m.name != self.state_handler.name and not m.nofailover and m.api_url]
        if members:
-            for member, reachable, _, _, tags in self.fetch_nodes_statuses(members):
+            for member, reachable, in_recovery, xlog_location, tags in self.fetch_nodes_statuses(members):
                if reachable and not tags.get('nofailover', False):
                    ret = True  # TODO: check xlog_location
                elif not reachable:
@@ -211,14 +247,14 @@ class Ha(object):

    def manual_failover_process_no_leader(self):
        failover = self.cluster.failover
-        if failover.candidate:  # manual failover to specific member
-            if failover.candidate == self.state_handler.name:  # manual failover to me
+        if failover.member:  # manual failover to specific member
+            if failover.member == self.state_handler.name:  # manual failover to me
                return True

            # find specific node and check that it is healthy
-            member = self.cluster.get_member(failover.candidate, fallback_to_leader=False)
-            if member:
-                member, reachable, _, _, tags = self.fetch_node_status(member)
+            members = [m for m in self.cluster.members if m.name == failover.member]
+            if members:
+                member, reachable, in_recovery, xlog_location, tags = self.fetch_node_status(members[0])
                if reachable and not tags.get('nofailover', False):  # node is healthy
                    logger.info('manual failover: to %s, i am %s', member.name, self.state_handler.name)
                    return False
@@ -229,13 +265,13 @@ class Ha(object):
                    logger.warning('manual failover: member %s is not allowed to promote', member.name)

            # at this point we should consider all members as a candidates for failover
-            # i.e. we assume that failover.candidate is None
+            # i.e. we assume that failover.member is None

        # try to pick some other members to failover and check that they are healthy
        if failover.leader:
            if self.state_handler.name == failover.leader:  # I was the leader
-                # exclude me and desired member which is unhealthy (failover.candidate can be None)
-                members = [m for m in self.cluster.members if m.name not in (failover.candidate, failover.leader)]
+                # exclude me and desired member which is unhealthy (failover.member can be None)
+                members = [m for m in self.cluster.members if m.name not in (failover.member, failover.leader)]
                if self.is_failover_possible(members):  # check that there are healthy members
                    return False
                else:  # I was the leader and it looks like currently I am the only healthy member
@@ -268,43 +304,20 @@ class Ha(object):
            self.dcs.delete_leader()
            self.touch_member()
            self.dcs.reset_cluster()
-            sleep(2)  # Give a time to somebody to promote
-            self.recover()
-        else:
-            self.state_handler.follow(None)
+        self.state_handler.follow_the_leader(None)

-    def process_manual_failover_from_leader(self):
+    def process_manual_failover_from_leader(self, bdr=False):
        failover = self.cluster.failover
-
-        if failover.scheduled_at:
-            # If the failover is in the far future, we shouldn't do anything and just return.
-            # If the failover is in the past, we consider the value to be stale and we remove
-            # the value.
-            # If the value is close to now, we initiate the failover
-            now = datetime.datetime.now(pytz.utc)
-            try:
-                delta = (failover.scheduled_at - now).total_seconds()
-
-                if delta > self.patroni.nap_time:
-                    logging.info('Awaiting failover at %s (in %.0f seconds)', failover.scheduled_at.isoformat(), delta)
-                    return
-                elif delta < - int(self.patroni.nap_time * 1.5):
-                    logger.warning('Found a stale failover value, cleaning up: %s', failover.scheduled_at)
-                    self.dcs.manual_failover('', '', index=self.cluster.failover.index)
-                    return
-
-                # The value is very close to now
-                sleep(max(delta, 0))
-                logger.info('Manual scheduled failover at {}'.format(failover.scheduled_at.isoformat()))
-            except TypeError:
-                logger.warning('Incorrect value in of scheduled_at: %s', failover.scheduled_at)
-
        if not failover.leader or failover.leader == self.state_handler.name:
-            if not failover.candidate or failover.candidate != self.state_handler.name:
-                members = [m for m in self.cluster.members if not failover.candidate or m.name == failover.candidate]
+            if not failover.member or failover.member != self.state_handler.name:
+                members = [m for m in self.cluster.members if not failover.member or m.name == failover.member]
                if self.is_failover_possible(members):  # check that there are healthy members
-                    self._async_executor.schedule('manual failover: demote')
-                    self._async_executor.run_async(self.demote)
+                    if not bdr:
+                        self._async_executor.schedule('manual failover: demote')
+                        self._async_executor.run_async(self.demote)
+                    else:
+                        self.dcs.delete_leader()
+                        self.dcs.reset_cluster()
                    return 'manual failover: demoting myself'
                else:
                    logger.warning('manual failover: no healthy members found, failover is not possible')
@@ -315,7 +328,10 @@ class Ha(object):
                           self.cluster.failover.leader, self.state_handler.name)

        logger.info('Trying to clean up failover key')
-        self.dcs.manual_failover('', '', index=self.cluster.failover.index)
+        self.dcs.manual_failover('', '', self.cluster.failover.index)
+
+    def process_manual_failover_from_leader_bdr(self):
+        self.process_manual_failover_from_leader(bdr=True)

    def process_unhealthy_cluster(self):
        if self.is_healthiest_node():
@@ -323,18 +339,18 @@ class Ha(object):
                if self.cluster.failover:
                    logger.info('Cleaning up failover key after acquiring leader lock...')
                    self.dcs.manual_failover('', '')
-                self.load_cluster_from_dcs()
+                self.dcs.get_cluster()
                return self.enforce_master_role('acquired session lock as a leader',
                                                'promoted self to leader by acquiring session lock')
            else:
-                return self.follow('demoted self after trying and failing to obtain lock',
-                                   'following new leader after trying and failing to obtain lock')
+                return self.follow_the_leader('demoted self due after trying and failing to obtain lock',
+                                              'following new leader after trying and failing to obtain lock')
        else:
            if self.patroni.nofailover:
-                return self.follow('demoting self because I am not allowed to become master',
-                                   'following a different leader because I am not allowed to promote')
-            return self.follow('demoting self because i am not the healthiest node',
-                               'following a different leader because i am not the healthiest node')
+                return self.follow_the_leader('demoting self because I am not allowed to become master',
+                                              'following a different leader because I am not allowed to promote')
+            return self.follow_the_leader('demoting self because i am not the healthiest node',
+                                          'following a different leader because i am not the healthiest node')

    def process_healthy_cluster(self):
        if self.has_lock():
@@ -352,8 +368,33 @@ class Ha(object):
                self.load_cluster_from_dcs()
        else:
            logger.info('does not have lock')
-        return self.follow('demoting self because i do not have the lock and i was a leader',
-                           'no action.  i am a secondary and i am following a leader', False)
+        return self.follow_the_leader('demoting self because i do not have the lock and i was a leader',
+                                      'no action.  i am a secondary and i am following a leader', False)
+
+    def process_unhealthy_cluster_bdr(self):
+        if self.acquire_lock():
+            if self.cluster.failover:
+                logger.info('Cleaning up failover key after acquiring leader lock...')
+                self.dcs.manual_failover('', '')
+            self.dcs.get_cluster()
+            return "acquired the session lock as a leader"
+        else:
+            return "lost the race to acquire the session lock"
+
+    def process_healthy_cluster_bdr(self):
+        if self.has_lock():
+            if self.cluster.failover:
+                msg = self.process_manual_failover_from_leader_bdr()
+                if msg is not None:
+                    return msg
+            if self.update_lock():
+                return "no action. I am the leader with the lock"
+            else:
+                logger.error("failed to update leader lock")
+                self.load_cluster_from_dcs()
+        else:
+            logger.info("does not have lock")
+        return "no action. I do not have the lock"

    def schedule(self, action):
        with self._async_executor:
@@ -381,10 +422,7 @@ class Ha(object):
    def reinitialize(self, cluster):
        self.state_handler.stop('immediate')
        self.state_handler.remove_data_directory()
-
-        clone_member = cluster.get_clone_member()
-        member_role = 'leader' if clone_member == cluster.leader else 'replica'
-        self.clone(clone_member, "from {0} '{1}'".format(member_role, clone_member.name))
+        self.copy_backup_from_leader(cluster.leader)

    def process_scheduled_action(self):
        if self.reinitialize_scheduled():
@@ -409,26 +447,23 @@ class Ha(object):
        else:
            return self._async_executor.scheduled_action + ' in progress'

-    @staticmethod
-    def sysid_valid(sysid):
+    def sysid_valid(self, sysid):
        # sysid does tv_sec << 32, where tv_sec is the number of seconds sine 1970,
        # so even 1 << 32 would have 10 digits.
        return str(sysid) and len(str(sysid)) >= 10 and str(sysid).isdigit()

-    def post_recover(self):
-        if not self.state_handler.is_running():
-            if self.has_lock():
-                self.dcs.delete_leader()
-                self.dcs.reset_cluster()
-                return 'removed leader key after trying and failing to start postgres'
-            return 'failed to start postgres'
-        return None
-
    def _run_cycle(self):
        try:
            self.load_cluster_from_dcs()

-            self.touch_member()
+            # if member key is missing - we may need to destory
+            # and re-create the BDR database unless we are the first node in the cluster
+            if self.use_bdr and len([m for m in self.cluster.members if m.name == self.state_handler.name]) == 0:
+                self.bdr_may_need_reinitialize = True
+
+            # Create a member key with a very short expiration time if we may need to reinitialize this member
+            # in order to avoid lossing the "may need to reinitialize status" in case Patroni is restarted
+            self.touch_member(ttl=None if not self.bdr_may_need_reinitialize else self.bdr.get('min_ttl', 5))

            # cluster has leader key but not initialize key
            if not self.cluster.is_unlocked() and not self.sysid_valid(self.cluster.initialize) and self.has_lock():
@@ -437,13 +472,6 @@ class Ha(object):
            if self._async_executor.busy:
                return self.handle_long_action_in_progress()

-            # we've got here, so any async action has finished. Check if we tried to recover and failed
-            if self.recovering:
-                self.recovering = False
-                msg = self.post_recover()
-                if msg is not None:
-                    return msg
-
            # currently it can trigger only reinitialize
            msg = self.process_scheduled_action()
            if msg is not None:
@@ -451,41 +479,50 @@ class Ha(object):

            # is data directory empty?
            if self.state_handler.data_directory_empty():
-                return self.bootstrap()  # new node
+                if self.use_bdr:
+                    # member key was missing, but it's a new database - no need to reinitialize
+                    self.bdr_may_need_reinitialize = False
+                    self.touch_member()
+                return self.bootstrap(self.use_bdr)  # new node
            # "bootstrap", but data directory is not empty
-            elif not self.sysid_valid(self.cluster.initialize) and self.cluster.is_unlocked():
+            elif not self.sysid_valid(self.cluster.initialize) and self.cluster.is_unlocked() and \
+                    not self.bdr_may_need_reinitialize:
                self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=self.state_handler.sysid)
            else:
                # check if we are allowed to join
-                if self.sysid_valid(self.cluster.initialize) and self.cluster.initialize != self.state_handler.sysid:
-                    logger.fatal("system ID mismatch, node %s belongs to a different cluster: %s != %s",
-                                 self.state_handler.name, self.cluster.initialize, self.state_handler.sysid)
+                if not self.use_bdr and \
+                   self.sysid_valid(self.cluster.initialize) and self.cluster.initialize != self.state_handler.sysid:
+                    logger.fatal("system ID mismatch, node {0} belongs to a different cluster".
+                                 format(self.state_handler.name))
                    sys.exit(1)

            # try to start dead postgres
-            if not self.state_handler.is_healthy():
-                msg = self.recover()
-                if msg is not None:
-                    return msg
+            if self.use_bdr:
+                if self.bdr_may_need_reinitialize or not self.state_handler.is_healthy():
+                    msg = self.recover_bdr()
+                    if msg is not None:
+                        return msg
+            else:
+                if not self.state_handler.is_healthy():
+                    msg = self.recover()
+                    if msg is not None:
+                        return msg

            try:
                if self.cluster.is_unlocked():
-                    return self.process_unhealthy_cluster()
+                    return self.process_unhealthy_cluster() if not self.use_bdr else self.process_unhealthy_cluster_bdr()
                else:
-                    return self.process_healthy_cluster()
+                    return self.process_healthy_cluster() if not self.use_bdr else self.process_healthy_cluster_bdr()
            finally:
-                # we might not have a valid PostgreSQL connection here if another thread
-                # stops PostgreSQL, therefore, we only reload replication slots if no
-                # asynchronous processes are running (should be always the case for the master)
-                if not self._async_executor.busy:
+                if not self.use_bdr:
                    self.state_handler.sync_replication_slots(self.cluster)
        except DCSError:
            logger.error('Error communicating with DCS')
-            if self.state_handler.is_running() and self.state_handler.is_leader():
+            if self.state_handler.is_running() and self.state_handler.is_leader() and not self.use_bdr:
                self.demote(delete_leader=False)
                return 'demoted self because DCS is not accessible and i was a leader'
        except (psycopg2.Error, PostgresConnectionException):
-            logger.exception('Error communicating with PostgreSQL. Will try again later')
+            logger.exception('Error communicating with Postgresql. Will try again later')

    def run_cycle(self):
        with self._async_executor:
@@ -1,6 +1,7 @@
 import logging
 import os
 import psycopg2
+import re
 import shlex
 import shutil
 import subprocess
@@ -39,48 +40,50 @@ def parseurl(url):
    return ret


-class Postgresql(object):
+class Postgresql:

-    def __init__(self, config):
+    def __init__(self, config, config_bdr={}):
        self.config = config
+        self.bdr = config_bdr
        self.name = config['name']
-        self.server_parameters = config.get('parameters') or {}
+        self.server_parameters = config.get('parameters', {})
        self.scope = config['scope']
        self.listen_addresses, self.port = config['listen'].split(':')
        self.data_dir = config['data_dir']
        self.replication = config['replication']
-        self.superuser = config.get('superuser') or {}
-        self.admin = config.get('admin') or {}
-
-        self.initdb_options = config.get('initdb') or []
-        self.pgpass = config.get('pgpass') or os.path.join(os.path.expanduser('~'), 'pgpass')
-        self.pg_rewind = config.get('pg_rewind') or {}
-        self.callback = config.get('callbacks') or {}
+        self.superuser = config['superuser']
+        self.admin = config['admin']
+        self.initdb_options = config.get('initdb', [])
+        self.pgpass = config.get('pgpass', None) or os.path.join(os.path.expanduser('~'), 'pgpass')
+        self.pg_rewind = config.get('pg_rewind', {})
+        self.callback = config.get('callbacks', {})
        self.use_slots = config.get('use_slots', True)
        self.schedule_load_slots = self.use_slots
        self.recovery_conf = os.path.join(self.data_dir, 'recovery.conf')
        self.configuration_to_save = (os.path.join(self.data_dir, 'pg_hba.conf'),
                                      os.path.join(self.data_dir, 'postgresql.conf'))
        self.postmaster_pid = os.path.join(self.data_dir, 'postmaster.pid')
-        self.trigger_file = config.get('recovery_conf', {}).get('trigger_file') or 'promote'
+        self.trigger_file = config.get('recovery_conf', {}).get('trigger_file', None) or 'promote'
        self.trigger_file = os.path.abspath(os.path.join(self.data_dir, self.trigger_file))

        self._pg_ctl = ['pg_ctl', '-w', '-D', self.data_dir]

        self.local_address = self.get_local_address()
-        connect_address = config.get('connect_address') or self.local_address
+        connect_address = config.get('connect_address', None) or self.local_address
        self.connection_string = 'postgres://{username}:{password}@{connect_address}/postgres'.format(
            connect_address=connect_address, **self.replication)

        self._connection = None
+        self._connection_db = None
        self._cursor_holder = None
+        self._need_rewind = False
        self._sysid = None
        self.replication_slots = []  # list of already existing replication slots
        self.retry = Retry(max_tries=-1, deadline=5, max_delay=1, retry_exceptions=PostgresConnectionException)

        self._state = 'stopped'
        self._state_lock = Lock()
-        self._role = self.get_postgres_role_from_data_directory()
+        self._role = 'replica'
        self._role_lock = Lock()

        if self.is_running():
@@ -115,6 +118,9 @@ class Postgresql(object):
            self._sysid = data.get('Database system identifier', "")
        return self._sysid

+    def require_rewind(self):
+        self._need_rewind = True
+
    def get_local_address(self):
        listen_addresses = self.listen_addresses.split(',')
        local_address = listen_addresses[0].strip()  # take first address from listen_addresses
@@ -125,40 +131,32 @@ class Postgresql(object):
                break
        return local_address + ':' + self.port

-    def get_postgres_role_from_data_directory(self):
-        return 'replica' if os.path.exists(self.recovery_conf) else 'master'
-
-    @property
-    def _connect_kwargs(self):
-        r = parseurl('postgres://{0}/postgres'.format(self.local_address))
-        if 'username' in self.superuser:
-            r['user'] = self.superuser['username']
-        if 'password' in self.superuser:
-            r['password'] = self.superuser['password']
-        return r
-
-    def connection(self):
+    def connection(self, dbname):
        if not self._connection or self._connection.closed != 0:
-            self._connection = psycopg2.connect(**self._connect_kwargs)
+            r = parseurl('postgres://{0}/{1}'.format(self.local_address, dbname))
+            self._connection = psycopg2.connect(**r)
+            self._connection_db = dbname
            self._connection.autocommit = True
-            self.server_version = self._connection.server_version
        return self._connection

-    def _cursor(self):
+    def _cursor(self, dbname='postgres'):
+        if self._connection_db != dbname:
+            self.close_connection()
        if not self._cursor_holder or self._cursor_holder.closed or self._cursor_holder.connection.closed != 0:
-            logger.info("establishing a new patroni connection to the postgres cluster")
-            self._cursor_holder = self.connection().cursor()
+            logger.info("established a new patroni connection to the {0} cluster".format(dbname))
+            self._cursor_holder = self.connection(dbname).cursor()
        return self._cursor_holder

    def close_connection(self):
        if self._cursor_holder and self._cursor_holder.connection and self._cursor_holder.connection.closed == 0:
            self._cursor_holder.connection.close()
-            logger.info("closed patroni connection to the postgresql cluster")
+            logger.info("closed patroni connection to the {0} cluster".format(self._connection_db))
+            self._connection_db = None

-    def _query(self, sql, *params):
+    def _query(self, sql, dbname, *params):
        cursor = None
        try:
-            cursor = self._cursor()
+            cursor = self._cursor(dbname)
            cursor.execute(sql, params)
            return cursor
        except psycopg2.Error as e:
@@ -168,9 +166,10 @@ class Postgresql(object):
                raise RetryFailedError('cluster is being restarted')
            raise PostgresConnectionException('connection problems')

-    def query(self, sql, *params):
+    def query(self, sql, *params, **kwargs):
+        dbname = kwargs.get('dbname', 'postgres')
        try:
-            return self.retry(self._query, sql, *params)
+            return self.retry(self._query, sql, dbname, *params)
        except RetryFailedError as e:
            raise PostgresConnectionException(str(e))

@@ -180,36 +179,32 @@ class Postgresql(object):
    @staticmethod
    def initdb_allowed_option(name):
        if name in ['pgdata', 'nosync', 'pwfile', 'sync-only']:
-            raise Exception('{0} option for initdb is not allowed'.format(name))
+            raise Exception('{} option for initdb is not allowed'.format(name))
        return True

    def get_initdb_options(self):
        options = []
        for o in self.initdb_options:
            if isinstance(o, string_types) and self.initdb_allowed_option(o):
-                options.append('--{0}'.format(o))
+                options.append('--{}'.format(o))
            elif isinstance(o, dict):
                keys = list(o.keys())
                if len(keys) != 1 or not isinstance(keys[0], string_types) or not self.initdb_allowed_option(keys[0]):
-                    raise Exception('Invalid option: {0}'.format(o))
-                options.append('--{0}={1}'.format(keys[0], o[keys[0]]))
+                    raise Exception('Invalid option: {}'.format(o))
+                options.append('--{}={}'.format(keys[0], o[keys[0]]))
            else:
-                raise Exception('Unknown type of initdb option: {0}'.format(o))
+                raise Exception('Unknown type of initdb option: {}'.format(o))
        return options

    def initialize(self):
        self.set_state('initalizing new cluster')
        options = self.get_initdb_options()
        pwfile = None
-
-        if self.superuser:
-            if 'username' in self.superuser:
-                options.append('--username={0}'.format(self.superuser['username']))
-            if 'password' in self.superuser:
-                (fd, pwfile) = tempfile.mkstemp()
-                os.write(fd, self.superuser['password'].encode('utf-8'))
-                os.close(fd)
-                options.append('--pwfile={0}'.format(pwfile))
+        if self.superuser and 'username' not in self.superuser and 'password' in self.superuser:
+            (fd, pwfile) = tempfile.mkstemp()
+            os.write(fd, self.superuser['password'].encode())
+            os.close(fd)
+            options.append('--pwfile={}'.format(pwfile))

        ret = subprocess.call(self._pg_ctl + ['initdb'] + (['-o', ' '.join(options)] if options else [])) == 0
        if pwfile:
@@ -220,9 +215,83 @@ class Postgresql(object):
            self.set_state('initdb failed')
        return ret

+    def initialize_bdr_database(self, leader=None, empty=True):
+        """ initialize BDR database. If empty = False, reinitialize an existing one.
+            If leader is None, create a new group, otherwise, join an exising one.
+            non-empty database with an empty leader is not supported
+        """
+        if not empty and not leader:
+            ret = False
+        else:
+            try:
+                ret = (self.initialize() and self.start_bdr()) if empty else\
+                    (self.drop_bdr_database() and self.start_bdr())
+            except:
+                logger.exception("exception")
+                ret = False
+        if ret:
+            # convert all connection URLs to name=value strings
+            leader_dsn = self.primary_conninfo(leader.conn_url, include_db=self.bdr['database']) if leader else None
+            self_dsn = self.primary_conninfo(self.connection_string, include_db=self.bdr['database'])
+            try:
+                self.create_replication_user(superuser=True)
+                ret = self.create_bdr_database() and (self.join_bdr_group(self_dsn, leader_dsn) if leader
+                                                      else self.create_bdr_group(self_dsn))
+            except:
+                logger.exception("exception")
+                ret = False
+        if not ret and not leader:
+            raise Exception("Could not bootstrap initial PostgreSQL BDR node")
+        return ret
+
+    def start_bdr(self):
+        logger.info("starting new BDR instance {0}".format(self.name))
+        return self.start(bdr=True)
+
+    def drop_bdr_database(self):
+        logger.info("Dropping BDR database for instance {0}".format(self.name))
+        # stop the database if it's running in order to turn off bdr
+        if self.is_running():
+            self.stop()
+        ret = self.start(bdr=False)
+        if ret:
+            self.query("DROP DATABASE IF EXISTS {0}".format(self.bdr['database']))
+        return ret and self.restart(bdr=True)
+
+    def create_bdr_database(self):
+        logger.info("Creating BDR database and extensions for instance {0}".format(self.name))
+        self.query("CREATE DATABASE {0}".format(self.bdr['database']))
+        self.query("CREATE EXTENSION IF NOT EXISTS btree_gist", dbname=self.bdr['database'])
+        self.query("CREATE EXTENSION IF NOT EXISTS bdr", dbname=self.bdr['database'])
+        return True
+
+    def join_bdr_group(self, self_dsn, join_dsn):
+        logger.info("Joining BDR group {0}, host {1}, dsn {2}".format(join_dsn, self.name, self_dsn))
+        self.query("SELECT bdr.bdr_group_join(local_node_name:=%s, join_using_dsn:=%s, node_external_dsn:=%s)",
+                   self.name, join_dsn, self_dsn, dbname=self.bdr['database'])
+        return True
+
+    def create_bdr_group(self, self_dsn):
+        logger.info("Creating new BDR group {0}, host {1}".format(self_dsn, self.name))
+        self.query("SELECT bdr.bdr_group_create(local_node_name:=%s, node_external_dsn:=%s)",
+                   self.name, self_dsn, dbname=self.bdr['database'])
+        return True
+
+    def remove_bdr_nodes(self, nodes):
+        logger.info("BDR node {0}, removing nodes {1}".format(self.name, nodes))
+        self.query("SELECT bdr.part_by_node_names(%s)", nodes, dbname=self.bdr['database'])
+
+    def remove_bdr_disconnected_members(self, cluster):
+        # get all members
+        result = self.query("SELECT node_name FROM bdr.bdr_nodes WHERE node_status = 'r'")
+        if result:
+            nodes_db = set([node[0] for node in result.fetchall()])
+            nodes_etcd = set([member.name for member in self.members])
+            nodes_remove = nodes_db.difference(nodes_etcd)
+            self.remove_bdr_nodes(list(nodes_remove))
+
    def delete_trigger_file(self):
-        if os.path.exists(self.trigger_file):
-            os.unlink(self.trigger_file)
+        os.path.exists(self.trigger_file) and os.unlink(self.trigger_file)

    def write_pgpass(self, record):
        with open(self.pgpass, 'w') as f:
@@ -233,13 +302,13 @@ class Postgresql(object):
        env['PGPASSFILE'] = self.pgpass
        return env

-    def sync_replica(self, clone_member):
-        # add the credentials to connect to the replica origin to pgpass.
-        env = self.write_pgpass(parseurl(clone_member.conn_url)) if clone_member else os.environ.copy()
-        if self.create_replica(clone_member, env) == 0:
-            self.delete_trigger_file()
-            return True
-        return False
+    def sync_from_leader(self, leader):
+        r = parseurl(leader.conn_url)
+
+        env = self.write_pgpass(r)
+        ret = self.create_replica(leader, env) == 0
+        ret and self.delete_trigger_file()
+        return ret

    @staticmethod
    def build_connstring(conn):
@@ -247,36 +316,22 @@ class Postgresql(object):
        >>> Postgresql.build_connstring({'host': '127.0.0.1', 'port': '5432'}) == 'host=127.0.0.1 port=5432'
        True
        """
-        return ' '.join('{0}={1}'.format(param, val) for param, val in sorted(conn.items()))
+        return ' '.join('{}={}'.format(param, val) for param, val in sorted(conn.items()))

-    def replica_method_can_work_without_replication_connection(self, method):
-        return method != 'basebackup' and self.config and self.config.get(method, {}).get('no_master')
-
-    def can_create_replica_without_replication_connection(self):
-        """ go through the replication methods to see if there are ones
-            that does not require a working replication connection.
-        """
-        replica_methods = self.config.get('create_replica_method', [])
-        return any(self.replica_method_can_work_without_replication_connection(replica_method)
-                   for replica_method in replica_methods)
-
-    def create_replica(self, clone_member, env):
+    def create_replica(self, leader, env):
        # create the replica according to the replica_method
        # defined by the user.  this is a list, so we need to
        # loop through all methods the user supplies
-        connstring = clone_member.conn_url if clone_member else ""
+        connstring = leader.conn_url
        # get list of replica methods from config.
        # If there is no configuration key, or no value is specified, use basebackup
        replica_methods = self.config.get('create_replica_method') or ['basebackup']
-        # if we don't have any source, leave only replica methods that work without it
-        replica_methods = replica_methods if clone_member else \
-            [r for r in replica_methods if self.replica_method_can_work_without_replication_connection(r)]
        # go through them in priority order
        ret = 1
        for replica_method in replica_methods:
            # if the method is basebackup, then use the built-in
            if replica_method == "basebackup":
-                ret = self.basebackup(clone_member, env)
+                ret = self.basebackup(leader, env)
                if ret == 0:
                    logger.info("replica has been created using basebackup")
                    # if basebackup succeeds, exit with success
@@ -323,7 +378,7 @@ class Postgresql(object):
        cmd = self.callback[cb_name]
        try:
            subprocess.Popen(shlex.split(cmd) + [cb_name, self.role, self.scope])
-        except OSError:
+        except:
            logger.exception('callback %s %s %s %s failed', cmd, cb_name, self.role, self.scope)
            return False
        return True
@@ -346,12 +401,15 @@ class Postgresql(object):
        with self._state_lock:
            self._state = value

-    def start(self, block_callbacks=False):
+    def start(self, block_callbacks=False, bdr=False):
        if self.is_running():
            logger.error('Cannot start PostgreSQL because one is already running.')
            return True

-        self.set_role(self.get_postgres_role_from_data_directory())
+        if not bdr:
+            self.set_role('replica' if os.path.exists(self.recovery_conf) else 'master')
+        else:
+            self.set_role('master')
        if os.path.exists(self.postmaster_pid):
            os.remove(self.postmaster_pid)
            logger.info('Removed %s', self.postmaster_pid)
@@ -359,36 +417,30 @@ class Postgresql(object):
        if not block_callbacks:
            self.set_state('starting')

-        env = {'PATH': os.environ.get('PATH')}
-        # pg_ctl will write a FATAL if the username is incorrect. exporting PGUSER if necessary
-        if 'username' in self.superuser and self.superuser['username'] != os.environ.get('USER'):
-            env['PGUSER'] = self.superuser['username']
-        ret = subprocess.call(self._pg_ctl + ['start', '-o', self.server_options()], env=env, preexec_fn=os.setsid) == 0
+        ret = subprocess.call(self._pg_ctl + ['start', '-o', self.server_options(bdr)]) == 0

        self.set_state('running' if ret else 'start failed')

-        self.schedule_load_slots = ret and self.use_slots
-        self.save_configuration_files()
+        if not bdr:
+            self.schedule_load_slots = ret and self.use_slots
+            self.save_configuration_files()
        # block_callbacks is used during restart to avoid
        # running start/stop callbacks in addition to restart ones
-        if ret and not block_callbacks:
-            self.call_nowait(ACTION_ON_START)
+        ret and not block_callbacks and self.call_nowait(ACTION_ON_START)
        return ret

-    def checkpoint(self, connect_kwargs=None):
-        connect_kwargs = connect_kwargs or self._connect_kwargs
-        for p in ['connect_timeout', 'options']:
-            connect_kwargs.pop(p, None)
+    def checkpoint(self, connstring=None):
        try:
-            with psycopg2.connect(**connect_kwargs) as conn:
+            connstring = connstring or 'postgres://{}/postgres'.format(self.local_address)
+            with psycopg2.connect(connstring) as conn:
                conn.autocommit = True
                with conn.cursor() as cur:
                    cur.execute("SET statement_timeout = 0")
                    cur.execute('CHECKPOINT')
-        except psycopg2.Error:
+        except:
            logging.exception('Exception during CHECKPOINT')

-    def stop(self, mode='fast', block_callbacks=False, checkpoint=True):
+    def stop(self, mode='fast', block_callbacks=False):
        # make sure we close all connections established against
        # the former node, otherwise, we might get a stalled one
        # after kill -9, which would report incorrect data to
@@ -400,10 +452,9 @@ class Postgresql(object):
                self.set_state('stopped')
            return True

-        if checkpoint:
+        if block_callbacks:
            self.checkpoint()
-
-        if not block_callbacks:
+        else:
            self.set_state('stopping')

        ret = subprocess.call(self._pg_ctl + ['stop', '-m', mode]) == 0
@@ -418,23 +469,31 @@ class Postgresql(object):

    def reload(self):
        ret = subprocess.call(self._pg_ctl + ['reload']) == 0
-        if ret:
-            self.call_nowait(ACTION_ON_RELOAD)
+        ret and self.call_nowait(ACTION_ON_RELOAD)
        return ret

-    def restart(self):
+    def restart(self, bdr=False):
        self.set_state('restarting')
-        ret = self.stop(block_callbacks=True) and self.start(block_callbacks=True)
+        ret = self.stop(block_callbacks=True) and self.start(block_callbacks=True, bdr=bdr)
        if ret:
            self.call_nowait(ACTION_ON_RESTART)
        else:
-            self.set_state('restart failed ({0})'.format(self.state))
+            self.set_state('restart failed ({})'.format(self.state))
        return ret

-    def server_options(self):
-        options = "--listen_addresses='{0}' --port={1}".format(self.listen_addresses, self.port)
+    def server_options(self, bdr=False):
+        bdr_set = False
+        options = "--listen_addresses='{}' --port={}".format(self.listen_addresses, self.port)
        for setting, value in self.server_parameters.items():
-            options += " --{0}='{1}'".format(setting, value)
+            if setting == 'shared_preload_libraries':
+                if not bdr and 'bdr' in value:
+                    value = ','.join([x for x in value.split(',') if x != 'bdr'])
+                elif bdr and 'bdr' not in value:
+                    bdr_set = True
+                    value = ','.join([x for x in value.split(',')]+['bdr'])
+            options += " --{}='{}'".format(setting, value)
+        if bdr and not bdr_set:
+            options += " --shared_preload_libraries='bdr'"
        return options

    def is_healthy(self):
@@ -444,16 +503,22 @@ class Postgresql(object):
        return True

    def check_replication_lag(self, last_leader_operation):
-        return (last_leader_operation or 0) - self.xlog_position() <= self.config.get('maximum_lag_on_failover', 0)
+        return (last_leader_operation if last_leader_operation else 0) - self.xlog_position() <=\
+            self.config.get('maximum_lag_on_failover', 0)

    def write_pg_hba(self):
        with open(os.path.join(self.data_dir, 'pg_hba.conf'), 'a') as f:
-            f.write('\n{}\n'.format('\n'.join(self.config.get('pg_hba', []))))
+            f.write('\nhost replication {username} {network} md5\n'.format(**self.replication))
+            for line in self.config.get('pg_hba', []):
+                if line.split()[0].strip() == 'hostssl' and self.server_parameters.get('ssl', 'off').lower() != 'on':
+                    continue
+                f.write(line + '\n')

    @staticmethod
-    def primary_conninfo(leader_url):
+    def primary_conninfo(leader_url, include_db=None):
        r = parseurl(leader_url)
-        return 'user={user} password={password} host={host} port={port} sslmode=prefer sslcompression=1'.format(**r)
+        ret = 'user={user} password={password} host={host} port={port} sslmode=prefer sslcompression=1'.format(**r)
+        return ret if not include_db else ret + ' dbname={0}'.format(include_db)

    def check_recovery_conf(self, leader):
        if not os.path.isfile(self.recovery_conf):
@@ -469,30 +534,31 @@ class Postgresql(object):

    def write_recovery_conf(self, leader):
        with open(self.recovery_conf, 'w') as f:
-            f.write("standby_mode = 'on'\nrecovery_target_timeline = 'latest'\n")
+            f.write("""standby_mode = 'on'
+recovery_target_timeline = 'latest'
+""")
            if leader and leader.conn_url:
-                f.write("primary_conninfo = '{0}'\n".format(self.primary_conninfo(leader.conn_url)))
+                f.write("""primary_conninfo = '{}'\n""".format(self.primary_conninfo(leader.conn_url)))
                if self.use_slots:
-                    f.write("primary_slot_name = '{0}'\n".format(self.name))
-            for name, value in self.config.get('recovery_conf', {}).items():
-                if name not in ('standby_mode', 'recovery_target_timeline', 'primary_conninfo', 'primary_slot_name'):
-                    f.write("{0} = '{1}'\n".format(name, value))
+                    f.write("""primary_slot_name = '{}'\n""".format(self.name))
+                for name, value in self.config.get('recovery_conf', {}).items():
+                    f.write("{} = '{}'\n".format(name, value))

    def rewind(self, leader):
        # prepare pg_rewind connection
        r = parseurl(leader.conn_url)
        r.update(self.pg_rewind)
-        r['user'] = r.pop('username')
+        r['user'] = r['username']
        env = self.write_pgpass(r)
        pc = "user={user} host={host} port={port} dbname=postgres sslmode=prefer sslcompression=1".format(**r)
        # first run a checkpoint on a promoted master in order
        # to make it store the new timeline ([email protected])
-        self.checkpoint(r)
-        logger.info("running pg_rewind from %s", pc)
+        self.checkpoint(pc)
+        logger.info("running pg_rewind from {}".format(pc))
        pg_rewind = ['pg_rewind', '-D', self.data_dir, '--source-server', pc]
        try:
-            ret = subprocess.call(pg_rewind, env=env) == 0
-        except OSError:
+            ret = (subprocess.call(pg_rewind, env=env) == 0)
+        except:
            ret = False
        if ret:
            self.write_recovery_conf(leader)
@@ -504,11 +570,12 @@ class Postgresql(object):
        try:
            data = subprocess.check_output(['pg_controldata', self.data_dir])
            if data:
-                data = data.decode('utf-8').splitlines()
+                data = data.decode().splitlines()
                result = {l.split(':')[0].replace('Current ', '', 1): l.split(':')[1].strip() for l in data if l}
        except subprocess.CalledProcessError:
            logger.exception("Error when calling pg_controldata")
-        return result
+        finally:
+            return result

    def read_postmaster_opts(self):
        """ returns the list of option names/values from postgres.opts, Empty dict if read failed or no file """
@@ -524,26 +591,26 @@ class Postgresql(object):
                        result[name] = val
        except IOError:
            logger.exception('Error when reading postmaster.opts')
-        return result
+        finally:
+            return result

-    def single_user_mode(self, command=None, options=None):
+    def single_user_mode(self, command=None, options={}):
        """ run a given command in a single-user mode. If the command is empty - then just start and stop """
        cmd = ['postgres', '--single', '-D', self.data_dir]
-        for opt, val in sorted((options or {}).items()):
-            cmd.extend(['-c', '{0}={1}'.format(opt, val)])
+        for opt in sorted(options):
+            cmd.extend(['-c', '{0}={1}'.format(opt, options[opt])])
        # need a database name to connect
        cmd.append('postgres')
        p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=open(os.devnull, 'w'), stderr=subprocess.STDOUT)
        if p:
-            if command:
-                p.communicate('{0}\n'.format(command))
+            command and p.communicate('{}\n'.format(command))
            p.stdin.close()
            return p.wait()
        return 1

    def cleanup_archive_status(self):
        status_dir = os.path.join(self.data_dir, 'pg_xlog', 'archive_status')
-        try:
+        if os.path.isdir(status_dir):
            for f in os.listdir(status_dir):
                path = os.path.join(status_dir, f)
                try:
@@ -551,48 +618,49 @@ class Postgresql(object):
                        os.unlink(path)
                    elif os.path.isfile(path):
                        os.remove(path)
-                except OSError:
-                    logger.exception("Unable to remove %s", path)
-        except OSError:
-            logger.exception("Unable to list %s", status_dir)
+                except:
+                    logger.exception("Unable to remove {}".format(path))

-    def follow(self, leader, recovery=False):
-        if self.check_recovery_conf(leader) and not recovery:
+    def follow_the_leader(self, leader, recovery=False):
+        if not self.check_recovery_conf(leader) or recovery:
+            change_role = (self.role == 'master')
+
+            self._need_rewind = (self._need_rewind or change_role) and self.can_rewind
+            if self._need_rewind:
+                logger.info("set the rewind flag after demote")
+            self.write_recovery_conf(leader)
+            if not leader or not self._need_rewind:  # do not rewind until the leader becomes available
+                ret = self.restart()
+            else:  # we have a leader and need to rewind
+                if self.is_running():
+                    self.stop()
+                # at present, pg_rewind only runs when the cluster is shut down cleanly
+                # and not shutdown in recovery. We have to remove the recovery.conf if present
+                # and start/shutdown in a single user mode to emulate this.
+                # XXX: if recovery.conf is linked, it will be written anew as a normal file.
+                if os.path.islink(self.recovery_conf):
+                    os.unlink(self.recovery_conf)
+                else:
+                    os.remove(self.recovery_conf)
+                # Archived segments might be useful to pg_rewind,
+                # clean the flags that tell we should remove them.
+                self.cleanup_archive_status()
+                # Start in a single user mode and stop to produce a clean shutdown
+                opts = self.read_postmaster_opts()
+                opts['archive_mode'] = 'on'
+                opts['archive_command'] = 'false'
+                self.single_user_mode(options=opts)
+                if self.rewind(leader):
+                    ret = self.start()
+                else:
+                    logger.error("unable to rewind the former master")
+                    self.remove_data_directory()
+                    ret = True
+                self._need_rewind = False
+            change_role and ret and self.call_nowait(ACTION_ON_ROLE_CHANGE)
+            return ret
+        else:
            return True
-        change_role = self.role == 'master'
-        need_rewind = change_role and self.can_rewind
-        if need_rewind:
-            logger.info("set the rewind flag after demote")
-        self.write_recovery_conf(leader)
-        if leader and need_rewind:  # we have a leader and need to rewind
-            if self.is_running():
-                self.stop()
-            # at present, pg_rewind only runs when the cluster is shut down cleanly
-            # and not shutdown in recovery. We have to remove the recovery.conf if present
-            # and start/shutdown in a single user mode to emulate this.
-            # XXX: if recovery.conf is linked, it will be written anew as a normal file.
-            if os.path.islink(self.recovery_conf):
-                os.unlink(self.recovery_conf)
-            else:
-                os.remove(self.recovery_conf)
-            # Archived segments might be useful to pg_rewind,
-            # clean the flags that tell we should remove them.
-            self.cleanup_archive_status()
-            # Start in a single user mode and stop to produce a clean shutdown
-            opts = self.read_postmaster_opts()
-            opts.update({'archive_mode': 'on', 'archive_command': 'false'})
-            self.single_user_mode(options=opts)
-            if self.rewind(leader):
-                ret = self.start()
-            else:
-                logger.error("unable to rewind the former master")
-                self.remove_data_directory()
-                ret = True
-        else:  # do not rewind until the leader becomes available
-            ret = self.restart()
-        if change_role and ret:
-            self.call_nowait(ACTION_ON_ROLE_CHANGE)
-        return ret

    def save_configuration_files(self):
        """
@@ -602,18 +670,16 @@ class Postgresql(object):
        """
        try:
            for f in self.configuration_to_save:
-                if os.path.isfile(f):
-                    shutil.copy(f, f + '.backup')
-        except IOError:
+                os.path.isfile(f) and shutil.copy(f, f + '.backup')
+        except:
            logger.exception('unable to create backup copies of configuration files')

    def restore_configuration_files(self):
        """ restore a previously saved postgresql.conf """
        try:
            for f in self.configuration_to_save:
-                if not os.path.isfile(f) and os.path.isfile(f + '.backup'):
-                    shutil.copy(f + '.backup', f)
-        except IOError:
+                not os.path.isfile(f) and os.path.isfile(f + '.backup') and shutil.copy(f + '.backup', f)
+        except:
            logger.exception('unable to restore configuration files from backup')

    def promote(self):
@@ -623,9 +689,13 @@ class Postgresql(object):
        if ret:
            self.set_role('master')
            logger.info("cleared rewind flag after becoming the leader")
+            self._need_rewind = False
            self.call_nowait(ACTION_ON_ROLE_CHANGE)
        return ret

+    def demote(self):
+        self.follow_the_leader(None)
+
    def create_or_update_role(self, name, password, options):
        self.query("""DO $$
 BEGIN
@@ -639,10 +709,13 @@ BEGIN
 END;
 $$""".format(name, options), name, password, password)

-    def create_replication_user(self):
-        self.create_or_update_role(self.replication['username'], self.replication['password'], 'REPLICATION')
+    def create_replication_user(self, superuser=False):
+        options = 'REPLICATION SUPERUSER' if superuser else 'REPLICATION'
+        self.create_or_update_role(self.replication['username'], self.replication['password'], '{0}'.format(options))

-    def create_connection_user(self):
+    def create_connection_users(self):
+        if 'username' in self.superuser:
+            self.create_or_update_role(self.superuser['username'], self.superuser['password'], 'SUPERUSER')
        if self.admin:
            self.create_or_update_role(self.admin['username'], self.admin['password'], 'CREATEDB CREATEROLE')

@@ -662,18 +735,7 @@ $$""".format(name, options), name, password, password)
        if self.use_slots:
            try:
                self.load_replication_slots()
-                # if the replicatefrom tag is set on the member - we should not create the replication slot for it on
-                # the current master, because that member would replicate from elsewhere. We still create the slot if
-                # the replicatefrom destination member is currently not a member of the cluster (fallback to the
-                # master), or if replicatefrom destination member happens to be the current master
-                if self.role == 'master':
-                    slots = [m.name for m in cluster.members if m.name != self.name and
-                             (m.replicatefrom is None or m.replicatefrom == self.name or
-                              not cluster.has_member(m.replicatefrom))]
-                else:
-                    # only manage slots for replicas that replicate from this one, except for the leader among them
-                    slots = [m.name for m in cluster.members if m.replicatefrom == self.name and
-                             m.name != cluster.leader.name]
+                slots = [m.name for m in cluster.members if m.name != self.name] if self.role == 'master' else []
                # drop unused slots
                for slot in set(self.replication_slots) - set(slots):
                    self.query("""SELECT pg_drop_replication_slot(%s)
@@ -687,45 +749,34 @@ $$""".format(name, options), name, password, password)
                                   WHERE slot_name = %s)""", slot, slot)

                self.replication_slots = slots
-            except psycopg2.Error:
+            except:
                logger.exception('Exception when changing replication slots')

    def last_operation(self):
        return str(self.xlog_position())

-    def bootstrap(self, cluster_initialized=False, clone_member=None):
+    def bootstrap(self, current_leader=None):
        """
-            Populate PostgreSQL data directory by doing one of the following:
-             - create with initdb if there is no master.
-             - initialize the replica from an existing member (master or replica)
-             - initialize the replica using the replica creation method that
-               works without the replication connection (i.e. restore from on-disk
-               base backup)
-
-            The choice between the last 2 is triggered by the initialize flag.
-            We should never try to initdb an already initialized cluster, nor
-            try to bootstrap the cluster that lacks the initialize key using the
-            master-less replica creation method (in the latter case, there is
-            no clear inidicator of the moment we should abandon our attempts and
-            swich to initdb).
-
-            Failure during initdb always leads to an exception, since there is
-            no point in continuing if initdb fails. For the rest of the cases,
-            the function returns False in order to inidicate a failed attempt
-            that should be retried in the future.
+            Initially bootstrap PostgreSQL, either by creating a data
+            directory with initdb, or by initalizing a replica from an
+            exiting leader. Failure in the first case always leads to
+            exception, since there is no point in continuing if initdb failed.
+            In the second case, however, a False is returned on failure, since
+            it is normal for the replica to retry a failed attempt to initialize
+            from the master.
        """
        ret = False
-        if not (cluster_initialized or clone_member):
+        if not current_leader:
            ret = self.initialize() and self.start()
            if ret:
                self.create_replication_user()
-                self.create_connection_user()
+                self.create_connection_users()
            else:
                raise PostgresException("Could not bootstrap master PostgreSQL")
        else:
-            if self.sync_replica(clone_member):
+            if self.sync_from_leader(current_leader):
                self.restore_configuration_files()
-                self.write_recovery_conf(clone_member)
+                self.write_recovery_conf(current_leader)
                ret = self.start()
        return ret

@@ -735,7 +786,7 @@ $$""".format(name, options), name, password, password)
                new_name = '{0}_{1}'.format(self.data_dir, time.strftime('%Y-%m-%d-%H-%M-%S'))
                logger.info('renaming data directory to %s', new_name)
                os.rename(self.data_dir, new_name)
-            except OSError:
+            except:
                logger.exception("Could not rename data directory %s", self.data_dir)

    def remove_data_directory(self):
@@ -749,16 +800,16 @@ $$""".format(name, options), name, password, password)
                os.remove(self.data_dir)
            elif os.path.isdir(self.data_dir):
                shutil.rmtree(self.data_dir)
-        except (IOError, OSError):
+        except:
            logger.exception('Could not remove data directory %s', self.data_dir)
            self.move_data_directory()

-    def basebackup(self, clone_member, env):
+    def basebackup(self, leader, env):
        # creates a replica data dir using pg_basebackup.
        # this is the default, built-in create_replica_method
        # tries twice, then returns failure (as 1)
        # uses "stream" as the xlog-method to avoid sync issues
-        master_connection = clone_member.conn_url
+        master_connection = leader.conn_url
        maxfailures = 2
        ret = 1
        for bbfailures in range(0, maxfailures):
@@ -9,7 +9,7 @@ import boto.ec2
 logger = logging.getLogger(__name__)


-class AWSConnection(object):
+class AWSConnection:
    def __init__(self, cluster_name):
        self.available = False
        self.cluster_name = cluster_name if cluster_name is not None else 'unknown'
@@ -56,7 +56,7 @@ class AWSConnection(object):
            conn = boto.ec2.connect_to_region(self.region)
            conn.create_tags([self.instance_id], tags)
        except Exception as e:
-            logger.info("could not set tags for EC2 instance %s: %s", self.instance_id, e)
+            logger.info("could not set tags for EC2 instance {}: {}".format(self.instance_id, e))
            return False
        return True

@@ -66,7 +66,6 @@ class AWSConnection(object):


 def main():
-    logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
    if len(sys.argv) == 4 and sys.argv[1] in ('on_start', 'on_stop', 'on_role_change'):
        AWSConnection(cluster_name=sys.argv[3]).on_role_change(sys.argv[2])
    else:
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python

 # sample script to clone new replicas using WAL-E restore
 # falls back to pg_basebackup if WAL-E restore fails, or if
@@ -36,12 +36,13 @@ import argparse
 if sys.hexversion >= 0x03000000:
    long = int

+logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
 logger = logging.getLogger(__name__)


 class WALERestore(object):

-    def __init__(self, scope, datadir, connstring, env_dir, threshold_mb, threshold_pct, use_iam, no_master):
+    def __init__(self, scope, datadir, connstring, env_dir, threshold_mb, threshold_pct, use_iam):
        self.scope = scope
        self.master_connection = connstring
        self.data_dir = datadir
@@ -50,7 +51,6 @@ class WALERestore(object):
        self.wal_e.threshold_mb = threshold_mb
        self.wal_e.threshold_pct = threshold_pct
        self.wal_e.iam_string = ' --aws-instance-profile ' if use_iam == 1 else ''
-        self.no_master = no_master
        self.wal_e.cmd = 'envdir {0} wal-e {1} '.format(self.wal_e.dir, self.wal_e.iam_string)
        self.init_error = (not os.path.exists(self.wal_e.dir))

@@ -104,23 +104,24 @@ class WALERestore(object):
        lsn_offset = hex((long(backup_start_segment[16:32], 16) << 24) + long(backup_start_offset))[2:-1]

        # construct the LSN from the segment and offset
-        backup_start_lsn = '{0}/{1}'.format(lsn_segment, lsn_offset)
+        backup_start_lsn = '{}/{}'.format(lsn_segment, lsn_offset)

+        conn = None
+        cursor = None
        diff_in_bytes = long(backup_size)
-        if not self.no_master:
-            try:
-                # get the difference in bytes between the current WAL location and the backup start offset
-                with psycopg2.connect(self.master_connection) as con:
-                    con.autocommit = True
-                    with con.cursor() as cur:
-                        cur.execute("SELECT pg_xlog_location_diff(pg_current_xlog_location(), %s)", (backup_start_lsn,))
-                        diff_in_bytes = long(cur.fetchone()[0])
-            except psycopg2.Error as e:
-                logger.error('could not determine difference with the master location: %s', e)
-                return False
-        else:
-            # always try to use WAL-E if base backup is available
-            diff_in_bytes = 0
+        try:
+            # get the difference in bytes between the current WAL location and the backup start offset
+            conn = psycopg2.connect(self.master_connection)
+            conn.autocommit = True
+            cursor = conn.cursor()
+            cursor.execute("SELECT pg_xlog_location_diff(pg_current_xlog_location(), %s)", (backup_start_lsn,))
+            diff_in_bytes = long(cursor.fetchone()[0])
+        except psycopg2.Error as e:
+            logger.error('could not determine difference with the master location: {}'.format(e))
+            return False
+        finally:
+            cursor and cursor.close()
+            conn and conn.close()

        # if the size of the accumulated WAL segments is more than a certan percentage of the backup size
        # or exceeds the pre-determined size - pg_basebackup is chosen instead.
@@ -139,7 +140,6 @@ class WALERestore(object):


 def main():
-    logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
    parser = argparse.ArgumentParser(description='Script to image replicas using WAL-E')
    parser.add_argument('--scope', required=True)
    parser.add_argument('--role', required=False)
@@ -150,15 +150,13 @@ def main():
    parser.add_argument('--threshold_megabytes', type=int, default=10240)
    parser.add_argument('--threshold_backup_size_percentage', type=int, default=30)
    parser.add_argument('--use_iam', type=int, default=0)
-    parser.add_argument('--no_master', type=int, default=0)
    args = parser.parse_args()

    # retry cloning in a loop
-    for _ in range(0, args.retries + 1):
+    for retry in range(0, args.retries + 1):
        restore = WALERestore(scope=args.scope, datadir=args.datadir, connstring=args.connstring,
                              env_dir=args.envdir, threshold_mb=args.threshold_megabytes,
-                              threshold_pct=args.threshold_backup_size_percentage, use_iam=args.use_iam,
-                              no_master=args.no_master)
+                              threshold_pct=args.threshold_backup_size_percentage, use_iam=args.use_iam)
        ret = restore.run()
        if ret == 0:
            break
@@ -1,65 +1,76 @@
 import datetime
 import os
 import random
+import re
 import signal
 import sys
 import time
-import pytz
-import dateutil.parser

 from patroni.exceptions import PatroniException

-__ignore_sigterm = False
-__interrupted_sleep = False
-__reap_children = False
+ignore_sigterm = False
+interrupted_sleep = False
+reap_children = False
+
+_DATE_TIME_RE = re.compile(r'''^
+(?P<year>\d{4})\-(?P<month>\d{2})\-(?P<day>\d{2})  # date
+T
+(?P<hour>\d{2}):(?P<minute>\d{2}):(?P<second>\d{2})\.(?P<microsecond>\d{6})  # time
+\d*Z$''', re.X)
+
+
+def parse_datetime(time_str):
+    """
+    >>> parse_datetime('2015-06-10T12:56:30.552539016Z')
+    datetime.datetime(2015, 6, 10, 12, 56, 30, 552539)
+    >>> parse_datetime('2015-06-10 12:56:30.552539016Z')
+    """
+    m = _DATE_TIME_RE.match(time_str)
+    if not m:
+        return None
+    p = dict((n, int(m.group(n))) for n in 'year month day hour minute second microsecond'.split(' '))
+    return datetime.datetime(**p)


 def calculate_ttl(expiration):
    """
    >>> calculate_ttl(None)
-    >>> calculate_ttl('2015-06-10 12:56:30.552539016Z') < 0
-    True
+    >>> calculate_ttl('2015-06-10 12:56:30.552539016Z')
    >>> calculate_ttl('2015-06-10T12:56:30.552539016Z') < 0
    True
-    >>> calculate_ttl('fail-06-10T12:56:30.552539016Z')
    """
    if not expiration:
        return None
-    try:
-        expiration = dateutil.parser.parse(expiration)
-    except (ValueError, TypeError):
+    expiration = parse_datetime(expiration)
+    if not expiration:
        return None
-    now = datetime.datetime.now(pytz.utc)
+    now = datetime.datetime.utcnow()
    return int((expiration - now).total_seconds())


-def set_ignore_sigterm(value=True):
-    global __ignore_sigterm
-    __ignore_sigterm = value
-
-
 def sigterm_handler(signo, stack_frame):
-    if not __ignore_sigterm:
-        set_ignore_sigterm()
+    global ignore_sigterm
+    if not ignore_sigterm:
+        ignore_sigterm = True
        sys.exit()


 def sigchld_handler(signo, stack_frame):
-    global __interrupted_sleep, __reap_children
-    __reap_children = __interrupted_sleep = True
+    global interrupted_sleep, reap_children
+    reap_children = interrupted_sleep = True


 def sleep(interval):
-    global __interrupted_sleep
+    global interrupted_sleep
    current_time = time.time()
    end_time = current_time + interval
    while current_time < end_time:
-        __interrupted_sleep = False
+        interrupted_sleep = False
        time.sleep(end_time - current_time)
-        if not __interrupted_sleep:  # we will ignore only sigchld
+        if not interrupted_sleep:  # we will ignore only sigchld
            break
        current_time = time.time()
-    __interrupted_sleep = False
+    interrupted_sleep = False


 def setup_signal_handlers():
@@ -68,8 +79,8 @@ def setup_signal_handlers():


 def reap_children():
-    global __reap_children
-    if __reap_children:
+    global reap_children
+    if reap_children:
        try:
            while True:
                ret = os.waitpid(-1, os.WNOHANG)
@@ -78,7 +89,7 @@ def reap_children():
        except OSError:
            pass
        finally:
-            __reap_children = False
+            reap_children = False


 class RetryFailedError(PatroniException):
@@ -86,7 +97,7 @@ class RetryFailedError(PatroniException):
    """Raised when retrying an operation ultimately failed, after retrying the maximum number of attempts."""


-class Retry(object):
+class Retry:

    """Helper for retrying a method in the face of retry-able exceptions"""

@@ -1 +1 @@
-__version__ = '0.90'
+__version__ = '0.76'
@@ -17,7 +17,7 @@ class ZooKeeperError(DCSError):
    pass


-class ExhibitorEnsembleProvider(object):
+class ExhibitorEnsembleProvider:

    TIMEOUT = 3.1

@@ -54,7 +54,7 @@ class ExhibitorEnsembleProvider(object):
    def _query_exhibitors(self, exhibitors):
        random.shuffle(exhibitors)
        for host in exhibitors:
-            uri = 'http://{0}:{1}{2}'.format(host, self._exhibitor_port, self._uri_path)
+            uri = 'http://{}:{}{}'.format(host, self._exhibitor_port, self._uri_path)
            try:
                response = requests.get(uri, timeout=self.TIMEOUT)
                return response.json()
@@ -83,29 +83,32 @@ class ZooKeeper(AbstractDCS):
            self.exhibitor = ExhibitorEnsembleProvider(exhibitor['hosts'], exhibitor['port'], poll_interval=interval)
            hosts = self.exhibitor.zookeeper_hosts

-        self._client = KazooClient(hosts=hosts, timeout=(config.get('session_timeout') or 30),
-                                   command_retry={'deadline': (config.get('reconnect_timeout') or 10),
-                                                  'max_delay': 1, 'max_tries': -1},
-                                   connection_retry={'max_delay': 1, 'max_tries': -1})
-        self._client.add_listener(self.session_listener)
+        self.client = KazooClient(hosts=hosts,
+                                  timeout=(config.get('session_timeout', None) or 30),
+                                  command_retry={
+                                      'deadline': (config.get('reconnect_timeout', None) or 10),
+                                      'max_delay': 1,
+                                      'max_tries': -1},
+                                  connection_retry={'max_delay': 1, 'max_tries': -1})
+        self.client.add_listener(self.session_listener)

        self._my_member_data = None
-        self._fetch_cluster = True
-        self._last_leader_operation = 0
+        self.fetch_cluster = True
+        self.last_leader_operation = 0

-        self._client.start()
+        self.client.start(None)

    def session_listener(self, state):
        if state in [KazooState.SUSPENDED, KazooState.LOST]:
            self.cluster_watcher(None)

    def cluster_watcher(self, event):
-        self._fetch_cluster = True
+        self.fetch_cluster = True
        self.event.set()

    def get_node(self, key, watch=None):
        try:
-            ret = self._client.get(key, watch)
+            ret = self.client.get(key, watch)
            return (ret[0].decode('utf-8'), ret[1])
        except NoNodeError:
            return None
@@ -116,7 +119,7 @@ class ZooKeeper(AbstractDCS):

    def get_children(self, key, watch=None):
        try:
-            return self._client.get_children(key, watch)
+            return self.client.get_children(key, watch)
        except NoNodeError:
            return []

@@ -129,11 +132,11 @@ class ZooKeeper(AbstractDCS):
        return members

    def _inner_load_cluster(self):
-        self._fetch_cluster = False
+        self.fetch_cluster = False
        self.event.clear()
        nodes = set(self.get_children(self.client_path(''), self.cluster_watcher))
        if not nodes:
-            self._fetch_cluster = True
+            self.fetch_cluster = True

        # get initialize flag
        initialize = (self.get_node(self.initialize_path) or [None])[0] if self._INITIALIZE in nodes else None
@@ -144,17 +147,17 @@ class ZooKeeper(AbstractDCS):
        # get leader
        leader = self.get_node(self.leader_path) if self._LEADER in nodes else None
        if leader:
-            client_id = self._client.client_id
+            client_id = self.client.client_id
            if leader[0] == self._name and client_id is not None and client_id[0] != leader[1].ephemeralOwner:
                logger.info('I am leader but not owner of the session. Removing leader node')
-                self._client.delete(self.leader_path)
+                self.client.delete(self.leader_path)
                leader = None

            if leader:
                member = Member(-1, leader[0], None, {})
                member = ([m for m in members if m.name == leader[0]] or [member])[0]
                leader = Leader(leader[1].version, leader[1].ephemeralOwner, member)
-                self._fetch_cluster = member.index == -1
+                self.fetch_cluster = member.index == -1

        # failover key
        failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
@@ -162,17 +165,17 @@ class ZooKeeper(AbstractDCS):
            failover = Failover.from_node(failover[1].version, failover[0])

        # get last leader operation
-        optime = self.get_node(self.leader_optime_path) if self._OPTIME in nodes and self._fetch_cluster else None
-        self._last_leader_operation = 0 if optime is None else int(optime[0])
-        self._cluster = Cluster(initialize, leader, self._last_leader_operation, members, failover)
+        optime = self.get_node(self.leader_optime_path) if self._OPTIME in nodes and self.fetch_cluster else None
+        self.last_leader_operation = 0 if optime is None else int(optime[0])
+        self._cluster = Cluster(initialize, leader, self.last_leader_operation, members, failover)

    def _load_cluster(self):
        if self.exhibitor and self.exhibitor.poll():
-            self._client.set_hosts(self.exhibitor.zookeeper_hosts)
+            self.client.set_hosts(self.exhibitor.zookeeper_hosts)

-        if self._fetch_cluster or self._cluster is None:
+        if self.fetch_cluster:
            try:
-                self._client.retry(self._inner_load_cluster)
+                self.client.retry(self._inner_load_cluster)
            except:
                logger.exception('get_cluster')
                self.session_listener(KazooState.LOST)
@@ -180,20 +183,19 @@ class ZooKeeper(AbstractDCS):

    def _create(self, path, value, **kwargs):
        try:
-            self._client.retry(self._client.create, path, value.encode('utf-8'), **kwargs)
+            self.client.retry(self.client.create, path, value.encode('utf-8'), **kwargs)
            return True
        except:
            return False

    def attempt_to_acquire_leader(self):
        ret = self._create(self.leader_path, self._name, makepath=True, ephemeral=True)
-        if not ret:
-            logger.info('Could not take out TTL lock')
+        ret or logger.info('Could not take out TTL lock')
        return ret

    def set_failover_value(self, value, index=None):
        try:
-            self._client.retry(self._client.set, self.failover_path, value.encode('utf-8'), version=index or -1)
+            self.client.retry(self.client.set, self.failover_path, value.encode('utf-8'), version=index or -1)
            return True
        except NoNodeError:
            return value == '' or (not index and self._create(self.failover_path, value))
@@ -203,35 +205,36 @@ class ZooKeeper(AbstractDCS):

    def initialize(self, create_new=True, sysid=""):
        return self._create(self.initialize_path, sysid, makepath=True) if create_new \
-            else self._client.retry(self._client.set, self.initialize_path,  sysid.encode("utf-8"))
+            else self.client.retry(self.client.set, self.initialize_path,  sysid.encode("utf-8"))

    def touch_member(self, data, ttl=None):
        cluster = self.cluster
-        member = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
+        me = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
        path = self.member_path
        data = data.encode('utf-8')
-        if member and self._client.client_id is not None and member.session != self._client.client_id[0]:
+        create = not me
+        if me and self.client.client_id is not None and me.session != self.client.client_id[0]:
            try:
-                self._client.retry(self._client.delete, path)
+                self.client.retry(self.client.delete, path)
            except NoNodeError:
                pass
            except:
                return False
-            member = None
+            create = True

-        if member and data == self._my_member_data:
+        if not create and data == self._my_member_data:
            return True

        try:
-            if member:
-                self._client.retry(self._client.set, path, data)
+            if create:
+                self.client.retry(self.client.create, path, data, makepath=True, ephemeral=True)
            else:
-                self._client.retry(self._client.create, path, data, makepath=True, ephemeral=True)
+                self.client.retry(self.client.set, path, data)
            self._my_member_data = data
            return True
        except NodeExistsError:
            try:
-                self._client.retry(self._client.set, path, data)
+                self.client.retry(self.client.set, path, data)
                self._my_member_data = data
                return True
            except:
@@ -245,14 +248,14 @@ class ZooKeeper(AbstractDCS):

    def write_leader_optime(self, last_operation):
        last_operation = last_operation.encode('utf-8')
-        if last_operation != self._last_leader_operation:
-            self._last_leader_operation = last_operation
+        if last_operation != self.last_leader_operation:
+            self.last_leader_operation = last_operation
            path = self.leader_optime_path
            try:
-                self._client.retry(self._client.set, path, last_operation)
+                self.client.retry(self.client.set, path, last_operation)
            except NoNodeError:
                try:
-                    self._client.retry(self._client.create, path, last_operation, makepath=True)
+                    self.client.retry(self.client.create, path, last_operation, makepath=True)
                except:
                    logger.exception('Failed to create %s', path)
            except:
@@ -262,28 +265,22 @@ class ZooKeeper(AbstractDCS):
        return True

    def delete_leader(self):
-        self._client.restart()
+        self.client.restart()
        self._my_member_data = None
        return True

    def _cancel_initialization(self):
        node = self.get_node(self.initialize_path)
        if node:
-            self._client.delete(self.initialize_path, version=node[1].version)
+            self.client.delete(self.initialize_path, version=node[1].version)

    def cancel_initialization(self):
        try:
-            self._client.retry(self._cancel_initialization)
+            self.client.retry(self._cancel_initialization)
        except:
            logger.exception("Unable to delete initialize key")

-    def delete_cluster(self):
-        try:
-            return self._client.retry(self._client.delete, self.client_path(''), recursive=True)
-        except NoNodeError:
-            return True
-
    def watch(self, timeout):
        if super(ZooKeeper, self).watch(timeout):
-            self._fetch_cluster = True
-        return self._fetch_cluster
+            self.fetch_cluster = True
+        return self.fetch_cluster
@@ -2,4 +2,4 @@
 from patroni.ctl import ctl

 if __name__ == '__main__':
-    ctl(None)
+    ctl()
@@ -4,7 +4,7 @@ scope: &scope batman
 restapi:
  listen: 127.0.0.1:8008
  connect_address: 127.0.0.1:8008
-#  auth: 'username:password'
+  #auth: 'username:password'
 #  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
 #  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
 etcd:
@@ -12,10 +12,6 @@ etcd:
  ttl: *ttl
  host: 127.0.0.1:4001
  #discovery_srv: my-etcd.domain
-#consul:
-#  scope: *scope
-#  ttl: *ttl
-#  host: 127.0.0.1:8500
 #zookeeper:
 #  scope: *scope
 #  session_timeout: *ttl
@@ -30,6 +26,9 @@ etcd:
 #      - host1
 #      - host2
 #      - host3
+bdr:
+  enable: "on"
+  database: "bdrtest"
 postgresql:
  name: postgresql0
  scope: *scope
@@ -60,12 +59,12 @@ postgresql:
    username: postgres
    password: zalando
  pg_hba:
-  - host replication replicator 127.0.0.1/32 md5
  - host all all 0.0.0.0/0 md5
-  # - hostssl all all 0.0.0.0/0 md5
+  - hostssl all all 0.0.0.0/0 md5
  replication:
    username: replicator
    password: rep-pass
+    network:  127.0.0.1/32
  superuser:
    username: postgres
    password: zalando
@@ -76,6 +75,7 @@ postgresql:
    - basebackup
 #   - wal_e
 # commented-out example for wal-e provisioning
+  #create_replica_method: wal_e, basebackup
  #wal_e:
    #command: /patroni/scripts/wale_restore.py
    #env_dir: /etc/wal-e.d/env
@@ -89,16 +89,19 @@ postgresql:
    restore_command: cp ../wal_archive/%f %p
  parameters:
    archive_mode: "on"
-    wal_level: hot_standby
+    wal_level: logical
    archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
    max_wal_senders: 10
    wal_keep_segments: 8
    archive_timeout: 1800s
    max_replication_slots: 10
+    max_worker_processes: 10
+    track_commit_timestamp: 'on'
+    shared_preload_libraries: 'bdr'
    hot_standby: "on"
    wal_log_hints: "on"
-    unix_socket_directories: '.'
 tags:
    nofailover: False
    noloadbalance: False
    clonefrom: False
+    replicatefrom: 127.0.0.1
@@ -4,7 +4,7 @@ scope: &scope batman
 restapi:
  listen: 127.0.0.1:8009
  connect_address: 127.0.0.1:8009
-#  auth: 'username:password'
+  #auth: 'username:password'
 #  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
 #  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
 etcd:
@@ -12,10 +12,6 @@ etcd:
  ttl: *ttl
  host: 127.0.0.1:4001
  #discovery_srv: my-etcd.domain
-#consul:
-#  scope: *scope
-#  ttl: *ttl
-#  host: 127.0.0.1:8500
 #zookeeper:
 #  scope: *scope
 #  session_timeout: *ttl
@@ -30,6 +26,9 @@ etcd:
 #      - host1
 #      - host2
 #      - host3
+bdr:
+    enable: "on"
+    database: "bdrtest"
 postgresql:
  name: postgresql1
  scope: *scope
@@ -60,14 +59,14 @@ postgresql:
    username: postgres
    password: zalando
  pg_hba:
-  - host replication replicator 127.0.0.1/32 md5
  - host all all 0.0.0.0/0 md5
-  # - hostssl all all 0.0.0.0/0 md5
+  - hostssl all all 0.0.0.0/0 md5
  replication:
    username: replicator
    password: rep-pass
+    network: 127.0.0.1/32
  superuser:
-    username: postgres
+    user: postgres
    password: zalando
  admin:
    username: admin
@@ -90,16 +89,19 @@ postgresql:
    restore_command: cp ../wal_archive/%f %p
  parameters:
    archive_mode: "on"
-    wal_level: hot_standby
+    wal_level: logical
    archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
    max_wal_senders: 10
+    max_worker_processes: 10
+    track_commit_timestamp: 'on'
+    shared_preload_libraries: 'bdr'
    wal_keep_segments: 8
    archive_timeout: 1800s
    max_replication_slots: 10
    hot_standby: "on"
    wal_log_hints: "on"
-    unix_socket_directories: '.'
 tags:
    nofailover: False
    noloadbalance: False
    clonefrom: False
+    replicatefrom: 127.0.0.1
@@ -1,106 +0,0 @@
-ttl: &ttl 30
-loop_wait: &loop_wait 10
-scope: &scope batman
-restapi:
-  listen: 127.0.0.1:8010
-  connect_address: 127.0.0.1:8010
-  auth: 'username:password'
-#  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
-#  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
-etcd:
-  scope: *scope
-  ttl: *ttl
-  host: 127.0.0.1:4001
-  #discovery_srv: my-etcd.domain
-#consul:
-#  scope: *scope
-#  ttl: *ttl
-#  host: 127.0.0.1:8500
-#zookeeper:
-#  scope: *scope
-#  session_timeout: *ttl
-#  reconnect_timeout: *loop_wait
-#  hosts:
-#    - 127.0.0.1:2181
-#    - 127.0.0.2:2181
-#  exhibitor:
-#    poll_interval: 300
-#    port: 8181
-#    hosts:
-#      - host1
-#      - host2
-#      - host3
-postgresql:
-  name: postgresql2
-  scope: *scope
-  listen: 127.0.0.1:5434
-  connect_address: 127.0.0.1:5434
-  data_dir: data/postgresql2
-  maximum_lag_on_failover: 1048576 # 1 megabyte in bytes
-  use_slots: True
-  pgpass: /tmp/pgpass2
-  initdb:  ## We allow the following options to be passed on to initdb
-  # - auth: authmethod
-  # - auth-host: authmethod
-  # - auth-local: authmethod
-  - encoding: UTF8
-  # - data-checksums  # When pg_rewind is needed on 9.3, this needs to be enabled
-  # - locale: locale
-  # - lc-collate: locale
-  # - lc-ctype: locale
-  # - lc-messages: locale
-  # - lc-monetary: locale
-  # - lc-numeric: locale
-  # - lc-time: locale
-  # - text-search-config: CFG
-  # - xlogdir: directory
-  # - debug
-  # - noclean
-  pg_rewind:
-    username: postgres
-    password: zalando
-  pg_hba:
-  - host replication replicator 127.0.0.1/32 md5
-  - host all all 0.0.0.0/0 md5
-  # - hostssl all all 0.0.0.0/0 md5
-  replication:
-    username: replicator
-    password: rep-pass
-  superuser:
-    username: postgres
-    password: zalando
-  admin:
-    username: admin
-    password: admin
-# commented-out example for wal-e provisioning
-  create_replica_method:
-    - basebackup
-#   - wal_e
-# commented-out example for wal-e provisioning
-  #wal_e:
-    #command: /patroni/scripts/wale_restore.py
-    #env_dir: /home/postgres/etc/wal-e.d/env
-    #threshold_megabytes: 10240
-    #threshold_backup_size_percentage: 30
-    #retries: 2
-    #use_iam: 1
-  #recovery_conf:
-    #restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1
-  recovery_conf:
-    restore_command: cp ../wal_archive/%f %p
-  parameters:
-    archive_mode: "on"
-    wal_level: hot_standby
-    archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
-    max_wal_senders: 10
-    wal_keep_segments: 8
-    archive_timeout: 1800s
-    max_replication_slots: 10
-    hot_standby: "on"
-    wal_log_hints: "on"
-    unix_socket_directories: '.'
-tags:
-    nofailover: False
-    noloadbalance: False
-    clonefrom: False
-    replicatefrom: postgresql1
@@ -1,12 +1,11 @@
 boto
+dnspython
+mock
 psycopg2>=2.6.1
 PyYAML
 requests
 six >= 1.7
 kazoo>=2.2.1
-python-etcd==0.4.3
-python-consul==0.6.0
+python-etcd>=0.4.1
 click>=4.1
 prettytable>=0.7
-tzlocal
-python-dateutil
@@ -0,0 +1,11 @@
+boto
+mock
+dnspython3
+psycopg2>=2.6.1
+PyYAML
+requests
+six
+kazoo>=2.2.1
+python-etcd>=0.4.1
+click>=4.1
+prettytable>=0.7
@@ -4,12 +4,13 @@
    Setup file for patroni
 """

-import inspect
-import os
 import sys
+import os
+import inspect

+import setuptools
 from setuptools.command.test import test as TestCommand
-from setuptools import find_packages, setup
+from setuptools import setup

 if sys.version_info < (2, 7, 0):
    sys.stderr.write('FATAL: patroni needs to be run with Python 2.7+\n')
@@ -50,8 +51,8 @@ CLASSIFIERS = [
    'Operating System :: POSIX :: Linux',
    'Programming Language :: Python',
    'Programming Language :: Python :: 2.7',
+    'Programming Language :: Python :: 3.3',
    'Programming Language :: Python :: 3.4',
-    'Programming Language :: Python :: 3.5',
    'Programming Language :: Python :: Implementation :: CPython',
 ]

@@ -75,7 +76,7 @@ class PyTest(TestCommand):
    def finalize_options(self):
        TestCommand.finalize_options(self)
        if self.cov_xml or self.cov_html:
-            self.cov = ['--cov', MAIN_PACKAGE, '--cov-report', 'term-missing']
+            self.cov = ['--cov', MAIN_PACKAGE, '--cov', MAIN_PACKAGE, '--cov-report', 'term-missing']
            if self.cov_xml:
                self.cov.extend(['--cov-report', 'xml'])
            if self.cov_html:
@@ -93,12 +94,7 @@ class PyTest(TestCommand):
            params['args'] += self.cov
        if self.junitxml:
            params['args'] += self.junitxml
-        params['args'] += ['--doctest-modules', MAIN_PACKAGE, '-vv']
-
-        import logging
-        silence = logging.WARNING
-        logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=os.getenv('LOGLEVEL', silence))
-        params['args'] += ['-s' if logging.getLogger().getEffectiveLevel() < silence else '--capture=fd']
+        params['args'] += ['--doctest-modules', MAIN_PACKAGE, '-s', '-vv']
        errno = pytest.main(**params)
        sys.exit(errno)

@@ -120,7 +116,8 @@ def setup_package():
    # Some helper variables
    version = os.getenv('GO_PIPELINE_LABEL', VERSION)

-    install_reqs = get_install_requirements('requirements.txt')
+    requirements = 'requirements-py2.txt' if sys.version_info[0] == 2 else 'requirements-py3.txt'
+    install_reqs = get_install_requirements(requirements)

    command_options = {'test': {'test_suite': ('setup.py', 'tests')}}
    if JUNIT_XML:
@@ -142,12 +139,12 @@ def setup_package():
        long_description=read('README.rst'),
        classifiers=CLASSIFIERS,
        test_suite='tests',
-        packages=find_packages(exclude=['tests', 'tests.*']),
+        packages=setuptools.find_packages(exclude=['tests', 'tests.*']),
        package_data={MAIN_PACKAGE: ["*.json"]},
        install_requires=install_reqs,
-        setup_requires=['flake8'],
+        setup_requires=['six', 'flake8'],
        cmdclass=cmdclass,
-        tests_require=['mock', 'pytest-cov', 'pytest'],
+        tests_require=['pytest-cov', 'pytest'],
        command_options=command_options,
        entry_points={'console_scripts': CONSOLE_SCRIPTS},
    )
@@ -11,53 +11,49 @@ import socket
 from test_postgresql import psycopg2_connect, MockCursor


-class MockPostgresql(object):
+class MockPostgresql(Mock):

    name = 'test'
    state = 'running'
    role = 'master'
-    server_version = '999999'
-    sysid = 'dummysysid'
-    scope = 'dummy'

-    @staticmethod
-    def connection():
+    def connection(self):
        return psycopg2_connect()

+    def is_running(self):
+        return True

-class MockHa(object):
+
+class MockHa(Mock):

    dcs = Mock()
    state_handler = MockPostgresql()

-    @staticmethod
-    def schedule_reinitialize():
+    def schedule_restart(self):
+        return 'restart'
+
+    def schedule_reinitialize(self):
        return 'reinitialize'

-    @staticmethod
-    def restart():
+    def restart(self):
        return (True, '')

-    @staticmethod
-    def restart_scheduled():
+    def restart_scheduled(self):
        return False

-    @staticmethod
-    def fetch_nodes_statuses(members):
+    def fetch_nodes_statuses(self, members):
        return [[None, True, None, None, {}]]


-class MockPatroni(object):
+class MockPatroni:

    postgresql = MockPostgresql()
    ha = MockHa()
    dcs = Mock()
    tags = {}
-    version = '0.00'
-    noloadbalance = Mock(return_value=False)


-class MockRequest(object):
+class MockRequest:

    def __init__(self, path):
        self.path = path
@@ -95,10 +91,10 @@ class TestRestApiHandler(unittest.TestCase):
            MockRestApiServer(RestApiHandler, b'GET /master')
        with patch.object(MockHa, 'restart_scheduled', Mock(return_value=True)):
            MockRestApiServer(RestApiHandler, b'GET /master')
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /master'))
+        MockRestApiServer(RestApiHandler, b'GET /master')

    def test_do_OPTIONS(self):
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0'))
+        MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0')

        with patch.object(BaseHTTPRequestHandler, 'handle_one_request') as mock_handle_request:
            mock_handle_request.side_effect = socket.error("foo")
@@ -107,20 +103,20 @@ class TestRestApiHandler(unittest.TestCase):
        # make sure socket.error gets propagated via wfile object in finalize()
        with patch.object(MockRequest, 'makefile') as makefile:
            makefile.return_value.closed = False
-            makefile.return_value.readline = Mock(return_value=b'foo')
-            makefile.return_value.flush = Mock(side_effect=socket.error('foo'))
+            makefile.return_value.readline.side_effect = lambda x: b"foo"
+            makefile.return_value.flush = Mock(side_effect=socket.error("foo"))
            MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0')

    def test_do_GET_patroni(self):
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /patroni'))
+        MockRestApiServer(RestApiHandler, b'GET /patroni')

    def test_basicauth(self):
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0'))
+        MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0')
        MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0\nAuthorization:')

    def test_do_POST_restart(self):
        request = b'POST /restart HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0'
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
+        MockRestApiServer(RestApiHandler, request)
        with patch.object(MockHa, 'restart', Mock(side_effect=Exception)):
            MockRestApiServer(RestApiHandler, request)

@@ -134,77 +130,39 @@ class TestRestApiHandler(unittest.TestCase):
        with patch.object(MockHa, 'schedule_reinitialize', Mock(return_value=None)):
            MockRestApiServer(RestApiHandler, request)
        cluster.leader.name = 'test'
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
+        MockRestApiServer(RestApiHandler, request)

    @patch('time.sleep', Mock())
    def test_RestApiServer_query(self):
        with patch.object(MockCursor, 'execute', Mock(side_effect=psycopg2.OperationalError)):
-            self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /patroni'))
+            MockRestApiServer(RestApiHandler, b'GET /patroni')
        with patch.object(MockPostgresql, 'connection', Mock(side_effect=psycopg2.OperationalError)):
-            self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /patroni'))
+            MockRestApiServer(RestApiHandler, b'GET /patroni')

    @patch('time.sleep', Mock())
    @patch.object(MockHa, 'dcs')
    def test_do_POST_failover(self, dcs):
        cluster = dcs.get_cluster.return_value
-
-        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\
-                  b'Content-Length: 0\n\n'
-        MockRestApiServer(RestApiHandler, request)
-
-        cluster.leader.name = 'postgresql1'
-        MockRestApiServer(RestApiHandler, request)
-
        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\
                  b'Content-Length: 25\n\n{"leader": "postgresql1"}'
        MockRestApiServer(RestApiHandler, request)
-
-        cluster.leader.name = 'postgresql2'
-        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\
-                  b'Content-Length: 53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
-        MockRestApiServer(RestApiHandler, request)
-
        cluster.leader.name = 'postgresql1'
        MockRestApiServer(RestApiHandler, request)
-
-        cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
-                           Member(0, 'postgresql2', 30, {'api_url': 'http'})]
+        cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'})]
        MockRestApiServer(RestApiHandler, request)
        with patch.object(MockPatroni, 'dcs') as d:
            cluster = d.get_cluster.return_value
            cluster.leader.name = 'postgresql0'
            MockRestApiServer(RestApiHandler, request)
-            cluster.leader.name = 'postgresql2'
-            MockRestApiServer(RestApiHandler, request)
            cluster.leader.name = 'postgresql1'
            cluster.failover = None
            MockRestApiServer(RestApiHandler, request)
-            d.get_cluster = Mock(side_effect=Exception)
+            d.get_cluster = Mock(side_effect=Exception())
            MockRestApiServer(RestApiHandler, request)
            d.manual_failover.return_value = False
            MockRestApiServer(RestApiHandler, request)
        with patch.object(MockHa, 'fetch_nodes_statuses', Mock(return_value=[])):
            MockRestApiServer(RestApiHandler, request)
-
-        # Valid future date
-        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\
-                  b'"postgresql1", "member": "postgresql2", "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}'
+        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\
+                  b'Content-Length: 50\n\n{"leader": "postgresql1", "member": "postgresql2"}'
        MockRestApiServer(RestApiHandler, request)
-        with patch.object(MockPatroni, 'dcs') as d:
-            d.manual_failover.return_value = False
-            MockRestApiServer(RestApiHandler, request)
-
-        # Exception: No timezone specified
-        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 97\n\n{"leader": ' +\
-                  b'"postgresql1", "member": "postgresql2", "scheduled_at": "6016-02-15T18:13:30.568224"}'
-        MockRestApiServer(RestApiHandler, request)
-
-        # Exception: Scheduled in the past
-        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\
-                  b'"postgresql1", "member": "postgresql2", "scheduled_at": "1016-02-15T18:13:30.568224+01:00"}'
-        MockRestApiServer(RestApiHandler, request)
-
-        # Invalid date
-        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\
-                  b'"postgresql1", "member": "postgresql2", "scheduled_at": "2010-02-29T18:13:30.568224+01:00"}'
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
@@ -1,15 +1,12 @@
-import boto.ec2
-import requests
-import sys
 import unittest
-
-from mock import Mock, patch
+import requests
+import boto.ec2
 from collections import namedtuple
-from patroni.scripts.aws import AWSConnection, main as _main
+from patroni.scripts.aws import AWSConnection
 from requests.exceptions import RequestException


-class MockEc2Connection(object):
+class MockEc2Connection:

    def __init__(self, error=False):
        self.error = error
@@ -26,7 +23,7 @@ class MockEc2Connection(object):
        return True


-class MockResponse(object):
+class MockResponse:

    def __init__(self, content):
        self.content = content
@@ -38,6 +35,15 @@ class MockResponse(object):

 class TestAWSConnection(unittest.TestCase):

+    def __init__(self, method_name='runTest'):
+        super(TestAWSConnection, self).__init__(method_name)
+
+    def set_error(self):
+        self.error = True
+
+    def set_json_error(self):
+        self.json_error = True
+
    def boto_ec2_connect_to_region(self, region):
        return MockEc2Connection(self.error)

@@ -68,27 +74,21 @@ class TestAWSConnection(unittest.TestCase):
        self.assertTrue(self.conn.on_role_change('master'))

    def test_non_aws(self):
-        self.error = True
+        self.set_error()
        conn = AWSConnection('test')
        self.assertFalse(conn.aws_available())
        self.assertFalse(conn._tag_ebs('master'))
        self.assertFalse(conn._tag_ec2('master'))

    def test_aws_bizare_response(self):
-        self.json_error = True
+        self.set_json_error()
        conn = AWSConnection('test')
        self.assertFalse(conn.aws_available())

    def test_aws_tag_ebs_error(self):
-        self.error = True
+        self.set_error()
        self.assertFalse(self.conn._tag_ebs("master"))

    def test_aws_tag_ec2_error(self):
-        self.error = True
+        self.set_error()
        self.assertFalse(self.conn._tag_ec2("master"))
-
-    @patch('sys.exit', Mock())
-    def test_main(self):
-        self.assertIsNone(_main())
-        sys.argv = ['aws.py', 'on_start', 'replica', 'foo']
-        self.assertIsNone(_main())
@@ -1,128 +0,0 @@
-import consul
-import unittest
-
-from patroni.dcs import AbstractDCS
-from mock import Mock, patch
-from patroni.consul import Cluster, Consul, ConsulError, ConsulException, HTTPClient, NotFound
-from test_etcd import SleepException
-
-
-def kv_get(self, key, **kwargs):
-    if key == 'service/test/members/postgresql1':
-        return '1', {'Session': 'fd4f44fe-2cac-bba5-a60b-304b51ff39b7'}
-    if key == 'service/test/':
-        return None, None
-    if key == 'service/good/leader':
-        return '1', None
-    if key == 'service/good/':
-        return ('6429',
-                [{'CreateIndex': 1334, 'Flags': 0, 'Key': key + 'failover', 'LockIndex': 0,
-                  'ModifyIndex': 1334, 'Value': b''},
-                 {'CreateIndex': 1334, 'Flags': 0, 'Key': key + 'initialize', 'LockIndex': 0,
-                  'ModifyIndex': 1334, 'Value': b'postgresql0'},
-                 {'CreateIndex': 2621, 'Flags': 0, 'Key': key + 'leader', 'LockIndex': 1,
-                  'ModifyIndex': 2621, 'Session': 'fd4f44fe-2cac-bba5-a60b-304b51ff39b7', 'Value': b'postgresql1'},
-                 {'CreateIndex': 6156, 'Flags': 0, 'Key': key + 'members/postgresql0', 'LockIndex': 1,
-                  'ModifyIndex': 6156, 'Session': '782e6da4-ed02-3aef-7963-99a90ed94b53',
-                  'Value': ('postgres://replicator:[email protected]:5432/postgres' +
-                            '?application_name=http://127.0.0.1:8008/patroni').encode('utf-8')},
-                 {'CreateIndex': 2630, 'Flags': 0, 'Key': key + 'members/postgresql1', 'LockIndex': 1,
-                  'ModifyIndex': 2630, 'Session': 'fd4f44fe-2cac-bba5-a60b-304b51ff39b7',
-                  'Value': ('postgres://replicator:[email protected]:5433/postgres' +
-                            '?application_name=http://127.0.0.1:8009/patroni').encode('utf-8')},
-                 {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'optime/leader', 'LockIndex': 0,
-                  'ModifyIndex': 6429, 'Value': b'4496294792'}])
-    raise ConsulException
-
-
-class TestHTTPClient(unittest.TestCase):
-
-    def test_get(self):
-        self.client = HTTPClient('127.0.0.1', '8500', 'http', False)
-        self.client.session.get = Mock()
-        self.client.get(Mock(), '')
-        self.client.get(Mock(), '', {'wait': '1s', 'index': 1})
-
-
-@patch.object(consul.Consul.KV, 'get', kv_get)
-class TestConsul(unittest.TestCase):
-
-    @patch.object(consul.Consul.Session, 'create', Mock(return_value='fd4f44fe-2cac-bba5-a60b-304b51ff39b7'))
-    @patch.object(consul.Consul.Session, 'renew', Mock(side_effect=NotFound))
-    @patch.object(consul.Consul.KV, 'get', kv_get)
-    @patch.object(consul.Consul.KV, 'delete', Mock())
-    def setUp(self):
-        self.c = Consul('postgresql1', {'ttl': 30, 'scope': 'test', 'host': 'localhost:1'})
-        self.c._base_path = '/service/good'
-        self.c._load_cluster()
-
-    @patch('time.sleep', Mock(side_effect=SleepException))
-    def test_create_or_restore_session(self):
-        self.c._session = None
-        self.assertRaises(SleepException, self.c.create_or_restore_session)
-
-    @patch.object(consul.Consul.Session, 'renew', Mock(side_effect=NotFound))
-    @patch.object(consul.Consul.Session, 'create', Mock(side_effect=ConsulException))
-    def test_referesh_session(self):
-        self.c._session = '1'
-        self.c._name = ''
-        self.assertRaises(ConsulError, self.c.refresh_session)
-
-    @patch.object(consul.Consul.KV, 'delete', Mock())
-    def test_get_cluster(self):
-        self.c._base_path = '/service/test'
-        self.assertIsInstance(self.c.get_cluster(), Cluster)
-        self.assertIsInstance(self.c.get_cluster(), Cluster)
-        self.c._base_path = '/service/fail'
-        self.assertRaises(ConsulError, self.c.get_cluster)
-        self.c._base_path = '/service/good'
-        self.c._session = 'fd4f44fe-2cac-bba5-a60b-304b51ff39b8'
-        self.assertIsInstance(self.c.get_cluster(), Cluster)
-
-    @patch.object(consul.Consul.KV, 'delete', Mock(side_effect=[ConsulException, True, True]))
-    @patch.object(consul.Consul.KV, 'put', Mock(side_effect=[True, ConsulException]))
-    def test_touch_member(self):
-        self.c.refresh_session = Mock(return_value=True)
-        self.c.touch_member('balbla')
-        self.c.touch_member('balbla')
-        self.c.touch_member('balbla')
-        self.c.refresh_session = Mock(return_value=False)
-        self.c.touch_member('balbla')
-
-    @patch.object(consul.Consul.KV, 'put', Mock(return_value=False))
-    def test_take_leader(self):
-        self.c.take_leader()
-
-    @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
-    def test_set_failover_value(self):
-        self.c.set_failover_value('')
-
-    @patch.object(consul.Consul.KV, 'put', Mock(side_effect=ConsulException))
-    def test_write_leader_optime(self):
-        self.c.write_leader_optime('')
-
-    def test_update_leader(self):
-        self.c.update_leader()
-
-    @patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
-    def test_delete_leader(self):
-        self.c.delete_leader()
-
-    @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
-    def test_initialize(self):
-        self.c.initialize()
-
-    @patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
-    def test_cancel_initialization(self):
-        self.c.cancel_initialization()
-
-    @patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
-    def test_delete_cluster(self):
-        self.c.delete_cluster()
-
-    @patch.object(AbstractDCS, 'watch', Mock())
-    def test_watch(self):
-        self.c._name = ''
-        self.c.watch(1)
-        with patch.object(consul.Consul.KV, 'get', Mock(side_effect=ConsulException)):
-            self.c.watch(1)
@@ -1,26 +1,31 @@
-import etcd
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 import os
 import pytest
-import requests.exceptions
 import unittest
+import psycopg2
+import requests
+import patroni.exceptions
+import etcd
+from mock import patch, Mock

 from click.testing import CliRunner
-from mock import patch, Mock
-from patroni.ctl import ctl, members, store_config, load_config, output_members, post_patroni, get_dcs, parse_dcs, \
-    wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException
+from patroni.ctl import ctl, members, store_config, load_config, output_members, post_patroni, get_dcs, \
+    wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure
+from patroni.ha import Ha
 from patroni.etcd import Etcd, Client
-from psycopg2 import OperationalError
-from test_etcd import etcd_read, requests_get, socket_getaddrinfo, MockResponse
 from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \
-    get_cluster_initialized_with_only_leader
+    get_cluster_initialized_with_only_leader, MockPostgresql, MockPatroni, run_async, \
+    get_cluster_not_initialized_without_leader
+from test_etcd import etcd_read, etcd_write, requests_get, MockResponse
 from test_postgresql import MockConnect, psycopg2_connect

 CONFIG_FILE_PATH = './test-ctl.yaml'

-
 def test_rw_config():
    runner = CliRunner()
-    config = {'a': 'b'}
+    config = {'a':'b'}
    with runner.isolated_filesystem():
        store_config(config, CONFIG_FILE_PATH + '/dummy')
        os.remove(CONFIG_FILE_PATH + '/dummy')
@@ -40,272 +45,332 @@ def test_rw_config():
    load_config(CONFIG_FILE_PATH, None)
    load_config(CONFIG_FILE_PATH, '0.0.0.0')

-    store_config({'dcs_api': None}, CONFIG_FILE_PATH)
-    load_config(CONFIG_FILE_PATH, None)
-
-
-@patch('patroni.ctl.load_config', Mock(return_value={'etcd': {'host': 'localhost:4001'}}))
+@patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'etcd', 'hostname': 'localhost', 'port': 4001}}))
 class TestCtl(unittest.TestCase):

-    @patch('socket.getaddrinfo', socket_getaddrinfo)
-    def setUp(self):
-        self.runner = CliRunner()
-        with patch.object(Client, 'machines') as mock_machines:
-            mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
-            self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'})
+    @patch.object(Client, 'machines')
+    def setUp(self, mock_machines):
+        mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
+        self.p = MockPostgresql()
+        self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'})
+        self.e.client.read = etcd_read
+        self.e.client.write = etcd_write
+        self.e.client.delete = Mock(side_effect=etcd.EtcdException())
+        self.ha = Ha(MockPatroni(self.p, self.e))
+        self.ha._async_executor.run_async = run_async
+        self.ha.old_cluster = self.e.get_cluster()
+        self.ha.cluster = get_cluster_not_initialized_without_leader()
+        self.ha.load_cluster_from_dcs = Mock()

    @patch('psycopg2.connect', psycopg2_connect)
    def test_get_cursor(self):
-        self.assertIsNone(get_cursor(get_cluster_initialized_without_leader(), role='master'))
+        c = get_cursor(get_cluster_initialized_without_leader(), role='master')
+        assert c is None

-        self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), role='master'))
+        c = get_cursor(get_cluster_initialized_with_leader(), role='master')
+        assert c is not None

-        # MockCursor returns pg_is_in_recovery as false
-        self.assertIsNone(get_cursor(get_cluster_initialized_with_leader(), role='replica'))
+        c = get_cursor(get_cluster_initialized_with_leader(), role='replica')
+        # # MockCursor returns pg_is_in_recovery as false
+        assert c is None

-        self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), role='any'))
-
-    def test_parse_dcs(self):
-        assert parse_dcs(None) is None
-        assert parse_dcs('localhost') == {'etcd': {'host': 'localhost:4001'}}
-        assert parse_dcs('') == {'etcd': {'host': 'localhost:4001'}}
-        assert parse_dcs('localhost:8500') == {'consul': {'host': 'localhost:8500'}}
-        assert parse_dcs('zookeeper://localhost') == {'zookeeper': {'hosts': ['localhost:2181']}}
-        assert parse_dcs('exhibitor://dummy') == {'zookeeper': {'exhibitor': {'hosts': ['dummy'], 'port': 8181}}}
-        assert parse_dcs('consul://localhost') == {'consul': {'host': 'localhost:8500'}}
-        self.assertRaises(PatroniCtlException, parse_dcs, 'invalid://test')
+        c = get_cursor(get_cluster_initialized_with_leader(), role='any')
+        assert c is not None

    def test_output_members(self):
        cluster = get_cluster_initialized_with_leader()
-        self.assertIsNone(output_members(cluster, name='abc', fmt='pretty'))
-        self.assertIsNone(output_members(cluster, name='abc', fmt='json'))
-        self.assertIsNone(output_members(cluster, name='abc', fmt='tsv'))
+        output_members(cluster, name='abc', format='pretty')
+        output_members(cluster, name='abc', format='json')
+        output_members(cluster, name='abc', format='tsv')

    @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
    @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
+    @patch('patroni.etcd.Etcd.set_failover_value', Mock(return_value=None))
+    @patch('patroni.ctl.wait_for_leader', Mock(return_value=get_cluster_initialized_with_leader()))
+    @patch('requests.get', requests_get)
+    @patch('requests.post', requests_get)
    @patch('patroni.ctl.post_patroni', Mock(return_value=MockResponse()))
    def test_failover(self):
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n\ny''')
-        assert 'leader' in result.output
+        runner = CliRunner()

-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n2100-01-01T12:23:00\ny''')
-        assert result.exit_code == 0
+        with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())):
+            result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
+other
+y''')
+            assert 'Failing over to new leader' in result.output

-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n2030-01-01T12:23:00\ny''')
-        assert result.exit_code == 0
+            result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
+other
+N''')
+            assert 'Aborting failover' in str(result.exception)

-        # Aborting failover,as we anser NO to the confirmation
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n\nN''')
-        assert result.exit_code == 1
+            result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
+leader
+y''')
+            assert 'target and source are the same' in str(result.exception)

-        # Target and source are equal
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nleader\n\ny''')
-        assert result.exit_code == 1
+            result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
+Reality
+y''')
+            assert 'Reality does not exist' in str(result.exception)

-        # Reality is not part of this cluster
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nReality\n\ny''')
-        assert result.exit_code == 1
+            result = runner.invoke(ctl, ['failover', 'dummy', '--force'])
+            assert 'Failing over to new leader' in result.output

-        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force'])
-        assert 'Member' in result.output
-
-        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
-        assert result.exit_code == 0
-
-        # Invalid timestamp
-        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', 'invalid'])
-        assert result.exit_code != 0
-
-        # Invalid timestamp
-        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2115-02-30T12:00:00+01:00'])
-        assert result.exit_code != 0
-
-        # Specifying wrong leader
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='dummy')
-        assert result.exit_code == 1
+            result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='dummy')
+            assert 'is not the leader of cluster' in str(result.exception)

        with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_only_leader())):
-            # No members available
-            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n\ny''')
-            assert result.exit_code == 1
+            result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
+other
+y''')
+            assert 'No candidates found to failover to' in str(result.exception)

        with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_without_leader())):
-            # No master available
-            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n\ny''')
-            assert result.exit_code == 1
+            result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
+other
+y''')
+            assert 'This cluster has no master' in str(result.exception)

-        with patch('patroni.ctl.post_patroni', Mock(side_effect=Exception)):
-            # Non-responding patroni
-            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n\ny''')
+        with patch('patroni.ctl.post_patroni', Mock(side_effect=Exception())):
+            result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
+other
+y''')
            assert 'falling back to DCS' in result.output
-
-        with patch('patroni.ctl.post_patroni') as mocked:
-            mocked.return_value.status_code = 500
-            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n\ny''')
            assert 'Failover failed' in result.output

-    def test_get_dcs(self):
-        self.assertRaises(PatroniCtlException, get_dcs, {'dummy': {}}, 'dummy')
-        with patch('patroni.Patroni.get_dcs', Mock(return_value=self.e)):
-            assert get_dcs({'etcd': {'host': 'none'}}, 'dummy').client_path('') == '/service/test/'
+        mocked = Mock()
+        mocked.return_value.status_code = 500
+        with patch('patroni.ctl.post_patroni', Mock(return_value=mocked)):
+            result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
+other
+y''')
+            assert 'Failover failed, details' in result.output
+
+#        with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())):
+#            result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='nonsense')
+#            assert 'is not the leader of cluster' in str(result.exception)
+
+            # result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8', '--master', 'nonsense'])
+            # assert 'is not the leader of cluster' in str(result.exception)
+
+            # result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='leader\nother\nn')
+            # assert 'Aborting failover' in str(result.exception)
+
+            # with patch('patroni.ctl.wait_for_leader', Mock(return_value = get_cluster_initialized_with_leader())):
+                # result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='leader\nother\nY')
+                # assert 'master did not change after' in result.output
+
+                # result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='leader\nother\nY')
+                # assert 'Failover failed' in result.output
+
+    def test_(self):
+        self.assertRaises(patroni.exceptions.PatroniCtlException, get_dcs, {'scheme': 'dummy'}, 'dummy')

    @patch('psycopg2.connect', psycopg2_connect)
    @patch('patroni.ctl.query_member', Mock(return_value=([['mock column']], None)))
-    @patch.object(etcd.Client, 'read', etcd_read)
    def test_query(self):
+        runner = CliRunner()
+
        with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)):
-            # Mutually exclusive
-            result = self.runner.invoke(ctl, ['query', 'alpha', '--member', 'abc', '--role', 'master'])
-            assert result.exit_code == 1
+            result = runner.invoke(ctl, [
+                'query',
+                'alpha',
+                '--member',
+                'abc',
+                '--role',
+                'master',
+            ])
+            assert 'mutually exclusive' in str(result.exception)

-            with self.runner.isolated_filesystem():
-                with open('dummy', 'w') as dummy_file:
-                    dummy_file.write('SELECT 1')
+            with runner.isolated_filesystem():
+                dummy_file = open('dummy', 'w')
+                dummy_file.write('SELECT 1')
+                dummy_file.close()

-                # Mutually exclusive
-                result = self.runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy', '--command', 'dummy'])
-                assert result.exit_code == 1
+                result = runner.invoke(ctl, [
+                    'query',
+                    'alpha',
+                    '--file',
+                    'dummy',
+                    '--command',
+                    'dummy',
+                ])
+                assert 'mutually exclusive' in str(result.exception)

-                result = self.runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy'])
-                assert result.exit_code == 0
+                result = runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy'])

                os.remove('dummy')

-            result = self.runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1'])
-            assert 'mock column' in result.output
-
-            # --command or --file is mandatory
-            result = self.runner.invoke(ctl, ['query', 'alpha'])
-            assert result.exit_code == 1
-
-            result = self.runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1', '--username', 'root',
-                                              '--password', '--dbname', 'postgres'], input='ab\nab')
+            result = runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1'])
            assert 'mock column' in result.output

+    @patch('patroni.ctl.get_cursor', Mock(return_value=MockConnect().cursor()))
    def test_query_member(self):
-        with patch('patroni.ctl.get_cursor', Mock(return_value=MockConnect().cursor())):
-            rows = query_member(None, None, None, 'master', 'SELECT pg_is_in_recovery()')
-            self.assertTrue('False' in str(rows))
+        rows = query_member(None, None, None, 'master', 'SELECT pg_is_in_recovery()')
+        assert 'False' in str(rows)

-            rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
-            self.assertEquals(rows, (None, None))
-
-            with patch('test_postgresql.MockCursor.execute', Mock(side_effect=OperationalError('bla'))):
-                rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
+        rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
+        assert rows == (None, None)

        with patch('patroni.ctl.get_cursor', Mock(return_value=None)):
            rows = query_member(None, None, None, None, 'SELECT pg_is_in_recovery()')
-            self.assertTrue('No connection to' in str(rows))
+            assert 'No connection to' in str(rows)

            rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
-            self.assertTrue('No connection to' in str(rows))
+            assert 'No connection to' in str(rows)

-        with patch('patroni.ctl.get_cursor', Mock(side_effect=OperationalError('bla'))):
+        with patch('patroni.ctl.get_cursor', Mock(side_effect=psycopg2.OperationalError('bla'))):
+            rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
+
+        with patch('test_postgresql.MockCursor.execute', Mock(side_effect=psycopg2.OperationalError('bla'))):
            rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')

    @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
    def test_dsn(self):
+        runner = CliRunner()
+
        with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)):
-            result = self.runner.invoke(ctl, ['dsn', 'alpha'])
+            result = runner.invoke(ctl, ['dsn', 'alpha', '--dcs', '8.8.8.8'])
            assert 'host=127.0.0.1 port=5435' in result.output

-            # Mutually exclusive options
-            result = self.runner.invoke(ctl, ['dsn', 'alpha', '--role', 'master', '--member', 'dummy'])
-            assert result.exit_code == 1
+            result = runner.invoke(ctl, [
+                'dsn',
+                'alpha',
+                '--role',
+                'master',
+                '--member',
+                'dummy',
+            ])
+            assert 'mutually exclusive' in str(result.exception)

-            # Non-existing member
-            result = self.runner.invoke(ctl, ['dsn', 'alpha', '--member', 'dummy'])
-            assert result.exit_code == 1
+            result = runner.invoke(ctl, ['dsn', 'alpha', '--member', 'dummy'])
+            assert 'Can not find' in str(result.exception)
+
+        # result = runner.invoke(ctl, ['dsn', 'alpha', '--dcs', '8.8.8.8', '--role', 'replica'])
+        # assert 'host=127.0.0.1 port=5436' in result.output

    @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
    @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
+    @patch('requests.get', requests_get)
    @patch('requests.post', requests_get)
    def test_restart_reinit(self):
-        result = self.runner.invoke(ctl, ['restart', 'alpha'], input='y')
-        assert 'restart failed for' in result.output
-        assert result.exit_code == 0
+        runner = CliRunner()

-        result = self.runner.invoke(ctl, ['reinit', 'alpha'], input='y')
-        assert result.exit_code == 1
+        result = runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='y')
+        result = runner.invoke(ctl, ['reinit', 'alpha', '--dcs', '8.8.8.8'], input='y')

-        # Aborted restart
-        result = self.runner.invoke(ctl, ['restart', 'alpha'], input='N')
-        assert result.exit_code == 1
-
-        # Not a member
-        result = self.runner.invoke(ctl, ['restart', 'alpha', 'dummy', '--any'], input='y')
-        assert result.exit_code == 1
+        result = runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='N')
+        result = runner.invoke(ctl, [
+            'restart',
+            'alpha',
+            '--dcs',
+            '8.8.8.8',
+            'dummy',
+            '--any',
+        ], input='y')
+        assert 'not a member' in str(result.exception)

        with patch('requests.post', Mock(return_value=MockResponse())):
-            result = self.runner.invoke(ctl, ['restart', 'alpha'], input='y')
-            assert result.exit_code == 0
+            result = runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='y')

    @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
-    @patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
+    @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
    def test_remove(self):
-        with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)):
-            result = self.runner.invoke(ctl, ['remove', 'alpha'], input='alpha\nslave')
-            assert 'Please confirm' in result.output
-            assert 'You are about to remove all' in result.output
-            # Not typing an exact confirmation
-            assert result.exit_code == 1
+        runner = CliRunner()

-            # master specified does not match master of cluster
-            result = self.runner.invoke(ctl, ['remove', 'alpha'], input='''alpha\nYes I am aware\nslave''')
-            assert result.exit_code == 1
+        result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='alpha\nslave')
+        assert 'Please confirm' in result.output
+        assert 'You are about to remove all' in result.output
+        assert 'You did not exactly type' in str(result.exception)

-            # cluster specified on cmdline does not match verification prompt
-            result = self.runner.invoke(ctl, ['remove', 'alpha'], input='beta\nleader')
-            assert result.exit_code == 1
+        result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='''alpha
+Yes I am aware
+slave''')
+        assert 'You did not specify the current master of the cluster' in str(result.exception)

-            result = self.runner.invoke(ctl, ['remove', 'alpha'], input='''alpha\nYes I am aware\nleader''')
-            assert result.exit_code == 0
+        result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='beta\nleader')
+        assert 'Cluster names specified do not match' in str(result.exception)
+
+        with patch('patroni.etcd.Etcd.get_cluster', get_cluster_initialized_with_leader):
+            result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'],
+                                   input='''alpha
+Yes I am aware
+leader''')
+            assert 'object has no attribute' in str(result.exception)
+
+        with patch('patroni.ctl.get_dcs', Mock(return_value=Mock())):
+            result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'],
+                                   input='''alpha
+Yes I am aware
+leader''')
+            assert 'We have not implemented this for DCS of type' in str(result.exception)

    @patch('patroni.etcd.Etcd.watch', Mock(return_value=None))
    @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
    def test_wait_for_leader(self):
-        self.assertRaises(PatroniCtlException, wait_for_leader, self.e, 0)
+        dcs = self.e
+        self.assertRaises(patroni.exceptions.PatroniCtlException, wait_for_leader, dcs, 0)

-        cluster = wait_for_leader(self.e, timeout=2)
+        cluster = wait_for_leader(dcs=dcs, timeout=2)
        assert cluster.leader.member.name == 'leader'

-    @patch('requests.post', Mock(side_effect=requests.exceptions.ConnectionError('foo')))
    def test_post_patroni(self):
        member = get_cluster_initialized_with_leader().leader.member
        self.assertRaises(requests.exceptions.ConnectionError, post_patroni, member, 'dummy', {})

    def test_ctl(self):
-        self.runner.invoke(ctl, ['list'])
+        runner = CliRunner()

-        result = self.runner.invoke(ctl, ['--help'])
+        runner.invoke(ctl, ['list'])
+
+        result = runner.invoke(ctl, ['--help'])
        assert 'Usage:' in result.output

    def test_get_any_member(self):
-        self.assertIsNone(get_any_member(get_cluster_initialized_without_leader(), role='master'))
+        m = get_any_member(get_cluster_initialized_without_leader(), role='master')
+        assert m is None

        m = get_any_member(get_cluster_initialized_with_leader(), role='master')
-        self.assertEquals(m.name, 'leader')
+        assert m.name == 'leader'

    def test_get_all_members(self):
-        self.assertEquals(list(get_all_members(get_cluster_initialized_without_leader(), role='master')), [])
+        r = list(get_all_members(get_cluster_initialized_without_leader(), role='master'))
+        assert len(r) == 0

        r = list(get_all_members(get_cluster_initialized_with_leader(), role='master'))
-        self.assertEquals(len(r), 1)
-        self.assertEquals(r[0].name, 'leader')
+        assert len(r) == 1
+        assert r[0].name == 'leader'

        r = list(get_all_members(get_cluster_initialized_with_leader(), role='replica'))
-        self.assertEquals(len(r), 1)
-        self.assertEquals(r[0].name, 'other')
+        assert len(r) == 1
+        assert r[0].name == 'other'

-        self.assertEquals(len(list(get_all_members(get_cluster_initialized_without_leader(), role='replica'))), 2)
+        r = list(get_all_members(get_cluster_initialized_without_leader(), role='replica'))
+        assert len(r) == 2

    @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
    @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
+    @patch('requests.get', requests_get)
+    @patch('requests.post', requests_get)
    def test_members(self):
-        result = self.runner.invoke(members, ['alpha'])
-        assert '127.0.0.1' in result.output
+        runner = CliRunner()
+
+        result = runner.invoke(members, ['alpha'])
        assert result.exit_code == 0

    def test_configure(self):
-        result = self.runner.invoke(configure, ['--dcs', 'abc', '-c', 'dummy', '-n', 'bla'])
+        runner = CliRunner()
+
+        result = runner.invoke(configure, [
+            '--dcs',
+            'abc',
+            '-c',
+            'dummy',
+            '-n',
+            'bla',
+        ])
+
        assert result.exit_code == 0
+
+
@@ -1,18 +1,17 @@
 import etcd
 import json
 import requests
+import urllib3
 import socket
 import unittest

 from dns.exception import DNSException
 from mock import Mock, patch
-from patroni.dcs import Cluster, AbstractDCS
+from patroni.dcs import Cluster, DCSError, Leader
 from patroni.etcd import Client, Etcd, EtcdError
-from patroni.exceptions import DCSError
-from urllib3.exceptions import ReadTimeoutError


-class MockResponse(object):
+class MockResponse:

    def __init__(self):
        self.status_code = 200
@@ -25,17 +24,26 @@ class MockResponse(object):

    @property
    def data(self):
-        return self.content.encode('utf-8')
+        if self.content == 'TimeoutError':
+            raise urllib3.exceptions.TimeoutError
+        if self.content == 'Exception':
+            raise Exception
+        return self.content

    @property
    def status(self):
        return self.status_code

-    @staticmethod
    def getheader(*args):
        return ''


+class MockPostgresql(Mock):
+
+    def last_operation(self):
+        return '0'
+
+
 def requests_get(url, **kwargs):
    members = '[{"id":14855829450254237642,"peerURLs":["http://localhost:2380","http://localhost:7001"],' +\
              '"name":"default","clientURLs":["http://localhost:2379","http://localhost:4001"]}]'
@@ -45,7 +53,10 @@ def requests_get(url, **kwargs):
    elif ':8011/patroni' in url:
        response.content = '{"role": "replica", "xlog": {"replayed_location": 0}, "tags": {}}'
    elif url.endswith('/members'):
-        response.content = '[{}]' if url.startswith('http://error') else members
+        if url.startswith('http://error'):
+            response.content = '[{}]'
+        else:
+            response.content = members
    elif url.startswith('http://exhibitor'):
        response.content = '{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}'
    else:
@@ -54,25 +65,29 @@ def requests_get(url, **kwargs):
    return response


-def etcd_watch(self, key, index=None, timeout=None, recursive=None):
+def etcd_watch(key, index=None, timeout=None, recursive=None):
    if timeout == 2.0:
-        raise etcd.EtcdWatchTimedOut
+        raise urllib3.exceptions.TimeoutError
    elif timeout == 5.0:
        return etcd.EtcdResult('delete', {})
    elif timeout == 10.0:
        raise etcd.EtcdException
+    elif index == 20729:
+        return etcd.EtcdResult('set', {'value': 'postgresql1', 'modifiedIndex': index + 1})
+    elif index == 20731:
+        return etcd.EtcdResult('set', {'value': 'postgresql2', 'modifiedIndex': index + 1})


-def etcd_write(self, key, value, **kwargs):
+def etcd_write(key, value, **kwargs):
    if key == '/service/exists/leader':
        raise etcd.EtcdAlreadyExist
-    if key in ['/service/test/leader', '/patroni/test/leader'] and \
-            (kwargs.get('prevValue') == 'foo' or not kwargs.get('prevExist', True)):
-        return True
+    if key == '/service/test/leader' or key == '/patroni/test/leader':
+        if kwargs.get('prevValue', None) == 'foo' or not kwargs.get('prevExist', True):
+            return True
    raise etcd.EtcdException


-def etcd_read(self, key, **kwargs):
+def etcd_read(key, **kwargs):
    if key == '/service/noleader/':
        raise DCSError('noleader')
    elif key == '/service/nocluster/':
@@ -109,12 +124,12 @@ class SleepException(Exception):
    pass


-class MockSRV(object):
+class MockSRV:
    port = 2380
    target = '127.0.0.1'


-def dns_query(name, _):
+def dns_query(name, type):
    if name == '_etcd-server._tcp.blabla':
        return []
    elif name == '_etcd-server._tcp.exception':
@@ -129,8 +144,6 @@ def socket_getaddrinfo(*args):


 def http_request(method, url, **kwargs):
-    if url == 'http://localhost:2379/timeout':
-        raise ReadTimeoutError(None, None, None)
    if url == 'http://localhost:2379/':
        return MockResponse()
    raise socket.error
@@ -148,27 +161,30 @@ class TestClient(unittest.TestCase):
            mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
            self.client = Client({'discovery_srv': 'test'})
            self.client.http.request = http_request
-            self.client.http.request_encode_body = http_request

    def test_api_execute(self):
        self.client._base_uri = 'http://localhost:4001'
        self.client._machines_cache = ['http://localhost:2379']
-        self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
-        self.client._update_machines_cache = False
-        self.client.api_execute('/', 'POST', timeout=0)
+        self.client.api_execute('/', 'GET')
        self.client._update_machines_cache = False
        self.client._base_uri = 'http://localhost:4001'
        self.client._machines_cache = []
        self.assertRaises(etcd.EtcdConnectionFailed, self.client.api_execute, '/', 'GET')
        self.assertTrue(self.client._update_machines_cache)
-        self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'GET')
-        self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '')
-        self.assertRaises(ValueError, self.client.api_execute, '', '')

    def test_get_srv_record(self):
        self.assertEquals(self.client.get_srv_record('blabla'), [])
        self.assertEquals(self.client.get_srv_record('exception'), [])

+    def test__result_from_response(self):
+        response = MockResponse()
+        response.content = 'TimeoutError'
+        self.assertRaises(urllib3.exceptions.TimeoutError, self.client._result_from_response, response)
+        response.content = 'Exception'
+        self.assertRaises(etcd.EtcdException, self.client._result_from_response, response)
+        response.content = b'{}'
+        self.assertRaises(etcd.EtcdException, self.client._result_from_response, response)
+
    def test__get_machines_cache_from_srv(self):
        self.client.get_srv_record = Mock(return_value=[('localhost', 2380)])
        self.client._get_machines_cache_from_srv('blabla')
@@ -184,15 +200,15 @@ class TestClient(unittest.TestCase):


@patch('requests.get', requests_get)
-@patch.object(etcd.Client, 'write', etcd_write)
-@patch.object(etcd.Client, 'read', etcd_read)
-@patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
 class TestEtcd(unittest.TestCase):

    def setUp(self):
        with patch.object(Client, 'machines') as mock_machines:
            mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
            self.etcd = Etcd('foo', {'namespace': '/patroni/', 'ttl': 30, 'host': 'localhost:2379', 'scope': 'test'})
+            self.etcd.client.write = etcd_write
+            self.etcd.client.read = etcd_read
+            self.etcd.client.delete = Mock(side_effect=etcd.EtcdException())

    def test_base_path(self):
        self.assertEquals(self.etcd._base_path, '/patroni/test')
@@ -210,8 +226,11 @@ class TestEtcd(unittest.TestCase):
        cluster = self.etcd.get_cluster()
        self.assertIsInstance(cluster, Cluster)
        self.assertIsNone(cluster.leader)
+
+    def test_current_leader(self):
+        self.assertIsInstance(self.etcd.current_leader(), Leader)
        self.etcd._base_path = '/service/noleader'
-        self.assertRaises(EtcdError, self.etcd.get_cluster)
+        self.assertIsNone(self.etcd.current_leader())

    def test_touch_member(self):
        self.assertFalse(self.etcd.touch_member('', ''))
@@ -240,14 +259,14 @@ class TestEtcd(unittest.TestCase):
    def test_delete_leader(self):
        self.assertFalse(self.etcd.delete_leader())

-    @patch.object(etcd.Client, 'watch', etcd_watch)
    def test_watch(self):
+        self.etcd.client.watch = etcd_watch
        self.etcd.watch(0)
        self.etcd.get_cluster()
        self.etcd.watch(1.5)
        self.etcd.watch(4.5)
-        with patch.object(AbstractDCS, 'watch', Mock()):
-            self.etcd.watch(9.5)
+        self.etcd.watch(9.5)
+        self.etcd.watch(100)

    @patch('patroni.etcd.Etcd.retry', Mock(side_effect=AttributeError("foo")))
    def test_other_exceptions(self):
@@ -1,14 +1,11 @@
 import etcd
 import unittest
-import datetime
-import pytz

 from mock import Mock, MagicMock, patch
 from patroni.dcs import Cluster, Failover, Leader, Member
 from patroni.etcd import Client, Etcd
 from patroni.exceptions import DCSError, PostgresException
 from patroni.ha import Ha
-from patroni.postgresql import Postgresql
 from test_etcd import socket_getaddrinfo, etcd_read, etcd_write, requests_get


@@ -21,7 +18,7 @@ def false(*args, **kwargs):


 def get_cluster(initialize, leader, members, failover):
-    return Cluster(initialize, leader, 10, members, failover)
+    return Cluster(initialize, leader, None, members, failover)


 def get_cluster_not_initialized_without_leader():
@@ -29,77 +26,92 @@ def get_cluster_not_initialized_without_leader():


 def get_cluster_initialized_without_leader(leader=False, failover=None):
-    m1 = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres',
-                                  'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location': 4})
-    l = Leader(0, 0, m1) if leader else None
-    m2 = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
-                                 'api_url': 'http://127.0.0.1:8011/patroni', 'tags': {'clonefrom': True}})
-    return get_cluster(True, l, [m1, m2], failover)
+    m = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres',
+                                 'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location':4})
+    l = Leader(0, 0, m) if leader else None
+    o = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
+                                'api_url': 'http://127.0.0.1:8011/patroni'})
+    return get_cluster(True, l, [m, o], failover)


 def get_cluster_initialized_with_leader(failover=None):
    return get_cluster_initialized_without_leader(leader=True, failover=failover)

-
 def get_cluster_initialized_with_only_leader(failover=None):
    l = get_cluster_initialized_without_leader(leader=True, failover=failover).leader
    return get_cluster(True, l, [l], failover)


-class MockPatroni(object):
+class MockPostgresql(Mock):
+
+    name = 'postgresql0'
+    role = 'replica'
+    state = 'running'
+    connection_string = 'postgres://foo@bar/postgres'
+
+    def is_healthy(self):
+        return True
+
+    def start(self):
+        return True
+
+    def is_healthiest_node(self, members):
+        return True
+
+    def is_leader(self):
+        return True
+
+    def xlog_position(self):
+        return 0
+
+    def last_operation(self):
+        return 0
+
+    def data_directory_empty(self):
+        return False
+
+    def bootstrap(self, *args, **kwargs):
+        return True
+
+    def check_replication_lag(self, last_leader_operation):
+        return True
+
+    def check_recovery_conf(self, leader):
+        return False
+
+
+class MockPatroni:

    def __init__(self, p, d):
        self.postgresql = p
        self.dcs = d
        self.api = Mock()
-        self.tags = {'foo': 'bar'}
+        self.tags = {}
        self.nofailover = None
-        self.nap_time = 10
-        self.replicatefrom = None
+        self.bdr = {}
        self.api.connection_string = 'http://127.0.0.1:8008'
-        self.clonefrom = None


 def run_async(func, args=()):
-    return func(*args) if args else func()
+    func(*args) if args else func()


-@patch.object(Postgresql, 'is_running', Mock(return_value=True))
-@patch.object(Postgresql, 'is_leader', Mock(return_value=True))
-@patch.object(Postgresql, 'xlog_position', Mock(return_value=0))
-@patch.object(Postgresql, 'call_nowait', Mock(return_value=True))
-@patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
-@patch.object(Postgresql, 'controldata', Mock(return_value={'Database system identifier': '1234567890'}))
-@patch.object(Postgresql, 'sync_replication_slots', Mock())
-@patch.object(Postgresql, 'write_pg_hba', Mock())
-@patch.object(Postgresql, 'write_pgpass', Mock())
-@patch.object(Postgresql, 'write_recovery_conf', Mock())
-@patch.object(Postgresql, 'query', Mock())
-@patch.object(Postgresql, 'checkpoint', Mock())
-@patch.object(etcd.Client, 'write', etcd_write)
-@patch.object(etcd.Client, 'read', etcd_read)
-@patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
-@patch('subprocess.call', Mock(return_value=0))
 class TestHa(unittest.TestCase):

    @patch('socket.getaddrinfo', socket_getaddrinfo)
-    @patch.object(etcd.Client, 'read', etcd_read)
-    def setUp(self):
-        with patch.object(Client, 'machines') as mock_machines:
-            mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
-            self.p = Postgresql({'name': 'postgresql0', 'scope': 'dummy', 'listen': '127.0.0.1:5432',
-                                 'data_dir': 'data/postgresql0', 'superuser': {}, 'admin': {},
-                                 'replication': {'username': '', 'password': '', 'network': ''}})
-            self.p.set_state('running')
-            self.p.set_role('replica')
-            self.p.check_replication_lag = true
-            self.p.can_create_replica_without_replication_connection = MagicMock(return_value=False)
-            self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'})
-            self.ha = Ha(MockPatroni(self.p, self.e))
-            self.ha._async_executor.run_async = run_async
-            self.ha.old_cluster = self.e.get_cluster()
-            self.ha.cluster = get_cluster_not_initialized_without_leader()
-            self.ha.load_cluster_from_dcs = Mock()
+    @patch.object(Client, 'machines')
+    def setUp(self, mock_machines):
+        mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
+        self.p = MockPostgresql()
+        self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'})
+        self.e.client.read = etcd_read
+        self.e.client.write = etcd_write
+        self.e.client.delete = Mock(side_effect=etcd.EtcdException())
+        self.ha = Ha(MockPatroni(self.p, self.e))
+        self.ha._async_executor.run_async = run_async
+        self.ha.old_cluster = self.e.get_cluster()
+        self.ha.cluster = get_cluster_not_initialized_without_leader()
+        self.ha.load_cluster_from_dcs = Mock()

    def test_update_lock(self):
        self.p.last_operation = Mock(side_effect=PostgresException(''))
@@ -111,25 +123,19 @@ class TestHa(unittest.TestCase):

    def test_start_as_replica(self):
        self.p.is_healthy = false
-        self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
+        self.assertEquals(self.ha.run_cycle(), 'started as a secondary')

    def test_recover_replica_failed(self):
        self.p.controldata = lambda: {'Database cluster state': 'in production'}
        self.p.is_healthy = false
-        self.p.is_running = false
-        self.p.follow = false
-        self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
+        self.p.follow_the_leader = false
        self.assertEquals(self.ha.run_cycle(), 'failed to start postgres')

    def test_recover_master_failed(self):
-        self.p.follow = false
+        self.p.follow_the_leader = false
        self.p.is_healthy = false
-        self.p.is_running = false
-        self.p.name = 'leader'
-        self.p.set_role('master')
-        self.p.controldata = lambda: {'Database cluster state': 'in production'}
-        self.ha.cluster = get_cluster_initialized_with_leader()
-        self.assertEquals(self.ha.run_cycle(), 'starting as readonly because i had the session lock')
+        self.ha.has_lock = true
+        self.assertEquals(self.ha.run_cycle(), 'removed leader key after trying and failing to start postgres')

    @patch('sys.exit', return_value=1)
    @patch('patroni.ha.Ha.sysid_valid', MagicMock(return_value=True))
@@ -139,8 +145,7 @@ class TestHa(unittest.TestCase):

    @patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
    def test_start_as_readonly(self):
-        self.p.is_leader = false
-        self.p.is_healthy = true
+        self.p.is_leader = self.p.is_healthy = false
        self.ha.has_lock = true
        self.assertEquals(self.ha.run_cycle(), 'promoted self to leader because i had the session lock')

@@ -154,7 +159,7 @@ class TestHa(unittest.TestCase):

    def test_demote_after_failing_to_obtain_lock(self):
        self.ha.acquire_lock = false
-        self.assertEquals(self.ha.run_cycle(), 'demoted self after trying and failing to obtain lock')
+        self.assertEquals(self.ha.run_cycle(), 'demoted self due after trying and failing to obtain lock')

    def test_follow_new_leader_after_failing_to_obtain_lock(self):
        self.ha.is_healthiest_node = true
@@ -192,30 +197,24 @@ class TestHa(unittest.TestCase):
        self.ha.update_lock = false
        self.assertEquals(self.ha.run_cycle(), 'demoting self because i do not have the lock and i was a leader')

-    def test_follow(self):
+    def test_follow_the_leader(self):
        self.ha.cluster.is_unlocked = false
        self.p.is_leader = false
        self.assertEquals(self.ha.run_cycle(), 'no action.  i am a secondary and i am following a leader')
-        self.ha.patroni.replicatefrom = "foo"
-        self.assertEquals(self.ha.run_cycle(), 'no action.  i am a secondary and i am following a leader')

    def test_no_etcd_connection_master_demote(self):
        self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
        self.assertEquals(self.ha.run_cycle(), 'demoted self because DCS is not accessible and i was a leader')

-    def test_bootstrap_from_another_member(self):
+    def test_bootstrap_from_leader(self):
        self.ha.cluster = get_cluster_initialized_with_leader()
-        self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap from replica \'other\'')
+        self.p.bootstrap = false
+        self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap from leader')

    def test_bootstrap_waiting_for_leader(self):
        self.ha.cluster = get_cluster_initialized_without_leader()
        self.assertEquals(self.ha.bootstrap(), 'waiting for leader to bootstrap')

-    def test_bootstrap_without_leader(self):
-        self.ha.cluster = get_cluster_initialized_without_leader()
-        self.p.can_create_replica_without_replication_connection = MagicMock(return_value=True)
-        self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap (without leader)')
-
    def test_bootstrap_initialize_lock_failed(self):
        self.ha.cluster = get_cluster_not_initialized_without_leader()
        self.assertEquals(self.ha.bootstrap(), 'failed to acquire initialize lock')
@@ -269,65 +268,40 @@ class TestHa(unittest.TestCase):
        self.assertEquals(self.ha.run_cycle(), 'failed to update leader lock during restart')

    @patch('requests.get', requests_get)
-    @patch('time.sleep', Mock())
    def test_manual_failover_from_leader(self):
        self.ha.has_lock = true
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None))
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', ''))
        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', self.p.name, None))
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', MockPostgresql.name))
        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'blabla', None))
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'blabla'))
        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
-        f = Failover(0, self.p.name, '', None)
+        f = Failover(0, MockPostgresql.name, '')
        self.ha.cluster = get_cluster_initialized_with_leader(f)
        self.assertEquals(self.ha.run_cycle(), 'manual failover: demoting myself')
        self.ha.fetch_node_status = lambda e: (e, True, True, 0, {'nofailover': 'True'})
        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
        # manual failover from the previous leader to us won't happen if we hold the nofailover flag
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, None))
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name))
        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')

-        # Failover scheduled time must include timezone
-        scheduled = datetime.datetime.now()
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
-        self.ha.run_cycle()
-
-        scheduled = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
-        self.assertEquals('no action.  i am the leader with the lock', self.ha.run_cycle())
-
-        scheduled = scheduled + datetime.timedelta(seconds=30)
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
-        self.assertEquals('no action.  i am the leader with the lock', self.ha.run_cycle())
-
-        scheduled = scheduled + datetime.timedelta(seconds=-600)
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
-        self.assertEquals('no action.  i am the leader with the lock', self.ha.run_cycle())
-
-        scheduled = None
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
-        self.assertEquals('no action.  i am the leader with the lock', self.ha.run_cycle())
-
    @patch('requests.get', requests_get)
    def test_manual_failover_process_no_leader(self):
        self.p.is_leader = false
-        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', self.p.name, None))
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', MockPostgresql.name))
        self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
-        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader', None))
-        self.p.set_role('replica')
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader'))
        self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
        self.ha.fetch_node_status = lambda e: (e, True, True, 0, {})  # accessible, in_recovery
        self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
-        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, self.p.name, '', None))
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, MockPostgresql.name, ''))
        self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
        self.ha.fetch_node_status = lambda e: (e, False, True, 0, {})  # inaccessible, in_recovery
-        self.p.set_role('replica')
        self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
        # set failover flag to True for all members of the cluster
        # this should elect the current member, as we are not going to call the API for it.
-        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other'))
        self.ha.fetch_node_status = lambda e: (e, True, True, 0, {'nofailover': 'True'})  # accessible, in_recovery
-        self.p.set_role('replica')
        self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
        # same as previous, but set the current member to nofailover. In no case it should be elected as a leader
        self.ha.patroni.nofailover = True
@@ -360,12 +334,3 @@ class TestHa(unittest.TestCase):
        self.ha.fetch_node_status(member)
        member = Member(0, 'test', 1, {'api_url': 'http://localhost:8011/patroni'})
        self.ha.fetch_node_status(member)
-
-    def test_post_recover(self):
-        self.p.is_running = false
-        self.ha.has_lock = true
-        self.assertEqual(self.ha.post_recover(), 'removed leader key after trying and failing to start postgres')
-        self.ha.has_lock = false
-        self.assertEqual(self.ha.post_recover(), 'failed to start postgres')
-        self.p.is_running = true
-        self.assertIsNone(self.ha.post_recover())
@@ -1,5 +1,3 @@
-import etcd
-import os
 import sys
 import time
 import unittest
@@ -8,9 +6,8 @@ import yaml
 from mock import Mock, patch
 from patroni.api import RestApiServer
 from patroni.async_executor import AsyncExecutor
-from patroni.consul import Consul
 from patroni.etcd import Etcd
-from patroni import Patroni, PatroniException, main as _main
+from patroni import Patroni, main
 from patroni.zookeeper import ZooKeeper
 from six.moves import BaseHTTPServer
 from test_etcd import Client, SleepException, etcd_read, etcd_write
@@ -18,6 +15,10 @@ from test_postgresql import Postgresql, psycopg2_connect
 from test_zookeeper import MockKazooClient


+def time_sleep(*args):
+    raise SleepException()
+
+
@patch('time.sleep', Mock())
@patch('subprocess.call', Mock(return_value=0))
@patch('psycopg2.connect', psycopg2_connect)
@@ -25,52 +26,46 @@ from test_zookeeper import MockKazooClient
@patch.object(Postgresql, 'write_recovery_conf', Mock())
@patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
@patch.object(AsyncExecutor, 'run', Mock())
-@patch.object(etcd.Client, 'write', etcd_write)
-@patch.object(etcd.Client, 'read', etcd_read)
 class TestPatroni(unittest.TestCase):

-    def setUp(self):
-        with patch.object(Client, 'machines') as mock_machines:
-            mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
-            self.touched = False
-            self.init_cancelled = False
-            RestApiServer._BaseServer__is_shut_down = Mock()
-            RestApiServer._BaseServer__shutdown_request = True
-            RestApiServer.socket = 0
-            with open('postgres0.yml', 'r') as f:
-                config = yaml.load(f)
-                self.p = Patroni(config)
+    @patch.object(Client, 'machines')
+    def setUp(self, mock_machines):
+        mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
+        self.touched = False
+        self.init_cancelled = False
+        RestApiServer._BaseServer__is_shut_down = Mock()
+        RestApiServer._BaseServer__shutdown_request = True
+        RestApiServer.socket = 0
+        with open('postgres0.yml', 'r') as f:
+            config = yaml.load(f)
+            self.p = Patroni(config)
+            self.p.ha.dcs.client.write = etcd_write
+            self.p.ha.dcs.client.read = etcd_read

    @patch('patroni.zookeeper.KazooClient', MockKazooClient())
-    @patch.object(Consul, 'create_or_restore_session', Mock())
    def test_get_dcs(self):
        self.assertIsInstance(self.p.get_dcs('', {'zookeeper': {'scope': '', 'hosts': ''}}), ZooKeeper)
-        self.assertIsInstance(self.p.get_dcs('', {'consul': {'scope': '', 'hosts': '127.0.0.1:1'}}), Consul)
-        self.assertRaises(PatroniException, self.p.get_dcs, '', {})
+        self.assertRaises(Exception, self.p.get_dcs, '', {})

-    @patch('time.sleep', Mock(side_effect=SleepException))
+    @patch('time.sleep', Mock(side_effect=SleepException()))
    @patch.object(Etcd, 'delete_leader', Mock())
    @patch.object(Client, 'machines')
    def test_patroni_main(self, mock_machines):
-        _main()
+        main()
        sys.argv = ['patroni.py', 'postgres0.yml']

        mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
-        with patch.object(Patroni, 'run', Mock(side_effect=SleepException)):
-            self.assertRaises(SleepException, _main)
-        with patch.object(Patroni, 'run', Mock(side_effect=KeyboardInterrupt())):
-            _main()
-        sys.argv = ['patroni.py']
-        # read the content of the yaml configuration file into the environment variable
-        # in order to test how does patroni handle the configuration passed from the environment.
-        with open('postgres0.yml', 'r') as f:
-            os.environ[Patroni.PATRONI_CONFIG_VARIABLE] = f.read()
        with patch.object(Patroni, 'run', Mock(side_effect=SleepException())):
-            self.assertRaises(SleepException, _main)
-        del os.environ[Patroni.PATRONI_CONFIG_VARIABLE]
+            self.assertRaises(SleepException, main)
+        with patch.object(Patroni, 'run', Mock(side_effect=KeyboardInterrupt())):
+            main()

+    @patch('time.sleep', Mock(side_effect=SleepException()))
    def test_run(self):
-        self.p.ha.dcs.watch = Mock(side_effect=SleepException)
+        self.p.ha.dcs.watch = time_sleep
+        self.assertRaises(SleepException, self.p.run)
+
+        self.p.ha.state_handler.is_leader = Mock(return_value=False)
        self.p.api.start = Mock()
        self.assertRaises(SleepException, self.p.run)

@@ -80,17 +75,8 @@ class TestPatroni(unittest.TestCase):
        self.p.next_run = time.time() - self.p.nap_time - 1
        self.p.schedule_next_run()

-    def test_noloadbalance(self):
-        self.p.tags['noloadbalance'] = True
-        self.assertTrue(self.p.noloadbalance)
-
    def test_nofailover(self):
        self.p.tags['nofailover'] = True
        self.assertTrue(self.p.nofailover)
        self.p.tags['nofailover'] = None
        self.assertFalse(self.p.nofailover)
-
-    def test_replicatefrom(self):
-        self.assertIsNone(self.p.replicatefrom)
-        self.p.tags['replicatefrom'] = 'foo'
-        self.assertEqual(self.p.replicatefrom, 'foo')
@@ -2,19 +2,24 @@ import mock  # for the mock.call method, importing it without a namespace breaks
 import os
 import psycopg2
 import shutil
-import subprocess
 import unittest

+from six.moves import builtins
 from mock import Mock, MagicMock, PropertyMock, patch, mock_open
 from patroni.dcs import Cluster, Leader, Member
 from patroni.exceptions import PostgresException, PostgresConnectionException
 from patroni.postgresql import Postgresql
 from patroni.utils import RetryFailedError
-from six.moves import builtins
 from test_ha import false
+import subprocess


-class MockCursor(object):
+def is_file_raise_on_backup(*args, **kwargs):
+    if args[0].endswith('.backup'):
+        raise Exception("foo")
+
+
+class MockCursor:

    def __init__(self, connection):
        self.connection = connection
@@ -33,7 +38,7 @@ class MockCursor(object):
        elif sql == 'SELECT pg_is_in_recovery()':
            self.results = [(False, )]
        elif sql.startswith('SELECT to_char(pg_postmaster_start_time'):
-            self.results = [('', True, '', '', '', '', False)]
+            self.results = [('', True, '', '', '', False)]
        else:
            self.results = [(
                None,
@@ -54,6 +59,9 @@ class MockCursor(object):
    def fetchall(self):
        return self.results

+    def close(self):
+        pass
+
    def __iter__(self):
        for i in self.results:
            yield i
@@ -65,9 +73,8 @@ class MockCursor(object):
        pass


-class MockConnect(object):
+class MockConnect(Mock):

-    server_version = '99999'
    autocommit = False
    closed = 0

@@ -80,10 +87,6 @@ class MockConnect(object):
    def __exit__(self, *args):
        pass

-    @staticmethod
-    def close():
-        pass
-

 def pg_controldata_string(*args, **kwargs):
    return b"""
@@ -151,12 +154,9 @@ def psycopg2_connect(*args, **kwargs):
    return MockConnect()


-def fake_listdir(path):
-    return ["a", "b", "c"] if path.endswith('pg_xlog/archive_status') else []
-
-
@patch('subprocess.call', Mock(return_value=0))
@patch('psycopg2.connect', psycopg2_connect)
+@patch('shutil.copy', Mock())
 class TestPostgresql(unittest.TestCase):

    @patch('subprocess.call', Mock(return_value=0))
@@ -164,14 +164,13 @@ class TestPostgresql(unittest.TestCase):
    def setUp(self):
        self.p = Postgresql({'name': 'test0', 'scope': 'batman', 'data_dir': 'data/test0',
                             'listen': '127.0.0.1, *:5432', 'connect_address': '127.0.0.2:5432',
-                             'pg_hba': ['host replication replicator 127.0.0.1/32 md5',
-                                        'hostssl all all 0.0.0.0/0 md5',
-                                        'host all all 0.0.0.0/0 md5'],
-                             'superuser': {'username': 'test', 'password': 'test'},
+                             'pg_hba': ['hostssl all all 0.0.0.0/0 md5', 'host all all 0.0.0.0/0 md5'],
+                             'superuser': {'password': 'test'},
                             'admin': {'username': 'admin', 'password': 'admin'},
                             'pg_rewind': {'username': 'admin', 'password': 'admin'},
                             'replication': {'username': 'replicator',
-                                             'password': 'rep-pass'},
+                                             'password': 'rep-pass',
+                                             'network': '127.0.0.1/32'},
                             'parameters': {'foo': 'bar'}, 'recovery_conf': {'foo': 'bar'},
                             'callbacks': {'on_start': 'true', 'on_stop': 'true',
                                           'on_restart': 'true', 'on_role_change': 'true',
@@ -182,8 +181,7 @@ class TestPostgresql(unittest.TestCase):
            os.makedirs(self.p.data_dir)
        self.leadermem = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres'})
        self.leader = Leader(-1, 28, self.leadermem)
-        self.other = Member(0, 'test1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres',
-                            'tags': {'replicatefrom': 'leader'}})
+        self.other = Member(0, 'test1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres'})
        self.me = Member(0, 'test0', 28, {'conn_url': 'postgres://replicator:[email protected]:5434/postgres'})

    def tearDown(self):
@@ -206,16 +204,6 @@ class TestPostgresql(unittest.TestCase):
        self.assertTrue(self.p.initialize())
        self.assertTrue(os.path.exists(os.path.join(self.p.data_dir, 'pg_hba.conf')))

-        with open(os.path.join(self.p.data_dir, 'pg_hba.conf')) as f:
-            lines = f.readlines()
-            assert 'host replication replicator 127.0.0.1/32 md5\n' in lines
-            assert 'host all all 0.0.0.0/0 md5\n' in lines
-
-    @patch('os.path.exists', Mock(return_value=True))
-    @patch('os.unlink', Mock())
-    def test_delete_trigger_file(self):
-        self.p.delete_trigger_file()
-
    def test_start(self):
        self.assertTrue(self.p.start())
        self.p.is_running = false
@@ -240,12 +228,10 @@ class TestPostgresql(unittest.TestCase):
        self.p.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo', 'password': 'bar'})

    @patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
-    def test_sync_replica(self):
-        self.assertTrue(self.p.sync_replica(self.leader))
-        self.p.create_replica = Mock(return_value=1)
-        self.assertFalse(self.p.sync_replica(self.leader))
+    def test_sync_from_leader(self):
+        self.assertTrue(self.p.sync_from_leader(self.leader))

-    @patch('subprocess.call', side_effect=OSError)
+    @patch('subprocess.call', side_effect=Exception("Test"))
    @patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
    def test_pg_rewind(self, mock_call):
        self.assertTrue(self.p.rewind(self.leader))
@@ -256,26 +242,26 @@ class TestPostgresql(unittest.TestCase):
    @patch('patroni.postgresql.Postgresql.remove_data_directory', MagicMock(return_value=True))
    @patch('patroni.postgresql.Postgresql.single_user_mode', MagicMock(return_value=1))
    @patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
-    @patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string))
-    def test_follow(self, mock_pg_rewind):
-        self.p.follow(None)
-        self.p.follow(self.leader)
-        self.p.follow(Leader(-1, 28, self.other))
+    def test_follow_the_leader(self, mock_pg_rewind):
+        self.p.demote()
+        self.p.follow_the_leader(None)
+        self.p.demote()
+        self.p.follow_the_leader(self.leader)
+        self.p.follow_the_leader(Leader(-1, 28, self.other))
        self.p.rewind = mock_pg_rewind
-        self.p.follow(self.leader)
+        self.p.follow_the_leader(self.leader)
+        self.p.require_rewind()
        with mock.patch('os.path.islink', MagicMock(return_value=True)):
            with mock.patch('patroni.postgresql.Postgresql.can_rewind', new_callable=PropertyMock(return_value=True)):
                with mock.patch('os.unlink', MagicMock(return_value=True)):
-                    self.p.follow(self.leader, recovery=True)
+                    self.p.follow_the_leader(self.leader, recovery=True)
+        self.p.require_rewind()
        with mock.patch('patroni.postgresql.Postgresql.can_rewind', new_callable=PropertyMock(return_value=True)):
            self.p.rewind.return_value = True
-            self.p.follow(self.leader, recovery=True)
+            self.p.follow_the_leader(self.leader, recovery=True)
            self.p.rewind.return_value = False
-            self.p.follow(self.leader, recovery=True)
-        with mock.patch('patroni.postgresql.Postgresql.check_recovery_conf', MagicMock(return_value=True)):
-            self.assertTrue(self.p.follow(None))
+            self.p.follow_the_leader(self.leader, recovery=True)

-    @patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string))
    def test_can_rewind(self):
        tmp = self.p.pg_rewind
        self.p.pg_rewind = None
@@ -283,16 +269,16 @@ class TestPostgresql(unittest.TestCase):
        self.p.pg_rewind = tmp
        with mock.patch('subprocess.call', MagicMock(return_value=1)):
            self.assertFalse(self.p.can_rewind)
-        with mock.patch('subprocess.call', side_effect=OSError):
+        with mock.patch('subprocess.call', side_effect=OSError("foo")):
            self.assertFalse(self.p.can_rewind)
-        tmp = self.p.controldata
+        tmp = self.p.controldata()
        self.p.controldata = lambda: {'wal_log_hints setting': 'on'}
        self.assertTrue(self.p.can_rewind)
        self.p.controldata = tmp

    @patch('time.sleep', Mock())
    def test_create_replica(self):
-        self.p.delete_trigger_file = Mock(side_effect=OSError)
+        self.p.delete_trigger_file = Mock(side_effect=OSError())
        with patch('subprocess.call', Mock(side_effect=[1, 0])):
            self.assertEquals(self.p.create_replica(self.leader, ''), 0)
        with patch('subprocess.call', Mock(side_effect=[Exception(), 0])):
@@ -308,6 +294,12 @@ class TestPostgresql(unittest.TestCase):
        with patch('subprocess.call', Mock(side_effect=Exception("foo"))):
            self.assertEquals(self.p.create_replica(self.leader, ''), 1)

+    def test_create_connection_users(self):
+        cfg = self.p.config
+        cfg['superuser']['username'] = 'test'
+        p = Postgresql(cfg)
+        p.create_connection_users()
+
    def test_sync_replication_slots(self):
        self.p.start()
        cluster = Cluster(True, self.leader, 0, [self.me, self.other, self.leadermem], None)
@@ -315,15 +307,12 @@ class TestPostgresql(unittest.TestCase):
        self.p.query = Mock(side_effect=psycopg2.OperationalError)
        self.p.schedule_load_slots = True
        self.p.sync_replication_slots(cluster)
-        self.p.schedule_load_slots = False
-        with mock.patch('patroni.postgresql.Postgresql.role', new_callable=PropertyMock(return_value='replica')):
-            self.p.sync_replication_slots(cluster)

    @patch.object(MockConnect, 'closed', 2)
    def test__query(self):
-        self.assertRaises(PostgresConnectionException, self.p._query, 'blabla')
+        self.assertRaises(PostgresConnectionException, self.p._query, 'blabla', 'postgres')
        self.p._state = 'restarting'
-        self.assertRaises(RetryFailedError, self.p._query, 'blabla')
+        self.assertRaises(RetryFailedError, self.p._query, 'blabla', 'postgres')

    def test_query(self):
        self.p.query('select 1')
@@ -349,7 +338,7 @@ class TestPostgresql(unittest.TestCase):
    def test_last_operation(self):
        self.assertEquals(self.p.last_operation(), '0')

-    @patch('subprocess.Popen', Mock(side_effect=OSError))
+    @patch('subprocess.Popen', Mock(side_effect=OSError()))
    def test_call_nowait(self):
        self.assertFalse(self.p.call_nowait('on_start'))

@@ -369,7 +358,7 @@ class TestPostgresql(unittest.TestCase):
    def test_move_data_directory(self):
        self.p.is_running = false
        self.p.move_data_directory()
-        with patch('os.rename', Mock(side_effect=OSError)):
+        with patch('os.rename', Mock(side_effect=OSError())):
            self.p.move_data_directory()

    @patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
@@ -377,8 +366,7 @@ class TestPostgresql(unittest.TestCase):
        with patch('subprocess.call', Mock(return_value=1)):
            self.assertRaises(PostgresException, self.p.bootstrap)
        self.p.bootstrap()
-        with patch('patroni.postgresql.Postgresql.sync_replica', MagicMock(return_value=True)):
-            self.p.bootstrap(self.leader)
+        self.p.bootstrap(self.leader)

    def test_remove_data_directory(self):
        self.p.data_dir = 'data_dir'
@@ -388,20 +376,26 @@ class TestPostgresql(unittest.TestCase):
        open(self.p.data_dir, 'w').close()
        self.p.remove_data_directory()
        os.symlink('unexisting', self.p.data_dir)
-        with patch('os.unlink', Mock(side_effect=OSError)):
+        with patch('os.unlink', Mock(side_effect=Exception)):
            self.p.remove_data_directory()
        self.p.remove_data_directory()

-    def test_controldata(self):
-        with patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string)):
-            data = self.p.controldata()
-            self.assertEquals(len(data), 50)
-            self.assertEquals(data['Database cluster state'], 'shut down in recovery')
-            self.assertEquals(data['wal_log_hints setting'], 'on')
-            self.assertEquals(int(data['Database block size']), 8192)
+    @patch('subprocess.check_output', MagicMock(return_value=0, side_effect=pg_controldata_string))
+    @patch('subprocess.check_output', side_effect=subprocess.CalledProcessError)
+    @patch('subprocess.check_output', side_effect=Exception('Failed'))
+    def test_controldata(self, check_output_call_error, check_output_generic_exception):
+        data = self.p.controldata()
+        self.assertEquals(len(data), 50)
+        self.assertEquals(data['Database cluster state'], 'shut down in recovery')
+        self.assertEquals(data['wal_log_hints setting'], 'on')
+        self.assertEquals(int(data['Database block size']), 8192)

-        with patch('subprocess.check_output', Mock(side_effect=subprocess.CalledProcessError(1, ''))):
-            self.assertEquals(self.p.controldata(), {})
+        subprocess.check_output = check_output_call_error
+        data = self.p.controldata()
+        self.assertEquals(data, dict())
+
+        subprocess.check_output = check_output_generic_exception
+        self.assertRaises(Exception, self.p.controldata())

    def test_read_postmaster_opts(self):
        m = mock_open(read_data=postmaster_opts_string())
@@ -411,10 +405,13 @@ class TestPostgresql(unittest.TestCase):
            self.assertEquals(int(data['max_replication_slots']), 5)
            self.assertEqual(data.get('D'), None)

-            m.side_effect = IOError
+            m.side_effect = IOError("foo")
            data = self.p.read_postmaster_opts()
            self.assertEqual(data, dict())

+            m.side_effect = Exception("foo")
+            self.assertRaises(Exception, self.p.read_postmaster_opts())
+
    @patch('subprocess.Popen')
    @patch.object(builtins, 'open', MagicMock(return_value=42))
    def test_single_user_mode(self, subprocess_popen_mock):
@@ -434,7 +431,13 @@ class TestPostgresql(unittest.TestCase):
        subprocess_popen_mock.return_value = None
        self.assertEquals(self.p.single_user_mode(), 1)

+    def fake_listdir(path):
+        if path.endswith(os.path.join('pg_xlog', 'archive_status')):
+            return ["a", "b", "c"]
+        return []
+
    @patch('os.listdir', MagicMock(side_effect=fake_listdir))
+    @patch('os.path.isdir', MagicMock(return_value=True))
    @patch('os.unlink', return_value=True)
    @patch('os.remove', return_value=True)
    @patch('os.path.islink', return_value=False)
@@ -456,8 +459,8 @@ class TestPostgresql(unittest.TestCase):
        mock_unlink.reset_mock()
        mock_remove.reset_mock()

-        mock_file.side_effect = OSError
-        mock_link.side_effect = OSError
+        mock_file.side_effect = Exception("foo")
+        mock_link.side_effect = Exception("foo")
        self.p.cleanup_archive_status()
        mock_unlink.assert_not_called()
        mock_remove.assert_not_called()
@@ -466,27 +469,14 @@ class TestPostgresql(unittest.TestCase):
    def test_sysid(self):
        self.assertEqual(self.p.sysid, "6200971513092291716")

-    @patch('os.path.isfile', Mock(return_value=True))
-    @patch('shutil.copy', Mock(side_effect=IOError))
-    def test_save_configuration_files(self):
+    @patch('os.path.isfile', MagicMock(return_value=True))
+    @patch('shutil.copy', side_effect=Exception)
+    def test_save_configuration_files(self, mock_copy):
+        shutil.copy = mock_copy
        self.p.save_configuration_files()

-    @patch('os.path.isfile', Mock(side_effect=[False, True]))
-    @patch('shutil.copy', Mock(side_effect=IOError))
-    def test_restore_configuration_files(self):
+    @patch('os.path.isfile', MagicMock(side_effect=is_file_raise_on_backup))
+    @patch('shutil.copy', side_effect=Exception)
+    def test_restore_configuration_files(self, mock_copy):
+        shutil.copy = mock_copy
        self.p.restore_configuration_files()
-
-    def test_can_create_replica_without_replication_connection(self):
-        self.p.config['create_replica_method'] = []
-        self.assertFalse(self.p.can_create_replica_without_replication_connection())
-        self.p.config['create_replica_method'] = ['wale', 'basebackup']
-        self.p.config['wale'] = {'command': 'foo', 'no_master': 1}
-        self.assertTrue(self.p.can_create_replica_without_replication_connection())
-
-    def test_replica_method_can_work_without_replication_connection(self):
-        self.assertFalse(self.p.replica_method_can_work_without_replication_connection('basebackup'))
-        self.assertFalse(self.p.replica_method_can_work_without_replication_connection('foobar'))
-        self.p.config['foo'] = {'command': 'bar', 'no_master': 1}
-        self.assertTrue(self.p.replica_method_can_work_without_replication_connection('foo'))
-        self.p.config['foo'] = {'command': 'bar'}
-        self.assertFalse(self.p.replica_method_can_work_without_replication_connection('foo'))
@@ -2,8 +2,7 @@ import unittest

 from mock import Mock, patch
 from patroni.exceptions import PatroniException
-from patroni.utils import reap_children, Retry, RetryFailedError, set_ignore_sigterm,\
-    sigchld_handler, sigterm_handler, sleep
+from patroni.utils import Retry, RetryFailedError, reap_children, sigchld_handler, sigterm_handler, sleep


 def time_sleep(_):
@@ -13,26 +12,24 @@ def time_sleep(_):
 class TestUtils(unittest.TestCase):

    def test_sigterm_handler(self):
-        set_ignore_sigterm(False)
        self.assertRaises(SystemExit, sigterm_handler, None, None)

    @patch('time.sleep', Mock())
    def test_reap_children(self):
-        self.assertIsNone(reap_children())
+        reap_children()
        with patch('os.waitpid', Mock(return_value=(0, 0))):
            sigchld_handler(None, None)
-            self.assertIsNone(reap_children())
+            reap_children()

    @patch('time.sleep', time_sleep)
    def test_sleep(self):
-        self.assertIsNone(sleep(0.01))
+        sleep(0.01)


@patch('time.sleep', Mock())
 class TestRetrySleeper(unittest.TestCase):

-    @staticmethod
-    def _fail(times=1):
+    def _fail(self, times=1):
        scope = dict(times=0)

        def inner():
@@ -43,33 +40,36 @@ class TestRetrySleeper(unittest.TestCase):
                raise PatroniException('Failed!')
        return inner

+    def _makeOne(self, *args, **kwargs):
+        return Retry(*args, **kwargs)
+
    def test_reset(self):
-        retry = Retry(delay=0, max_tries=2)
+        retry = self._makeOne(delay=0, max_tries=2)
        retry(self._fail())
        self.assertEquals(retry._attempts, 1)
        retry.reset()
        self.assertEquals(retry._attempts, 0)

    def test_too_many_tries(self):
-        retry = Retry(delay=0)
+        retry = self._makeOne(delay=0)
        self.assertRaises(RetryFailedError, retry, self._fail(times=999))
        self.assertEquals(retry._attempts, 1)

    def test_maximum_delay(self):
-        retry = Retry(delay=10, max_tries=100)
+        retry = self._makeOne(delay=10, max_tries=100)
        retry(self._fail(times=10))
        self.assertTrue(retry._cur_delay < 4000, retry._cur_delay)
        # gevent's sleep function is picky about the type
        self.assertEquals(type(retry._cur_delay), float)

    def test_deadline(self):
-        retry = Retry(deadline=0.0001)
+        retry = self._makeOne(deadline=0.0001)
        self.assertRaises(RetryFailedError, retry, self._fail(times=100))

    def test_copy(self):
        def _sleep(t):
-            pass
+            None

-        retry = Retry(sleep_func=_sleep)
+        retry = self._makeOne(sleep_func=_sleep)
        rcopy = retry.copy()
        self.assertTrue(rcopy.sleep_func is _sleep)
@@ -1,9 +1,25 @@
+import unittest
+from mock import MagicMock, patch, PropertyMock
+import os
 import psycopg2
 import subprocess
-import unittest
+from patroni.scripts.wale_restore import WALERestore

-from mock import MagicMock, patch, PropertyMock
-from patroni.scripts.wale_restore import WALERestore, main as _main
+
+def fake_cursor_fetchone(*args, **kwargs):
+    return ('16777216',)
+
+
+def fake_call_fail_for_wal_e(*args, **kwargs):
+    if len(args) > 0 and 'backup-fetch' in args[0]:
+        return 1
+    return 0
+
+
+def fake_call_fail_for_base_backup(*args, **kwargs):
+    if len(args) > 0 and 'backup-fetch' in args[0]:
+        return 0
+    return 1


 def fake_backup_data(self, *args, **kwargs):
@@ -12,19 +28,16 @@ def fake_backup_data(self, *args, **kwargs):
 base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z 167772160   00000001000000000000007F    00000040 00000001000000000000007F    00000240
 """

-
 def fake_backup_data_2(self, *args, **kwargs):
    """ return the fake result of WAL-E backup-list"""
    return """name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop """

-
 def fake_backup_data_3(self, *args, **kwargs):
    """ return the fake result of WAL-E backup-list"""
    return """name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop
 base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z 167772160   00000001000000000000007F    00000040 00000001000000000000007F    00000240
 """

-
 def fake_backup_data_4(self, *args, **kwargs):
    """ return the fake result of WAL-E backup-list"""
    return """name    last_modified   expanded_size_foo wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop
@@ -36,6 +49,7 @@ base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z 167772160   000
@patch('os.makedirs', MagicMock(return_value=True))
@patch('os.path.exists', MagicMock(return_value=True))
@patch('os.path.isdir', MagicMock(return_value=True))
+@patch('psycopg2.extensions.cursor.fetchone', MagicMock(side_effect=fake_cursor_fetchone))
@patch('psycopg2.extensions.cursor', MagicMock(autospec=True))
@patch('psycopg2.extensions.connection', MagicMock(autospec=True))
@patch('psycopg2.connect', MagicMock(autospec=True))
@@ -43,7 +57,11 @@ base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z 167772160   000
 class TestWALERestore(unittest.TestCase):

    def setUp(self):
-        self.wale_restore = WALERestore("batman", "/data", "host=batman port=5432 user=batman", "/etc", 100, 100, 1, 0)
+        self.wale_restore = WALERestore("batman", "/data",
+                                        "host=batman port=5432 user=batman", "/etc", 100, 100, 1)
+
+    def tearDown(self):
+        pass

    def test_should_use_s3_to_create_replica(self):
        with patch('psycopg2.connect', MagicMock(side_effect=psycopg2.Error("foo"))):
@@ -58,8 +76,6 @@ class TestWALERestore(unittest.TestCase):
            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())

        self.wale_restore.should_use_s3_to_create_replica()
-        self.wale_restore.no_master = 1
-        self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())

    def test_create_replica_with_s3(self):
        with patch('subprocess.call', MagicMock(return_value=0)):
@@ -73,8 +89,3 @@ class TestWALERestore(unittest.TestCase):
        with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', MagicMock(return_value=True)):
            with patch.object(self.wale_restore, 'create_replica_with_s3', MagicMock(return_value=0)):
                self.assertEqual(self.wale_restore.run(), 0)
-
-    @patch('sys.exit', MagicMock())
-    @patch.object(WALERestore, 'run', MagicMock(return_value=0))
-    def test_main(self):
-        self.assertEqual(_main(), None)
@@ -14,13 +14,13 @@ class MockKazooClient(Mock):

    leader = False
    exists = True
+    handler = Mock()

    @property
    def client_id(self):
        return (-1, '')

-    @staticmethod
-    def retry(func, *args, **kwargs):
+    def retry(self, func, *args, **kwargs):
        func(*args, **kwargs)

    def get(self, path, watch=None):
@@ -33,6 +33,8 @@ class MockKazooClient(Mock):
                b'postgres://repuser:rep-pass@localhost:5434/postgres?application_name=http://127.0.0.1:8009/patroni',
                ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0 if self.exists else -1, 0, 0, 0)
            )
+        elif path.endswith('/optime/leader'):
+            return (b'1', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))
        elif path.endswith('/leader'):
            if self.leader:
                return (b'foo', ZnodeStat(0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0))
@@ -41,8 +43,7 @@ class MockKazooClient(Mock):
            return (b'foo', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))
        return (b'', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))

-    @staticmethod
-    def get_children(path, watch=None, include_data=False):
+    def get_children(self, path, watch=None, include_data=False):
        if not isinstance(path, six.string_types):
            raise TypeError("Invalid type for 'path' (string expected)")
        if path.startswith('/no_node'):
@@ -61,16 +62,16 @@ class MockKazooClient(Mock):
        elif value == b'retry' or (value == b'exists' and self.exists):
            raise NodeExistsError

-    @staticmethod
-    def set(path, value, version=-1):
+    def set(self, path, value, version=-1):
        if not isinstance(path, six.string_types):
            raise TypeError("Invalid type for 'path' (string expected)")
        if not isinstance(value, (six.binary_type,)):
            raise TypeError("Invalid type for 'value' (must be a byte string)")
        if path == '/service/bla/optime/leader':
            raise Exception
-        if path == '/service/test/members/bar' and value == b'retry':
-            return
+        if path == '/service/test/members/bar':
+            if value == b'retry':
+                return
        if path == '/service/test/failover':
            if value == b'Exception':
                raise Exception
@@ -83,11 +84,13 @@ class MockKazooClient(Mock):
            raise TypeError("Invalid type for 'path' (string expected)")
        self.exists = False
        if path == '/service/test/leader':
+            if self.leader:
+                return
            self.leader = True
            raise Exception
        elif path == '/service/test/members/buzz':
            raise Exception
-        elif path.endswith('/') or path.endswith('/initialize') or path == '/service/test/members/bar':
+        elif path.endswith('/initialize') or path == '/service/test/members/bar':
            raise NoNodeError


@@ -147,18 +150,16 @@ class TestZooKeeper(unittest.TestCase):
        self.zk._name = 'bar'
        self.zk.touch_member('new')
        self.zk._name = 'na'
-        self.zk._client.exists = 1
+        self.zk.client.exists = 1
        self.zk.touch_member('exists')
        self.zk._name = 'bar'
        self.zk.touch_member('retry')
-        self.zk._fetch_cluster = True
+        self.zk.fetch_cluster = True
        self.zk.get_cluster()
        self.zk.touch_member('retry')

    def test_take_leader(self):
        self.zk.take_leader()
-        with patch.object(MockKazooClient, 'create', Mock(side_effect=Exception)):
-            self.zk.take_leader()

    def test_update_leader(self):
        self.assertTrue(self.zk.update_leader())
@@ -169,9 +170,6 @@ class TestZooKeeper(unittest.TestCase):
        self.zk._base_path = self.zk._base_path.replace('test', 'bla')
        self.zk.write_leader_optime('2')

-    def test_delete_cluster(self):
-        self.assertTrue(self.zk.delete_cluster())
-
    def test_watch(self):
        self.zk.watch(0)
        self.zk.event.isSet = lambda: True