Fix the README in order to show BDR-related PostgreSQL options.

Currently, BDR and physical replication at the same time
is not supported. BDR requires additional postgresql
configuration and a patched version of PostgreSQL 9.4 + BDR plugin.
Included is the Docker image to try it locally.

.travis.yml

@@ -1,32 +1,14 @@
-sudo: required
 language: python
-addons:
-  postgresql: "9.5"
-env:
-  global:
-    - BOTO_CONFIG='' ETCDVERSION=2.2.5
-  matrix:
-    - TEST_SUITE="python setup.py test"
-    - TEST_SUITE="behave"
 python:
   - "2.7"
+  - "3.3"
   - "3.4"
-  - "3.5"
 install:
-  - sudo /etc/init.d/postgresql stop
+  - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py2.txt --use-mirrors; fi
-  - sudo apt-get -y remove --purge postgresql-9.1 postgresql-9.2 postgresql-9.3 postgresql-9.4
+  - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r requirements-py3.txt --use-mirrors; fi
-  - sudo apt-get -y autoremove
+  - pip install coveralls
-  - sudo apt-key adv --keyserver keys.gnupg.net --recv-keys 7FCC7D46ACCC4CF8
-  - sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main 9.5" >> /etc/apt/sources.list.d/postgresql.list'
-  - sudo apt-get update
-  - sudo apt-get -y install postgresql-9.5
-  - sudo /etc/init.d/postgresql stop
-  - pip install -r requirements.txt
-  - curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C . --strip=1 --wildcards --no-anchored etcd
-  - pip install behave codacy-coverage coverage coveralls 
 script:
-  - PATH=.:$PATH $TEST_SUITE
+  - python setup.py test
   - python setup.py flake8
 after_success:
   - coveralls
-  - if [[ -f coverage.xml ]]; then python-codacy-coverage -r coverage.xml; fi

.zappr.yml

@@ -1,12 +0,0 @@
-approvals:
-  # PR needs at least 4 approvals
-  minimum: 1
-  # approval = comment that matches this regex
-  pattern: "^:?\\+1:?$"
-  from:
-    # commenter must be either one of:
-    # a public zalando org member
-    orgs:
-      - zalando
-    # a collaborator of the repo
-    collaborators: true

Dockerfile

@@ -6,28 +6,30 @@ MAINTAINER Feike Steenbergen <[email protected]>
 # We need curl
 RUN apt-get update -y && apt-get install curl -y
 
-# Add PGDG repositories
+# Add PGDG and BDR repositories
 RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
+RUN echo "deb http://packages.2ndquadrant.com/bdr/apt/ $(lsb_release -cs)-2ndquadrant main" >> /etc/apt/sources.list.d/pgdg.list
 RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+# import the BDR key
+RUN curl -s -o - http://packages.2ndquadrant.com/bdr/apt/AA7A6805.asc | sudo apt-key add -
+
 RUN apt-get update -y
 RUN apt-get upgrade -y
 
-ENV PGVERSION 9.5
+ENV PGVERSION 9.4
-RUN apt-get install postgresql-${PGVERSION} postgresql-server-dev-${PGVERSION} -y
+ENV PGTYPE postgresql-bdr
-RUN apt-get install python python-dev python-pip -y
+ENV PGCOMPATIBLETYPE postgresql
-ADD requirements-py2.txt /requirements-py2.txt
+
-RUN pip install -r /requirements-py2.txt
+RUN apt-get install python python-yaml python-requests python-boto ${PGTYPE}-${PGVERSION} ${PGTYPE}-${PGVERSION}-bdr-plugin python-dnspython python-kazoo python-pip -y
+RUN apt-get install python-dev ${PGTYPE}-server-dev-${PGVERSION} -y
+RUN pip install python-etcd psycopg2
 
 ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:$PATH
 
 ADD patroni.py /patroni.py
-ADD patronictl.py /patronictl.py
 ADD patroni/ /patroni
 
-RUN ln -s /patroni.py /usr/local/bin/patroni
+ENV ETCDVERSION 2.0.13
-RUN ln -s /patronictl.py /usr/local/bin/patronictl
-
-ENV ETCDVERSION 2.2.5
 RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C /bin --strip=1 --wildcards --no-anchored etcd etcdctl
 
 ### Setting up a simple script that will serve as an entrypoint

MAINTAINERS

@@ -1,3 +0,0 @@
-Alexander Kukushkin <[email protected]>
-Feike Steenbergen <[email protected]>
-Oleksii Kliukin <[email protected]>

README.rst

@@ -75,10 +75,13 @@ For an example file, see ``postgres0.yml``. Regarding settings:
         -  *port*: Exhibitor port.
         -  *hosts*: initial list of Exhibitor (ZooKeeper) nodes in format: ['host1', 'host2', 'etc...' ]. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes.
 
+-  *bdr*:
+    - *enable*: on if you want to enable BDR
+    - *database*: database name to support BDR (only a single database is supported)
+
 -  *postgresql*:
     -  *name*: the name of the Postgres host. Must be unique for the cluster.
-    -  *listen*: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. The first address from this list will be used by Patroni to establish local connections to the PostgreSQL node. 
+    -  *listen*: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication.
-    
     -  *connect\_address*: IP address + port through which Postgres is accessible from other nodes and applications.
     -  *data\_dir*: file path to initialize and store Postgres data files.
     -  *maximum\_lag\_on\_failover*: the maximum bytes a follower may lag.
@@ -164,6 +167,31 @@ Choosing your replication schema is dependent on your business
 considerations. Investigate both async and sync replication, as well as other
 HA solutions, to determine which solution is best for you.
 
+You can also use BDR (bi-directional replication) if you have a compatible
+PostgreSQL version with the BDR plugin installed (see http://bdr-project.org/docs/next/installation.html).
+It will require adding a BDR shared_library to your configuration, as well as
+setting the following options for postgresql (see http://bdr-project.org/docs/next/settings-prerequisite.html):
+
+.. code:: YAML
+
+        max_worker_processes: 10
+        max_replication_slots: 10
+        max_wal_senders: 10
+        shared_preload_libraries: 'bdr'
+        track_commit_timestamp: 'on'
+        wal_level: 'logical'
+
+At the moment Patroni BDR is not compatible with a streaming replication,
+if BDR is enabled normal replica node won't be able to join. This is not
+a principal limitation of BDR, and we might resolve this in the future
+(although 'promotion' will only work between nodes running a physical
+replication, i.e. a replica won't be able to attach to a different
+multimaster node).
+
+Another limitation is that only one database is supported at the moment.
+BDR requires the replication user to be also a superuser, so you might
+want to excersie extra caution when choosing the password for this user.
+
 Applications Should Not Use Superusers
 --------------------------------------
 

TODO.md

@@ -1,16 +1,4 @@
 Failover
 ========
-- When determining who should become master, include the minor version of PostgreSQL in the decision.
+- When determining who should become master, include the minor version of PostgreSQL in the decision
-- Create a way to disable governance of a cluster, something like the existence of a "nogover" or "admin" file in PGDATA will stop patroni from changing the cluster state.
+- Create a way to disable governance of a cluster, something like the existence of a "nogover" or "admin" file in PGDATA will stop governor from changing the cluster state
-
-Configuration
-==============
-- Provide a way to change postgresql.conf and pg_hba.conf of a running cluster on the Patroni level, without changing individual nodes.
-- Provide hooks to store and retrieve cluster-wide passwords without exposing them in a plain-text form to unauthorized users.
-- Implement patronictl command to create initial configuration of the cluster with leader and member keys fixed to the user-supplied values in order to simplify migrations.
-- Implement support for consul in addtion to etcd and zookeeper
-- Complete zookeeper support in patronictl
-
-Documentation
-==============
-- Document how to run cascading replication and possibly initialize the cluster without an access to the master node.

docker/entrypoint.sh

@@ -79,12 +79,7 @@ then
     ETCD_CLUSTER="127.0.0.1:4001"
 fi
 
-mkdir -p ~postgres/.config/patroni
+cat > /patroni/postgres.yml <<__EOF__
-cat > ~postgres/.config/patroni/patronictl.yaml <<__EOF__
-{dcs_api: 'etcd://${ETCD_CLUSTER}', namespace: /service/}
-__EOF__
-
-cat > /patroni/postgres.yaml <<__EOF__
 
 ttl: &ttl 30
 loop_wait: &loop_wait 10
@@ -97,6 +92,9 @@ etcd:
   scope: *scope
   ttl: *ttl
   host: ${ETCD_CLUSTER}
+bdr:
+   enable: 'on'
+   database: 'bdrtest'
 postgresql:
   name: ${HOSTNAME}
   scope: *scope
@@ -124,22 +122,24 @@ postgresql:
     archive_command: 'true'
     max_wal_senders: 20
     listen_addresses: 0.0.0.0
-    max_wal_size: 1GB
+    checkpoint_segments: 64
-    min_wal_size: 128MB
     wal_keep_segments: 64
     archive_timeout: 1800s
     max_replication_slots: 20
     hot_standby: "on"
+    max_worker_processes: 10
+    max_replication_slots: 10
+    max_wal_senders: 10
+    shared_preload_libraries: 'bdr'
+    track_commit_timestamp: true
+    wal_level: 'logical'
 __EOF__
 
-cat /patroni/postgres.yaml
+cat /patroni/postgres.yml
 
 if [ ! -z $CHEAT ]
 then
-    while :
+    exec bash
-    do
-        sleep 60
-    done
 else
-    exec python /patroni.py /patroni/postgres.yaml
+    exec python /patroni.py /patroni/postgres.yml
 fi

extras/startup-scripts/README.md

@@ -8,6 +8,3 @@ Scripts supplied:
 ### patroni.upstart.conf
 
 Upstart job for Ubuntu 12.04 or 14.04.  Requires Upstart > 1.4. Intended for systems where Patroni has been installed on a base system, rather than in Docker.
-
-### patroni.service
-Systemd service file, to be copied to /etc/systemd/system/patroni.service, tested on Centos 7.1 with Patroni installed from pip. 

extras/startup-scripts/patroni.service

@@ -1,28 +0,0 @@
-# This is an example systemd config file for Patroni
-# You can copy it to "/etc/systemd/system/patroni.service",
-
-[Unit]
-Description=Runners to orchestrate a high-availability PostgreSQL
-After=syslog.target network.target
-
-[Service]
-Type=simple
-
-User=postgres
-Group=postgres
-
-# Where to send early-startup messages from the server
-# This is normally controlled by the global default set by systemd
-# StandardOutput=syslog
-
-ExecStart=/bin/patroni /etc/patroni.yml
-
-# Give a reasonable amount of time for the server to start up/shut down
-TimeoutSec=10
-
-# Do not restart the service if it crashes, we want to manually inspect database on failure
-Restart=no
-
-[Install]
-WantedBy=multi-user.target
-

features/basic_replication.feature

@@ -1,17 +0,0 @@
-Feature: basic replication
-  We should check that the basic bootstrapping, replication and failover works.
-
-  Scenario: check replication of a single table
-    Given I start postgres0
-    And postgres0 is a leader after 10 seconds
-    And I start postgres1
-    When I add the table foo to postgres0
-    Then table foo is present on postgres1 after 15 seconds
-
-  Scenario: check the basic failover
-    When I kill postgres0
-    Then postgres1 role is the primary after 30 seconds
-    When I start postgres0
-    Then postgres0 role is the secondary after 15 seconds
-    When I add the table bar to postgres1
-    Then table bar is present on postgres0 after 10 seconds

features/cascading_replication.feature

@@ -1,13 +0,0 @@
-Feature: cascading replication
-	We should check that patroni can do base backup and streaming from the replica
-
-Scenario: check a base backup from the replica
-	Given I start postgres0
-	And postgres0 is a leader after 10 seconds
-	And I start postgres1
-	And replication works from postgres0 to postgres1 after 15 seconds
-	And I create label with "postgres0" in postgres0 data directory
-	And I create label with "postgres1" in postgres1 data directory
-	And I configure and start postgres2 with a tag clonefrom postgres1
-	Then replication works from postgres0 to postgres2 after 30 seconds
-	And there is a label with "postgres1" in postgres2 data directory

features/environment.py

@@ -1,305 +0,0 @@
-import os
-import psycopg2
-import requests
-import shutil
-import subprocess
-import tempfile
-import time
-import yaml
-
-
-class PatroniController(object):
-    PATRONI_CONFIG = '{}.yml'
-    """ starts and stops individual patronis"""
-
-    def __init__(self):
-        self._output_dir = None
-        self._patroni_path = None
-        self._connections = {}
-        self._config = {}
-        self._connstring = {}
-        self._cursors = {}
-        self._log = {}
-        self._processes = {}
-
-    @property
-    def patroni_path(self):
-        if self._patroni_path is None:
-            cwd = os.path.realpath(__file__)
-            while True:
-                path, entry = os.path.split(cwd)
-                cwd = path
-                if entry == 'features' or cwd == '/':
-                    break
-            self._patroni_path = cwd
-        return self._patroni_path
-
-    def data_dir(self, pg_name):
-        return os.path.join(self.patroni_path, 'data', pg_name)
-
-    def write_label(self, pg_name, content):
-        with open(os.path.join(self.data_dir(pg_name), 'label'), 'w') as f:
-            f.write(content)
-
-    def read_label(self, pg_name):
-        content = None
-        try:
-            with open(os.path.join(self.data_dir(pg_name), 'label'), 'r') as f:
-                content = f.read()
-        except IOError:
-            return None
-        return content.strip()
-
-    def start(self, pg_name, max_wait_limit=20, tags=None):
-        if not self._is_running(pg_name):
-            if pg_name in self._processes:
-                del self._processes[pg_name]
-            cwd = self.patroni_path
-            self._log[pg_name] = open(os.path.join(self._output_dir, 'patroni_{0}.log'.format(pg_name)), 'a')
-
-            self._config[pg_name] = self._make_patroni_test_config(pg_name, tags=tags)
-
-            p = subprocess.Popen(['coverage', 'run', '--branch', '--source=patroni', '-p', 'patroni.py', self._config[pg_name]],
-                                 stdout=self._log[pg_name], stderr=subprocess.STDOUT, cwd=cwd)
-            if not (p and p.pid and p.poll() is None):
-                assert False, "PostgreSQL {0} is not running after being started".format(pg_name)
-            self._processes[pg_name] = p
-        # wait while patroni is available for queries, but not more than 10 seconds.
-        for _ in range(max_wait_limit):
-            if self.query(pg_name, "SELECT 1", fail_ok=True) is not None:
-                break
-            time.sleep(1)
-        else:
-            assert False,\
-                "Patroni instance is not available for queries after {0} seconds".format(max_wait_limit)
-
-    def stop(self, pg_name, kill=False, timeout=15):
-        start_time = time.time()
-        while self._is_running(pg_name):
-            if not kill:
-                self._processes[pg_name].terminate()
-            else:
-                self._processes[pg_name].kill()
-            time.sleep(1)
-            if not kill and time.time() - start_time > timeout:
-                kill = True
-        if self._log.get('pg_name') and not self._log['pg_name'].closed:
-            self._log[pg_name].close()
-        if pg_name in self._processes:
-            del self._processes[pg_name]
-
-    def query(self, pg_name, query, fail_ok=False):
-        try:
-            cursor = self._cursor(pg_name)
-            cursor.execute(query)
-            return cursor
-        except psycopg2.Error:
-            if fail_ok:
-                return None
-            else:
-                raise
-
-    def check_role_has_changed_to(self, pg_name, new_role, timeout=10):
-        bound_time = time.time() + timeout
-        recovery_status = False if new_role == 'primary' else True
-        role_has_changed = False
-        while not role_has_changed:
-            cur = self.query(pg_name, "SELECT pg_is_in_recovery()", fail_ok=True)
-            if cur:
-                row = cur.fetchone()
-                if row and len(row) > 0 and row[0] == recovery_status:
-                    role_has_changed = True
-            if time.time() > bound_time:
-                break
-            time.sleep(1)
-        return role_has_changed
-
-    def stop_all(self):
-        for patroni in self._processes.copy():
-            self.stop(patroni)
-
-    def create_and_set_output_directory(self, feature_name):
-        feature_dir = os.path.join(self.patroni_path, "features", "output",
-                                   feature_name.replace(' ', '_'))
-        if os.path.exists(feature_dir):
-            shutil.rmtree(feature_dir)
-        os.makedirs(feature_dir)
-        self._output_dir = feature_dir
-
-    def _is_running(self, pg_name):
-        return pg_name in self._processes and self._processes[pg_name].pid and (self._processes[pg_name].poll() is None)
-
-    def _make_patroni_test_config(self, pg_name, tags=None):
-        patroni_config_name = PatroniController.PATRONI_CONFIG.format(pg_name)
-        patroni_config_path = os.path.join(self._output_dir, patroni_config_name)
-
-        with open(patroni_config_name) as f:
-            config = yaml.load(f)
-        postgresql = config['postgresql']
-        postgresql['name'] = pg_name
-        postgresql['data_dir'] = 'data/{0}'.format(pg_name)
-        postgresql_params = postgresql['parameters']
-        postgresql_params['logging_collector'] = 'on'
-        postgresql_params['log_destination'] = 'csvlog'
-        postgresql_params['log_directory'] = self._output_dir
-        postgresql_params['log_filename'] = '{0}.log'.format(pg_name)
-        postgresql_params['log_statement'] = 'all'
-        postgresql_params['log_min_messages'] = 'debug1'
-        postgresql_params['unix_socket_directories'] = '.'
-
-        if tags:
-            config['tags'] = tags
-
-        with open(patroni_config_path, 'w') as f:
-            yaml.dump(config, f, default_flow_style=False)
-
-        return patroni_config_path
-
-    def _make_connstring(self, pg_name):
-        if pg_name in self._connstring:
-            return self._connstring[pg_name]
-        try:
-            patroni_path = self.patroni_path
-            with open(os.path.join(patroni_path, PatroniController.PATRONI_CONFIG.format(pg_name)), 'r') as f:
-                config = yaml.load(f)
-        except IOError:
-            return None
-        connstring = config['postgresql']['connect_address']
-        if ':' in connstring:
-            address, port = connstring.split(':')
-        else:
-            address = connstring
-            port = '5432'
-        user = "postgres"
-        dbname = "postgres"
-        self._connstring[pg_name] = "host={0} port={1} dbname={2} user={3}".format(address, port, dbname, user)
-        return self._connstring[pg_name]
-
-    def _connection(self, pg_name):
-        if pg_name not in self._connections or self._connections[pg_name].closed:
-            conn = psycopg2.connect(self._make_connstring(pg_name))
-            conn.autocommit = True
-            self._connections[pg_name] = conn
-        return self._connections[pg_name]
-
-    def _cursor(self, pg_name):
-        if pg_name not in self._cursors or self._cursors[pg_name].closed:
-            cursor = self._connection(pg_name).cursor()
-            self._cursors[pg_name] = cursor
-        return self._cursors[pg_name]
-
-
-class EtcdController(object):
-
-    """ handles all etcd related tasks, used for the tests setup and cleanup """
-    ETCD_VERSION_URL = 'http://127.0.0.1:2379/version'
-    ETCD_CLEANUP_URL = 'http://127.0.0.1:2379/v2/keys/service/batman?recursive=true'
-
-    def __init__(self, log_directory):
-        self.handle = None
-        self.work_directory = None
-        self.log_directory = log_directory
-        self.log_file = None
-        self.pid = None
-        self.start_timeout = 5
-
-    def start(self):
-        """ start etcd if it's not already running """
-        if self._is_running():
-            return True
-        self.work_directory = tempfile.mkdtemp()
-        # etcd is running throughout the tests, no need to append to the log
-        output_dir = os.path.join(self.log_directory, "features", "output")
-        if not os.path.exists(output_dir):
-            os.makedirs(output_dir)
-        self.log_file = open(os.path.join(output_dir, 'etcd.log'), 'w')
-        self.handle =\
-            subprocess.Popen(["etcd", "--debug", "--data-dir", self.work_directory],
-                             stdout=self.log_file, stderr=subprocess.STDOUT)
-        start_time = time.time()
-        while (not self._is_running()):
-            if time.time() - start_time > self.start_timeout:
-                assert False, "Failed to start etcd"
-            time.sleep(1)
-        return True
-
-    @staticmethod
-    def query(key):
-        """ query etcd for a value of a given key """
-        r = requests.get("http://127.0.0.1:2379/v2/keys/service/batman/{0}".format(key))
-        if r.ok:
-            content = r.json()
-            if content:
-                return content.get('node', {}).get('value')
-        return None
-
-    def stop_and_remove_work_directory(self, timeout=15):
-        """ terminate etcd and wipe out the temp work directory, but only if we actually started it"""
-        kill = False
-        start_time = time.time()
-        while self._is_running() and self.handle:
-            if not kill:
-                self.handle.terminate()
-            else:
-                self.handle.kill()
-            time.sleep(1)
-            if not kill and time.time() - start_time > timeout:
-                kill = True
-        self.handle = None
-        if self.log_file and not self.log_file.closed:
-            self.log_file.close()
-        if self.work_directory:
-            shutil.rmtree(self.work_directory)
-            self.work_directory = None
-
-    @staticmethod
-    def cleanup_service_tree():
-        """ clean all contents stored in the tree used for the tests """
-        r = None
-        try:
-            r = requests.delete(EtcdController.ETCD_CLEANUP_URL)
-            if r and not r.ok:
-                assert False,\
-                    "request to cleanup the etcd contents was not successfull: status code {0}".format(r.status_code)
-        except requests.exceptions.RequestException as e:
-            assert False, "exception when cleaning up etcd contents: {0}".format(e)
-
-    @staticmethod
-    def _is_running():
-        # if etcd is running, but we didn't start it
-        try:
-            r = requests.get(EtcdController.ETCD_VERSION_URL)
-            running = (r and r.ok and b'etcdserver' in r.content)
-        except requests.ConnectionError:
-            running = False
-        return running
-
-
-# actions to execute on start/stop of the tests and before running invidual features
-def before_all(context):
-    context.pctl = PatroniController()
-    context.etcd_ctl = EtcdController(context.pctl.patroni_path)
-    context.etcd_ctl.start()
-    try:
-        context.etcd_ctl.cleanup_service_tree()
-    except AssertionError:  # after.all handlers won't be executed in before.all
-        context.etcd_ctl.stop_and_remove_work_directory()
-        raise
-
-
-def after_all(context):
-    context.etcd_ctl.stop_and_remove_work_directory()
-    subprocess.call(['coverage', 'combine'])
-    subprocess.call(['coverage', 'report'])
-
-
-def before_feature(context, feature):
-    """ create per-feature output directory to collect Patroni and PostgreSQL logs """
-    context.pctl.create_and_set_output_directory(feature.name)
-
-
-def after_feature(context, feature):
-    """ stop all Patronis, remove their data directory and cleanup the keys in etcd """
-    context.pctl.stop_all()
-    shutil.rmtree(os.path.join(context.pctl.patroni_path, 'data'))
-    context.etcd_ctl.cleanup_service_tree()

features/patroni_api.feature

@@ -1,48 +0,0 @@
-Feature: patroni api
-	We should check that patroni correctly responds to valid and not-valid API requests.
-
-Scenario: check API requests on a stand-alone server
-	Given I start postgres0
-	And postgres0 is a leader after 10 seconds
-	When I issue a GET request to http://127.0.0.1:8008/
-	Then I receive a response code 200
-	And I receive a response state running
-	And I receive a response role master
-	When I issue a GET request to http://127.0.0.1:8008/replica
-	Then I receive a response code 503
-	When I issue an empty POST request to http://127.0.0.1:8008/reinitialize
-	Then I receive a response code 503
-	And I receive a response text "I am the leader, can not reinitialize"
-	When I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0
-	Then I receive a response code 500
-	And I receive a response text "failover is not possible: cluster does not have members except leader"
-	When I issue an empty POST request to http://127.0.0.1:8008/failover
-	Then I receive a response code 400
-	And I receive a response text "No values given for required parameters leader and member"
-
-Scenario: check API requests for the primary-replica pair
-	Given I start postgres1
-	And replication works from postgres0 to postgres1 after 15 seconds
-	When I issue a GET request to http://127.0.0.1:8009/replica
-	Then I receive a response code 200
-	And I receive a response state running
-	And I receive a response role replica
-	When I issue an empty POST request to http://127.0.0.1:8009/reinitialize
-	Then I receive a response code 200
-	Given replication works from postgres0 to postgres1 after 10 seconds
-	When I issue an empty POST request to http://127.0.0.1:8008/restart
-	Then I receive a response code 200
-	And postgres0 is a leader after 5 seconds
-
-Scenario: check the failover via the API
-	Given I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0,candidate=postgres1
-	Then I receive a response code 200
-	And postgres1 is a leader after 5 seconds
-	And replication works from postgres1 to postgres0 after 15 seconds
-
-Scenario: check the scheduled failover
-	Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgresq0 in 10 seconds
-	Then I receive a response code 200
-	And postgres0 is a leader after 15 seconds
-	And replication works from postgres0 to postgres1 after 25 seconds
-

features/steps/basic_replication.py

@@ -1,55 +0,0 @@
-import psycopg2 as pg
-
-from behave import step, then
-from time import sleep, time
-
-
-@step('I start {name:w}')
-def start_patroni(context, name):
-    return context.pctl.start(name)
-
-
-@step('I shut down {name:w}')
-def stop_patroni(context, name):
-    return context.pctl.stop(name)
-
-
-@step('I kill {name:w}')
-def kill_patroni(context, name):
-    return context.pctl.stop(name, kill=True)
-
-
-@step('I add the table {table_name:w} to {pg_name:w}')
-def add_table(context, table_name, pg_name):
-    # parse the configuration file and get the port
-    try:
-        context.pctl.query(pg_name, "CREATE TABLE {0}()".format(table_name))
-    except pg.Error as e:
-        assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e)
-
-
-@then('Table {table_name:w} is present on {pg_name:w} after {max_replication_delay:d} seconds')
-def table_is_present_on(context, table_name, pg_name, max_replication_delay):
-    for _ in range(int(max_replication_delay)):
-        if context.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
-            break
-        sleep(1)
-    else:
-        assert False,\
-            "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay)
-
-
-@then('{pg_name:w} role is the {pg_role:w} after {max_promotion_timeout:d} seconds')
-def check_role(context, pg_name, pg_role, max_promotion_timeout):
-    if not context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)):
-        assert False,\
-            "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
-
-
-@step('replication works from {master:w} to {replica:w} after {time_limit:d} seconds')
-@then('replication works from {master:w} to {replica:w} after {time_limit:d} seconds')
-def replication_works(context, master, replica, time_limit):
-    context.execute_steps(u"""
-        When I add the table test_{0} to {1}
-        Then table test_{0} is present on {2} after {3} seconds
-    """.format(int(time()), master, replica, time_limit))

features/steps/cascading_replication.py

@@ -1,17 +0,0 @@
-from behave import step, then
-
-
-@step('I configure and start {name:w} with a tag {tag_name:w} {tag_value:w}')
-def start_patroni_with_a_name_value_tag(context, name, tag_name, tag_value):
-    return context.pctl.start(name, tags={tag_name: tag_value})
-
-
-@then('There is a label with "{content:w}" in {name:w} data directory')
-def check_label(context, content, name):
-    label = context.pctl.read_label(name)
-    assert label == content, "{0} is not equal to {1}".format(label, content)
-
-
-@step('I create label with "{content:w}" in {name:w} data directory')
-def write_label(context, content, name):
-    context.pctl.write_label(name, content)

features/steps/patroni_api.py

@@ -1,101 +0,0 @@
-import parse
-import pytz
-import requests
-import time
-
-from behave import register_type, step, then
-from datetime import datetime, timedelta
-
-
-@parse.with_pattern(r'https?://(?:\w|\.|:|/)+')
-def parse_url(text):
-    return text
-
-
-@parse.with_pattern(r'(?:\w+=(?:\w|\.|:|-|\+|\s)+,?)+')
-def parse_data(text):
-    return text
-
-
-register_type(url=parse_url, data=parse_data)
-
-
-# there is no way we can find out if the node has already
-# started as a leader without checking the DCS. We cannot
-# just rely on the database availability, since there is
-# a short gap between the time PostgreSQL becomes available
-# and Patroni assuming the leader role.
-@step('{name:w} is a leader after {time_limit:d} seconds')
-@then('{name:w} is a leader after {time_limit:d} seconds')
-def is_a_leader(context, name, time_limit):
-    max_time = time.time() + int(time_limit)
-    while (context.etcd_ctl.query("leader") != name):
-        time.sleep(1)
-        if time.time() > max_time:
-            assert False, "{0} is not a leader in etcd after {1} seconds".format(name, time_limit)
-
-
-@step('I sleep for {value:d} seconds')
-def sleep_for_n_seconds(context, value):
-    time.sleep(int(value))
-
-
-@step('I issue a GET request to {url:url}')
-def do_get(context, url):
-    try:
-        r = requests.get(url)
-    except requests.exceptions.RequestException:
-        context.status_code = None
-        context.response = None
-    else:
-        context.status_code = r.status_code
-        try:
-            context.response = r.json()
-        except ValueError:
-            context.response = r.content.decode('utf-8')
-
-
-@step('I issue an empty POST request to {url:url}')
-def do_post_empty(context, url):
-    do_post(context, url, None)
-
-
-@step('I issue a POST request to {url:url} with {data:data}')
-def do_post(context, url, data):
-    post_data = {}
-    if data:
-        post_components = data.split(',')
-        for pc in post_components:
-            if '=' in pc:
-                k, v = pc.split('=', 2)
-                post_data[k.strip()] = v.strip()
-    try:
-        r = requests.post(url, json=post_data)
-    except requests.exceptions.RequestException:
-        context.status_code = None
-        context.response = None
-    else:
-        context.status_code = r.status_code
-        try:
-            context.response = r.json()
-        except ValueError:
-            context.response = r.content.decode('utf-8')
-
-
-@then('I receive a response {component:w} {data}')
-def check_response(context, component, data):
-    if component == 'code':
-        assert context.status_code == int(data),\
-            "status code {0} != {1}, response: {2}".format(context.status_code, int(data), context.response)
-    elif component == 'text':
-        assert context.response == data.strip('"'), "response {0} does not contain {1}".format(context.response, data)
-    else:
-        assert component in context.response, "{0} is not part of the response".format(component)
-        assert context.response[component] == data, "{0} does not contain {1}".format(component, data)
-
-
-@step('I issue a scheduled failover at {at_url:url} from {from_host:w} to {to_host:w} in {in_seconds:d} seconds')
-def scheduled_failover(context, at_url, from_host, to_host, in_seconds):
-    context.execute_steps(u"""
-        Given I issue a POST request to {0}/failover with leader={1},candidate={2},scheduled_at={3}
-    """.format(at_url, from_host, to_host, datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds))))

patroni/__init__.py

@@ -10,19 +10,18 @@ from patroni.ha import Ha
 from patroni.postgresql import Postgresql
 from patroni.utils import setup_signal_handlers, reap_children
 from patroni.zookeeper import ZooKeeper
-from .version import __version__
 
 logger = logging.getLogger(__name__)
 
 
-class Patroni(object):
+class Patroni:
 
     def __init__(self, config):
         self.nap_time = config['loop_wait']
         self.tags = config.get('tags', dict())
-        self.postgresql = Postgresql(config['postgresql'])
+        self.bdr = config.get('bdr', dict())
+        self.postgresql = Postgresql(config['postgresql'], self.bdr)
         self.dcs = self.get_dcs(self.postgresql.name, config)
-        self.version = __version__
         self.api = RestApiServer(self, config['restapi'])
         self.ha = Ha(self)
         self.next_run = time.time()
@@ -31,14 +30,6 @@ class Patroni(object):
     def nofailover(self):
         return self.tags.get('nofailover', False)
 
-    @property
-    def replicatefrom(self):
-        return self.tags.get('replicatefrom')
-
-    @property
-    def clonefrom(self):
-        return self.tags.get('clonefrom')
-
     @staticmethod
     def get_dcs(name, config):
         if 'etcd' in config:
@@ -72,7 +63,7 @@ def main():
     setup_signal_handlers()
 
     if len(sys.argv) < 2 or not os.path.isfile(sys.argv[1]):
-        print('Usage: {0} config.yml'.format(sys.argv[0]))
+        print('Usage: {} config.yml'.format(sys.argv[0]))
         return
 
     with open(sys.argv[1], 'r') as f:

patroni/api.py

@@ -5,9 +5,6 @@ import logging
 import psycopg2
 import socket
 import time
-import dateutil
-import datetime
-import pytz
 
 from patroni.exceptions import PostgresConnectionException
 from patroni.utils import Retry, RetryFailedError
@@ -95,7 +92,6 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_GET_patroni(self):
         response = self.get_postgresql_status(True)
         response.update(self.get_tags())
-        response['patroni'] = {'version': self.server.patroni.version, 'scope': self.server.patroni.postgresql.scope}
 
         self.send_response(200)
         self.send_header('Content-Type', 'application/json')
@@ -104,7 +100,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
     @check_auth
     def do_POST_restart(self):
-        status_code = 500
+        status_code = 503
         data = b'restart failed'
         try:
             status, msg = self.server.patroni.ha.restart()
@@ -143,7 +139,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         self.wfile.write(data)
 
     def poll_failover_result(self, leader, member):
-        for _ in range(0, 15):
+        for a in range(0, 15):
             time.sleep(1)
             try:
                 cluster = self.server.patroni.dcs.get_cluster()
@@ -166,7 +162,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
             members = [m for m in cluster.members if m.name != cluster.leader.name and m.api_url]
             if not members:
                 return b'failover is not possible: cluster does not have members except leader'
-        for member, reachable, _, xlog_location, tags in self.server.patroni.ha.fetch_nodes_statuses(members):
+        for member, reachable, in_recovery, xlog_location, tags in self.server.patroni.ha.fetch_nodes_statuses(members):
             if reachable and not tags.get('nofailover', False):
                 return None
         return b'failover is not possible: no good candidates have been found'
@@ -174,48 +170,18 @@ class RestApiHandler(BaseHTTPRequestHandler):
     @check_auth
     def do_POST_failover(self):
         content_length = int(self.headers.get('content-length', 0))
-        try:
+        request = json.loads(self.rfile.read(content_length).decode('utf-8'))
-            request = json.loads(self.rfile.read(content_length).decode('utf-8'))
+        leader = request.get('leader', None)
-        except ValueError:
+        member = request.get('member', None)
-            request = {}
-        leader = request.get('leader')
-        member = request.get('member')
         cluster = self.server.patroni.ha.dcs.get_cluster()
-        status_code = 500
+        status_code = 503
-
+        data = self.is_failover_possible(cluster, leader, member)
-        logger.info("received failover request with leader {0} member {1} scheduled_at {2}".
+        if not data:
-                    format(leader, member, request.get("scheduled_at")))
+            if not self.server.patroni.dcs.manual_failover(leader, member):
-
+                data = b'failed to write failover key into DCS'
-        data = b''
-        if leader or member:
-            if request.get('scheduled_at'):
-                try:
-                    scheduled_at = dateutil.parser.parse(request['scheduled_at'])
-                    if scheduled_at.tzinfo is None:
-                        data = b'Timezone information is mandatory for scheduled_at'
-                        status_code = 400
-                    elif scheduled_at < datetime.datetime.now(pytz.utc):
-                        data = b'Cannot schedule failover in the past'
-                        status_code = 422
-                    elif self.server.patroni.dcs.manual_failover(leader, member, scheduled_at):
-                        data = b'Failover scheduled'
-                        status_code = 200
-                except (ValueError, TypeError):
-                    logger.exception('Invalid scheduled failover time: {}'.format(request['scheduled_at']))
-                    data = b'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601'
-                    status_code = 422
             else:
-                data = self.is_failover_possible(cluster, leader, member)
+                self.server.patroni.dcs.event.set()
-                if not data:
+                status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, member)
-                    if not self.server.patroni.dcs.manual_failover(leader, member):
-                        data = b'failed to write failover key into DCS'
-                        status_code = 503
-                    else:
-                        self.server.patroni.dcs.event.set()
-                        status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, member)
-        else:
-            status_code = 400
-            data = b'No values given for required parameters leader and member'
 
         self.send_response(status_code)
         self.send_header('Content-Type', 'text/html')
@@ -262,18 +228,15 @@ class RestApiHandler(BaseHTTPRequestHandler):
                                        END,
                                        pg_xlog_location_diff(pg_last_xlog_receive_location(), '0/0')::bigint,
                                        pg_xlog_location_diff(pg_last_xlog_replay_location(), '0/0')::bigint,
-                                       to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
                                        pg_is_in_recovery() AND pg_is_xlog_replay_paused()""", retry=retry)[0]
             return {
                 'state': self.server.patroni.postgresql.state,
                 'postmaster_start_time': row[0],
                 'role': 'replica' if row[1] else 'master',
-                'server_version': self.server.patroni.postgresql.server_version,
                 'xlog': ({
                     'received_location': row[3],
                     'replayed_location': row[4],
-                    'replayed_timestamp': row[5],
+                    'paused': row[5]} if row[1] else {
-                    'paused': row[6]} if row[1] else {
                     'location': row[2]
                 })
             }
@@ -287,8 +250,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def get_tags(self):
         return {'tags': self.server.patroni.tags}
 
-    def log_message(self, fmt, *args):
+    def log_message(self, format, *args):
-        logger.debug("API thread: %s - - [%s] %s", self.client_address[0], self.log_date_time_string(), fmt % args)
+        logger.debug("API thread: " + format % args)
 
 
 class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
@@ -305,12 +268,12 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         # wrap socket with ssl if 'certfile' is defined in a config.yaml
         # Sometime it's also needed to pass reference to a 'keyfile'.
         options = {option: config[option] for option in ['certfile', 'keyfile'] if option in config}
-        if options.get('certfile'):
+        if options.get('certfile', None):
             import ssl
             self.socket = ssl.wrap_socket(self.socket, server_side=True, **options)
             protocol = 'https'
 
-        self.connection_string = '{0}://{1}/patroni'.format(protocol, config.get('connect_address', config['listen']))
+        self.connection_string = '{}://{}/patroni'.format(protocol, config.get('connect_address', config['listen']))
 
         self.patroni = patroni
         self.daemon = True
@@ -318,7 +281,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
     def query(self, sql, *params):
         cursor = None
         try:
-            with self.patroni.postgresql.connection().cursor() as cursor:
+            with self.patroni.postgresql.connection('postgres').cursor() as cursor:
                 cursor.execute(sql, params)
                 return [r for r in cursor]
         except psycopg2.Error as e:

patroni/async_executor.py

@@ -4,9 +4,10 @@ from threading import Lock, Thread
 logger = logging.getLogger(__name__)
 
 
-class AsyncExecutor(object):
+class AsyncExecutor:
 
     def __init__(self):
+        Lock.__init__(self)
         self._busy = False
         self._thread_lock = Lock()
         self._scheduled_action = None
@@ -50,5 +51,5 @@ class AsyncExecutor(object):
     def __enter__(self):
         self._thread_lock.acquire()
 
-    def __exit__(self, *args):
+    def __exit__(self, type, value, traceback):
         self._thread_lock.release()

patroni/ctl.py

@@ -14,11 +14,8 @@ import datetime
 from prettytable import PrettyTable
 from six.moves.urllib_parse import urlparse
 import logging
-import dateutil
-import tzlocal
 
 from .etcd import Etcd
-from .zookeeper import ZooKeeper
 from .exceptions import PatroniCtlException
 from .postgresql import parseurl
 
@@ -47,24 +44,24 @@ def parse_dcs(dcs):
         parsed = urlparse('//' + dcs)
 
     if scheme == '':
-        default_schemes = {'2181': 'zookeeper', '8181': 'exhibitor', '8500': 'consul'}
+        default_schemes = {'2181': 'zookeeper', '8500': 'consul'}
         scheme = default_schemes.get(str(parsed.port), 'etcd')
 
     port = parsed.port
     if port is None:
-        default_ports = {'consul': 8500, 'zookeeper': 2181, 'exhibitor': 8181}
+        default_ports = {'consul': 8500, 'zookeeper': 2181}
         port = default_ports.get(str(scheme), 4001)
 
     return {'scheme': str(scheme), 'hostname': str(parsed.hostname), 'port': int(port)}
 
 
 def load_config(path, dcs):
-    logging.debug('Loading configuration from file %s', path)
+    logging.debug('Loading configuration from file {}'.format(path))
     config = dict()
     try:
         with open(path, 'rb') as fd:
             config = yaml.safe_load(fd)
-    except (IOError, yaml.YAMLError):
+    except:
         logging.exception('Could not load configuration file')
 
     if dcs:
@@ -77,14 +74,15 @@ def load_config(path, dcs):
 
 def store_config(config, path):
     dir_path = os.path.dirname(path)
-    if dir_path and not os.path.isdir(dir_path):
+    if dir_path:
-        os.makedirs(dir_path)
+        if not os.path.isdir(dir_path):
+            os.makedirs(dir_path)
     with open(path, 'w') as fd:
         yaml.dump(config, fd)
 
 
 option_config_file = click.option('--config-file', '-c', help='Configuration file', default=CONFIG_FILE_PATH)
-option_format = click.option('--format', '-f', 'fmt', help='Output format (pretty, json)', default='pretty')
+option_format = click.option('--format', '-f', help='Output format (pretty, json)', default='pretty')
 option_dcs = click.option('--dcs', '-d', help='Use this DCS', envvar='DCS')
 option_watchrefresh = click.option('-w', '--watch', type=float, help='Auto update the screen every X seconds')
 option_watch = click.option('-W', is_flag=True, help='Auto update the screen every 2 seconds')
@@ -104,28 +102,20 @@ def get_dcs(config, scope):
     scheme, hostname, port = map(config.get('dcs', {}).get, ('scheme', 'hostname', 'port'))
 
     if scheme == 'etcd':
-        return Etcd(name=scope, config={'scope': scope, 'host': '{0}:{1}'.format(hostname, port)})
+        return Etcd(name=scope, config={'scope': scope, 'host': '{}:{}'.format(hostname, port)})
-
-    if scheme == 'zookeeper':
-        return ZooKeeper(name=scope, config={'scope': scope, 'hosts': [hostname], 'port': port})
-
-    if scheme == 'exhibitor':
-        return ZooKeeper(name=scope, config={'scope': scope, 'exhibitor': {'hosts': [hostname], 'port': port}})
 
     raise PatroniCtlException('Can not find suitable configuration of distributed configuration store')
 
 
-def post_patroni(member, endpoint, content, headers=None):
+def post_patroni(member, endpoint, content, headers={'Content-Type': 'application/json'}):
     url = urlparse(member.api_url)
     logging.debug(url)
-    return requests.post('{0}://{1}/{2}'.format(url.scheme, url.netloc, endpoint),
+    return requests.post('{}://{}/{}'.format(url.scheme, url.netloc, endpoint), headers=headers,
-                         headers=headers or {'Content-Type': 'application/json'},
                          data=json.dumps(content), timeout=60)
 
 
-def print_output(columns, rows=None, alignment=None, fmt='pretty', header=True, delimiter='\t'):
+def print_output(columns, rows=[], alignment=None, format='pretty', header=True, delimiter='\t'):
-    rows = rows or []
+    if format == 'pretty':
-    if fmt == 'pretty':
         t = PrettyTable(columns)
         for k, v in (alignment or {}).items():
             t.align[k] = v
@@ -134,18 +124,18 @@ def print_output(columns, rows=None, alignment=None, fmt='pretty', header=True,
         click.echo(t)
         return
 
-    if fmt == 'json':
+    if format == 'json':
         elements = list()
         for r in rows:
             elements.append(dict(zip(columns, r)))
 
         click.echo(json.dumps(elements))
 
-    if fmt == 'tsv':
+    if format == 'tsv':
         if columns is not None and header:
             click.echo(delimiter.join(columns) + '\n')
 
-        for r in rows:
+        for r in rows or []:
             c = [str(c) for c in r]
             click.echo(delimiter.join(c))
 
@@ -178,8 +168,8 @@ def watching(w, watch, max_count=None, clear=True):
         yield 0
 
 
-def build_connect_parameters(conn_url, connect_parameters=None):
+def build_connect_parameters(conn_url, connect_parameters={}):
-    params = (connect_parameters or {}).copy()
+    params = connect_parameters.copy()
     parsed = parseurl(conn_url)
     params['host'] = parsed['host']
     params['port'] = parsed['port']
@@ -210,12 +200,12 @@ def get_any_member(cluster, role='master', member=None):
     return None
 
 
-def get_cursor(cluster, role='master', member=None, connect_parameters=None):
+def get_cursor(cluster, role='master', member=None, connect_parameters={}):
     member = get_any_member(cluster=cluster, role=role, member=member)
     if member is None:
         return None
 
-    params = build_connect_parameters(member.conn_url, connect_parameters)
+    params = build_connect_parameters(member.conn_url, connect_parameters=connect_parameters)
 
     conn = psycopg2.connect(**params)
     conn.autocommit = True
@@ -247,23 +237,21 @@ def dsn(cluster_name, config_file, dcs, role, member):
     if member is None and role is None:
         role = 'master'
 
-    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
     m = get_any_member(cluster=cluster, role=role, member=member)
     if m is None:
         raise PatroniCtlException('Can not find a suitable member')
 
     params = build_connect_parameters(m.conn_url)
-    click.echo('host={host} port={port}'.format(**params))
+    click.echo('host={} port={}'.format(params['host'], params['port']))
 
 
 @ctl.command('query', help='Query a Patroni PostgreSQL member')
 @click.argument('cluster_name')
 @option_config_file
 @option_format
-@click.option('--format', 'fmt', help='Output format (pretty, json)', default='tsv')
+@click.option('--format', help='Output format (pretty, json)', default='tsv')
-@click.option('--file', '-f', 'p_file', help='Execute the SQL commands from this file', type=click.File('rb'))
+@click.option('--file', '-f', help='Execute the SQL commands from this file', type=click.File('rb'))
-@click.option('--password', help='force password prompt', is_flag=True)
-@click.option('-U', '--username', help='database user name', type=str)
 @option_dcs
 @option_watch
 @option_watchrefresh
@@ -272,7 +260,6 @@ def dsn(cluster_name, config_file, dcs, role, member):
 @click.option('--member', '-m', help='Query a specific member', type=str)
 @click.option('--delimiter', help='The column delimiter', default='\t')
 @click.option('--command', '-c', help='The SQL commands to execute')
-@click.option('-d', '--dbname', help='database name to connect to', type=str)
 def query(
     cluster_name,
     config_file,
@@ -283,57 +270,42 @@ def query(
     watch,
     delimiter,
     command,
-    p_file,
+    file,
-    password,
+    format='tsv',
-    username,
-    dbname,
-    fmt='tsv',
 ):
     if role is not None and member is not None:
         raise PatroniCtlException('--role and --member are mutually exclusive options')
     if member is None and role is None:
         role = 'master'
 
-    if p_file is not None and command is not None:
+    if file is not None and command is not None:
         raise PatroniCtlException('--file and --command are mutually exclusive options')
 
-    if p_file is None and command is None:
+    if file is not None:
-        raise PatroniCtlException('You need to specify either --command or --file')
+        command = file.read()
-
-    connect_parameters = dict()
-    if username:
-        connect_parameters['user'] = username
-    if password:
-        connect_parameters['password'] = click.prompt('Password', hide_input=True, type=str)
-    if dbname:
-        connect_parameters['database'] = dbname
-
-    if p_file is not None:
-        command = p_file.read()
 
     config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
 
     cursor = None
     for _ in watching(w, watch, clear=False):
 
-        output, cursor = query_member(cluster=cluster, cursor=cursor, member=member, role=role, command=command,
+        output, cursor = query_member(cluster=cluster, cursor=cursor, member=member, role=role, command=command)
-                                      connect_parameters=connect_parameters)
+        print_output(None, output, format=format, delimiter=delimiter)
-        print_output(None, output, fmt=fmt, delimiter=delimiter)
 
         if cursor is None:
             cluster = dcs.get_cluster()
 
 
-def query_member(cluster, cursor, member, role, command, connect_parameters=None):
+def query_member(cluster, cursor, member, role, command):
     try:
         if cursor is None:
-            cursor = get_cursor(cluster, role=role, member=member, connect_parameters=connect_parameters)
+            cursor = get_cursor(cluster, role=role, member=member)
 
         if cursor is None:
             if role is None:
-                message = 'No connection to member {0} is available'.format(member)
+                message = 'No connection to member {} is available'.format(member)
             else:
-                message = 'No connection to role={0} is available'.format(role)
+                message = 'No connection to role={} is available'.format(role)
             logging.debug(message)
             return [[timestamp(0), message]], None
 
@@ -352,7 +324,7 @@ def query_member(cluster, cursor, member, role, command, connect_parameters=None
             cursor.connection.close()
         message = oe.pgcode or oe.pgerror or str(oe)
         message = message.replace('\n', ' ')
-        return [[timestamp(0), 'ERROR, SQLSTATE: {0}'.format(message)]], None
+        return [[timestamp(0), 'ERROR, SQLSTATE: {}'.format(message)]], None
 
 
 @ctl.command('remove', help='Remove cluster from DCS')
@@ -360,13 +332,13 @@ def query_member(cluster, cursor, member, role, command, connect_parameters=None
 @option_config_file
 @option_format
 @option_dcs
-def remove(config_file, cluster_name, fmt, dcs):
+def remove(config_file, cluster_name, format, dcs):
     config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
 
     if not isinstance(dcs, Etcd):
-        raise PatroniCtlException('We have not implemented this for DCS of type {0}'.format(type(dcs)))
+        raise PatroniCtlException('We have not implemented this for DCS of type {}'.format(type(dcs)))
 
-    output_members(cluster, fmt=fmt)
+    output_members(cluster, format=format)
 
     confirm = click.prompt('Please confirm the cluster name to remove', type=str)
     if confirm != cluster_name:
@@ -374,17 +346,17 @@ def remove(config_file, cluster_name, fmt, dcs):
 
     message = 'Yes I am aware'
     confirm = \
-        click.prompt('You are about to remove all information in DCS for {0}, please type: "{1}"'.format(cluster_name,
+        click.prompt('You are about to remove all information in DCS for {}, please type: "{}"'.format(cluster_name,
                      message), type=str)
     if message != confirm:
-        raise PatroniCtlException('You did not exactly type "{0}"'.format(message))
+        raise PatroniCtlException('You did not exactly type "{}"'.format(message))
 
     if cluster.leader:
         confirm = click.prompt('This cluster currently is healthy. Please specify the master name to continue')
         if confirm != cluster.leader.name:
             raise PatroniCtlException('You did not specify the current master of the cluster')
 
-    dcs.client.delete(dcs.client_path(''), recursive=True)
+    dcs.client.delete(dcs._base_path, recursive=True)
 
 
 def wait_for_leader(dcs, timeout=30):
@@ -406,25 +378,25 @@ def empty_post_to_members(cluster, member_names, force, endpoint):
     for m in cluster.members:
         candidates[m.name] = m
 
-    if not member_names:
+    if len(member_names) == 0:
-        member_names = [click.prompt('Which member do you want to {0} [{1}]?'.format(endpoint,
+        member_names = [click.prompt('Which member do you want to {} [{}]?'.format(endpoint,
                         ', '.join(candidates.keys())), type=str, default='')]
 
     for mn in member_names:
         if mn not in candidates.keys():
-            raise PatroniCtlException('{0} is not a member of cluster'.format(mn))
+            raise PatroniCtlException('{} is not a member of cluster'.format(mn))
 
     if not force:
-        confirm = click.confirm('Are you sure you want to {0} members {1}?'.format(endpoint, ', '.join(member_names)))
+        confirm = click.confirm('Are you sure you want to {} members {}?'.format(endpoint, ', '.join(member_names)))
         if not confirm:
-            raise PatroniCtlException('Aborted {0}'.format(endpoint))
+            raise PatroniCtlException('Aborted {}'.format(endpoint))
 
     for mn in member_names:
         r = post_patroni(candidates[mn], endpoint, '')
         if r.status_code != 200:
-            click.echo('{0} failed for member {1}, status code={2}, ({3})'.format(endpoint, mn, r.status_code, r.text))
+            click.echo('{} failed for member {}, status code={}, ({})'.format(endpoint, mn, r.status_code, r.text))
         else:
-            click.echo('Succesful {0} on member {1}'.format(endpoint, mn))
+            click.echo('Succesful {} on member {}'.format(endpoint, mn))
 
 
 def ctl_load_config(cluster_name, config_file, dcs):
@@ -440,21 +412,21 @@ def ctl_load_config(cluster_name, config_file, dcs):
 @click.argument('member_names', nargs=-1)
 @click.option('--role', '-r', help='Restart only members with this role', default='any',
               type=click.Choice(['master', 'replica', 'any']))
-@click.option('--any', 'p_any', help='Restart a single member only', is_flag=True)
+@click.option('--any', help='Restart a single member only', is_flag=True)
 @option_config_file
 @option_force
 @option_dcs
-def restart(cluster_name, member_names, config_file, dcs, force, role, p_any):
+def restart(cluster_name, member_names, config_file, dcs, force, role, any):
     config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
 
     role_names = [m.name for m in get_all_members(cluster=cluster, role=role)]
 
-    if member_names:
+    if len(member_names) > 0:
         member_names = list(set(member_names) & set(role_names))
     else:
         member_names = role_names
 
-    if p_any:
+    if any:
         random.shuffle(member_names)
         member_names = member_names[:1]
 
@@ -477,12 +449,10 @@ def reinit(cluster_name, member_names, config_file, dcs, force):
 @click.argument('cluster_name')
 @click.option('--master', help='The name of the current master', default=None)
 @click.option('--candidate', help='The name of the candidate', default=None)
-@click.option('--scheduled', help='Timestamp of a scheduled failover in unambiguous format (e.g. ISO 8601)',
-              default=None)
 @click.option('--force', is_flag=True)
 @option_config_file
 @option_dcs
-def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled):
+def failover(config_file, cluster_name, master, candidate, force, dcs):
     """
         We want to trigger a failover for the specified cluster name.
 
@@ -502,13 +472,13 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
             master = click.prompt('Master', type=str, default=cluster.leader.member.name)
 
     if cluster.leader.member.name != master:
-        raise PatroniCtlException('Member {0} is not the leader of cluster {1}'.format(master, cluster_name))
+        raise PatroniCtlException('Member {} is not the leader of cluster {}'.format(master, cluster_name))
 
     candidate_names = [str(m.name) for m in cluster.members if m.name != master]
     # We sort the names for consistent output to the client
     candidate_names.sort()
 
-    if not candidate_names:
+    if len(candidate_names) == 0:
         raise PatroniCtlException('No candidates found to failover to')
 
     if candidate is None and not force:
@@ -518,26 +488,7 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
         raise PatroniCtlException('Failover target and source are the same.')
 
     if candidate and candidate not in candidate_names:
-        raise PatroniCtlException('Member {0} does not exist in cluster {1}'.format(candidate, cluster_name))
+        raise PatroniCtlException('Member {} does not exist in cluster {}'.format(candidate, cluster_name))
-
-    if scheduled is None and not force:
-        scheduled = click.prompt('When should the failover take place (e.g. 2015-10-01T14:30) ', type=str,
-                                 default='now')
-
-    if (scheduled or 'now') == 'now':
-        scheduled_at = None
-    else:
-        try:
-            scheduled_at = dateutil.parser.parse(scheduled)
-            if scheduled_at.tzinfo is None:
-                scheduled_at = tzlocal.get_localzone().localize(scheduled_at)
-        except (ValueError, TypeError):
-            message = 'Unable to parse scheduled timestamp ({}). It should be in an unambiguous format (e.g. ISO 8601)'
-            raise PatroniCtlException(message.format(scheduled))
-        scheduled_at = scheduled_at.isoformat()
-
-    failover_value = {'leader': master, 'member': candidate, 'scheduled_at': scheduled_at}
-    logging.debug(failover_value)
 
     # By now we have established that the leader exists and the candidate exists
     click.echo('Current cluster topology')
@@ -545,33 +496,44 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
 
     if not force:
         a = \
-            click.confirm('Are you sure you want to failover cluster {0}, demoting current master {1}?'.format(
+            click.confirm('Are you sure you want to failover cluster {}, demoting current master {}?'.format(
                 cluster_name, master))
         if not a:
             raise PatroniCtlException('Aborting failover')
 
+    failover_value = '{}:{}'.format(master, candidate or '')
+
+    t_started = time.time()
     r = None
     try:
-        r = post_patroni(cluster.leader.member, 'failover', failover_value)
+        r = post_patroni(cluster.leader.member, 'failover', {'leader': master, 'member': candidate or ''})
         if r.status_code == 200:
             logging.debug(r)
+            logging.debug(r.text)
             cluster = dcs.get_cluster()
-            logging.debug(cluster)
+            click.echo(timestamp() + ' Failing over to new leader: {}'.format(cluster.leader.member.name))
-            click.echo('{0} {1}'.format(timestamp(), r.text))
         else:
-            click.echo('Failover failed, details: {0}, {1}'.format(r.status_code, r.text))
+            click.echo('Failover failed, details: {}, {}'.format(r.status_code, r.text))
             return
     except:
         logging.exception(r)
         logging.warning('Failing over to DCS')
         click.echo(timestamp() + ' Could not failover using Patroni api, falling back to DCS')
-        click.echo(timestamp() + ' Initializing failover from master {0}'.format(master))
+        dcs.set_failover_value(failover_value)
-        dcs.manual_failover(leader=master, member=candidate, scheduled_at=failover_value)
+        click.echo(timestamp() + ' Initialized failover from master {}'.format(master))
+        # The failover process should within a minute update the failover key, we will keep watching it until it changes
+        # or we timeout
+        cluster = wait_for_leader(dcs, timeout=60)
+        if cluster.leader.member.name == master:
+            click.echo('Failover failed, master did not change after {:0.1f} seconds'.format(time.time() - t_started))
+            return
 
+    click.echo(timestamp() + ' Failover completed in {:0.1f} seconds, new leader is {}'.format(time.time() - t_started,
+               str(cluster.leader.member.name)))
     output_members(cluster, name=cluster_name)
 
 
-def output_members(cluster, name=None, fmt='pretty'):
+def output_members(cluster, name=None, format='pretty'):
     rows = []
     logging.debug(cluster)
     leader_name = None
@@ -591,9 +553,10 @@ def output_members(cluster, name=None, fmt='pretty'):
 
         host = build_connect_parameters(m.conn_url)['host']
 
-        xlog_location = m.data.get('xlog_location') or 0
+        xlog_location = m.data.get('xlog_location')
-        lag = ''
+        if xlog_location is None or (xlog_location_cluster < xlog_location):
-        if (xlog_location_cluster >= xlog_location):
+            lag = ''
+        else:
             lag = round((xlog_location_cluster - xlog_location)/1024/1024)
 
         rows.append([
@@ -615,7 +578,7 @@ def output_members(cluster, name=None, fmt='pretty'):
     ]
     alignment = {'Cluster': 'l', 'Member': 'l', 'Host': 'l', 'Lag in MB': 'r'}
 
-    print_output(columns, rows, alignment, fmt)
+    print_output(columns, rows, alignment, format)
 
 
 @ctl.command('list', help='List the Patroni members for a given Patroni')
@@ -625,8 +588,8 @@ def output_members(cluster, name=None, fmt='pretty'):
 @option_watch
 @option_watchrefresh
 @option_dcs
-def members(config_file, cluster_names, fmt, watch, w, dcs):
+def members(config_file, cluster_names, format, watch, w, dcs):
-    if not cluster_names:
+    if len(cluster_names) == 0:
         logging.warning('Listing members: No cluster names were provided')
         return
 
@@ -635,7 +598,7 @@ def members(config_file, cluster_names, fmt, watch, w, dcs):
         dcs = get_dcs(config, cn)
 
         for _ in watching(w, watch):
-            output_members(dcs.get_cluster(), name=cn, fmt=fmt)
+            output_members(dcs.get_cluster(), name=cn, format=format)
 
 
 def timestamp(precision=6):

patroni/dcs.py

@@ -1,8 +1,8 @@
 import abc
 import json
-import dateutil
 
 from collections import namedtuple
+from patroni.exceptions import DCSError
 from six.moves.urllib_parse import urlparse, urlunparse, parse_qsl
 from threading import Event, Lock
 
@@ -51,26 +51,22 @@ class Member(namedtuple('Member', 'index,name,session,data')):
         else:
             try:
                 data = json.loads(data)
-            except (TypeError, ValueError):
+            except:
                 data = {}
         return Member(index, name, session, data)
 
     @property
     def conn_url(self):
-        return self.data.get('conn_url')
+        return self.data.get('conn_url', None)
 
     @property
     def api_url(self):
-        return self.data.get('api_url')
+        return self.data.get('api_url', None)
 
     @property
     def nofailover(self):
         return self.data.get('tags', {}).get('nofailover', False)
 
-    @property
-    def replicatefrom(self):
-        return self.data.get('tags', {}).get('replicatefrom')
-
 
 class Leader(namedtuple('Leader', 'index,session,member')):
 
@@ -89,44 +85,12 @@ class Leader(namedtuple('Leader', 'index,session,member')):
         return self.member.conn_url
 
 
-class Failover(namedtuple('Failover', 'index,leader,member,scheduled_at')):
+class Failover(namedtuple('Failover', 'index,leader,member')):
 
-    """
-    >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader"}'))
-    True
-    >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader", "member": "cluster:member"}'))
-    True
-    >>> Failover.from_node(1, 'null') is None
-    True
-    >>> n = '{"leader": "cluster_leader", "member": "cluster:member", "scheduled_at": "2016-01-14T10:09:57.1394Z"}'
-    >>> 'tzinfo=' in str(Failover.from_node(1, n))
-    True
-    >>> Failover.from_node(1, None) is None
-    True
-    >>> Failover.from_node(1, '{}') is None
-    True
-    >>> 'abc' in Failover.from_node(1, 'abc:def')
-    True
-    """
     @staticmethod
     def from_node(index, value):
-        if not value:
+        t = [a.strip() for a in value.split(':')] + ['']
-            return None
+        return Failover(index, t[0], t[1]) if t[0] or t[1] else None
-
-        try:
-            data = json.loads(value)
-            if not data:
-                return None
-        except ValueError:
-            t = [a.strip() for a in value.split(':')]
-            leader = t[0]
-            candidate = t[1] if len(t) > 1 else None
-            return Failover(index, leader, candidate, None) if leader or candidate else None
-
-        if data.get('scheduled_at'):
-            data['scheduled_at'] = dateutil.parser.parse(data['scheduled_at'])
-
-        return Failover(index, data.get('leader'), data.get('member'), data.get('scheduled_at'))
 
 
 class Cluster(namedtuple('Cluster', 'initialize,leader,last_leader_operation,members,failover')):
@@ -143,14 +107,8 @@ class Cluster(namedtuple('Cluster', 'initialize,leader,last_leader_operation,mem
     def is_unlocked(self):
         return not (self.leader and self.leader.name)
 
-    def has_member(self, member_name):
-        return any(m for m in self.members if m.name == member_name)
 
-    def get_member(self, member_name):
+class AbstractDCS:
-        return ([m for m in self.members if m.name == member_name] or [None])[0]
-
-
-class AbstractDCS(object):
 
     __metaclass__ = abc.ABCMeta
 
@@ -168,7 +126,7 @@ class AbstractDCS(object):
             i.e.: `zookeeper` for zookeeper, `etcd` for etcd, etc...
         """
         self._name = name
-        self._namespace = '/{0}'.format(config.get('namespace', '/service/').strip('/'))
+        self._namespace = '/{}'.format(config.get('namespace', '/service/').strip('/'))
         self._base_path = '/'.join([self._namespace, config['scope']])
 
         self._cluster = None
@@ -258,18 +216,15 @@ class AbstractDCS(object):
     def set_failover_value(self, value, index=None):
         """Create or update `/failover` key"""
 
-    def manual_failover(self, leader, member, scheduled_at=None, index=None):
+    def manual_failover(self, leader, member, index=None):
-        failover_value = dict()
+        return self.set_failover_value(leader + (':' + member if member else ''), index)
-        if leader:
-            failover_value['leader'] = leader
 
-        if member:
+    def current_leader(self):
-            failover_value['member'] = member
+        try:
-
+            cluster = self.get_cluster()
-        if scheduled_at:
+            return None if cluster.is_unlocked() else cluster.leader
-            failover_value['scheduled_at'] = scheduled_at.isoformat()
+        except DCSError:
-
+            return None
-        return self.set_failover_value(json.dumps(failover_value), index)
 
     @abc.abstractmethod
     def touch_member(self, connection_string, ttl=None):

patroni/etcd.py

@@ -14,7 +14,6 @@ from patroni.dcs import AbstractDCS, Cluster, Failover, Leader, Member
 from patroni.exceptions import DCSError
 from patroni.utils import Retry, RetryFailedError, sleep
 from requests.exceptions import RequestException
-from six.moves.http_client import HTTPException
 
 logger = logging.getLogger(__name__)
 
@@ -50,57 +49,11 @@ class Client(etcd.Client):
             self._update_machines_cache = True
             return [self._base_uri]
 
-    def _do_http_request(self, request_executor, method, url, fields=None, **kwargs):
+    def api_execute(self, path, method, **kwargs):
-        try:
-            response = request_executor(method, url, fields=fields, **kwargs)
-            response.data.decode('utf-8')
-            self._check_cluster_id(response)
-        except (urllib3.exceptions.HTTPError, HTTPException, socket.error) as e:
-            if (isinstance(fields, dict) and fields.get("wait") == "true" and
-                    isinstance(e, urllib3.exceptions.ReadTimeoutError)):
-                logger.debug("Watch timed out.")
-                raise etcd.EtcdWatchTimedOut("Watch timed out: {0}".format(e), cause=e)
-            logger.error("Request to server %s failed: %r", self._base_uri, e)
-            logger.info("Reconnection allowed, looking for another server.")
-            self._base_uri = self._next_server(cause=e)
-            response = False
-        return response
-
-    def api_execute(self, path, method, params=None, timeout=None):
-        if not path.startswith('/'):
-            raise ValueError('Path does not start with /')
-
-        if timeout is None:
-            timeout = self.read_timeout
-
-        if timeout == 0:
-            timeout = None
-
-        kwargs = {'timeout': timeout, 'fields': params, 'redirect': self.allow_redirect,
-                  'headers': self._get_headers(), 'preload_content': False}
-
-        if method in [self._MGET, self._MDELETE]:
-            request_executor = self.http.request
-        elif method in [self._MPUT, self._MPOST]:
-            request_executor = self.http.request_encode_body
-            kwargs['encode_multipart'] = False
-        else:
-            raise etcd.EtcdException('HTTP method {0} not supported'.format(method))
-
         # Update machines_cache if previous attempt of update has failed
-        if self._update_machines_cache:
+        self._update_machines_cache and self._load_machines_cache()
-            self._load_machines_cache()
-
-        response = False
-
         try:
-            while not response:
+            return super(Client, self).api_execute(path, method, **kwargs)
-                response = self._do_http_request(request_executor, method, self._base_uri + path, **kwargs)
-
-                if response is False and not self._use_proxies:
-                    self._machines_cache = self.machines
-                    self._machines_cache.remove(self._base_uri)
-            return self._handle_server_response(response)
         except etcd.EtcdConnectionFailed:
             self._update_machines_cache = True
             raise
@@ -113,6 +66,16 @@ class Client(etcd.Client):
             logger.exception('Can not resolve SRV for %s', host)
         return []
 
+    # try to workarond bug in python-etcd: https://github.com/jplana/python-etcd/issues/81
+    def _result_from_response(self, response):
+        try:
+            response.data.decode('utf-8')
+        except urllib3.exceptions.TimeoutError:
+            raise
+        except Exception as e:
+            raise etcd.EtcdException('Unable to decode server response: %s' % e)
+        return super(Client, self)._result_from_response(response)
+
     def _get_machines_cache_from_srv(self, discovery_srv):
         """Fetch list of etcd-cluster member by resolving _etcd-server._tcp. SRV record.
         This record should contain list of host and peer ports which could be used to run
@@ -120,7 +83,7 @@ class Client(etcd.Client):
 
         ret = []
         for host, port in self.get_srv_record(discovery_srv):
-            url = '{0}://{1}:{2}/members'.format(self._protocol, host, port)
+            url = '{}://{}:{}/members'.format(self._protocol, host, port)
             try:
                 response = requests.get(url, timeout=5)
                 if response.ok:
@@ -138,10 +101,10 @@ class Client(etcd.Client):
         host, port = addr.split(':')
         try:
             for r in set(socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)):
-                ret.append('{0}://{1}:{2}'.format(self._protocol, r[4][0], r[4][1]))
+                ret.append('{}://{}:{}'.format(self._protocol, r[4][0], r[4][1]))
         except socket.error:
             logger.exception('Can not resolve %s', host)
-        return list(set(ret)) if ret else ['{0}://{1}:{2}'.format(self._protocol, host, port)]
+        return list(set(ret)) if ret else ['{}://{}:{}'.format(self._protocol, host, port)]
 
     def _load_machines_cache(self):
         """This method should fill up `_machines_cache` from scratch.
@@ -169,9 +132,7 @@ class Client(etcd.Client):
         # After filling up initial list of machines_cache we should ask etcd-cluster about actual list
         self._base_uri = self._machines_cache.pop(0)
         self._machines_cache = self.machines
-
+        self._base_uri in self._machines_cache and self._machines_cache.remove(self._base_uri)
-        if self._base_uri in self._machines_cache:
-            self._machines_cache.remove(self._base_uri)
 
         self._update_machines_cache = False
 
@@ -179,7 +140,7 @@ class Client(etcd.Client):
 def catch_etcd_errors(func):
     def wrapper(*args, **kwargs):
         try:
-            return func(*args, **kwargs) is not None
+            return not func(*args, **kwargs) is None
         except (RetryFailedError, etcd.EtcdException):
             return False
         except:
@@ -204,8 +165,7 @@ class Etcd(AbstractDCS):
     def retry(self, *args, **kwargs):
         return self._retry.copy()(*args, **kwargs)
 
-    @staticmethod
+    def get_etcd_client(self, config):
-    def get_etcd_client(config):
         client = None
         while not client:
             try:
@@ -225,25 +185,25 @@ class Etcd(AbstractDCS):
             nodes = {os.path.relpath(node.key, result.key): node for node in result.leaves}
 
             # get initialize flag
-            initialize = nodes.get(self._INITIALIZE)
+            initialize = nodes.get(self._INITIALIZE, None)
             initialize = initialize and initialize.value
 
             # get last leader operation
-            last_leader_operation = nodes.get(self._LEADER_OPTIME)
+            last_leader_operation = nodes.get(self._LEADER_OPTIME, None)
             last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation.value)
 
             # get list of members
             members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
 
             # get leader
-            leader = nodes.get(self._LEADER)
+            leader = nodes.get(self._LEADER, None)
             if leader:
                 member = Member(-1, leader.value, None, {})
                 member = ([m for m in members if m.name == leader.value] or [member])[0]
                 leader = Leader(leader.modifiedIndex, leader.ttl, member)
 
             # failover key
-            failover = nodes.get(self._FAILOVER)
+            failover = nodes.get(self._FAILOVER, None)
             if failover:
                 failover = Failover.from_node(failover.modifiedIndex, failover.value)
 
@@ -308,7 +268,7 @@ class Etcd(AbstractDCS):
                     # Synchronous work of all cluster members with etcd is less expensive
                     # than reestablishing http connection every time from every replica.
                     return True
-                except etcd.EtcdWatchTimedOut:
+                except urllib3.exceptions.TimeoutError:
                     self.client.http.clear()
                     return False
                 except etcd.EtcdException:

patroni/exceptions.py

@@ -1,6 +1,3 @@
-from click import ClickException
-
-
 class PatroniException(Exception):
 
     """Parent class for all kind of exceptions related to selected distributed configuration store"""
@@ -16,7 +13,7 @@ class PatroniException(Exception):
         return repr(self.value)
 
 
-class PatroniCtlException(ClickException):
+class PatroniCtlException(Exception):
     pass
 
 

patroni/ha.py

@@ -3,18 +3,15 @@ import logging
 import psycopg2
 import requests
 import sys
-import datetime
-import pytz
 
-from multiprocessing.pool import ThreadPool
 from patroni.async_executor import AsyncExecutor
 from patroni.exceptions import DCSError, PostgresConnectionException
-from patroni.utils import sleep
+from multiprocessing.pool import ThreadPool
 
 logger = logging.getLogger(__name__)
 
 
-class Ha(object):
+class Ha:
 
     def __init__(self, patroni):
         self.patroni = patroni
@@ -22,13 +19,17 @@ class Ha(object):
         self.dcs = patroni.dcs
         self.cluster = None
         self.old_cluster = None
-        self.recovering = False
         self._async_executor = AsyncExecutor()
+        self.bdr = patroni.bdr
+        self.use_bdr = self.bdr and\
+            self.bdr.get('enable') and\
+            (self.bdr.get('database') is not None)
+        self.bdr_may_need_reinitialize = False
 
     def load_cluster_from_dcs(self):
         cluster = self.dcs.get_cluster()
 
-        # We want to keep the state of cluster when it was healthy
+        # We want to keep the state of cluster when it was healhy
         if not cluster.is_unlocked() or not self.old_cluster:
             self.old_cluster = cluster
         self.cluster = cluster
@@ -50,7 +51,7 @@ class Ha(object):
         logger.info('Lock owner: %s; I am %s', lock_owner, self.state_handler.name)
         return lock_owner == self.state_handler.name
 
-    def touch_member(self):
+    def touch_member(self, ttl=None):
         data = {
             'conn_url': self.state_handler.connection_string,
             'api_url': self.patroni.api.connection_string,
@@ -63,31 +64,39 @@ class Ha(object):
                 data['xlog_location'] = self.state_handler.xlog_position()
             except:
                 pass
-        self.dcs.touch_member(json.dumps(data, separators=(',', ':')))
+        self.dcs.touch_member(json.dumps(data, separators=(',', ':')), ttl=ttl)
 
-    def clone(self, clone_member, clone_member_name="leader"):
+    def copy_backup_from_leader(self, leader):
-        if self.state_handler.bootstrap(cluster_initialized=True, clone_member=clone_member):
+        if self.state_handler.bootstrap(leader):
-            logger.info('bootstrapped from {0}'.format(clone_member_name)
+            logger.info('bootstrapped from leader')
-                        if clone_member else 'bootstrapped without leader')
         else:
             self.state_handler.stop('immediate')
             self.state_handler.remove_data_directory()
-            logger.error('failed to bootstrap from {0}'.format(clone_member_name)
+            logger.error('failed to bootstrap from leader')
-                         if clone_member else 'failed to bootstrap (without leader)')
 
-    def bootstrap(self):
+    def initialize_bdr(self, leader):
+        if self.state_handler.initialize_bdr_database(leader):
+            logger.info("bootstrapped from leader")
+        else:
+            self.state_handler.stop('immediate')
+            self.state_handler.remove_data_directory()
+            logger.error('failed to bootstrap from leader')
+
+    def bootstrap(self, bdr=False):
         if not self.cluster.is_unlocked():  # cluster already has leader
-            clonefrom = self.patroni.clonefrom
+            self._async_executor.schedule('bootstrap from leader')
-            clone_member = self.cluster.get_member(clonefrom)\
+            if not bdr:
-                if self.cluster.has_member(clonefrom) else self.cluster.leader
+                self._async_executor.run_async(self.copy_backup_from_leader, args=(self.cluster.leader, ))
-            clone_member_name = 'leader' if clone_member == self.cluster.leader else 'replica \'{0}\''.format(clonefrom)
+            else:
-            self._async_executor.schedule('bootstrap from {0}'.format(clone_member_name))
+                self._async_executor.run_async(self.initialize_bdr, args=(self.cluster.leader, ))
-            self._async_executor.run_async(self.clone, args=(clone_member, clone_member_name))
+            return 'trying to bootstrap from leader'
-            return 'trying to bootstrap from {0}'.format(clone_member_name)
         elif not self.cluster.initialize and not self.patroni.nofailover:  # no initialize key
             if self.dcs.initialize(create_new=True):  # race for initialization
                 try:
-                    self.state_handler.bootstrap()
+                    if not bdr:
+                        self.state_handler.bootstrap()
+                    else:
+                        self.state_handler.initialize_bdr_database()
                     self.dcs.initialize(create_new=False, sysid=self.state_handler.sysid)
                 except:  # initdb or start failed
                     # remove initialization key and give a chance to other members
@@ -97,48 +106,64 @@ class Ha(object):
                     self.state_handler.move_data_directory()
                     raise
                 self.dcs.take_leader()
-                self.load_cluster_from_dcs()
                 return 'initialized a new cluster'
             else:
                 return 'failed to acquire initialize lock'
         else:
-            if self.state_handler.can_create_replica_without_replication_connection():
-                self._async_executor.run_async(self.clone, args=(None, ))
-                return "trying to bootstrap without leader"
             return 'waiting for leader to bootstrap'
 
+    def bootstrap_bdr(self):
+        return self.bootstrap(bdr=True)
+
     def recover(self):
+        has_lock = self.has_lock()
+
         # try to see if we are the former master that crashed. If so - we likely need to run pg_rewind
         # in order to join the former standby being promoted.
         pg_controldata = self.state_handler.controldata()
-        if (self.state_handler.role == 'master') and pg_controldata and\
+        if not has_lock and pg_controldata and\
                 pg_controldata.get('Database cluster state', '') == 'in production':  # crashed master
             self.state_handler.require_rewind()
-        self.recovering = True
-        return self.follow("started as readonly because i had the session lock",
-                           "started as a secondary",
-                           refresh=True, recovery=True)
 
-    def follow(self, demote_reason, follow_reason, refresh=True, recovery=False):
+        # XXX: follow the leader calls stop, which might take quite some time.
-        if refresh:
+        # perhaps we should run sync asynchronously
-            self.load_cluster_from_dcs()
+        # (we still need the exit code from follow_the_leader)
+        ret = self.state_handler.follow_the_leader(None if has_lock else self.cluster.leader, recovery=True)
+        if not ret:
+            if not has_lock:
+                return 'failed to start postgres'
+            self.dcs.delete_leader()
+            self.dcs.reset_cluster()
+            return 'removed leader key after trying and failing to start postgres'
+        if not has_lock:
+            return 'started as a secondary'
+        logger.info('started as readonly because i had the session lock')
+        self.load_cluster_from_dcs()
 
-        if not recovery and self.state_handler.is_leader() or recovery and self.state_handler.role == 'master':
+    def recover_bdr(self):
-            ret = demote_reason
+        if not self.bdr_may_need_reinitialize:
-        else:
+            self.state_handler.start_bdr()
-            ret = follow_reason
+            return "started as a member of BDR group"
+        has_lock = self.has_lock()
+        if self.cluster.is_unlocked():  # no leader yet, and we need to reinitialize from the leader
+            return "waiting for another member of the BDR group"
+        if has_lock:
+            self.dcs.delete_leader()
+            return "released the lock to another member of the BDR group"
+        if self.state_handler.initialize_bdr_database(self.cluster.leader, empty=False):
+            # we reinitialized this node, make sure the cluster is aware of it.
+            self.touch_member()
+            self.bdr_may_need_reinitialize = False
+            return "reinitialized and started as a member of the BDR group"
 
-        # determine the node to follow. If replicatefrom tag is set,
+    def follow_the_leader(self, demote_reason, follow_reason, refresh=True):
-        # try to follow the node mentioned there, otherwise, follow the leader.
+        refresh and self.load_cluster_from_dcs()
-        if self.patroni.replicatefrom:
+        ret = demote_reason if self.state_handler.is_leader() else follow_reason
-            node_to_follow = [m for m in self.cluster.members if m.name == self.patroni.replicatefrom]
+        leader = self.cluster.leader
-            node_to_follow = node_to_follow[0] if node_to_follow else self.cluster.leader
+        leader = None if (leader and leader.name) == self.state_handler.name else leader
-        else:
+        if not self.state_handler.check_recovery_conf(leader):
-            node_to_follow = self.cluster.leader
-        node_to_follow = None if node_to_follow and node_to_follow.name == self.state_handler.name else node_to_follow
-        if not self.state_handler.check_recovery_conf(node_to_follow) or recovery:
             self._async_executor.schedule('changing primary_conninfo and restarting')
-            self._async_executor.run_async(self.state_handler.follow, (node_to_follow, recovery))
+            self._async_executor.run_async(self.state_handler.follow_the_leader, (leader, ))
         return ret
 
     def enforce_master_role(self, message, promote_message):
@@ -209,7 +234,7 @@ class Ha(object):
         ret = False
         members = [m for m in members if m.name != self.state_handler.name and not m.nofailover and m.api_url]
         if members:
-            for member, reachable, _, _, tags in self.fetch_nodes_statuses(members):
+            for member, reachable, in_recovery, xlog_location, tags in self.fetch_nodes_statuses(members):
                 if reachable and not tags.get('nofailover', False):
                     ret = True  # TODO: check xlog_location
                 elif not reachable:
@@ -229,7 +254,7 @@ class Ha(object):
             # find specific node and check that it is healthy
             members = [m for m in self.cluster.members if m.name == failover.member]
             if members:
-                member, reachable, _, _, tags = self.fetch_node_status(members[0])
+                member, reachable, in_recovery, xlog_location, tags = self.fetch_node_status(members[0])
                 if reachable and not tags.get('nofailover', False):  # node is healthy
                     logger.info('manual failover: to %s, i am %s', member.name, self.state_handler.name)
                     return False
@@ -279,40 +304,20 @@ class Ha(object):
             self.dcs.delete_leader()
             self.touch_member()
             self.dcs.reset_cluster()
-        self.state_handler.follow(None)
+        self.state_handler.follow_the_leader(None)
 
-    def process_manual_failover_from_leader(self):
+    def process_manual_failover_from_leader(self, bdr=False):
         failover = self.cluster.failover
-
-        if failover.scheduled_at:
-            # If the failover is in the far future, we shouldn't do anything and just return.
-            # If the failover is in the past, we consider the value to be stale and we remove
-            # the value.
-            # If the value is close to now, we initiate the failover
-            now = datetime.datetime.now(pytz.utc)
-            try:
-                delta = (failover.scheduled_at - now).total_seconds()
-
-                if delta > self.patroni.nap_time:
-                    logging.info('Awaiting failover at %s (in %.0f seconds)', failover.scheduled_at.isoformat(), delta)
-                    return
-                elif delta < - int(self.patroni.nap_time * 1.5):
-                    logger.warning('Found a stale failover value, cleaning up: %s', failover.scheduled_at)
-                    self.dcs.manual_failover('', '', self.cluster.failover.index)
-                    return
-
-                # The value is very close to now
-                sleep(max(delta, 0))
-                logger.info('Manual scheduled failover at {}'.format(failover.scheduled_at.isoformat()))
-            except TypeError:
-                logger.warning('Incorrect value in of scheduled_at: %s', failover.scheduled_at)
-
         if not failover.leader or failover.leader == self.state_handler.name:
             if not failover.member or failover.member != self.state_handler.name:
                 members = [m for m in self.cluster.members if not failover.member or m.name == failover.member]
                 if self.is_failover_possible(members):  # check that there are healthy members
-                    self._async_executor.schedule('manual failover: demote')
+                    if not bdr:
-                    self._async_executor.run_async(self.demote)
+                        self._async_executor.schedule('manual failover: demote')
+                        self._async_executor.run_async(self.demote)
+                    else:
+                        self.dcs.delete_leader()
+                        self.dcs.reset_cluster()
                     return 'manual failover: demoting myself'
                 else:
                     logger.warning('manual failover: no healthy members found, failover is not possible')
@@ -325,6 +330,9 @@ class Ha(object):
         logger.info('Trying to clean up failover key')
         self.dcs.manual_failover('', '', self.cluster.failover.index)
 
+    def process_manual_failover_from_leader_bdr(self):
+        self.process_manual_failover_from_leader(bdr=True)
+
     def process_unhealthy_cluster(self):
         if self.is_healthiest_node():
             if self.acquire_lock():
@@ -335,14 +343,14 @@ class Ha(object):
                 return self.enforce_master_role('acquired session lock as a leader',
                                                 'promoted self to leader by acquiring session lock')
             else:
-                return self.follow('demoted self after trying and failing to obtain lock',
+                return self.follow_the_leader('demoted self due after trying and failing to obtain lock',
-                                   'following new leader after trying and failing to obtain lock')
+                                              'following new leader after trying and failing to obtain lock')
         else:
             if self.patroni.nofailover:
-                return self.follow('demoting self because I am not allowed to become master',
+                return self.follow_the_leader('demoting self because I am not allowed to become master',
-                                   'following a different leader because I am not allowed to promote')
+                                              'following a different leader because I am not allowed to promote')
-            return self.follow('demoting self because i am not the healthiest node',
+            return self.follow_the_leader('demoting self because i am not the healthiest node',
-                               'following a different leader because i am not the healthiest node')
+                                          'following a different leader because i am not the healthiest node')
 
     def process_healthy_cluster(self):
         if self.has_lock():
@@ -360,8 +368,33 @@ class Ha(object):
                 self.load_cluster_from_dcs()
         else:
             logger.info('does not have lock')
-        return self.follow('demoting self because i do not have the lock and i was a leader',
+        return self.follow_the_leader('demoting self because i do not have the lock and i was a leader',
-                           'no action.  i am a secondary and i am following a leader', False)
+                                      'no action.  i am a secondary and i am following a leader', False)
+
+    def process_unhealthy_cluster_bdr(self):
+        if self.acquire_lock():
+            if self.cluster.failover:
+                logger.info('Cleaning up failover key after acquiring leader lock...')
+                self.dcs.manual_failover('', '')
+            self.dcs.get_cluster()
+            return "acquired the session lock as a leader"
+        else:
+            return "lost the race to acquire the session lock"
+
+    def process_healthy_cluster_bdr(self):
+        if self.has_lock():
+            if self.cluster.failover:
+                msg = self.process_manual_failover_from_leader_bdr()
+                if msg is not None:
+                    return msg
+            if self.update_lock():
+                return "no action. I am the leader with the lock"
+            else:
+                logger.error("failed to update leader lock")
+                self.load_cluster_from_dcs()
+        else:
+            logger.info("does not have lock")
+        return "no action. I do not have the lock"
 
     def schedule(self, action):
         with self._async_executor:
@@ -389,7 +422,7 @@ class Ha(object):
     def reinitialize(self, cluster):
         self.state_handler.stop('immediate')
         self.state_handler.remove_data_directory()
-        self.clone(cluster.leader)
+        self.copy_backup_from_leader(cluster.leader)
 
     def process_scheduled_action(self):
         if self.reinitialize_scheduled():
@@ -414,26 +447,23 @@ class Ha(object):
         else:
             return self._async_executor.scheduled_action + ' in progress'
 
-    @staticmethod
+    def sysid_valid(self, sysid):
-    def sysid_valid(sysid):
         # sysid does tv_sec << 32, where tv_sec is the number of seconds sine 1970,
         # so even 1 << 32 would have 10 digits.
         return str(sysid) and len(str(sysid)) >= 10 and str(sysid).isdigit()
 
-    def post_recover(self):
-        if not self.state_handler.is_running():
-            if self.has_lock():
-                self.dcs.delete_leader()
-                self.dcs.reset_cluster()
-                return 'removed leader key after trying and failing to start postgres'
-            return 'failed to start postgres'
-        return None
-
     def _run_cycle(self):
         try:
             self.load_cluster_from_dcs()
 
-            self.touch_member()
+            # if member key is missing - we may need to destory
+            # and re-create the BDR database unless we are the first node in the cluster
+            if self.use_bdr and len([m for m in self.cluster.members if m.name == self.state_handler.name]) == 0:
+                self.bdr_may_need_reinitialize = True
+
+            # Create a member key with a very short expiration time if we may need to reinitialize this member
+            # in order to avoid lossing the "may need to reinitialize status" in case Patroni is restarted
+            self.touch_member(ttl=None if not self.bdr_may_need_reinitialize else self.bdr.get('min_ttl', 5))
 
             # cluster has leader key but not initialize key
             if not self.cluster.is_unlocked() and not self.sysid_valid(self.cluster.initialize) and self.has_lock():
@@ -442,13 +472,6 @@ class Ha(object):
             if self._async_executor.busy:
                 return self.handle_long_action_in_progress()
 
-            # we've got here, so any async action has finished. Check if we tried to recover and failed
-            if self.recovering:
-                self.recovering = False
-                msg = self.post_recover()
-                if msg is not None:
-                    return msg
-
             # currently it can trigger only reinitialize
             msg = self.process_scheduled_action()
             if msg is not None:
@@ -456,41 +479,50 @@ class Ha(object):
 
             # is data directory empty?
             if self.state_handler.data_directory_empty():
-                return self.bootstrap()  # new node
+                if self.use_bdr:
+                    # member key was missing, but it's a new database - no need to reinitialize
+                    self.bdr_may_need_reinitialize = False
+                    self.touch_member()
+                return self.bootstrap(self.use_bdr)  # new node
             # "bootstrap", but data directory is not empty
-            elif not self.sysid_valid(self.cluster.initialize) and self.cluster.is_unlocked():
+            elif not self.sysid_valid(self.cluster.initialize) and self.cluster.is_unlocked() and \
+                    not self.bdr_may_need_reinitialize:
                 self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=self.state_handler.sysid)
             else:
                 # check if we are allowed to join
-                if self.sysid_valid(self.cluster.initialize) and self.cluster.initialize != self.state_handler.sysid:
+                if not self.use_bdr and \
-                    logger.fatal("system ID mismatch, node %s belongs to a different cluster: %s != %s",
+                   self.sysid_valid(self.cluster.initialize) and self.cluster.initialize != self.state_handler.sysid:
-                                 self.state_handler.name, self.cluster.initialize, self.state_handler.sysid)
+                    logger.fatal("system ID mismatch, node {0} belongs to a different cluster".
+                                 format(self.state_handler.name))
                     sys.exit(1)
 
             # try to start dead postgres
-            if not self.state_handler.is_healthy():
+            if self.use_bdr:
-                msg = self.recover()
+                if self.bdr_may_need_reinitialize or not self.state_handler.is_healthy():
-                if msg is not None:
+                    msg = self.recover_bdr()
-                    return msg
+                    if msg is not None:
+                        return msg
+            else:
+                if not self.state_handler.is_healthy():
+                    msg = self.recover()
+                    if msg is not None:
+                        return msg
 
             try:
                 if self.cluster.is_unlocked():
-                    return self.process_unhealthy_cluster()
+                    return self.process_unhealthy_cluster() if not self.use_bdr else self.process_unhealthy_cluster_bdr()
                 else:
-                    return self.process_healthy_cluster()
+                    return self.process_healthy_cluster() if not self.use_bdr else self.process_healthy_cluster_bdr()
             finally:
-                # we might not have a valid PostgreSQL connection here if another thread
+                if not self.use_bdr:
-                # stops PostgreSQL, therefore, we only reload replication slots if no
-                # asynchronous processes are running (should be always the case for the master)
-                if not self._async_executor.busy:
                     self.state_handler.sync_replication_slots(self.cluster)
         except DCSError:
             logger.error('Error communicating with DCS')
-            if self.state_handler.is_running() and self.state_handler.is_leader():
+            if self.state_handler.is_running() and self.state_handler.is_leader() and not self.use_bdr:
                 self.demote(delete_leader=False)
                 return 'demoted self because DCS is not accessible and i was a leader'
         except (psycopg2.Error, PostgresConnectionException):
-            logger.exception('Error communicating with PostgreSQL. Will try again later')
+            logger.exception('Error communicating with Postgresql. Will try again later')
 
     def run_cycle(self):
         with self._async_executor:

patroni/postgresql.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import psycopg2
+import re
 import shlex
 import shutil
 import subprocess
@@ -39,10 +40,11 @@ def parseurl(url):
     return ret
 
 
-class Postgresql(object):
+class Postgresql:
 
-    def __init__(self, config):
+    def __init__(self, config, config_bdr={}):
         self.config = config
+        self.bdr = config_bdr
         self.name = config['name']
         self.server_parameters = config.get('parameters', {})
         self.scope = config['scope']
@@ -52,7 +54,7 @@ class Postgresql(object):
         self.superuser = config['superuser']
         self.admin = config['admin']
         self.initdb_options = config.get('initdb', [])
-        self.pgpass = config.get('pgpass') or os.path.join(os.path.expanduser('~'), 'pgpass')
+        self.pgpass = config.get('pgpass', None) or os.path.join(os.path.expanduser('~'), 'pgpass')
         self.pg_rewind = config.get('pg_rewind', {})
         self.callback = config.get('callbacks', {})
         self.use_slots = config.get('use_slots', True)
@@ -61,17 +63,18 @@ class Postgresql(object):
         self.configuration_to_save = (os.path.join(self.data_dir, 'pg_hba.conf'),
                                       os.path.join(self.data_dir, 'postgresql.conf'))
         self.postmaster_pid = os.path.join(self.data_dir, 'postmaster.pid')
-        self.trigger_file = config.get('recovery_conf', {}).get('trigger_file') or 'promote'
+        self.trigger_file = config.get('recovery_conf', {}).get('trigger_file', None) or 'promote'
         self.trigger_file = os.path.abspath(os.path.join(self.data_dir, self.trigger_file))
 
         self._pg_ctl = ['pg_ctl', '-w', '-D', self.data_dir]
 
         self.local_address = self.get_local_address()
-        connect_address = config.get('connect_address') or self.local_address
+        connect_address = config.get('connect_address', None) or self.local_address
         self.connection_string = 'postgres://{username}:{password}@{connect_address}/postgres'.format(
             connect_address=connect_address, **self.replication)
 
         self._connection = None
+        self._connection_db = None
         self._cursor_holder = None
         self._need_rewind = False
         self._sysid = None
@@ -128,37 +131,32 @@ class Postgresql(object):
                 break
         return local_address + ':' + self.port
 
-    @property
+    def connection(self, dbname):
-    def _connect_kwargs(self):
-        r = parseurl('postgres://{0}/postgres'.format(self.local_address))
-        if 'username' in self.superuser:
-            r['user'] = self.superuser['username']
-        if 'password' in self.superuser:
-            r['password'] = self.superuser['password']
-        return r
-
-    def connection(self):
         if not self._connection or self._connection.closed != 0:
-            self._connection = psycopg2.connect(**self._connect_kwargs)
+            r = parseurl('postgres://{0}/{1}'.format(self.local_address, dbname))
+            self._connection = psycopg2.connect(**r)
+            self._connection_db = dbname
             self._connection.autocommit = True
-            self.server_version = self._connection.server_version
         return self._connection
 
-    def _cursor(self):
+    def _cursor(self, dbname='postgres'):
+        if self._connection_db != dbname:
+            self.close_connection()
         if not self._cursor_holder or self._cursor_holder.closed or self._cursor_holder.connection.closed != 0:
-            logger.info("establishing a new patroni connection to the postgres cluster")
+            logger.info("established a new patroni connection to the {0} cluster".format(dbname))
-            self._cursor_holder = self.connection().cursor()
+            self._cursor_holder = self.connection(dbname).cursor()
         return self._cursor_holder
 
     def close_connection(self):
         if self._cursor_holder and self._cursor_holder.connection and self._cursor_holder.connection.closed == 0:
             self._cursor_holder.connection.close()
-            logger.info("closed patroni connection to the postgresql cluster")
+            logger.info("closed patroni connection to the {0} cluster".format(self._connection_db))
+            self._connection_db = None
 
-    def _query(self, sql, *params):
+    def _query(self, sql, dbname, *params):
         cursor = None
         try:
-            cursor = self._cursor()
+            cursor = self._cursor(dbname)
             cursor.execute(sql, params)
             return cursor
         except psycopg2.Error as e:
@@ -168,9 +166,10 @@ class Postgresql(object):
                 raise RetryFailedError('cluster is being restarted')
             raise PostgresConnectionException('connection problems')
 
-    def query(self, sql, *params):
+    def query(self, sql, *params, **kwargs):
+        dbname = kwargs.get('dbname', 'postgres')
         try:
-            return self.retry(self._query, sql, *params)
+            return self.retry(self._query, sql, dbname, *params)
         except RetryFailedError as e:
             raise PostgresConnectionException(str(e))
 
@@ -180,36 +179,32 @@ class Postgresql(object):
     @staticmethod
     def initdb_allowed_option(name):
         if name in ['pgdata', 'nosync', 'pwfile', 'sync-only']:
-            raise Exception('{0} option for initdb is not allowed'.format(name))
+            raise Exception('{} option for initdb is not allowed'.format(name))
         return True
 
     def get_initdb_options(self):
         options = []
         for o in self.initdb_options:
             if isinstance(o, string_types) and self.initdb_allowed_option(o):
-                options.append('--{0}'.format(o))
+                options.append('--{}'.format(o))
             elif isinstance(o, dict):
                 keys = list(o.keys())
                 if len(keys) != 1 or not isinstance(keys[0], string_types) or not self.initdb_allowed_option(keys[0]):
-                    raise Exception('Invalid option: {0}'.format(o))
+                    raise Exception('Invalid option: {}'.format(o))
-                options.append('--{0}={1}'.format(keys[0], o[keys[0]]))
+                options.append('--{}={}'.format(keys[0], o[keys[0]]))
             else:
-                raise Exception('Unknown type of initdb option: {0}'.format(o))
+                raise Exception('Unknown type of initdb option: {}'.format(o))
         return options
 
     def initialize(self):
         self.set_state('initalizing new cluster')
         options = self.get_initdb_options()
         pwfile = None
-
+        if self.superuser and 'username' not in self.superuser and 'password' in self.superuser:
-        if self.superuser:
+            (fd, pwfile) = tempfile.mkstemp()
-            if 'username' in self.superuser:
+            os.write(fd, self.superuser['password'].encode())
-                options.append('--username={0}'.format(self.superuser['username']))
+            os.close(fd)
-            if 'password' in self.superuser:
+            options.append('--pwfile={}'.format(pwfile))
-                (fd, pwfile) = tempfile.mkstemp()
-                os.write(fd, self.superuser['password'].encode('utf-8'))
-                os.close(fd)
-                options.append('--pwfile={0}'.format(pwfile))
 
         ret = subprocess.call(self._pg_ctl + ['initdb'] + (['-o', ' '.join(options)] if options else [])) == 0
         if pwfile:
@@ -220,9 +215,83 @@ class Postgresql(object):
             self.set_state('initdb failed')
         return ret
 
+    def initialize_bdr_database(self, leader=None, empty=True):
+        """ initialize BDR database. If empty = False, reinitialize an existing one.
+            If leader is None, create a new group, otherwise, join an exising one.
+            non-empty database with an empty leader is not supported
+        """
+        if not empty and not leader:
+            ret = False
+        else:
+            try:
+                ret = (self.initialize() and self.start_bdr()) if empty else\
+                    (self.drop_bdr_database() and self.start_bdr())
+            except:
+                logger.exception("exception")
+                ret = False
+        if ret:
+            # convert all connection URLs to name=value strings
+            leader_dsn = self.primary_conninfo(leader.conn_url, include_db=self.bdr['database']) if leader else None
+            self_dsn = self.primary_conninfo(self.connection_string, include_db=self.bdr['database'])
+            try:
+                self.create_replication_user(superuser=True)
+                ret = self.create_bdr_database() and (self.join_bdr_group(self_dsn, leader_dsn) if leader
+                                                      else self.create_bdr_group(self_dsn))
+            except:
+                logger.exception("exception")
+                ret = False
+        if not ret and not leader:
+            raise Exception("Could not bootstrap initial PostgreSQL BDR node")
+        return ret
+
+    def start_bdr(self):
+        logger.info("starting new BDR instance {0}".format(self.name))
+        return self.start(bdr=True)
+
+    def drop_bdr_database(self):
+        logger.info("Dropping BDR database for instance {0}".format(self.name))
+        # stop the database if it's running in order to turn off bdr
+        if self.is_running():
+            self.stop()
+        ret = self.start(bdr=False)
+        if ret:
+            self.query("DROP DATABASE IF EXISTS {0}".format(self.bdr['database']))
+        return ret and self.restart(bdr=True)
+
+    def create_bdr_database(self):
+        logger.info("Creating BDR database and extensions for instance {0}".format(self.name))
+        self.query("CREATE DATABASE {0}".format(self.bdr['database']))
+        self.query("CREATE EXTENSION IF NOT EXISTS btree_gist", dbname=self.bdr['database'])
+        self.query("CREATE EXTENSION IF NOT EXISTS bdr", dbname=self.bdr['database'])
+        return True
+
+    def join_bdr_group(self, self_dsn, join_dsn):
+        logger.info("Joining BDR group {0}, host {1}, dsn {2}".format(join_dsn, self.name, self_dsn))
+        self.query("SELECT bdr.bdr_group_join(local_node_name:=%s, join_using_dsn:=%s, node_external_dsn:=%s)",
+                   self.name, join_dsn, self_dsn, dbname=self.bdr['database'])
+        return True
+
+    def create_bdr_group(self, self_dsn):
+        logger.info("Creating new BDR group {0}, host {1}".format(self_dsn, self.name))
+        self.query("SELECT bdr.bdr_group_create(local_node_name:=%s, node_external_dsn:=%s)",
+                   self.name, self_dsn, dbname=self.bdr['database'])
+        return True
+
+    def remove_bdr_nodes(self, nodes):
+        logger.info("BDR node {0}, removing nodes {1}".format(self.name, nodes))
+        self.query("SELECT bdr.part_by_node_names(%s)", nodes, dbname=self.bdr['database'])
+
+    def remove_bdr_disconnected_members(self, cluster):
+        # get all members
+        result = self.query("SELECT node_name FROM bdr.bdr_nodes WHERE node_status = 'r'")
+        if result:
+            nodes_db = set([node[0] for node in result.fetchall()])
+            nodes_etcd = set([member.name for member in self.members])
+            nodes_remove = nodes_db.difference(nodes_etcd)
+            self.remove_bdr_nodes(list(nodes_remove))
+
     def delete_trigger_file(self):
-        if os.path.exists(self.trigger_file):
+        os.path.exists(self.trigger_file) and os.unlink(self.trigger_file)
-            os.unlink(self.trigger_file)
 
     def write_pgpass(self, record):
         with open(self.pgpass, 'w') as f:
@@ -233,13 +302,13 @@ class Postgresql(object):
         env['PGPASSFILE'] = self.pgpass
         return env
 
-    def sync_replica(self, clone_member):
+    def sync_from_leader(self, leader):
-        # add the credentials to connect to the replica origin to pgpass.
+        r = parseurl(leader.conn_url)
-        env = self.write_pgpass(parseurl(clone_member.conn_url)) if clone_member else os.environ.copy()
+
-        if self.create_replica(clone_member, env) == 0:
+        env = self.write_pgpass(r)
-            self.delete_trigger_file()
+        ret = self.create_replica(leader, env) == 0
-            return True
+        ret and self.delete_trigger_file()
-        return False
+        return ret
 
     @staticmethod
     def build_connstring(conn):
@@ -247,37 +316,22 @@ class Postgresql(object):
         >>> Postgresql.build_connstring({'host': '127.0.0.1', 'port': '5432'}) == 'host=127.0.0.1 port=5432'
         True
         """
-        return ' '.join('{0}={1}'.format(param, val) for param, val in sorted(conn.items()))
+        return ' '.join('{}={}'.format(param, val) for param, val in sorted(conn.items()))
 
-    def replica_method_can_work_without_replication_connection(self, method):
+    def create_replica(self, leader, env):
-        return method != 'basebackup' and self.config and self.config.get(method, {}).get('no_master')
-
-    def can_create_replica_without_replication_connection(self):
-        """ go through the replication methods to see if there are ones
-            that does not require a working replication connection.
-        """
-        replica_methods = self.config.get('create_replica_method', [])
-        return any(self.replica_method_can_work_without_replication_connection(replica_method)
-                   for replica_method in replica_methods)
-
-    def create_replica(self, clone_member, env):
         # create the replica according to the replica_method
         # defined by the user.  this is a list, so we need to
         # loop through all methods the user supplies
-        connstring = clone_member.conn_url if clone_member else ""
+        connstring = leader.conn_url
         # get list of replica methods from config.
         # If there is no configuration key, or no value is specified, use basebackup
         replica_methods = self.config.get('create_replica_method') or ['basebackup']
-        # if we don't have any source, leave only replica methods that work without it
-        replica_methods = \
-            [r for r in replica_methods if self.replica_method_can_work_without_replication_connection(r)]\
-            if not clone_member else replica_methods
         # go through them in priority order
         ret = 1
         for replica_method in replica_methods:
             # if the method is basebackup, then use the built-in
             if replica_method == "basebackup":
-                ret = self.basebackup(clone_member, env)
+                ret = self.basebackup(leader, env)
                 if ret == 0:
                     logger.info("replica has been created using basebackup")
                     # if basebackup succeeds, exit with success
@@ -324,7 +378,7 @@ class Postgresql(object):
         cmd = self.callback[cb_name]
         try:
             subprocess.Popen(shlex.split(cmd) + [cb_name, self.role, self.scope])
-        except OSError:
+        except:
             logger.exception('callback %s %s %s %s failed', cmd, cb_name, self.role, self.scope)
             return False
         return True
@@ -347,12 +401,15 @@ class Postgresql(object):
         with self._state_lock:
             self._state = value
 
-    def start(self, block_callbacks=False):
+    def start(self, block_callbacks=False, bdr=False):
         if self.is_running():
             logger.error('Cannot start PostgreSQL because one is already running.')
             return True
 
-        self.set_role('replica' if os.path.exists(self.recovery_conf) else 'master')
+        if not bdr:
+            self.set_role('replica' if os.path.exists(self.recovery_conf) else 'master')
+        else:
+            self.set_role('master')
         if os.path.exists(self.postmaster_pid):
             os.remove(self.postmaster_pid)
             logger.info('Removed %s', self.postmaster_pid)
@@ -360,32 +417,27 @@ class Postgresql(object):
         if not block_callbacks:
             self.set_state('starting')
 
-        env = os.environ.copy()
+        ret = subprocess.call(self._pg_ctl + ['start', '-o', self.server_options(bdr)]) == 0
-        if 'username' in self.superuser:
-            env['PGUSER'] = self.superuser['username']
-        ret = subprocess.call(self._pg_ctl + ['start', '-o', self.server_options()], env=env, preexec_fn=os.setsid) == 0
 
         self.set_state('running' if ret else 'start failed')
 
-        self.schedule_load_slots = ret and self.use_slots
+        if not bdr:
-        self.save_configuration_files()
+            self.schedule_load_slots = ret and self.use_slots
+            self.save_configuration_files()
         # block_callbacks is used during restart to avoid
         # running start/stop callbacks in addition to restart ones
-        if ret and not block_callbacks:
+        ret and not block_callbacks and self.call_nowait(ACTION_ON_START)
-            self.call_nowait(ACTION_ON_START)
         return ret
 
-    def checkpoint(self, connect_kwargs=None):
+    def checkpoint(self, connstring=None):
-        connect_kwargs = connect_kwargs or self._connect_kwargs
-        for p in ['connect_timeout', 'options']:
-            connect_kwargs.pop(p, None)
         try:
-            with psycopg2.connect(**connect_kwargs) as conn:
+            connstring = connstring or 'postgres://{}/postgres'.format(self.local_address)
+            with psycopg2.connect(connstring) as conn:
                 conn.autocommit = True
                 with conn.cursor() as cur:
                     cur.execute("SET statement_timeout = 0")
                     cur.execute('CHECKPOINT')
-        except psycopg2.Error:
+        except:
             logging.exception('Exception during CHECKPOINT')
 
     def stop(self, mode='fast', block_callbacks=False):
@@ -417,23 +469,31 @@ class Postgresql(object):
 
     def reload(self):
         ret = subprocess.call(self._pg_ctl + ['reload']) == 0
-        if ret:
+        ret and self.call_nowait(ACTION_ON_RELOAD)
-            self.call_nowait(ACTION_ON_RELOAD)
         return ret
 
-    def restart(self):
+    def restart(self, bdr=False):
         self.set_state('restarting')
-        ret = self.stop(block_callbacks=True) and self.start(block_callbacks=True)
+        ret = self.stop(block_callbacks=True) and self.start(block_callbacks=True, bdr=bdr)
         if ret:
             self.call_nowait(ACTION_ON_RESTART)
         else:
-            self.set_state('restart failed ({0})'.format(self.state))
+            self.set_state('restart failed ({})'.format(self.state))
         return ret
 
-    def server_options(self):
+    def server_options(self, bdr=False):
-        options = "--listen_addresses='{0}' --port={1}".format(self.listen_addresses, self.port)
+        bdr_set = False
+        options = "--listen_addresses='{}' --port={}".format(self.listen_addresses, self.port)
         for setting, value in self.server_parameters.items():
-            options += " --{0}='{1}'".format(setting, value)
+            if setting == 'shared_preload_libraries':
+                if not bdr and 'bdr' in value:
+                    value = ','.join([x for x in value.split(',') if x != 'bdr'])
+                elif bdr and 'bdr' not in value:
+                    bdr_set = True
+                    value = ','.join([x for x in value.split(',')]+['bdr'])
+            options += " --{}='{}'".format(setting, value)
+        if bdr and not bdr_set:
+            options += " --shared_preload_libraries='bdr'"
         return options
 
     def is_healthy(self):
@@ -443,7 +503,8 @@ class Postgresql(object):
         return True
 
     def check_replication_lag(self, last_leader_operation):
-        return (last_leader_operation or 0) - self.xlog_position() <= self.config.get('maximum_lag_on_failover', 0)
+        return (last_leader_operation if last_leader_operation else 0) - self.xlog_position() <=\
+            self.config.get('maximum_lag_on_failover', 0)
 
     def write_pg_hba(self):
         with open(os.path.join(self.data_dir, 'pg_hba.conf'), 'a') as f:
@@ -454,9 +515,10 @@ class Postgresql(object):
                 f.write(line + '\n')
 
     @staticmethod
-    def primary_conninfo(leader_url):
+    def primary_conninfo(leader_url, include_db=None):
         r = parseurl(leader_url)
-        return 'user={user} password={password} host={host} port={port} sslmode=prefer sslcompression=1'.format(**r)
+        ret = 'user={user} password={password} host={host} port={port} sslmode=prefer sslcompression=1'.format(**r)
+        return ret if not include_db else ret + ' dbname={0}'.format(include_db)
 
     def check_recovery_conf(self, leader):
         if not os.path.isfile(self.recovery_conf):
@@ -470,34 +532,33 @@ class Postgresql(object):
                     return pattern and (pattern in line)
         return not pattern
 
-    def write_recovery_conf(self, leader, bootstrap=False):
+    def write_recovery_conf(self, leader):
         with open(self.recovery_conf, 'w') as f:
             f.write("""standby_mode = 'on'
 recovery_target_timeline = 'latest'
 """)
             if leader and leader.conn_url:
-                f.write("""primary_conninfo = '{0}'\n""".format(self.primary_conninfo(leader.conn_url)))
+                f.write("""primary_conninfo = '{}'\n""".format(self.primary_conninfo(leader.conn_url)))
                 if self.use_slots:
-                    f.write("""primary_slot_name = '{0}'\n""".format(self.name))
+                    f.write("""primary_slot_name = '{}'\n""".format(self.name))
-            if (leader and leader.conn_url) or bootstrap:
                 for name, value in self.config.get('recovery_conf', {}).items():
-                    f.write("{0} = '{1}'\n".format(name, value))
+                    f.write("{} = '{}'\n".format(name, value))
 
     def rewind(self, leader):
         # prepare pg_rewind connection
         r = parseurl(leader.conn_url)
         r.update(self.pg_rewind)
-        r['user'] = r.pop('username')
+        r['user'] = r['username']
         env = self.write_pgpass(r)
         pc = "user={user} host={host} port={port} dbname=postgres sslmode=prefer sslcompression=1".format(**r)
         # first run a checkpoint on a promoted master in order
         # to make it store the new timeline ([email protected])
-        self.checkpoint(r)
+        self.checkpoint(pc)
-        logger.info("running pg_rewind from %s", pc)
+        logger.info("running pg_rewind from {}".format(pc))
         pg_rewind = ['pg_rewind', '-D', self.data_dir, '--source-server', pc]
         try:
-            ret = subprocess.call(pg_rewind, env=env) == 0
+            ret = (subprocess.call(pg_rewind, env=env) == 0)
-        except OSError:
+        except:
             ret = False
         if ret:
             self.write_recovery_conf(leader)
@@ -509,11 +570,12 @@ recovery_target_timeline = 'latest'
         try:
             data = subprocess.check_output(['pg_controldata', self.data_dir])
             if data:
-                data = data.decode('utf-8').splitlines()
+                data = data.decode().splitlines()
                 result = {l.split(':')[0].replace('Current ', '', 1): l.split(':')[1].strip() for l in data if l}
         except subprocess.CalledProcessError:
             logger.exception("Error when calling pg_controldata")
-        return result
+        finally:
+            return result
 
     def read_postmaster_opts(self):
         """ returns the list of option names/values from postgres.opts, Empty dict if read failed or no file """
@@ -529,26 +591,26 @@ recovery_target_timeline = 'latest'
                         result[name] = val
         except IOError:
             logger.exception('Error when reading postmaster.opts')
-        return result
+        finally:
+            return result
 
-    def single_user_mode(self, command=None, options=None):
+    def single_user_mode(self, command=None, options={}):
         """ run a given command in a single-user mode. If the command is empty - then just start and stop """
         cmd = ['postgres', '--single', '-D', self.data_dir]
-        for opt, val in sorted((options or {}).items()):
+        for opt in sorted(options):
-            cmd.extend(['-c', '{0}={1}'.format(opt, val)])
+            cmd.extend(['-c', '{0}={1}'.format(opt, options[opt])])
         # need a database name to connect
         cmd.append('postgres')
         p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=open(os.devnull, 'w'), stderr=subprocess.STDOUT)
         if p:
-            if command:
+            command and p.communicate('{}\n'.format(command))
-                p.communicate('{0}\n'.format(command))
             p.stdin.close()
             return p.wait()
         return 1
 
     def cleanup_archive_status(self):
         status_dir = os.path.join(self.data_dir, 'pg_xlog', 'archive_status')
-        try:
+        if os.path.isdir(status_dir):
             for f in os.listdir(status_dir):
                 path = os.path.join(status_dir, f)
                 try:
@@ -556,51 +618,50 @@ recovery_target_timeline = 'latest'
                         os.unlink(path)
                     elif os.path.isfile(path):
                         os.remove(path)
-                except OSError:
+                except:
-                    logger.exception("Unable to remove %s", path)
+                    logger.exception("Unable to remove {}".format(path))
-        except OSError:
-            logger.exception("Unable to list %s", status_dir)
 
-    def follow(self, leader, recovery=False):
+    def follow_the_leader(self, leader, recovery=False):
-        if self.check_recovery_conf(leader) and not recovery:
+        if not self.check_recovery_conf(leader) or recovery:
+            change_role = (self.role == 'master')
+
+            self._need_rewind = (self._need_rewind or change_role) and self.can_rewind
+            if self._need_rewind:
+                logger.info("set the rewind flag after demote")
+            self.write_recovery_conf(leader)
+            if not leader or not self._need_rewind:  # do not rewind until the leader becomes available
+                ret = self.restart()
+            else:  # we have a leader and need to rewind
+                if self.is_running():
+                    self.stop()
+                # at present, pg_rewind only runs when the cluster is shut down cleanly
+                # and not shutdown in recovery. We have to remove the recovery.conf if present
+                # and start/shutdown in a single user mode to emulate this.
+                # XXX: if recovery.conf is linked, it will be written anew as a normal file.
+                if os.path.islink(self.recovery_conf):
+                    os.unlink(self.recovery_conf)
+                else:
+                    os.remove(self.recovery_conf)
+                # Archived segments might be useful to pg_rewind,
+                # clean the flags that tell we should remove them.
+                self.cleanup_archive_status()
+                # Start in a single user mode and stop to produce a clean shutdown
+                opts = self.read_postmaster_opts()
+                opts['archive_mode'] = 'on'
+                opts['archive_command'] = 'false'
+                self.single_user_mode(options=opts)
+                if self.rewind(leader):
+                    ret = self.start()
+                else:
+                    logger.error("unable to rewind the former master")
+                    self.remove_data_directory()
+                    ret = True
+                self._need_rewind = False
+            change_role and ret and self.call_nowait(ACTION_ON_ROLE_CHANGE)
+            return ret
+        else:
             return True
 
-        change_role = self.role == 'master'
-        self._need_rewind = (self._need_rewind or change_role) and self.can_rewind
-        if self._need_rewind:
-            logger.info("set the rewind flag after demote")
-        self.write_recovery_conf(leader)
-        if leader and self._need_rewind:  # we have a leader and need to rewind
-            if self.is_running():
-                self.stop()
-            # at present, pg_rewind only runs when the cluster is shut down cleanly
-            # and not shutdown in recovery. We have to remove the recovery.conf if present
-            # and start/shutdown in a single user mode to emulate this.
-            # XXX: if recovery.conf is linked, it will be written anew as a normal file.
-            if os.path.islink(self.recovery_conf):
-                os.unlink(self.recovery_conf)
-            else:
-                os.remove(self.recovery_conf)
-            # Archived segments might be useful to pg_rewind,
-            # clean the flags that tell we should remove them.
-            self.cleanup_archive_status()
-            # Start in a single user mode and stop to produce a clean shutdown
-            opts = self.read_postmaster_opts()
-            opts.update({'archive_mode': 'on', 'archive_command': 'false'})
-            self.single_user_mode(options=opts)
-            if self.rewind(leader):
-                ret = self.start()
-            else:
-                logger.error("unable to rewind the former master")
-                self.remove_data_directory()
-                ret = True
-            self._need_rewind = False
-        else:  # do not rewind until the leader becomes available
-            ret = self.restart()
-        if change_role and ret:
-            self.call_nowait(ACTION_ON_ROLE_CHANGE)
-        return ret
-
     def save_configuration_files(self):
         """
             copy postgresql.conf to postgresql.conf.backup to be able to retrive configuration files
@@ -609,18 +670,16 @@ recovery_target_timeline = 'latest'
         """
         try:
             for f in self.configuration_to_save:
-                if os.path.isfile(f):
+                os.path.isfile(f) and shutil.copy(f, f + '.backup')
-                    shutil.copy(f, f + '.backup')
+        except:
-        except IOError:
             logger.exception('unable to create backup copies of configuration files')
 
     def restore_configuration_files(self):
         """ restore a previously saved postgresql.conf """
         try:
             for f in self.configuration_to_save:
-                if not os.path.isfile(f) and os.path.isfile(f + '.backup'):
+                not os.path.isfile(f) and os.path.isfile(f + '.backup') and shutil.copy(f + '.backup', f)
-                    shutil.copy(f + '.backup', f)
+        except:
-        except IOError:
             logger.exception('unable to restore configuration files from backup')
 
     def promote(self):
@@ -634,6 +693,9 @@ recovery_target_timeline = 'latest'
             self.call_nowait(ACTION_ON_ROLE_CHANGE)
         return ret
 
+    def demote(self):
+        self.follow_the_leader(None)
+
     def create_or_update_role(self, name, password, options):
         self.query("""DO $$
 BEGIN
@@ -647,10 +709,13 @@ BEGIN
 END;
 $$""".format(name, options), name, password, password)
 
-    def create_replication_user(self):
+    def create_replication_user(self, superuser=False):
-        self.create_or_update_role(self.replication['username'], self.replication['password'], 'REPLICATION')
+        options = 'REPLICATION SUPERUSER' if superuser else 'REPLICATION'
+        self.create_or_update_role(self.replication['username'], self.replication['password'], '{0}'.format(options))
 
-    def create_connection_user(self):
+    def create_connection_users(self):
+        if 'username' in self.superuser:
+            self.create_or_update_role(self.superuser['username'], self.superuser['password'], 'SUPERUSER')
         if self.admin:
             self.create_or_update_role(self.admin['username'], self.admin['password'], 'CREATEDB CREATEROLE')
 
@@ -670,18 +735,7 @@ $$""".format(name, options), name, password, password)
         if self.use_slots:
             try:
                 self.load_replication_slots()
-                # if the replicatefrom tag is set on the member - we should not create the replication slot for it on
+                slots = [m.name for m in cluster.members if m.name != self.name] if self.role == 'master' else []
-                # the current master, because that member would replicate from elsewhere. We still create the slot if
-                # the replicatefrom destination member is currently not a member of the cluster (fallback to the
-                # master), or if replicatefrom destination member happens to be the current master
-                if self.role == 'master':
-                    slots = [m.name for m in cluster.members if m.name != self.name and
-                             (m.replicatefrom is None or m.replicatefrom == self.name or
-                              not cluster.has_member(m.replicatefrom))]
-                else:
-                    # only manage slots for replicas that replicate from this one, except for the leader among them
-                    slots = [m.name for m in cluster.members if m.replicatefrom == self.name and
-                             m.name != cluster.leader.name]
                 # drop unused slots
                 for slot in set(self.replication_slots) - set(slots):
                     self.query("""SELECT pg_drop_replication_slot(%s)
@@ -695,45 +749,34 @@ $$""".format(name, options), name, password, password)
                                    WHERE slot_name = %s)""", slot, slot)
 
                 self.replication_slots = slots
-            except psycopg2.Error:
+            except:
                 logger.exception('Exception when changing replication slots')
 
     def last_operation(self):
         return str(self.xlog_position())
 
-    def bootstrap(self, cluster_initialized=False, clone_member=None):
+    def bootstrap(self, current_leader=None):
         """
-            Populate PostgreSQL data directory by doing one of the following:
+            Initially bootstrap PostgreSQL, either by creating a data
-             - create with initdb if there is no master.
+            directory with initdb, or by initalizing a replica from an
-             - initialize the replica from an existing member (master or replica)
+            exiting leader. Failure in the first case always leads to
-             - initialize the replica using the replica creation method that
+            exception, since there is no point in continuing if initdb failed.
-               works without the replication connection (i.e. restore from on-disk
+            In the second case, however, a False is returned on failure, since
-               base backup)
+            it is normal for the replica to retry a failed attempt to initialize
-
+            from the master.
-            The choice between the last 2 is triggered by the initialize flag.
-            We should never try to initdb an already initialized cluster, nor
-            try to bootstrap the cluster that lacks the initialize key using the
-            master-less replica creation method (in the latter case, there is
-            no clear inidicator of the moment we should abandon our attempts and
-            swich to initdb).
-
-            Failure during initdb always leads to an exception, since there is
-            no point in continuing if initdb fails. For the rest of the cases,
-            the function returns False in order to inidicate a failed attempt
-            that should be retried in the future.
         """
         ret = False
-        if not (cluster_initialized or clone_member):
+        if not current_leader:
             ret = self.initialize() and self.start()
             if ret:
                 self.create_replication_user()
-                self.create_connection_user()
+                self.create_connection_users()
             else:
                 raise PostgresException("Could not bootstrap master PostgreSQL")
         else:
-            if self.sync_replica(clone_member):
+            if self.sync_from_leader(current_leader):
                 self.restore_configuration_files()
-                self.write_recovery_conf(clone_member, True)
+                self.write_recovery_conf(current_leader)
                 ret = self.start()
         return ret
 
@@ -743,7 +786,7 @@ $$""".format(name, options), name, password, password)
                 new_name = '{0}_{1}'.format(self.data_dir, time.strftime('%Y-%m-%d-%H-%M-%S'))
                 logger.info('renaming data directory to %s', new_name)
                 os.rename(self.data_dir, new_name)
-            except OSError:
+            except:
                 logger.exception("Could not rename data directory %s", self.data_dir)
 
     def remove_data_directory(self):
@@ -757,16 +800,16 @@ $$""".format(name, options), name, password, password)
                 os.remove(self.data_dir)
             elif os.path.isdir(self.data_dir):
                 shutil.rmtree(self.data_dir)
-        except (IOError, OSError):
+        except:
             logger.exception('Could not remove data directory %s', self.data_dir)
             self.move_data_directory()
 
-    def basebackup(self, clone_member, env):
+    def basebackup(self, leader, env):
         # creates a replica data dir using pg_basebackup.
         # this is the default, built-in create_replica_method
         # tries twice, then returns failure (as 1)
         # uses "stream" as the xlog-method to avoid sync issues
-        master_connection = clone_member.conn_url
+        master_connection = leader.conn_url
         maxfailures = 2
         ret = 1
         for bbfailures in range(0, maxfailures):

patroni/scripts/aws.py

@@ -9,7 +9,7 @@ import boto.ec2
 logger = logging.getLogger(__name__)
 
 
-class AWSConnection(object):
+class AWSConnection:
     def __init__(self, cluster_name):
         self.available = False
         self.cluster_name = cluster_name if cluster_name is not None else 'unknown'
@@ -56,7 +56,7 @@ class AWSConnection(object):
             conn = boto.ec2.connect_to_region(self.region)
             conn.create_tags([self.instance_id], tags)
         except Exception as e:
-            logger.info("could not set tags for EC2 instance %s: %s", self.instance_id, e)
+            logger.info("could not set tags for EC2 instance {}: {}".format(self.instance_id, e))
             return False
         return True
 
@@ -66,7 +66,6 @@ class AWSConnection(object):
 
 
 def main():
-    logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
     if len(sys.argv) == 4 and sys.argv[1] in ('on_start', 'on_stop', 'on_role_change'):
         AWSConnection(cluster_name=sys.argv[3]).on_role_change(sys.argv[2])
     else:

patroni/scripts/wale_restore.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
 
 # sample script to clone new replicas using WAL-E restore
 # falls back to pg_basebackup if WAL-E restore fails, or if
@@ -36,12 +36,13 @@ import argparse
 if sys.hexversion >= 0x03000000:
     long = int
 
+logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
 class WALERestore(object):
 
-    def __init__(self, scope, datadir, connstring, env_dir, threshold_mb, threshold_pct, use_iam, no_master):
+    def __init__(self, scope, datadir, connstring, env_dir, threshold_mb, threshold_pct, use_iam):
         self.scope = scope
         self.master_connection = connstring
         self.data_dir = datadir
@@ -50,7 +51,6 @@ class WALERestore(object):
         self.wal_e.threshold_mb = threshold_mb
         self.wal_e.threshold_pct = threshold_pct
         self.wal_e.iam_string = ' --aws-instance-profile ' if use_iam == 1 else ''
-        self.no_master = no_master
         self.wal_e.cmd = 'envdir {0} wal-e {1} '.format(self.wal_e.dir, self.wal_e.iam_string)
         self.init_error = (not os.path.exists(self.wal_e.dir))
 
@@ -104,23 +104,24 @@ class WALERestore(object):
         lsn_offset = hex((long(backup_start_segment[16:32], 16) << 24) + long(backup_start_offset))[2:-1]
 
         # construct the LSN from the segment and offset
-        backup_start_lsn = '{0}/{1}'.format(lsn_segment, lsn_offset)
+        backup_start_lsn = '{}/{}'.format(lsn_segment, lsn_offset)
 
+        conn = None
+        cursor = None
         diff_in_bytes = long(backup_size)
-        if not self.no_master:
+        try:
-            try:
+            # get the difference in bytes between the current WAL location and the backup start offset
-                # get the difference in bytes between the current WAL location and the backup start offset
+            conn = psycopg2.connect(self.master_connection)
-                with psycopg2.connect(self.master_connection) as con:
+            conn.autocommit = True
-                    con.autocommit = True
+            cursor = conn.cursor()
-                    with con.cursor() as cur:
+            cursor.execute("SELECT pg_xlog_location_diff(pg_current_xlog_location(), %s)", (backup_start_lsn,))
-                        cur.execute("SELECT pg_xlog_location_diff(pg_current_xlog_location(), %s)", (backup_start_lsn,))
+            diff_in_bytes = long(cursor.fetchone()[0])
-                        diff_in_bytes = long(cur.fetchone()[0])
+        except psycopg2.Error as e:
-            except psycopg2.Error as e:
+            logger.error('could not determine difference with the master location: {}'.format(e))
-                logger.error('could not determine difference with the master location: %s', e)
+            return False
-                return False
+        finally:
-        else:
+            cursor and cursor.close()
-            # always try to use WAL-E if base backup is available
+            conn and conn.close()
-            diff_in_bytes = 0
 
         # if the size of the accumulated WAL segments is more than a certan percentage of the backup size
         # or exceeds the pre-determined size - pg_basebackup is chosen instead.
@@ -139,7 +140,6 @@ class WALERestore(object):
 
 
 def main():
-    logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
     parser = argparse.ArgumentParser(description='Script to image replicas using WAL-E')
     parser.add_argument('--scope', required=True)
     parser.add_argument('--role', required=False)
@@ -150,15 +150,13 @@ def main():
     parser.add_argument('--threshold_megabytes', type=int, default=10240)
     parser.add_argument('--threshold_backup_size_percentage', type=int, default=30)
     parser.add_argument('--use_iam', type=int, default=0)
-    parser.add_argument('--no_master', type=int, default=0)
     args = parser.parse_args()
 
     # retry cloning in a loop
-    for _ in range(0, args.retries + 1):
+    for retry in range(0, args.retries + 1):
         restore = WALERestore(scope=args.scope, datadir=args.datadir, connstring=args.connstring,
                               env_dir=args.envdir, threshold_mb=args.threshold_megabytes,
-                              threshold_pct=args.threshold_backup_size_percentage, use_iam=args.use_iam,
+                              threshold_pct=args.threshold_backup_size_percentage, use_iam=args.use_iam)
-                              no_master=args.no_master)
         ret = restore.run()
         if ret == 0:
             break

patroni/utils.py

@@ -1,61 +1,76 @@
 import datetime
 import os
 import random
+import re
 import signal
 import sys
 import time
-import pytz
-import dateutil.parser
 
 from patroni.exceptions import PatroniException
 
-__ignore_sigterm = False
+ignore_sigterm = False
-__interrupted_sleep = False
+interrupted_sleep = False
-__reap_children = False
+reap_children = False
+
+_DATE_TIME_RE = re.compile(r'''^
+(?P<year>\d{4})\-(?P<month>\d{2})\-(?P<day>\d{2})  # date
+T
+(?P<hour>\d{2}):(?P<minute>\d{2}):(?P<second>\d{2})\.(?P<microsecond>\d{6})  # time
+\d*Z$''', re.X)
+
+
+def parse_datetime(time_str):
+    """
+    >>> parse_datetime('2015-06-10T12:56:30.552539016Z')
+    datetime.datetime(2015, 6, 10, 12, 56, 30, 552539)
+    >>> parse_datetime('2015-06-10 12:56:30.552539016Z')
+    """
+    m = _DATE_TIME_RE.match(time_str)
+    if not m:
+        return None
+    p = dict((n, int(m.group(n))) for n in 'year month day hour minute second microsecond'.split(' '))
+    return datetime.datetime(**p)
 
 
 def calculate_ttl(expiration):
     """
     >>> calculate_ttl(None)
-    >>> calculate_ttl('2015-06-10 12:56:30.552539016Z') < 0
+    >>> calculate_ttl('2015-06-10 12:56:30.552539016Z')
-    True
     >>> calculate_ttl('2015-06-10T12:56:30.552539016Z') < 0
     True
-    >>> calculate_ttl('fail-06-10T12:56:30.552539016Z')
     """
     if not expiration:
         return None
-    try:
+    expiration = parse_datetime(expiration)
-        expiration = dateutil.parser.parse(expiration)
+    if not expiration:
-    except (ValueError, TypeError):
         return None
-    now = datetime.datetime.now(pytz.utc)
+    now = datetime.datetime.utcnow()
     return int((expiration - now).total_seconds())
 
 
 def sigterm_handler(signo, stack_frame):
-    global __ignore_sigterm
+    global ignore_sigterm
-    if not __ignore_sigterm:
+    if not ignore_sigterm:
-        __ignore_sigterm = True
+        ignore_sigterm = True
         sys.exit()
 
 
 def sigchld_handler(signo, stack_frame):
-    global __interrupted_sleep, __reap_children
+    global interrupted_sleep, reap_children
-    __reap_children = __interrupted_sleep = True
+    reap_children = interrupted_sleep = True
 
 
 def sleep(interval):
-    global __interrupted_sleep
+    global interrupted_sleep
     current_time = time.time()
     end_time = current_time + interval
     while current_time < end_time:
-        __interrupted_sleep = False
+        interrupted_sleep = False
         time.sleep(end_time - current_time)
-        if not __interrupted_sleep:  # we will ignore only sigchld
+        if not interrupted_sleep:  # we will ignore only sigchld
             break
         current_time = time.time()
-    __interrupted_sleep = False
+    interrupted_sleep = False
 
 
 def setup_signal_handlers():
@@ -64,8 +79,8 @@ def setup_signal_handlers():
 
 
 def reap_children():
-    global __reap_children
+    global reap_children
-    if __reap_children:
+    if reap_children:
         try:
             while True:
                 ret = os.waitpid(-1, os.WNOHANG)
@@ -74,7 +89,7 @@ def reap_children():
         except OSError:
             pass
         finally:
-            __reap_children = False
+            reap_children = False
 
 
 class RetryFailedError(PatroniException):
@@ -82,7 +97,7 @@ class RetryFailedError(PatroniException):
     """Raised when retrying an operation ultimately failed, after retrying the maximum number of attempts."""
 
 
-class Retry(object):
+class Retry:
 
     """Helper for retrying a method in the face of retry-able exceptions"""
 

patroni/version.py

@@ -1 +1 @@
-__version__ = '0.80'
+__version__ = '0.76'

patroni/zookeeper.py

@@ -17,7 +17,7 @@ class ZooKeeperError(DCSError):
     pass
 
 
-class ExhibitorEnsembleProvider(object):
+class ExhibitorEnsembleProvider:
 
     TIMEOUT = 3.1
 
@@ -54,7 +54,7 @@ class ExhibitorEnsembleProvider(object):
     def _query_exhibitors(self, exhibitors):
         random.shuffle(exhibitors)
         for host in exhibitors:
-            uri = 'http://{0}:{1}{2}'.format(host, self._exhibitor_port, self._uri_path)
+            uri = 'http://{}:{}{}'.format(host, self._exhibitor_port, self._uri_path)
             try:
                 response = requests.get(uri, timeout=self.TIMEOUT)
                 return response.json()
@@ -84,9 +84,9 @@ class ZooKeeper(AbstractDCS):
             hosts = self.exhibitor.zookeeper_hosts
 
         self.client = KazooClient(hosts=hosts,
-                                  timeout=(config.get('session_timeout') or 30),
+                                  timeout=(config.get('session_timeout', None) or 30),
                                   command_retry={
-                                      'deadline': (config.get('reconnect_timeout') or 10),
+                                      'deadline': (config.get('reconnect_timeout', None) or 10),
                                       'max_delay': 1,
                                       'max_tries': -1},
                                   connection_retry={'max_delay': 1, 'max_tries': -1})
@@ -190,8 +190,7 @@ class ZooKeeper(AbstractDCS):
 
     def attempt_to_acquire_leader(self):
         ret = self._create(self.leader_path, self._name, makepath=True, ephemeral=True)
-        if ret:
+        ret or logger.info('Could not take out TTL lock')
-            logger.info('Could not take out TTL lock')
         return ret
 
     def set_failover_value(self, value, index=None):

patronictl.py

@@ -2,4 +2,4 @@
 from patroni.ctl import ctl
 
 if __name__ == '__main__':
-    ctl(None)
+    ctl()

postgres0.yml

@@ -4,7 +4,7 @@ scope: &scope batman
 restapi:
   listen: 127.0.0.1:8008
   connect_address: 127.0.0.1:8008
-#  auth: 'username:password'
+  #auth: 'username:password'
 #  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
 #  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
 etcd:
@@ -26,6 +26,9 @@ etcd:
 #      - host1
 #      - host2
 #      - host3
+bdr:
+  enable: "on"
+  database: "bdrtest"
 postgresql:
   name: postgresql0
   scope: *scope
@@ -72,6 +75,7 @@ postgresql:
     - basebackup
 #   - wal_e
 # commented-out example for wal-e provisioning
+  #create_replica_method: wal_e, basebackup
   #wal_e:
     #command: /patroni/scripts/wale_restore.py
     #env_dir: /etc/wal-e.d/env
@@ -85,15 +89,19 @@ postgresql:
     restore_command: cp ../wal_archive/%f %p
   parameters:
     archive_mode: "on"
-    wal_level: hot_standby
+    wal_level: logical
     archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
     max_wal_senders: 10
     wal_keep_segments: 8
     archive_timeout: 1800s
     max_replication_slots: 10
+    max_worker_processes: 10
+    track_commit_timestamp: 'on'
+    shared_preload_libraries: 'bdr'
     hot_standby: "on"
     wal_log_hints: "on"
 tags:
     nofailover: False
     noloadbalance: False
     clonefrom: False
+    replicatefrom: 127.0.0.1

postgres1.yml

@@ -4,7 +4,7 @@ scope: &scope batman
 restapi:
   listen: 127.0.0.1:8009
   connect_address: 127.0.0.1:8009
-#  auth: 'username:password'
+  #auth: 'username:password'
 #  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
 #  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
 etcd:
@@ -26,6 +26,9 @@ etcd:
 #      - host1
 #      - host2
 #      - host3
+bdr:
+    enable: "on"
+    database: "bdrtest"
 postgresql:
   name: postgresql1
   scope: *scope
@@ -63,7 +66,7 @@ postgresql:
     password: rep-pass
     network: 127.0.0.1/32
   superuser:
-    username: postgres
+    user: postgres
     password: zalando
   admin:
     username: admin
@@ -86,9 +89,12 @@ postgresql:
     restore_command: cp ../wal_archive/%f %p
   parameters:
     archive_mode: "on"
-    wal_level: hot_standby
+    wal_level: logical
     archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
     max_wal_senders: 10
+    max_worker_processes: 10
+    track_commit_timestamp: 'on'
+    shared_preload_libraries: 'bdr'
     wal_keep_segments: 8
     archive_timeout: 1800s
     max_replication_slots: 10
@@ -98,3 +104,4 @@ tags:
     nofailover: False
     noloadbalance: False
     clonefrom: False
+    replicatefrom: 127.0.0.1

postgres2.yml

@@ -1,101 +0,0 @@
-ttl: &ttl 30
-loop_wait: &loop_wait 10
-scope: &scope batman
-restapi:
-  listen: 127.0.0.1:8010
-  connect_address: 127.0.0.1:8010
-  auth: 'username:password'
-#  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
-#  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
-etcd:
-  scope: *scope
-  ttl: *ttl
-  host: 127.0.0.1:4001
-  #discovery_srv: my-etcd.domain
-#zookeeper:
-#  scope: *scope
-#  session_timeout: *ttl
-#  reconnect_timeout: *loop_wait
-#  hosts:
-#    - 127.0.0.1:2181
-#    - 127.0.0.2:2181
-#  exhibitor:
-#    poll_interval: 300
-#    port: 8181
-#    hosts:
-#      - host1
-#      - host2
-#      - host3
-postgresql:
-  name: postgresql2
-  scope: *scope
-  listen: 127.0.0.1:5434
-  connect_address: 127.0.0.1:5434
-  data_dir: data/postgresql2
-  maximum_lag_on_failover: 1048576 # 1 megabyte in bytes
-  use_slots: True
-  pgpass: /tmp/pgpass2
-  initdb:  ## We allow the following options to be passed on to initdb
-  # - auth: authmethod
-  # - auth-host: authmethod
-  # - auth-local: authmethod
-  - encoding: UTF8
-  # - data-checksums  # When pg_rewind is needed on 9.3, this needs to be enabled
-  # - locale: locale
-  # - lc-collate: locale
-  # - lc-ctype: locale
-  # - lc-messages: locale
-  # - lc-monetary: locale
-  # - lc-numeric: locale
-  # - lc-time: locale
-  # - text-search-config: CFG
-  # - xlogdir: directory
-  # - debug
-  # - noclean
-  pg_rewind:
-    username: postgres
-    password: zalando
-  pg_hba:
-  - host all all 0.0.0.0/0 md5
-  - hostssl all all 0.0.0.0/0 md5
-  replication:
-    username: replicator
-    password: rep-pass
-    network: 127.0.0.1/32
-  superuser:
-    username: postgres
-    password: zalando
-  admin:
-    username: admin
-    password: admin
-# commented-out example for wal-e provisioning
-  create_replica_method:
-    - basebackup
-#   - wal_e
-# commented-out example for wal-e provisioning
-  #wal_e:
-    #command: /patroni/scripts/wale_restore.py
-    #env_dir: /home/postgres/etc/wal-e.d/env
-    #threshold_megabytes: 10240
-    #threshold_backup_size_percentage: 30
-    #retries: 2
-    #use_iam: 1
-  #recovery_conf:
-    #restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1
-  recovery_conf:
-    restore_command: cp ../wal_archive/%f %p
-  parameters:
-    archive_mode: "on"
-    wal_level: hot_standby
-    archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
-    max_wal_senders: 10
-    wal_keep_segments: 8
-    archive_timeout: 1800s
-    max_replication_slots: 10
-    hot_standby: "on"
-    wal_log_hints: "on"
-tags:
-    nofailover: False
-    noloadbalance: False
-    clonefrom: False
-    replicatefrom: postgresql1

requirements-py2.txt

@@ -1,11 +1,11 @@
 boto
+dnspython
+mock
 psycopg2>=2.6.1
 PyYAML
 requests
 six >= 1.7
 kazoo>=2.2.1
-python-etcd==0.4.3
+python-etcd>=0.4.1
 click>=4.1
 prettytable>=0.7
-tzlocal
-python-dateutil

requirements-py3.txt

@@ -0,0 +1,11 @@
+boto
+mock
+dnspython3
+psycopg2>=2.6.1
+PyYAML
+requests
+six
+kazoo>=2.2.1
+python-etcd>=0.4.1
+click>=4.1
+prettytable>=0.7

setup.py

@@ -51,8 +51,8 @@ CLASSIFIERS = [
     'Operating System :: POSIX :: Linux',
     'Programming Language :: Python',
     'Programming Language :: Python :: 2.7',
+    'Programming Language :: Python :: 3.3',
     'Programming Language :: Python :: 3.4',
-    'Programming Language :: Python :: 3.5',
     'Programming Language :: Python :: Implementation :: CPython',
 ]
 
@@ -76,7 +76,7 @@ class PyTest(TestCommand):
     def finalize_options(self):
         TestCommand.finalize_options(self)
         if self.cov_xml or self.cov_html:
-            self.cov = ['--cov', MAIN_PACKAGE, '--cov-report', 'term-missing']
+            self.cov = ['--cov', MAIN_PACKAGE, '--cov', MAIN_PACKAGE, '--cov-report', 'term-missing']
             if self.cov_xml:
                 self.cov.extend(['--cov-report', 'xml'])
             if self.cov_html:
@@ -116,7 +116,8 @@ def setup_package():
     # Some helper variables
     version = os.getenv('GO_PIPELINE_LABEL', VERSION)
 
-    install_reqs = get_install_requirements('requirements.txt')
+    requirements = 'requirements-py2.txt' if sys.version_info[0] == 2 else 'requirements-py3.txt'
+    install_reqs = get_install_requirements(requirements)
 
     command_options = {'test': {'test_suite': ('setup.py', 'tests')}}
     if JUNIT_XML:
@@ -141,9 +142,9 @@ def setup_package():
         packages=setuptools.find_packages(exclude=['tests', 'tests.*']),
         package_data={MAIN_PACKAGE: ["*.json"]},
         install_requires=install_reqs,
-        setup_requires=['flake8'],
+        setup_requires=['six', 'flake8'],
         cmdclass=cmdclass,
-        tests_require=['mock', 'pytest-cov', 'pytest'],
+        tests_require=['pytest-cov', 'pytest'],
         command_options=command_options,
         entry_points={'console_scripts': CONSOLE_SCRIPTS},
     )

tests/test_api.py

@@ -16,15 +16,11 @@ class MockPostgresql(Mock):
     name = 'test'
     state = 'running'
     role = 'master'
-    server_version = '999999'
-    scope = 'dummy'
 
-    @staticmethod
+    def connection(self):
-    def connection():
         return psycopg2_connect()
 
-    @staticmethod
+    def is_running(self):
-    def is_running():
         return True
 
 
@@ -33,37 +29,31 @@ class MockHa(Mock):
     dcs = Mock()
     state_handler = MockPostgresql()
 
-    @staticmethod
+    def schedule_restart(self):
-    def schedule_restart():
         return 'restart'
 
-    @staticmethod
+    def schedule_reinitialize(self):
-    def schedule_reinitialize():
         return 'reinitialize'
 
-    @staticmethod
+    def restart(self):
-    def restart():
         return (True, '')
 
-    @staticmethod
+    def restart_scheduled(self):
-    def restart_scheduled():
         return False
 
-    @staticmethod
+    def fetch_nodes_statuses(self, members):
-    def fetch_nodes_statuses(members):
         return [[None, True, None, None, {}]]
 
 
-class MockPatroni(Mock):
+class MockPatroni:
 
     postgresql = MockPostgresql()
     ha = MockHa()
     dcs = Mock()
     tags = {}
-    version = '0.00'
 
 
-class MockRequest(object):
+class MockRequest:
 
     def __init__(self, path):
         self.path = path
@@ -101,10 +91,10 @@ class TestRestApiHandler(unittest.TestCase):
             MockRestApiServer(RestApiHandler, b'GET /master')
         with patch.object(MockHa, 'restart_scheduled', Mock(return_value=True)):
             MockRestApiServer(RestApiHandler, b'GET /master')
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /master'))
+        MockRestApiServer(RestApiHandler, b'GET /master')
 
     def test_do_OPTIONS(self):
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0'))
+        MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0')
 
         with patch.object(BaseHTTPRequestHandler, 'handle_one_request') as mock_handle_request:
             mock_handle_request.side_effect = socket.error("foo")
@@ -118,15 +108,15 @@ class TestRestApiHandler(unittest.TestCase):
             MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0')
 
     def test_do_GET_patroni(self):
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /patroni'))
+        MockRestApiServer(RestApiHandler, b'GET /patroni')
 
     def test_basicauth(self):
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0'))
+        MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0')
         MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0\nAuthorization:')
 
     def test_do_POST_restart(self):
         request = b'POST /restart HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0'
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
+        MockRestApiServer(RestApiHandler, request)
         with patch.object(MockHa, 'restart', Mock(side_effect=Exception)):
             MockRestApiServer(RestApiHandler, request)
 
@@ -140,23 +130,19 @@ class TestRestApiHandler(unittest.TestCase):
         with patch.object(MockHa, 'schedule_reinitialize', Mock(return_value=None)):
             MockRestApiServer(RestApiHandler, request)
         cluster.leader.name = 'test'
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
+        MockRestApiServer(RestApiHandler, request)
 
     @patch('time.sleep', Mock())
     def test_RestApiServer_query(self):
         with patch.object(MockCursor, 'execute', Mock(side_effect=psycopg2.OperationalError)):
-            self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /patroni'))
+            MockRestApiServer(RestApiHandler, b'GET /patroni')
         with patch.object(MockPostgresql, 'connection', Mock(side_effect=psycopg2.OperationalError)):
-            self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /patroni'))
+            MockRestApiServer(RestApiHandler, b'GET /patroni')
 
     @patch('time.sleep', Mock())
     @patch.object(MockHa, 'dcs')
     def test_do_POST_failover(self, dcs):
         cluster = dcs.get_cluster.return_value
-
-        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\
-                  b'Content-Length: 0\n\n'
-        MockRestApiServer(RestApiHandler, request)
         request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\
                   b'Content-Length: 25\n\n{"leader": "postgresql1"}'
         MockRestApiServer(RestApiHandler, request)
@@ -180,23 +166,3 @@ class TestRestApiHandler(unittest.TestCase):
         request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\
                   b'Content-Length: 50\n\n{"leader": "postgresql1", "member": "postgresql2"}'
         MockRestApiServer(RestApiHandler, request)
-
-        # Valid future date
-        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\
-                  b'"postgresql1", "member": "postgresql2", "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}'
-        MockRestApiServer(RestApiHandler, request)
-
-        # Exception: No timezone specified
-        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 97\n\n{"leader": ' +\
-                  b'"postgresql1", "member": "postgresql2", "scheduled_at": "6016-02-15T18:13:30.568224"}'
-        MockRestApiServer(RestApiHandler, request)
-
-        # Exception: Scheduled in the past
-        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\
-                  b'"postgresql1", "member": "postgresql2", "scheduled_at": "1016-02-15T18:13:30.568224+01:00"}'
-        MockRestApiServer(RestApiHandler, request)
-
-        # Invalid date
-        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\
-                  b'"postgresql1", "member": "postgresql2", "scheduled_at": "2010-02-29T18:13:30.568224+01:00"}'
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))

tests/test_aws.py

@@ -1,15 +1,12 @@
-import boto.ec2
-import requests
-import sys
 import unittest
-
+import requests
-from mock import Mock, patch
+import boto.ec2
 from collections import namedtuple
-from patroni.scripts.aws import AWSConnection, main as _main
+from patroni.scripts.aws import AWSConnection
 from requests.exceptions import RequestException
 
 
-class MockEc2Connection(object):
+class MockEc2Connection:
 
     def __init__(self, error=False):
         self.error = error
@@ -26,7 +23,7 @@ class MockEc2Connection(object):
         return True
 
 
-class MockResponse(object):
+class MockResponse:
 
     def __init__(self, content):
         self.content = content
@@ -38,6 +35,15 @@ class MockResponse(object):
 
 class TestAWSConnection(unittest.TestCase):
 
+    def __init__(self, method_name='runTest'):
+        super(TestAWSConnection, self).__init__(method_name)
+
+    def set_error(self):
+        self.error = True
+
+    def set_json_error(self):
+        self.json_error = True
+
     def boto_ec2_connect_to_region(self, region):
         return MockEc2Connection(self.error)
 
@@ -68,27 +74,21 @@ class TestAWSConnection(unittest.TestCase):
         self.assertTrue(self.conn.on_role_change('master'))
 
     def test_non_aws(self):
-        self.error = True
+        self.set_error()
         conn = AWSConnection('test')
         self.assertFalse(conn.aws_available())
         self.assertFalse(conn._tag_ebs('master'))
         self.assertFalse(conn._tag_ec2('master'))
 
     def test_aws_bizare_response(self):
-        self.json_error = True
+        self.set_json_error()
         conn = AWSConnection('test')
         self.assertFalse(conn.aws_available())
 
     def test_aws_tag_ebs_error(self):
-        self.error = True
+        self.set_error()
         self.assertFalse(self.conn._tag_ebs("master"))
 
     def test_aws_tag_ec2_error(self):
-        self.error = True
+        self.set_error()
         self.assertFalse(self.conn._tag_ec2("master"))
-
-    @patch('sys.exit', Mock())
-    def test_main(self):
-        self.assertIsNone(_main())
-        sys.argv = ['aws.py', 'on_start', 'replica', 'foo']
-        self.assertIsNone(_main())

tests/test_ctl.py

@@ -1,28 +1,31 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 import os
 import pytest
-import requests.exceptions
 import unittest
+import psycopg2
+import requests
+import patroni.exceptions
+import etcd
+from mock import patch, Mock
 
 from click.testing import CliRunner
-from etcd import EtcdException
-from mock import patch, Mock, MagicMock
 from patroni.ctl import ctl, members, store_config, load_config, output_members, post_patroni, get_dcs, \
     wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure
+from patroni.ha import Ha
 from patroni.etcd import Etcd, Client
-from patroni.exceptions import PatroniCtlException
-from psycopg2 import OperationalError
-from test_etcd import etcd_read, etcd_write, requests_get, socket_getaddrinfo, MockResponse
-from test_zookeeper import MockKazooClient
 from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \
-    get_cluster_initialized_with_only_leader
+    get_cluster_initialized_with_only_leader, MockPostgresql, MockPatroni, run_async, \
+    get_cluster_not_initialized_without_leader
+from test_etcd import etcd_read, etcd_write, requests_get, MockResponse
 from test_postgresql import MockConnect, psycopg2_connect
 
 CONFIG_FILE_PATH = './test-ctl.yaml'
 
-
 def test_rw_config():
     runner = CliRunner()
-    config = {'a': 'b'}
+    config = {'a':'b'}
     with runner.isolated_filesystem():
         store_config(config, CONFIG_FILE_PATH + '/dummy')
         os.remove(CONFIG_FILE_PATH + '/dummy')
@@ -42,36 +45,43 @@ def test_rw_config():
     load_config(CONFIG_FILE_PATH, None)
     load_config(CONFIG_FILE_PATH, '0.0.0.0')
 
-
 @patch('patroni.ctl.load_config', Mock(return_value={'dcs': {'scheme': 'etcd', 'hostname': 'localhost', 'port': 4001}}))
 class TestCtl(unittest.TestCase):
 
-    @patch('socket.getaddrinfo', socket_getaddrinfo)
+    @patch.object(Client, 'machines')
-    def setUp(self):
+    def setUp(self, mock_machines):
-        self.runner = CliRunner()
+        mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
-        with patch.object(Client, 'machines') as mock_machines:
+        self.p = MockPostgresql()
-            mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
+        self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'})
-            self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'})
+        self.e.client.read = etcd_read
-            self.e.client.read = etcd_read
+        self.e.client.write = etcd_write
-            self.e.client.write = etcd_write
+        self.e.client.delete = Mock(side_effect=etcd.EtcdException())
-            self.e.client.delete = Mock(side_effect=EtcdException)
+        self.ha = Ha(MockPatroni(self.p, self.e))
+        self.ha._async_executor.run_async = run_async
+        self.ha.old_cluster = self.e.get_cluster()
+        self.ha.cluster = get_cluster_not_initialized_without_leader()
+        self.ha.load_cluster_from_dcs = Mock()
 
     @patch('psycopg2.connect', psycopg2_connect)
     def test_get_cursor(self):
-        self.assertIsNone(get_cursor(get_cluster_initialized_without_leader(), role='master'))
+        c = get_cursor(get_cluster_initialized_without_leader(), role='master')
+        assert c is None
 
-        self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), role='master'))
+        c = get_cursor(get_cluster_initialized_with_leader(), role='master')
+        assert c is not None
 
-        # MockCursor returns pg_is_in_recovery as false
+        c = get_cursor(get_cluster_initialized_with_leader(), role='replica')
-        self.assertIsNone(get_cursor(get_cluster_initialized_with_leader(), role='replica'))
+        # # MockCursor returns pg_is_in_recovery as false
+        assert c is None
 
-        self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), role='any'))
+        c = get_cursor(get_cluster_initialized_with_leader(), role='any')
+        assert c is not None
 
     def test_output_members(self):
         cluster = get_cluster_initialized_with_leader()
-        self.assertIsNone(output_members(cluster, name='abc', fmt='pretty'))
+        output_members(cluster, name='abc', format='pretty')
-        self.assertIsNone(output_members(cluster, name='abc', fmt='json'))
+        output_members(cluster, name='abc', format='json')
-        self.assertIsNone(output_members(cluster, name='abc', fmt='tsv'))
+        output_members(cluster, name='abc', format='tsv')
 
     @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
     @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
@@ -81,112 +91,89 @@ class TestCtl(unittest.TestCase):
     @patch('requests.post', requests_get)
     @patch('patroni.ctl.post_patroni', Mock(return_value=MockResponse()))
     def test_failover(self):
+        runner = CliRunner()
+
         with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())):
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
+            result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
 other
-
 y''')
-            assert 'leader' in result.output
+            assert 'Failing over to new leader' in result.output
 
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
+            result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
 other
-2100-01-01T12:23:00
-y''')
-            assert result.exit_code == 0
-
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
-other
-2030-01-01T12:23:00
-y''')
-            assert result.exit_code == 0
-
-            # Aborting failover,as we anser NO to the confirmation
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
-other
-
 N''')
-            assert result.exit_code == 1
+            assert 'Aborting failover' in str(result.exception)
 
-            # Target and source are equal
+            result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
 leader
-
 y''')
-            assert result.exit_code == 1
+            assert 'target and source are the same' in str(result.exception)
 
-            # Reality is not part of this cluster
+            result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
 Reality
-
 y''')
-            assert result.exit_code == 1
+            assert 'Reality does not exist' in str(result.exception)
 
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--force'])
+            result = runner.invoke(ctl, ['failover', 'dummy', '--force'])
-            assert 'Member' in result.output
+            assert 'Failing over to new leader' in result.output
 
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--force',
+            result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='dummy')
-                                              '--scheduled', '2015-01-01T12:00:00+01:00'])
+            assert 'is not the leader of cluster' in str(result.exception)
-            assert result.exit_code == 0
-
-            # Invalid timestamp
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', 'invalid'])
-            assert result.exit_code != 0
-
-            # Invalid timestamp
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--force',
-                                              '--scheduled', '2115-02-30T12:00:00+01:00'])
-            assert result.exit_code != 0
-
-            # Specifying wrong leader
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='dummy')
-            assert result.exit_code == 1
 
         with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_only_leader())):
-            # No members available
+            result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
 other
-
 y''')
-            assert result.exit_code == 1
+            assert 'No candidates found to failover to' in str(result.exception)
 
         with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_without_leader())):
-            # No master available
+            result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
 other
-
 y''')
-            assert result.exit_code == 1
+            assert 'This cluster has no master' in str(result.exception)
 
         with patch('patroni.ctl.post_patroni', Mock(side_effect=Exception())):
-            # Non-responding patroni
+            result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
 other
-
 y''')
             assert 'falling back to DCS' in result.output
+            assert 'Failover failed' in result.output
 
         mocked = Mock()
         mocked.return_value.status_code = 500
         with patch('patroni.ctl.post_patroni', Mock(return_value=mocked)):
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
+            result = runner.invoke(ctl, ['failover', 'dummy', '--dcs', '8.8.8.8'], input='''leader
 other
-
 y''')
-            assert 'Failover failed' in result.output
+            assert 'Failover failed, details' in result.output
 
-    @patch('patroni.zookeeper.KazooClient', MockKazooClient)
+#        with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())):
-    @patch('requests.get', requests_get)
+#            result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='nonsense')
-    def test_get_dcs(self):
+#            assert 'is not the leader of cluster' in str(result.exception)
-        self.assertIsNotNone(get_dcs({'dcs': {'scheme': 'zookeeper', 'hostname': 'foo', 'port': 2181}}, 'dummy'))
+
-        self.assertIsNotNone(get_dcs({'dcs': {'scheme': 'exhibitor', 'hostname': 'exhibitor', 'port': 8181}}, 'dummy'))
+            # result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8', '--master', 'nonsense'])
-        self.assertRaises(PatroniCtlException, get_dcs, {'scheme': 'dummy'}, 'dummy')
+            # assert 'is not the leader of cluster' in str(result.exception)
+
+            # result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='leader\nother\nn')
+            # assert 'Aborting failover' in str(result.exception)
+
+            # with patch('patroni.ctl.wait_for_leader', Mock(return_value = get_cluster_initialized_with_leader())):
+                # result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='leader\nother\nY')
+                # assert 'master did not change after' in result.output
+
+                # result = runner.invoke(ctl, ['failover', 'alpha', '--dcs', '8.8.8.8'], input='leader\nother\nY')
+                # assert 'Failover failed' in result.output
+
+    def test_(self):
+        self.assertRaises(patroni.exceptions.PatroniCtlException, get_dcs, {'scheme': 'dummy'}, 'dummy')
 
     @patch('psycopg2.connect', psycopg2_connect)
     @patch('patroni.ctl.query_member', Mock(return_value=([['mock column']], None)))
     def test_query(self):
+        runner = CliRunner()
+
         with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)):
-            # Mutually exclusive
+            result = runner.invoke(ctl, [
-            result = self.runner.invoke(ctl, [
                 'query',
                 'alpha',
                 '--member',
@@ -194,14 +181,14 @@ y''')
                 '--role',
                 'master',
             ])
-            assert result.exit_code == 1
+            assert 'mutually exclusive' in str(result.exception)
 
-            with self.runner.isolated_filesystem():
+            with runner.isolated_filesystem():
-                with open('dummy', 'w') as dummy_file:
+                dummy_file = open('dummy', 'w')
-                    dummy_file.write('SELECT 1')
+                dummy_file.write('SELECT 1')
+                dummy_file.close()
 
-                # Mutually exclusive
+                result = runner.invoke(ctl, [
-                result = self.runner.invoke(ctl, [
                     'query',
                     'alpha',
                     '--file',
@@ -209,52 +196,45 @@ y''')
                     '--command',
                     'dummy',
                 ])
-                assert result.exit_code == 1
+                assert 'mutually exclusive' in str(result.exception)
 
-                result = self.runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy'])
+                result = runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy'])
 
                 os.remove('dummy')
 
-            result = self.runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1'])
+            result = runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1'])
-            assert 'mock column' in result.output
-
-            # --command or --file is mandatory
-            result = self.runner.invoke(ctl, ['query', 'alpha'])
-            assert result.exit_code == 1
-
-            result = self.runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1', '--username', 'root',
-                                              '--password', '--dbname', 'postgres'], input='ab\nab')
             assert 'mock column' in result.output
 
     @patch('patroni.ctl.get_cursor', Mock(return_value=MockConnect().cursor()))
     def test_query_member(self):
         rows = query_member(None, None, None, 'master', 'SELECT pg_is_in_recovery()')
-        self.assertTrue('False' in str(rows))
+        assert 'False' in str(rows)
 
         rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
-        self.assertEquals(rows, (None, None))
+        assert rows == (None, None)
 
         with patch('patroni.ctl.get_cursor', Mock(return_value=None)):
             rows = query_member(None, None, None, None, 'SELECT pg_is_in_recovery()')
-            self.assertTrue('No connection to' in str(rows))
+            assert 'No connection to' in str(rows)
 
             rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
-            self.assertTrue('No connection to' in str(rows))
+            assert 'No connection to' in str(rows)
 
-        with patch('patroni.ctl.get_cursor', Mock(side_effect=OperationalError('bla'))):
+        with patch('patroni.ctl.get_cursor', Mock(side_effect=psycopg2.OperationalError('bla'))):
             rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
 
-        with patch('test_postgresql.MockCursor.execute', Mock(side_effect=OperationalError('bla'))):
+        with patch('test_postgresql.MockCursor.execute', Mock(side_effect=psycopg2.OperationalError('bla'))):
             rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
 
     @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
     def test_dsn(self):
+        runner = CliRunner()
+
         with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)):
-            result = self.runner.invoke(ctl, ['dsn', 'alpha', '--dcs', '8.8.8.8'])
+            result = runner.invoke(ctl, ['dsn', 'alpha', '--dcs', '8.8.8.8'])
             assert 'host=127.0.0.1 port=5435' in result.output
 
-            # Mutually exclusive options
+            result = runner.invoke(ctl, [
-            result = self.runner.invoke(ctl, [
                 'dsn',
                 'alpha',
                 '--role',
@@ -262,29 +242,26 @@ y''')
                 '--member',
                 'dummy',
             ])
-            assert result.exit_code == 1
+            assert 'mutually exclusive' in str(result.exception)
 
-            # Non-existing member
+            result = runner.invoke(ctl, ['dsn', 'alpha', '--member', 'dummy'])
-            result = self.runner.invoke(ctl, ['dsn', 'alpha', '--member', 'dummy'])
+            assert 'Can not find' in str(result.exception)
-            assert result.exit_code == 1
+
+        # result = runner.invoke(ctl, ['dsn', 'alpha', '--dcs', '8.8.8.8', '--role', 'replica'])
+        # assert 'host=127.0.0.1 port=5436' in result.output
 
     @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
     @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
     @patch('requests.get', requests_get)
     @patch('requests.post', requests_get)
     def test_restart_reinit(self):
-        result = self.runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='y')
+        runner = CliRunner()
-        assert result.exit_code == 0
 
-        result = self.runner.invoke(ctl, ['reinit', 'alpha', '--dcs', '8.8.8.8'], input='y')
+        result = runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='y')
-        assert result.exit_code == 1
+        result = runner.invoke(ctl, ['reinit', 'alpha', '--dcs', '8.8.8.8'], input='y')
 
-        # Aborted restart
+        result = runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='N')
-        result = self.runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='N')
+        result = runner.invoke(ctl, [
-        assert result.exit_code == 1
-
-        # Not a member
-        result = self.runner.invoke(ctl, [
             'restart',
             'alpha',
             '--dcs',
@@ -292,93 +269,100 @@ y''')
             'dummy',
             '--any',
         ], input='y')
-        assert result.exit_code == 1
+        assert 'not a member' in str(result.exception)
 
         with patch('requests.post', Mock(return_value=MockResponse())):
-            result = self.runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='y')
+            result = runner.invoke(ctl, ['restart', 'alpha', '--dcs', '8.8.8.8'], input='y')
 
     @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
     @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
     def test_remove(self):
-        result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='alpha\nslave')
+        runner = CliRunner()
+
+        result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='alpha\nslave')
         assert 'Please confirm' in result.output
         assert 'You are about to remove all' in result.output
-        # Not typing an exact confirmation
+        assert 'You did not exactly type' in str(result.exception)
-        assert result.exit_code == 1
 
-        # master specified does not match master of cluster
+        result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='''alpha
-        result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='''alpha
 Yes I am aware
 slave''')
-        assert result.exit_code == 1
+        assert 'You did not specify the current master of the cluster' in str(result.exception)
 
-        # cluster specified on cmdline does not match verification prompt
+        result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='beta\nleader')
-        result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='beta\nleader')
+        assert 'Cluster names specified do not match' in str(result.exception)
-        assert result.exit_code == 1
 
         with patch('patroni.etcd.Etcd.get_cluster', get_cluster_initialized_with_leader):
-            result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'],
+            result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'],
-                                        input='''alpha
+                                   input='''alpha
 Yes I am aware
 leader''')
             assert 'object has no attribute' in str(result.exception)
 
         with patch('patroni.ctl.get_dcs', Mock(return_value=Mock())):
-            # Not implemented DCS
+            result = runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'],
-            result = self.runner.invoke(ctl, ['remove', 'alpha', '--dcs', '8.8.8.8'], input='''alpha
+                                   input='''alpha
 Yes I am aware
 leader''')
-            assert result.exit_code == 1
+            assert 'We have not implemented this for DCS of type' in str(result.exception)
 
     @patch('patroni.etcd.Etcd.watch', Mock(return_value=None))
     @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
     def test_wait_for_leader(self):
         dcs = self.e
-        self.assertRaises(PatroniCtlException, wait_for_leader, dcs, 0)
+        self.assertRaises(patroni.exceptions.PatroniCtlException, wait_for_leader, dcs, 0)
 
         cluster = wait_for_leader(dcs=dcs, timeout=2)
         assert cluster.leader.member.name == 'leader'
 
     def test_post_patroni(self):
-        with patch('requests.post', MagicMock(side_effect=requests.exceptions.ConnectionError('foo'))):
+        member = get_cluster_initialized_with_leader().leader.member
-            member = get_cluster_initialized_with_leader().leader.member
+        self.assertRaises(requests.exceptions.ConnectionError, post_patroni, member, 'dummy', {})
-            self.assertRaises(requests.exceptions.ConnectionError, post_patroni, member, 'dummy', {})
 
     def test_ctl(self):
-        self.runner.invoke(ctl, ['list'])
+        runner = CliRunner()
 
-        result = self.runner.invoke(ctl, ['--help'])
+        runner.invoke(ctl, ['list'])
+
+        result = runner.invoke(ctl, ['--help'])
         assert 'Usage:' in result.output
 
     def test_get_any_member(self):
-        self.assertIsNone(get_any_member(get_cluster_initialized_without_leader(), role='master'))
+        m = get_any_member(get_cluster_initialized_without_leader(), role='master')
+        assert m is None
 
         m = get_any_member(get_cluster_initialized_with_leader(), role='master')
-        self.assertEquals(m.name, 'leader')
+        assert m.name == 'leader'
 
     def test_get_all_members(self):
-        self.assertEquals(list(get_all_members(get_cluster_initialized_without_leader(), role='master')), [])
+        r = list(get_all_members(get_cluster_initialized_without_leader(), role='master'))
+        assert len(r) == 0
 
         r = list(get_all_members(get_cluster_initialized_with_leader(), role='master'))
-        self.assertEquals(len(r), 1)
+        assert len(r) == 1
-        self.assertEquals(r[0].name, 'leader')
+        assert r[0].name == 'leader'
 
         r = list(get_all_members(get_cluster_initialized_with_leader(), role='replica'))
-        self.assertEquals(len(r), 1)
+        assert len(r) == 1
-        self.assertEquals(r[0].name, 'other')
+        assert r[0].name == 'other'
 
-        self.assertEquals(len(list(get_all_members(get_cluster_initialized_without_leader(), role='replica'))), 2)
+        r = list(get_all_members(get_cluster_initialized_without_leader(), role='replica'))
+        assert len(r) == 2
 
     @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
     @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
     @patch('requests.get', requests_get)
     @patch('requests.post', requests_get)
     def test_members(self):
-        result = self.runner.invoke(members, ['alpha'])
+        runner = CliRunner()
+
+        result = runner.invoke(members, ['alpha'])
         assert result.exit_code == 0
 
     def test_configure(self):
-        result = self.runner.invoke(configure, [
+        runner = CliRunner()
+
+        result = runner.invoke(configure, [
             '--dcs',
             'abc',
             '-c',
@@ -388,3 +372,5 @@ leader''')
         ])
 
         assert result.exit_code == 0
+
+

tests/test_etcd.py

@@ -7,12 +7,11 @@ import unittest
 
 from dns.exception import DNSException
 from mock import Mock, patch
-from patroni.dcs import Cluster
+from patroni.dcs import Cluster, DCSError, Leader
 from patroni.etcd import Client, Etcd, EtcdError
-from patroni.exceptions import DCSError
 
 
-class MockResponse(object):
+class MockResponse:
 
     def __init__(self):
         self.status_code = 200
@@ -25,24 +24,23 @@ class MockResponse(object):
 
     @property
     def data(self):
-        return self.content.encode('utf-8')
+        if self.content == 'TimeoutError':
+            raise urllib3.exceptions.TimeoutError
+        if self.content == 'Exception':
+            raise Exception
+        return self.content
 
     @property
     def status(self):
         return self.status_code
 
-    @staticmethod
     def getheader(*args):
         return ''
 
 
 class MockPostgresql(Mock):
 
-    server_version = '999999'
+    def last_operation(self):
-    scope = 'dummy'
-
-    @staticmethod
-    def last_operation():
         return '0'
 
 
@@ -55,7 +53,10 @@ def requests_get(url, **kwargs):
     elif ':8011/patroni' in url:
         response.content = '{"role": "replica", "xlog": {"replayed_location": 0}, "tags": {}}'
     elif url.endswith('/members'):
-        response.content = '[{}]' if url.startswith('http://error') else members
+        if url.startswith('http://error'):
+            response.content = '[{}]'
+        else:
+            response.content = members
     elif url.startswith('http://exhibitor'):
         response.content = '{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}'
     else:
@@ -66,7 +67,7 @@ def requests_get(url, **kwargs):
 
 def etcd_watch(key, index=None, timeout=None, recursive=None):
     if timeout == 2.0:
-        raise etcd.EtcdWatchTimedOut
+        raise urllib3.exceptions.TimeoutError
     elif timeout == 5.0:
         return etcd.EtcdResult('delete', {})
     elif timeout == 10.0:
@@ -80,9 +81,9 @@ def etcd_watch(key, index=None, timeout=None, recursive=None):
 def etcd_write(key, value, **kwargs):
     if key == '/service/exists/leader':
         raise etcd.EtcdAlreadyExist
-    if key in ['/service/test/leader', '/patroni/test/leader'] and \
+    if key == '/service/test/leader' or key == '/patroni/test/leader':
-            (kwargs.get('prevValue') == 'foo' or not kwargs.get('prevExist', True)):
+        if kwargs.get('prevValue', None) == 'foo' or not kwargs.get('prevExist', True):
-        return True
+            return True
     raise etcd.EtcdException
 
 
@@ -123,12 +124,12 @@ class SleepException(Exception):
     pass
 
 
-class MockSRV(object):
+class MockSRV:
     port = 2380
     target = '127.0.0.1'
 
 
-def dns_query(name, _):
+def dns_query(name, type):
     if name == '_etcd-server._tcp.blabla':
         return []
     elif name == '_etcd-server._tcp.exception':
@@ -143,8 +144,6 @@ def socket_getaddrinfo(*args):
 
 
 def http_request(method, url, **kwargs):
-    if url == 'http://localhost:2379/timeout':
-        raise urllib3.exceptions.ReadTimeoutError(None, None, None)
     if url == 'http://localhost:2379/':
         return MockResponse()
     raise socket.error
@@ -162,27 +161,30 @@ class TestClient(unittest.TestCase):
             mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
             self.client = Client({'discovery_srv': 'test'})
             self.client.http.request = http_request
-            self.client.http.request_encode_body = http_request
 
     def test_api_execute(self):
         self.client._base_uri = 'http://localhost:4001'
         self.client._machines_cache = ['http://localhost:2379']
-        self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
+        self.client.api_execute('/', 'GET')
-        self.client._update_machines_cache = False
-        self.client.api_execute('/', 'POST', timeout=0)
         self.client._update_machines_cache = False
         self.client._base_uri = 'http://localhost:4001'
         self.client._machines_cache = []
         self.assertRaises(etcd.EtcdConnectionFailed, self.client.api_execute, '/', 'GET')
         self.assertTrue(self.client._update_machines_cache)
-        self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'GET')
-        self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '')
-        self.assertRaises(ValueError, self.client.api_execute, '', '')
 
     def test_get_srv_record(self):
         self.assertEquals(self.client.get_srv_record('blabla'), [])
         self.assertEquals(self.client.get_srv_record('exception'), [])
 
+    def test__result_from_response(self):
+        response = MockResponse()
+        response.content = 'TimeoutError'
+        self.assertRaises(urllib3.exceptions.TimeoutError, self.client._result_from_response, response)
+        response.content = 'Exception'
+        self.assertRaises(etcd.EtcdException, self.client._result_from_response, response)
+        response.content = b'{}'
+        self.assertRaises(etcd.EtcdException, self.client._result_from_response, response)
+
     def test__get_machines_cache_from_srv(self):
         self.client.get_srv_record = Mock(return_value=[('localhost', 2380)])
         self.client._get_machines_cache_from_srv('blabla')
@@ -224,8 +226,11 @@ class TestEtcd(unittest.TestCase):
         cluster = self.etcd.get_cluster()
         self.assertIsInstance(cluster, Cluster)
         self.assertIsNone(cluster.leader)
+
+    def test_current_leader(self):
+        self.assertIsInstance(self.etcd.current_leader(), Leader)
         self.etcd._base_path = '/service/noleader'
-        self.assertRaises(EtcdError, self.etcd.get_cluster)
+        self.assertIsNone(self.etcd.current_leader())
 
     def test_touch_member(self):
         self.assertFalse(self.etcd.touch_member('', ''))

tests/test_ha.py

@@ -1,14 +1,11 @@
+import etcd
 import unittest
-import datetime
-import pytz
 
-from etcd import EtcdException
 from mock import Mock, MagicMock, patch
 from patroni.dcs import Cluster, Failover, Leader, Member
 from patroni.etcd import Client, Etcd
 from patroni.exceptions import DCSError, PostgresException
 from patroni.ha import Ha
-from patroni.postgresql import Postgresql
 from test_etcd import socket_getaddrinfo, etcd_read, etcd_write, requests_get
 
 
@@ -21,7 +18,7 @@ def false(*args, **kwargs):
 
 
 def get_cluster(initialize, leader, members, failover):
-    return Cluster(initialize, leader, 10, members, failover)
+    return Cluster(initialize, leader, None, members, failover)
 
 
 def get_cluster_not_initialized_without_leader():
@@ -29,24 +26,61 @@ def get_cluster_not_initialized_without_leader():
 
 
 def get_cluster_initialized_without_leader(leader=False, failover=None):
-    m1 = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres',
+    m = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres',
-                                  'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location': 4})
+                                 'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location':4})
-    l = Leader(0, 0, m1) if leader else None
+    l = Leader(0, 0, m) if leader else None
-    m2 = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
+    o = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
-                                 'api_url': 'http://127.0.0.1:8011/patroni'})
+                                'api_url': 'http://127.0.0.1:8011/patroni'})
-    return get_cluster(True, l, [m1, m2], failover)
+    return get_cluster(True, l, [m, o], failover)
 
 
 def get_cluster_initialized_with_leader(failover=None):
     return get_cluster_initialized_without_leader(leader=True, failover=failover)
 
-
 def get_cluster_initialized_with_only_leader(failover=None):
     l = get_cluster_initialized_without_leader(leader=True, failover=failover).leader
     return get_cluster(True, l, [l], failover)
 
 
-class MockPatroni(object):
+class MockPostgresql(Mock):
+
+    name = 'postgresql0'
+    role = 'replica'
+    state = 'running'
+    connection_string = 'postgres://foo@bar/postgres'
+
+    def is_healthy(self):
+        return True
+
+    def start(self):
+        return True
+
+    def is_healthiest_node(self, members):
+        return True
+
+    def is_leader(self):
+        return True
+
+    def xlog_position(self):
+        return 0
+
+    def last_operation(self):
+        return 0
+
+    def data_directory_empty(self):
+        return False
+
+    def bootstrap(self, *args, **kwargs):
+        return True
+
+    def check_replication_lag(self, last_leader_operation):
+        return True
+
+    def check_recovery_conf(self, leader):
+        return False
+
+
+class MockPatroni:
 
     def __init__(self, p, d):
         self.postgresql = p
@@ -54,50 +88,30 @@ class MockPatroni(object):
         self.api = Mock()
         self.tags = {}
         self.nofailover = None
-        self.nap_time = 10
+        self.bdr = {}
-        self.replicatefrom = None
         self.api.connection_string = 'http://127.0.0.1:8008'
-        self.clonefrom = None
 
 
 def run_async(func, args=()):
-    return func(*args) if args else func()
+    func(*args) if args else func()
 
 
-@patch.object(Postgresql, 'is_running', Mock(return_value=True))
-@patch.object(Postgresql, 'is_leader', Mock(return_value=True))
-@patch.object(Postgresql, 'xlog_position', Mock(return_value=0))
-@patch.object(Postgresql, 'call_nowait', Mock(return_value=True))
-@patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
-@patch.object(Postgresql, 'controldata', Mock(return_value={'Database system identifier': '1234567890'}))
-@patch.object(Postgresql, 'sync_replication_slots', Mock())
-@patch.object(Postgresql, 'write_pg_hba', Mock())
-@patch.object(Postgresql, 'write_pgpass', Mock())
-@patch.object(Postgresql, 'write_recovery_conf', Mock())
-@patch.object(Postgresql, 'query', Mock())
-@patch.object(Postgresql, 'checkpoint', Mock())
-@patch('subprocess.call', Mock(return_value=0))
 class TestHa(unittest.TestCase):
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
-    def setUp(self):
+    @patch.object(Client, 'machines')
-        with patch.object(Client, 'machines') as mock_machines:
+    def setUp(self, mock_machines):
-            mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
+        mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
-            self.p = Postgresql({'name': 'postgresql0', 'scope': 'dummy', 'listen': '127.0.0.1:5432',
+        self.p = MockPostgresql()
-                                 'data_dir': 'data/postgresql0', 'superuser': {}, 'admin': {},
+        self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'})
-                                 'replication': {'username': '', 'password': '', 'network': ''}})
+        self.e.client.read = etcd_read
-            self.p.set_state('running')
+        self.e.client.write = etcd_write
-            self.p.check_replication_lag = true
+        self.e.client.delete = Mock(side_effect=etcd.EtcdException())
-            self.p.can_create_replica_without_replication_connection = MagicMock(return_value=False)
+        self.ha = Ha(MockPatroni(self.p, self.e))
-            self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'})
+        self.ha._async_executor.run_async = run_async
-            self.e.client.read = etcd_read
+        self.ha.old_cluster = self.e.get_cluster()
-            self.e.client.write = etcd_write
+        self.ha.cluster = get_cluster_not_initialized_without_leader()
-            self.e.client.delete = Mock(side_effect=EtcdException())
+        self.ha.load_cluster_from_dcs = Mock()
-            self.ha = Ha(MockPatroni(self.p, self.e))
-            self.ha._async_executor.run_async = run_async
-            self.ha.old_cluster = self.e.get_cluster()
-            self.ha.cluster = get_cluster_not_initialized_without_leader()
-            self.ha.load_cluster_from_dcs = Mock()
 
     def test_update_lock(self):
         self.p.last_operation = Mock(side_effect=PostgresException(''))
@@ -114,19 +128,13 @@ class TestHa(unittest.TestCase):
     def test_recover_replica_failed(self):
         self.p.controldata = lambda: {'Database cluster state': 'in production'}
         self.p.is_healthy = false
-        self.p.is_running = false
+        self.p.follow_the_leader = false
-        self.p.follow = false
-        self.assertEquals(self.ha.run_cycle(), 'started as a secondary')
         self.assertEquals(self.ha.run_cycle(), 'failed to start postgres')
 
     def test_recover_master_failed(self):
-        self.p.follow = false
+        self.p.follow_the_leader = false
         self.p.is_healthy = false
-        self.p.is_running = false
         self.ha.has_lock = true
-        self.p.set_role('master')
-        self.p.controldata = lambda: {'Database cluster state': 'in production'}
-        self.assertEquals(self.ha.run_cycle(), 'started as readonly because i had the session lock')
         self.assertEquals(self.ha.run_cycle(), 'removed leader key after trying and failing to start postgres')
 
     @patch('sys.exit', return_value=1)
@@ -137,8 +145,7 @@ class TestHa(unittest.TestCase):
 
     @patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
     def test_start_as_readonly(self):
-        self.p.is_leader = false
+        self.p.is_leader = self.p.is_healthy = false
-        self.p.is_healthy = true
         self.ha.has_lock = true
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader because i had the session lock')
 
@@ -152,7 +159,7 @@ class TestHa(unittest.TestCase):
 
     def test_demote_after_failing_to_obtain_lock(self):
         self.ha.acquire_lock = false
-        self.assertEquals(self.ha.run_cycle(), 'demoted self after trying and failing to obtain lock')
+        self.assertEquals(self.ha.run_cycle(), 'demoted self due after trying and failing to obtain lock')
 
     def test_follow_new_leader_after_failing_to_obtain_lock(self):
         self.ha.is_healthiest_node = true
@@ -190,12 +197,10 @@ class TestHa(unittest.TestCase):
         self.ha.update_lock = false
         self.assertEquals(self.ha.run_cycle(), 'demoting self because i do not have the lock and i was a leader')
 
-    def test_follow(self):
+    def test_follow_the_leader(self):
         self.ha.cluster.is_unlocked = false
         self.p.is_leader = false
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am a secondary and i am following a leader')
-        self.ha.patroni.replicatefrom = "foo"
-        self.assertEquals(self.ha.run_cycle(), 'no action.  i am a secondary and i am following a leader')
 
     def test_no_etcd_connection_master_demote(self):
         self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
@@ -206,20 +211,10 @@ class TestHa(unittest.TestCase):
         self.p.bootstrap = false
         self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap from leader')
 
-    def test_bootstrap_from_another_member(self):
-        self.ha.cluster = get_cluster_initialized_with_leader()
-        self.ha.patroni.clonefrom = 'other'
-        self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap from replica \'other\'')
-
     def test_bootstrap_waiting_for_leader(self):
         self.ha.cluster = get_cluster_initialized_without_leader()
         self.assertEquals(self.ha.bootstrap(), 'waiting for leader to bootstrap')
 
-    def test_bootstrap_without_leader(self):
-        self.ha.cluster = get_cluster_initialized_without_leader()
-        self.p.can_create_replica_without_replication_connection = MagicMock(return_value=True)
-        self.assertEquals(self.ha.bootstrap(), "trying to bootstrap without leader")
-
     def test_bootstrap_initialize_lock_failed(self):
         self.ha.cluster = get_cluster_not_initialized_without_leader()
         self.assertEquals(self.ha.bootstrap(), 'failed to acquire initialize lock')
@@ -275,62 +270,38 @@ class TestHa(unittest.TestCase):
     @patch('requests.get', requests_get)
     def test_manual_failover_from_leader(self):
         self.ha.has_lock = true
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None))
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', ''))
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', self.p.name, None))
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', MockPostgresql.name))
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'blabla', None))
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', 'blabla'))
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
-        f = Failover(0, self.p.name, '', None)
+        f = Failover(0, MockPostgresql.name, '')
         self.ha.cluster = get_cluster_initialized_with_leader(f)
         self.assertEquals(self.ha.run_cycle(), 'manual failover: demoting myself')
         self.ha.fetch_node_status = lambda e: (e, True, True, 0, {'nofailover': 'True'})
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
         # manual failover from the previous leader to us won't happen if we hold the nofailover flag
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, None))
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', MockPostgresql.name))
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
 
-        # Failover scheduled time must include timezone
-        scheduled = datetime.datetime.now()
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
-        self.ha.run_cycle()
-
-        scheduled = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
-        self.assertEquals('no action.  i am the leader with the lock', self.ha.run_cycle())
-
-        scheduled = scheduled + datetime.timedelta(seconds=30)
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
-        self.assertEquals('no action.  i am the leader with the lock', self.ha.run_cycle())
-
-        scheduled = scheduled + datetime.timedelta(seconds=-600)
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
-        self.assertEquals('no action.  i am the leader with the lock', self.ha.run_cycle())
-
-        scheduled = None
-        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
-        self.assertEquals('no action.  i am the leader with the lock', self.ha.run_cycle())
-
     @patch('requests.get', requests_get)
     def test_manual_failover_process_no_leader(self):
         self.p.is_leader = false
-        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', self.p.name, None))
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', MockPostgresql.name))
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
-        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader', None))
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader'))
-        self.p.set_role('replica')
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
         self.ha.fetch_node_status = lambda e: (e, True, True, 0, {})  # accessible, in_recovery
         self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
-        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, self.p.name, '', None))
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, MockPostgresql.name, ''))
         self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
         self.ha.fetch_node_status = lambda e: (e, False, True, 0, {})  # inaccessible, in_recovery
-        self.p.set_role('replica')
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
         # set failover flag to True for all members of the cluster
         # this should elect the current member, as we are not going to call the API for it.
-        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other'))
         self.ha.fetch_node_status = lambda e: (e, True, True, 0, {'nofailover': 'True'})  # accessible, in_recovery
-        self.p.set_role('replica')
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
         # same as previous, but set the current member to nofailover. In no case it should be elected as a leader
         self.ha.patroni.nofailover = True
@@ -363,12 +334,3 @@ class TestHa(unittest.TestCase):
         self.ha.fetch_node_status(member)
         member = Member(0, 'test', 1, {'api_url': 'http://localhost:8011/patroni'})
         self.ha.fetch_node_status(member)
-
-    def test_post_recover(self):
-        self.p.is_running = false
-        self.ha.has_lock = true
-        self.assertEqual(self.ha.post_recover(), 'removed leader key after trying and failing to start postgres')
-        self.ha.has_lock = false
-        self.assertEqual(self.ha.post_recover(), 'failed to start postgres')
-        self.p.is_running = true
-        self.assertIsNone(self.ha.post_recover())

tests/test_patroni.py

@@ -7,7 +7,7 @@ from mock import Mock, patch
 from patroni.api import RestApiServer
 from patroni.async_executor import AsyncExecutor
 from patroni.etcd import Etcd
-from patroni import Patroni, main as _main
+from patroni import Patroni, main
 from patroni.zookeeper import ZooKeeper
 from six.moves import BaseHTTPServer
 from test_etcd import Client, SleepException, etcd_read, etcd_write
@@ -15,6 +15,10 @@ from test_postgresql import Postgresql, psycopg2_connect
 from test_zookeeper import MockKazooClient
 
 
+def time_sleep(*args):
+    raise SleepException()
+
+
 @patch('time.sleep', Mock())
 @patch('subprocess.call', Mock(return_value=0))
 @patch('psycopg2.connect', psycopg2_connect)
@@ -24,19 +28,19 @@ from test_zookeeper import MockKazooClient
 @patch.object(AsyncExecutor, 'run', Mock())
 class TestPatroni(unittest.TestCase):
 
-    def setUp(self):
+    @patch.object(Client, 'machines')
-        with patch.object(Client, 'machines') as mock_machines:
+    def setUp(self, mock_machines):
-            mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
+        mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
-            self.touched = False
+        self.touched = False
-            self.init_cancelled = False
+        self.init_cancelled = False
-            RestApiServer._BaseServer__is_shut_down = Mock()
+        RestApiServer._BaseServer__is_shut_down = Mock()
-            RestApiServer._BaseServer__shutdown_request = True
+        RestApiServer._BaseServer__shutdown_request = True
-            RestApiServer.socket = 0
+        RestApiServer.socket = 0
-            with open('postgres0.yml', 'r') as f:
+        with open('postgres0.yml', 'r') as f:
-                config = yaml.load(f)
+            config = yaml.load(f)
-                self.p = Patroni(config)
+            self.p = Patroni(config)
-                self.p.ha.dcs.client.write = etcd_write
+            self.p.ha.dcs.client.write = etcd_write
-                self.p.ha.dcs.client.read = etcd_read
+            self.p.ha.dcs.client.read = etcd_read
 
     @patch('patroni.zookeeper.KazooClient', MockKazooClient())
     def test_get_dcs(self):
@@ -47,18 +51,18 @@ class TestPatroni(unittest.TestCase):
     @patch.object(Etcd, 'delete_leader', Mock())
     @patch.object(Client, 'machines')
     def test_patroni_main(self, mock_machines):
-        _main()
+        main()
         sys.argv = ['patroni.py', 'postgres0.yml']
 
         mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
         with patch.object(Patroni, 'run', Mock(side_effect=SleepException())):
-            self.assertRaises(SleepException, _main)
+            self.assertRaises(SleepException, main)
         with patch.object(Patroni, 'run', Mock(side_effect=KeyboardInterrupt())):
-            _main()
+            main()
 
     @patch('time.sleep', Mock(side_effect=SleepException()))
     def test_run(self):
-        self.p.ha.dcs.watch = Mock(side_effect=SleepException())
+        self.p.ha.dcs.watch = time_sleep
         self.assertRaises(SleepException, self.p.run)
 
         self.p.ha.state_handler.is_leader = Mock(return_value=False)
@@ -76,8 +80,3 @@ class TestPatroni(unittest.TestCase):
         self.assertTrue(self.p.nofailover)
         self.p.tags['nofailover'] = None
         self.assertFalse(self.p.nofailover)
-
-    def test_replicatefrom(self):
-        self.assertIsNone(self.p.replicatefrom)
-        self.p.tags['replicatefrom'] = 'foo'
-        self.assertEqual(self.p.replicatefrom, 'foo')

tests/test_postgresql.py

@@ -2,19 +2,24 @@ import mock  # for the mock.call method, importing it without a namespace breaks
 import os
 import psycopg2
 import shutil
-import subprocess
 import unittest
 
+from six.moves import builtins
 from mock import Mock, MagicMock, PropertyMock, patch, mock_open
 from patroni.dcs import Cluster, Leader, Member
 from patroni.exceptions import PostgresException, PostgresConnectionException
 from patroni.postgresql import Postgresql
 from patroni.utils import RetryFailedError
-from six.moves import builtins
 from test_ha import false
+import subprocess
 
 
-class MockCursor(object):
+def is_file_raise_on_backup(*args, **kwargs):
+    if args[0].endswith('.backup'):
+        raise Exception("foo")
+
+
+class MockCursor:
 
     def __init__(self, connection):
         self.connection = connection
@@ -33,7 +38,7 @@ class MockCursor(object):
         elif sql == 'SELECT pg_is_in_recovery()':
             self.results = [(False, )]
         elif sql.startswith('SELECT to_char(pg_postmaster_start_time'):
-            self.results = [('', True, '', '', '', '', False)]
+            self.results = [('', True, '', '', '', False)]
         else:
             self.results = [(
                 None,
@@ -54,8 +59,7 @@ class MockCursor(object):
     def fetchall(self):
         return self.results
 
-    @staticmethod
+    def close(self):
-    def close():
         pass
 
     def __iter__(self):
@@ -150,12 +154,9 @@ def psycopg2_connect(*args, **kwargs):
     return MockConnect()
 
 
-def fake_listdir(path):
-    return ["a", "b", "c"] if path.endswith('pg_xlog/archive_status') else []
-
-
 @patch('subprocess.call', Mock(return_value=0))
 @patch('psycopg2.connect', psycopg2_connect)
+@patch('shutil.copy', Mock())
 class TestPostgresql(unittest.TestCase):
 
     @patch('subprocess.call', Mock(return_value=0))
@@ -164,7 +165,7 @@ class TestPostgresql(unittest.TestCase):
         self.p = Postgresql({'name': 'test0', 'scope': 'batman', 'data_dir': 'data/test0',
                              'listen': '127.0.0.1, *:5432', 'connect_address': '127.0.0.2:5432',
                              'pg_hba': ['hostssl all all 0.0.0.0/0 md5', 'host all all 0.0.0.0/0 md5'],
-                             'superuser': {'username': 'test', 'password': 'test'},
+                             'superuser': {'password': 'test'},
                              'admin': {'username': 'admin', 'password': 'admin'},
                              'pg_rewind': {'username': 'admin', 'password': 'admin'},
                              'replication': {'username': 'replicator',
@@ -180,8 +181,7 @@ class TestPostgresql(unittest.TestCase):
             os.makedirs(self.p.data_dir)
         self.leadermem = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres'})
         self.leader = Leader(-1, 28, self.leadermem)
-        self.other = Member(0, 'test1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres',
+        self.other = Member(0, 'test1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres'})
-                            'tags': {'replicatefrom': 'leader'}})
         self.me = Member(0, 'test0', 28, {'conn_url': 'postgres://replicator:[email protected]:5434/postgres'})
 
     def tearDown(self):
@@ -204,11 +204,6 @@ class TestPostgresql(unittest.TestCase):
         self.assertTrue(self.p.initialize())
         self.assertTrue(os.path.exists(os.path.join(self.p.data_dir, 'pg_hba.conf')))
 
-    @patch('os.path.exists', Mock(return_value=True))
-    @patch('os.unlink', Mock())
-    def test_delete_trigger_file(self):
-        self.p.delete_trigger_file()
-
     def test_start(self):
         self.assertTrue(self.p.start())
         self.p.is_running = false
@@ -233,12 +228,10 @@ class TestPostgresql(unittest.TestCase):
         self.p.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo', 'password': 'bar'})
 
     @patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
-    def test_sync_replica(self):
+    def test_sync_from_leader(self):
-        self.assertTrue(self.p.sync_replica(self.leader))
+        self.assertTrue(self.p.sync_from_leader(self.leader))
-        self.p.create_replica = Mock(return_value=1)
-        self.assertFalse(self.p.sync_replica(self.leader))
 
-    @patch('subprocess.call', side_effect=OSError)
+    @patch('subprocess.call', side_effect=Exception("Test"))
     @patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
     def test_pg_rewind(self, mock_call):
         self.assertTrue(self.p.rewind(self.leader))
@@ -249,28 +242,26 @@ class TestPostgresql(unittest.TestCase):
     @patch('patroni.postgresql.Postgresql.remove_data_directory', MagicMock(return_value=True))
     @patch('patroni.postgresql.Postgresql.single_user_mode', MagicMock(return_value=1))
     @patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
-    @patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string))
+    def test_follow_the_leader(self, mock_pg_rewind):
-    def test_follow(self, mock_pg_rewind):
+        self.p.demote()
-        self.p.follow(None)
+        self.p.follow_the_leader(None)
-        self.p.follow(self.leader)
+        self.p.demote()
-        self.p.follow(Leader(-1, 28, self.other))
+        self.p.follow_the_leader(self.leader)
+        self.p.follow_the_leader(Leader(-1, 28, self.other))
         self.p.rewind = mock_pg_rewind
-        self.p.follow(self.leader)
+        self.p.follow_the_leader(self.leader)
         self.p.require_rewind()
         with mock.patch('os.path.islink', MagicMock(return_value=True)):
             with mock.patch('patroni.postgresql.Postgresql.can_rewind', new_callable=PropertyMock(return_value=True)):
                 with mock.patch('os.unlink', MagicMock(return_value=True)):
-                    self.p.follow(self.leader, recovery=True)
+                    self.p.follow_the_leader(self.leader, recovery=True)
         self.p.require_rewind()
         with mock.patch('patroni.postgresql.Postgresql.can_rewind', new_callable=PropertyMock(return_value=True)):
             self.p.rewind.return_value = True
-            self.p.follow(self.leader, recovery=True)
+            self.p.follow_the_leader(self.leader, recovery=True)
             self.p.rewind.return_value = False
-            self.p.follow(self.leader, recovery=True)
+            self.p.follow_the_leader(self.leader, recovery=True)
-        with mock.patch('patroni.postgresql.Postgresql.check_recovery_conf', MagicMock(return_value=True)):
-            self.assertTrue(self.p.follow(None))
 
-    @patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string))
     def test_can_rewind(self):
         tmp = self.p.pg_rewind
         self.p.pg_rewind = None
@@ -278,16 +269,16 @@ class TestPostgresql(unittest.TestCase):
         self.p.pg_rewind = tmp
         with mock.patch('subprocess.call', MagicMock(return_value=1)):
             self.assertFalse(self.p.can_rewind)
-        with mock.patch('subprocess.call', side_effect=OSError):
+        with mock.patch('subprocess.call', side_effect=OSError("foo")):
             self.assertFalse(self.p.can_rewind)
-        tmp = self.p.controldata
+        tmp = self.p.controldata()
         self.p.controldata = lambda: {'wal_log_hints setting': 'on'}
         self.assertTrue(self.p.can_rewind)
         self.p.controldata = tmp
 
     @patch('time.sleep', Mock())
     def test_create_replica(self):
-        self.p.delete_trigger_file = Mock(side_effect=OSError)
+        self.p.delete_trigger_file = Mock(side_effect=OSError())
         with patch('subprocess.call', Mock(side_effect=[1, 0])):
             self.assertEquals(self.p.create_replica(self.leader, ''), 0)
         with patch('subprocess.call', Mock(side_effect=[Exception(), 0])):
@@ -303,6 +294,12 @@ class TestPostgresql(unittest.TestCase):
         with patch('subprocess.call', Mock(side_effect=Exception("foo"))):
             self.assertEquals(self.p.create_replica(self.leader, ''), 1)
 
+    def test_create_connection_users(self):
+        cfg = self.p.config
+        cfg['superuser']['username'] = 'test'
+        p = Postgresql(cfg)
+        p.create_connection_users()
+
     def test_sync_replication_slots(self):
         self.p.start()
         cluster = Cluster(True, self.leader, 0, [self.me, self.other, self.leadermem], None)
@@ -310,15 +307,12 @@ class TestPostgresql(unittest.TestCase):
         self.p.query = Mock(side_effect=psycopg2.OperationalError)
         self.p.schedule_load_slots = True
         self.p.sync_replication_slots(cluster)
-        self.p.schedule_load_slots = False
-        with mock.patch('patroni.postgresql.Postgresql.role', new_callable=PropertyMock(return_value='replica')):
-            self.p.sync_replication_slots(cluster)
 
     @patch.object(MockConnect, 'closed', 2)
     def test__query(self):
-        self.assertRaises(PostgresConnectionException, self.p._query, 'blabla')
+        self.assertRaises(PostgresConnectionException, self.p._query, 'blabla', 'postgres')
         self.p._state = 'restarting'
-        self.assertRaises(RetryFailedError, self.p._query, 'blabla')
+        self.assertRaises(RetryFailedError, self.p._query, 'blabla', 'postgres')
 
     def test_query(self):
         self.p.query('select 1')
@@ -344,7 +338,7 @@ class TestPostgresql(unittest.TestCase):
     def test_last_operation(self):
         self.assertEquals(self.p.last_operation(), '0')
 
-    @patch('subprocess.Popen', Mock(side_effect=OSError))
+    @patch('subprocess.Popen', Mock(side_effect=OSError()))
     def test_call_nowait(self):
         self.assertFalse(self.p.call_nowait('on_start'))
 
@@ -364,7 +358,7 @@ class TestPostgresql(unittest.TestCase):
     def test_move_data_directory(self):
         self.p.is_running = false
         self.p.move_data_directory()
-        with patch('os.rename', Mock(side_effect=OSError)):
+        with patch('os.rename', Mock(side_effect=OSError())):
             self.p.move_data_directory()
 
     @patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
@@ -372,8 +366,7 @@ class TestPostgresql(unittest.TestCase):
         with patch('subprocess.call', Mock(return_value=1)):
             self.assertRaises(PostgresException, self.p.bootstrap)
         self.p.bootstrap()
-        with patch('patroni.postgresql.Postgresql.sync_replica', MagicMock(return_value=True)):
+        self.p.bootstrap(self.leader)
-            self.p.bootstrap(self.leader)
 
     def test_remove_data_directory(self):
         self.p.data_dir = 'data_dir'
@@ -383,20 +376,26 @@ class TestPostgresql(unittest.TestCase):
         open(self.p.data_dir, 'w').close()
         self.p.remove_data_directory()
         os.symlink('unexisting', self.p.data_dir)
-        with patch('os.unlink', Mock(side_effect=OSError)):
+        with patch('os.unlink', Mock(side_effect=Exception)):
             self.p.remove_data_directory()
         self.p.remove_data_directory()
 
-    def test_controldata(self):
+    @patch('subprocess.check_output', MagicMock(return_value=0, side_effect=pg_controldata_string))
-        with patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string)):
+    @patch('subprocess.check_output', side_effect=subprocess.CalledProcessError)
-            data = self.p.controldata()
+    @patch('subprocess.check_output', side_effect=Exception('Failed'))
-            self.assertEquals(len(data), 50)
+    def test_controldata(self, check_output_call_error, check_output_generic_exception):
-            self.assertEquals(data['Database cluster state'], 'shut down in recovery')
+        data = self.p.controldata()
-            self.assertEquals(data['wal_log_hints setting'], 'on')
+        self.assertEquals(len(data), 50)
-            self.assertEquals(int(data['Database block size']), 8192)
+        self.assertEquals(data['Database cluster state'], 'shut down in recovery')
+        self.assertEquals(data['wal_log_hints setting'], 'on')
+        self.assertEquals(int(data['Database block size']), 8192)
 
-        with patch('subprocess.check_output', Mock(side_effect=subprocess.CalledProcessError(1, ''))):
+        subprocess.check_output = check_output_call_error
-            self.assertEquals(self.p.controldata(), {})
+        data = self.p.controldata()
+        self.assertEquals(data, dict())
+
+        subprocess.check_output = check_output_generic_exception
+        self.assertRaises(Exception, self.p.controldata())
 
     def test_read_postmaster_opts(self):
         m = mock_open(read_data=postmaster_opts_string())
@@ -406,10 +405,13 @@ class TestPostgresql(unittest.TestCase):
             self.assertEquals(int(data['max_replication_slots']), 5)
             self.assertEqual(data.get('D'), None)
 
-            m.side_effect = IOError
+            m.side_effect = IOError("foo")
             data = self.p.read_postmaster_opts()
             self.assertEqual(data, dict())
 
+            m.side_effect = Exception("foo")
+            self.assertRaises(Exception, self.p.read_postmaster_opts())
+
     @patch('subprocess.Popen')
     @patch.object(builtins, 'open', MagicMock(return_value=42))
     def test_single_user_mode(self, subprocess_popen_mock):
@@ -429,7 +431,13 @@ class TestPostgresql(unittest.TestCase):
         subprocess_popen_mock.return_value = None
         self.assertEquals(self.p.single_user_mode(), 1)
 
+    def fake_listdir(path):
+        if path.endswith(os.path.join('pg_xlog', 'archive_status')):
+            return ["a", "b", "c"]
+        return []
+
     @patch('os.listdir', MagicMock(side_effect=fake_listdir))
+    @patch('os.path.isdir', MagicMock(return_value=True))
     @patch('os.unlink', return_value=True)
     @patch('os.remove', return_value=True)
     @patch('os.path.islink', return_value=False)
@@ -451,8 +459,8 @@ class TestPostgresql(unittest.TestCase):
         mock_unlink.reset_mock()
         mock_remove.reset_mock()
 
-        mock_file.side_effect = OSError
+        mock_file.side_effect = Exception("foo")
-        mock_link.side_effect = OSError
+        mock_link.side_effect = Exception("foo")
         self.p.cleanup_archive_status()
         mock_unlink.assert_not_called()
         mock_remove.assert_not_called()
@@ -461,27 +469,14 @@ class TestPostgresql(unittest.TestCase):
     def test_sysid(self):
         self.assertEqual(self.p.sysid, "6200971513092291716")
 
-    @patch('os.path.isfile', Mock(return_value=True))
+    @patch('os.path.isfile', MagicMock(return_value=True))
-    @patch('shutil.copy', Mock(side_effect=IOError))
+    @patch('shutil.copy', side_effect=Exception)
-    def test_save_configuration_files(self):
+    def test_save_configuration_files(self, mock_copy):
+        shutil.copy = mock_copy
         self.p.save_configuration_files()
 
-    @patch('os.path.isfile', Mock(side_effect=[False, True]))
+    @patch('os.path.isfile', MagicMock(side_effect=is_file_raise_on_backup))
-    @patch('shutil.copy', Mock(side_effect=IOError))
+    @patch('shutil.copy', side_effect=Exception)
-    def test_restore_configuration_files(self):
+    def test_restore_configuration_files(self, mock_copy):
+        shutil.copy = mock_copy
         self.p.restore_configuration_files()
-
-    def test_can_create_replica_without_replication_connection(self):
-        self.p.config['create_replica_method'] = []
-        self.assertFalse(self.p.can_create_replica_without_replication_connection())
-        self.p.config['create_replica_method'] = ['wale', 'basebackup']
-        self.p.config['wale'] = {'command': 'foo', 'no_master': 1}
-        self.assertTrue(self.p.can_create_replica_without_replication_connection())
-
-    def test_replica_method_can_work_without_replication_connection(self):
-        self.assertFalse(self.p.replica_method_can_work_without_replication_connection('basebackup'))
-        self.assertFalse(self.p.replica_method_can_work_without_replication_connection('foobar'))
-        self.p.config['foo'] = {'command': 'bar', 'no_master': 1}
-        self.assertTrue(self.p.replica_method_can_work_without_replication_connection('foo'))
-        self.p.config['foo'] = {'command': 'bar'}
-        self.assertFalse(self.p.replica_method_can_work_without_replication_connection('foo'))

tests/test_utils.py

@@ -16,21 +16,20 @@ class TestUtils(unittest.TestCase):
 
     @patch('time.sleep', Mock())
     def test_reap_children(self):
-        self.assertIsNone(reap_children())
+        reap_children()
         with patch('os.waitpid', Mock(return_value=(0, 0))):
             sigchld_handler(None, None)
-            self.assertIsNone(reap_children())
+            reap_children()
 
     @patch('time.sleep', time_sleep)
     def test_sleep(self):
-        self.assertIsNone(sleep(0.01))
+        sleep(0.01)
 
 
 @patch('time.sleep', Mock())
 class TestRetrySleeper(unittest.TestCase):
 
-    @staticmethod
+    def _fail(self, times=1):
-    def _fail(times=1):
         scope = dict(times=0)
 
         def inner():
@@ -41,33 +40,36 @@ class TestRetrySleeper(unittest.TestCase):
                 raise PatroniException('Failed!')
         return inner
 
+    def _makeOne(self, *args, **kwargs):
+        return Retry(*args, **kwargs)
+
     def test_reset(self):
-        retry = Retry(delay=0, max_tries=2)
+        retry = self._makeOne(delay=0, max_tries=2)
         retry(self._fail())
         self.assertEquals(retry._attempts, 1)
         retry.reset()
         self.assertEquals(retry._attempts, 0)
 
     def test_too_many_tries(self):
-        retry = Retry(delay=0)
+        retry = self._makeOne(delay=0)
         self.assertRaises(RetryFailedError, retry, self._fail(times=999))
         self.assertEquals(retry._attempts, 1)
 
     def test_maximum_delay(self):
-        retry = Retry(delay=10, max_tries=100)
+        retry = self._makeOne(delay=10, max_tries=100)
         retry(self._fail(times=10))
         self.assertTrue(retry._cur_delay < 4000, retry._cur_delay)
         # gevent's sleep function is picky about the type
         self.assertEquals(type(retry._cur_delay), float)
 
     def test_deadline(self):
-        retry = Retry(deadline=0.0001)
+        retry = self._makeOne(deadline=0.0001)
         self.assertRaises(RetryFailedError, retry, self._fail(times=100))
 
     def test_copy(self):
         def _sleep(t):
-            pass
+            None
 
-        retry = Retry(sleep_func=_sleep)
+        retry = self._makeOne(sleep_func=_sleep)
         rcopy = retry.copy()
         self.assertTrue(rcopy.sleep_func is _sleep)

tests/test_wale_restore.py

@@ -1,9 +1,9 @@
+import unittest
+from mock import MagicMock, patch, PropertyMock
+import os
 import psycopg2
 import subprocess
-import unittest
+from patroni.scripts.wale_restore import WALERestore
-
-from mock import MagicMock, patch, PropertyMock
-from patroni.scripts.wale_restore import WALERestore, main as _main
 
 
 def fake_cursor_fetchone(*args, **kwargs):
@@ -28,19 +28,16 @@ def fake_backup_data(self, *args, **kwargs):
 base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z 167772160   00000001000000000000007F    00000040 00000001000000000000007F    00000240
 """
 
-
 def fake_backup_data_2(self, *args, **kwargs):
     """ return the fake result of WAL-E backup-list"""
     return """name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop """
 
-
 def fake_backup_data_3(self, *args, **kwargs):
     """ return the fake result of WAL-E backup-list"""
     return """name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop
 base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z 167772160   00000001000000000000007F    00000040 00000001000000000000007F    00000240
 """
 
-
 def fake_backup_data_4(self, *args, **kwargs):
     """ return the fake result of WAL-E backup-list"""
     return """name    last_modified   expanded_size_foo wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop
@@ -61,7 +58,7 @@ class TestWALERestore(unittest.TestCase):
 
     def setUp(self):
         self.wale_restore = WALERestore("batman", "/data",
-                                        "host=batman port=5432 user=batman", "/etc", 100, 100, 1, 0)
+                                        "host=batman port=5432 user=batman", "/etc", 100, 100, 1)
 
     def tearDown(self):
         pass
@@ -79,8 +76,6 @@ class TestWALERestore(unittest.TestCase):
             self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
 
         self.wale_restore.should_use_s3_to_create_replica()
-        self.wale_restore.no_master = 1
-        self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
 
     def test_create_replica_with_s3(self):
         with patch('subprocess.call', MagicMock(return_value=0)):
@@ -94,8 +89,3 @@ class TestWALERestore(unittest.TestCase):
         with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', MagicMock(return_value=True)):
             with patch.object(self.wale_restore, 'create_replica_with_s3', MagicMock(return_value=0)):
                 self.assertEqual(self.wale_restore.run(), 0)
-
-    @patch('sys.exit', MagicMock())
-    @patch.object(WALERestore, 'run', MagicMock(return_value=0))
-    def test_main(self):
-        self.assertEqual(_main(), None)

tests/test_zookeeper.py

@@ -20,8 +20,7 @@ class MockKazooClient(Mock):
     def client_id(self):
         return (-1, '')
 
-    @staticmethod
+    def retry(self, func, *args, **kwargs):
-    def retry(func, *args, **kwargs):
         func(*args, **kwargs)
 
     def get(self, path, watch=None):
@@ -44,8 +43,7 @@ class MockKazooClient(Mock):
             return (b'foo', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))
         return (b'', ZnodeStat(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0))
 
-    @staticmethod
+    def get_children(self, path, watch=None, include_data=False):
-    def get_children(path, watch=None, include_data=False):
         if not isinstance(path, six.string_types):
             raise TypeError("Invalid type for 'path' (string expected)")
         if path.startswith('/no_node'):
@@ -64,16 +62,16 @@ class MockKazooClient(Mock):
         elif value == b'retry' or (value == b'exists' and self.exists):
             raise NodeExistsError
 
-    @staticmethod
+    def set(self, path, value, version=-1):
-    def set(path, value, version=-1):
         if not isinstance(path, six.string_types):
             raise TypeError("Invalid type for 'path' (string expected)")
         if not isinstance(value, (six.binary_type,)):
             raise TypeError("Invalid type for 'value' (must be a byte string)")
         if path == '/service/bla/optime/leader':
             raise Exception
-        if path == '/service/test/members/bar' and value == b'retry':
+        if path == '/service/test/members/bar':
-            return
+            if value == b'retry':
+                return
         if path == '/service/test/failover':
             if value == b'Exception':
                 raise Exception