Bump up to 1.2.3 (#386)

Change hostnames by ip addresses was causing certificate verification to
fail. Instead of doing it we will better monkey patch urllib3
functionality which does name resolution. It should work without
problems even for https connection.

.gitignore

@@ -1,12 +1,53 @@
-data/*
-*.pyc
-*.egg/
-*.egg-info/
+*.py[cod]
+
+# vi(m) swap files:
+*.sw?
+
+# C extensions
+*.so
+
+# Packages
 .cache/
+*.egg
+*.eggs
+*.egg-info
+dist
+build
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+lib
+lib64
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
 .coverage
-.eggs/
-build/
+.tox
+nosetests.xml
 coverage.xml
+htmlcov
 junit.xml
+features/output
+dummy
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
 pgpass
 scm-source.json
+
+# Sphinx-generated documentation
+docs/build/
+docs/source/_static/
+docs/source/_templates/

.travis.yml

@@ -1,15 +1,12 @@
 sudo: false
+dist: trusty
 language: python
 python:
-- "3.5"
-addons:
-  apt:
-    packages:
-    - postgresql-contrib-9.5
-  postgresql: "9.5"
+- "3.4"  # 2.7 and 3.5 are preinstalled by default
 env:
   global:
-  - ETCDVERSION=2.3.2 ZKVERSION=3.4.6 CONSULVERSION=0.6.4
+  - ETCDVERSION=3.0.15 ZKVERSION=3.4.9 CONSULVERSION=0.7.2
+  - PYVERSIONS="2.7 3.4 3.5"
   matrix:
   - TEST_SUITE="python setup.py"
   - DCS="etcd" TEST_SUITE="behave"
@@ -17,62 +14,97 @@ env:
   - DCS="consul" TEST_SUITE="behave"
 cache:
   directories:
-  - $HOME/virtualenv/python2.7.9
-  - $HOME/virtualenv/python3.4.2
-  - $HOME/virtualenv/python3.5.0
+  - $HOME/mycache
+before_cache:
+- |
+  rm -fr $HOME/mycache/python*
+  for pv in $PYVERSIONS; do
+      if [[ $TEST_SUITE != "behave" || $pv != "3.4" ]]; then
+          fpv=$(basename $(readlink $HOME/virtualenv/python${pv}))
+          mv $HOME/virtualenv/${fpv} $HOME/mycache/${fpv}
+      fi
+  done
 install:
   - |
     set -e
 
     if [[ $TEST_SUITE == "behave" ]]; then
-        if [[ $DCS == "consul" ]]; then
-            curl -L https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip \
-                | gunzip > consul
-            chmod +x consul
-        fi
+        function get_consul() {
+            CC=~/mycache/consul_${CONSULVERSION}
+            if [[ ! -x $CC ]]; then
+                curl -L https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip \
+                    | gunzip > $CC
+                [[ ${PIPESTATUS[0]} == 0 ]] || return 1
+                chmod +x $CC
+            fi
+            ln -s $CC consul
+        }
 
-        if [[ $DCS == "etcd" ]]; then
-            curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
-                | tar xz -C . --strip=1 --wildcards --no-anchored etcd
-        fi
+        function get_etcd() {
+            EC=~/mycache/etcd_${ETCDVERSION}
+            if [[ ! -x $EC ]]; then
+                curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
+                    | tar xz -C . --strip=1 --wildcards --no-anchored etcd
+                [[ ${PIPESTATUS[0]} == 0 ]] || return 1
+                mv etcd $EC
+            fi
+            ln -s $EC etcd
+        }
 
-        if [[ $DCS == "exhibitor" ]]; then
-            curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz
-            mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg
-            zookeeper-${ZKVERSION}/bin/zkServer.sh start
+        function get_exhibitor() {
+            ZC=~/mycache/zookeeper-${ZKVERSION}
+            if [[ ! -d $ZC ]]; then
+                curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz
+                [[ ${PIPESTATUS[0]} == 0 ]] || return 1
+                mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg
+                mv zookeeper-${ZKVERSION} $ZC
+            fi
+            $ZC/bin/zkServer.sh start
             # following lines are 'emulating' exhibitor REST API
             while true; do
                 echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \
                     | nc -l 8181 &> /dev/null
             done&
-        fi
+        }
+
+        attempt_num=1
+        until get_${DCS}; do
+            [[ $attempt_num -ge 3 ]] && exit 1
+            echo "Attempt $attempt_num failed! Trying again in $attempt_num seconds..."
+            sleep $(( attempt_num++ ))
+        done
     fi
 
-    for pv in "2.7" "3.4" "3.5"; do
-        source ~/virtualenv/python${pv}/bin/activate
-        # explicitly install all needed python modules to cache them
-        for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8 mock>=2.0.0 pytest-cov pytest'; do
-            pip install $p
-        done
+    for pv in $PYVERSIONS; do
+        if [[ $TEST_SUITE != "behave" || $pv != "3.4" ]]; then
+            fpv=$(basename $(readlink $HOME/virtualenv/python$pv))
+            if [[ -d ~/mycache/${fpv} ]]; then
+                mv ~/virtualenv/${fpv} ~/virtualenv/${fpv}.bckp
+                mv ~/mycache/${fpv} ~/virtualenv/${fpv}
+            fi
+            source ~/virtualenv/python${pv}/bin/activate
+            # explicitly install all needed python modules to cache them
+            for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8 mock pytest-cov pytest setuptools'; do
+                pip install $p --upgrade
+            done
+        fi
     done
 script:
   - |
-    for pv in "2.7" "3.4" "3.5"; do
+    for pv in $PYVERSIONS; do
         source ~/virtualenv/python${pv}/bin/activate
 
-        if [[ $TEST_SUITE == "behave" ]]; then
-            if [[ $pv != "3.4" ]]; then
-                echo Running acceptance tests using python${pv}
-                if ! PATH=.:$PATH $TEST_SUITE; then
-                    # output all log files when tests are failing
-                    grep . features/output/*/*postgres?.*
-                    exit 1
-                fi
-            fi
-        else
+        if [[ $TEST_SUITE != "behave" ]]; then
             echo Running unit tests using python${pv}
             $TEST_SUITE test
             $TEST_SUITE flake8
+        elif [[ $pv != "3.4" ]]; then
+            echo Running acceptance tests using python${pv}
+            if ! PATH=.:/usr/lib/postgresql/9.6/bin:$PATH $TEST_SUITE; then
+                # output all log files when tests are failing
+                grep . features/output/*/*postgres?.*
+                exit 1
+            fi
         fi
     done
 

.zappr.yaml

@@ -0,0 +1,13 @@
+# for github.com
+approvals:
+  groups:
+    zalando:
+      minimum: 2
+      from:
+        orgs:
+          - "zalando"
+# team should be valid team id in team service https://teams.auth.zalando.com/api/teams/:id
+X-Zalando-Team: "acid"
+# type should be one of [code, doc, config, tools, secrets]
+# code will be the default value, if X-Zalando-Type is not found in .zappr.yml
+X-Zalando-Type: code

.zappr.yml

@@ -1,12 +0,0 @@
-approvals:
-  # PR needs at least 4 approvals
-  minimum: 1
-  # approval = comment that matches this regex
-  pattern: "^\\s*(:?\\+1:?|👍)\\s*$"
-  from:
-    # commenter must be either one of:
-    # a public zalando org member
-    orgs:
-      - zalando
-    # a collaborator of the repo
-    collaborators: true

Dockerfile

@@ -1,26 +1,41 @@
 ## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine
 ## It has all the necessary components to play/debug with a single node appliance, running etcd
 FROM ubuntu:16.04
-MAINTAINER Feike Steenbergen <feike.steenberge[email protected]>
+MAINTAINER Alexander Kukushkin <alexander.kukushki[email protected]>
 
 RUN echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/01norecommend \
     && echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/01norecommend
 
-ENV PGVERSION 9.5
+ENV PGVERSION 9.6
 ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:$PATH
 RUN apt-get update -y \
     && apt-get upgrade -y \
-    && apt-get install -y curl jq haproxy zookeeper postgresql-${PGVERSION} python-psycopg2 python-yaml \
-        python-requests python-six python-click python-dateutil python-tzlocal python-urllib3 \
-        python-dnspython python-pip python-setuptools python-kazoo python-prettytable python \
-    && pip install python-etcd==0.4.3 python-consul==0.6.0 --upgrade \
+    && apt-get install -y curl jq haproxy zookeeper python-psycopg2 python-yaml python-requests \
+        python-six python-click python-dateutil python-tzlocal python-urllib3 python-dnspython \
+        python-pip python-setuptools python-kazoo python-prettytable python-wheel python \
+
+    && export DISTRIB_CODENAME=$(sed -n 's/DISTRIB_CODENAME=//p' /etc/lsb-release) \
+    && echo "deb http://apt.postgresql.org/pub/repos/apt/ ${DISTRIB_CODENAME}-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
+    && echo "deb-src http://apt.postgresql.org/pub/repos/apt/ ${DISTRIB_CODENAME}-pgdg main" >> /etc/apt/sources.list.d/pgdg.list \
+    && curl -s -o - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
+
+    ## Make sure we have a en_US.UTF-8 locale available
+    && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
+
+    && apt-get update -y \
+    && apt-get install -y postgresql-contrib-${PGVERSION} \
+
+    # Remove the default cluster, which Debian stupidly starts right after installation of the packages
+    && pg_dropcluster --stop ${PGVERSION} main \
+    && pip install python-etcd==0.4.3 python-consul==0.6.1 --upgrade \
+
+    # Clean up
     && apt-get remove -y python-pip python-setuptools \
     && apt-get autoremove -y \
-        # Clean up
     && apt-get clean -y \
     && rm -rf /var/lib/apt/lists/* /root/.cache
 
-ENV ETCDVERSION 2.3.6
+ENV ETCDVERSION 3.0.15
 RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
     | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl
 

README.rst

@@ -2,13 +2,17 @@
 
 Patroni: A Template for PostgreSQL HA with ZooKeeper, etcd or Consul
 ------------------------------------------------------------
+
+You can find a version of this documentation that is searchable and also easier to navigagate at `patroni.readthedocs.io <https://patroni.readthedocs.io>`__.
+
+
 There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
 
-Patroni is a template for you to create your own customized, high-availability solution using Python and — for maximum accessibility — a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__ or `Consul <https://github.com/hashicorp/consul>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter—or anywhere else—will hopefully find it useful.
+Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__ or `Consul <https://github.com/hashicorp/consul>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
 
 We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely.
 
-**Note to Kubernetes users**: We're currently developing Patroni to be as useful as possible for teams running Kubernetes on top of Google Compute Engine; Patroni can be the HA solution for Postgres in such an environment. Please contact us via our Issues Tracker if this describes your team's current setup, and we'll follow up.
+**Note to Kubernetes users**: We're currently developing Patroni to be as useful as possible for teams running Kubernetes on top of Google Compute Engine; Patroni can be the HA solution for Postgres in such an environment. To this end, we've created a `Helm Chart <https://github.com/kubernetes/charts/tree/master/incubator/patroni>`__ that enables you to deploy a five-node Patroni cluster using a Kubernetes PetSet.
 
 .. contents::
     :local:
@@ -19,7 +23,7 @@ We call Patroni a "template" because it is far from being a one-size-fits-all or
 How Patroni Works
 ==============
 
-Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features. 
+Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features.
 
 For an example of a Docker-based deployment with Patroni, see `Spilo <https://github.com/zalando/spilo>`__, currently in use at Zalando.
 
@@ -32,7 +36,9 @@ For additional background info, see:
 Development Status
 ================
 
-Patroni is in active development and accepts contributions. See our `Contributing <https://github.com/zalando/patroni/blob/master/README.rst#contributing>`__ section below for more details. 
+Patroni is in active development and accepts contributions. See our `Contributing <https://github.com/zalando/patroni/blob/master/docs/CONTRIBUTING.rst>`__ section below for more details.
+
+We report new releases information `here <https://github.com/zalando/patroni/releases>`__.
 
 ===========================
 Technical Requirements/Installation
@@ -58,7 +64,7 @@ To get started, do the following from different terminals:
     > ./patroni.py postgres0.yml
     > ./patroni.py postgres1.yml
 
-You will then see a high-availability cluster start up. Test different settings in the YAML files to see how the cluster’s behavior changes. Kill some of the components to see how the system behaves.
+You will then see a high-availability cluster start up. Test different settings in the YAML files to see how the cluster's behavior changes. Kill some of the components to see how the system behaves.
 
 Add more ``postgres*.yml`` files to create an even larger cluster.
 
@@ -77,7 +83,7 @@ run:
 YAML Configuration
 ===============
 
-Go `here <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__. 
+Go `here <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
 
 =========================
 Environment Configuration
@@ -89,24 +95,7 @@ Go `here <https://github.com/zalando/patroni/blob/master/docs/ENVIRONMENT.rst>`_
 Replication Choices
 ===============
 
-Patroni uses Postgres' streaming replication, which is asynchronous by default. For more information, see the `Postgres documentation on streaming replication <http://www.postgresql.org/docs/current/static/warm-standby.html#STREAMING-REPLICATION>`__.
-
-Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the follower. This setting should be increased or decreased based on business requirements.
-
-When asynchronous replication is not optimal for your use case, investigate Postgres's `synchronous replication <http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION>`__. Synchronous replication ensures consistency across a cluster by confirming that writes are written to a secondary before returning to the connecting client with a success. The cost of synchronous replication: reduced throughput on writes. This throughput will be entirely based on network performance. 
-
-In hosted datacenter environments (like AWS, Rackspace, or any network you do not control), synchronous replication significantly increases the variability of write performance. If followers become inaccessible from the leader, the leader effectively becomes read-only.
-
-To enable a simple synchronous replication test, add the follow lines to the ``parameters`` section of your YAML configuration files:
-
-.. code:: YAML
-
-        synchronous_commit: "on"
-        synchronous_standby_names: "*"
-
-When using synchronous replication, use at least three Postgres data nodes to ensure write availability if one host fails.
-
-Choosing your replication schema is dependent on your business considerations. Investigate both async and sync replication, as well as other HA solutions, to determine which solution is best for you.
+Patroni uses Postgres' streaming replication, which is asynchronous by default. Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the leader. This setting should be increased or decreased based on business requirements. It's also possible to use synchronous replication for better durability guarantees. See `replication modes documentation <https://github.com/zalando/patroni/blob/master/docs/replication_modes.rst>` for details.
 
 ===============================
 Applications Should Not Use Superusers
@@ -114,14 +103,6 @@ Applications Should Not Use Superusers
 
 When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
 
-================
-Contributing
-================
-Patroni accepts contributions from the open-source community; see the `Issues Tracker <https://github.com/zalando/patroni/issues>`__ for current needs. 
-
-Before making a contribution, please let us know by posting a comment to the relevant issue. 
-If you would like to propose a new feature, please first file a new issue explaining the feature you’d like to create.
-
 .. |Build Status| image:: https://travis-ci.org/zalando/patroni.svg?branch=master
    :target: https://travis-ci.org/zalando/patroni
 .. |Coverage Status| image:: https://coveralls.io/repos/zalando/patroni/badge.svg?branch=master

docs/CONTRIBUTING.rst

@@ -0,0 +1,24 @@
+.. _contributing:
+
+Contributing guidelines
+=======================
+
+Wanna contribute to Patroni? Yay - here is how!
+
+Reporting issues
+----------------
+
+If you have a question about patroni or have a problem using it, please read the :ref:`README <readme>` before filing an issue.
+Also double check with the current issues on our `Issues Tracker <https://github.com/zalando/patroni/issues>`__.
+
+Contributing a pull request
+---------------------------
+
+1) Submit a comment to the relevant issue or create a new issue describing your proposed change.
+2) Do a fork, develop and test your code changes.
+3) Include documentation
+4) Submit a pull request.
+
+You'll get feedback about your pull request as soon as possible.
+
+Happy Patroni hacking ;-)

docs/ENVIRONMENT.rst

@@ -1,3 +1,5 @@
+.. _environment:
+
 ==================================
 Environment Configuration Settings
 ==================================
@@ -27,6 +29,12 @@ Consul
 Etcd
 ----
 -  **PATRONI\_ETCD\_HOST**: the host:port for the etcd endpoint.
+-  **PATRONI\_ETCD\_URL**: url for the etcd, in format: http(s)://(username:password@)host:port
+-  **PATRONI\_ETCD\_PROXY**: proxy url for the etcd. If you are connecting to the etcd using proxy, use this parameter instead of **PATRONI\_ETCD\_URL**
+-  **PATRONI\_ETCD\_SRV**: Domain to search the SRV record(s) for cluster autodiscovery.
+-  **PATRONI\_ETCD\_CACERT**: The ca certificate. If pressent it will enable validation.
+-  **PATRONI\_ETCD\_CERT**: File with the client certificate
+-  **PATRONI\_ETCD\_KEY**: File with the client key. Can be empty if the key is part of certificate.
 
 Exhibitor
 ---------
@@ -38,6 +46,7 @@ PostgreSQL
 -  **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
 -  **PATRONI\_POSTGRESQL\_CONNECT\_ADDRESS**: IP address + port through which Postgres is accessible from other nodes and applications.
 -  **PATRONI\_POSTGRESQL\_DATA\_DIR**: The location of the Postgres data directory, either existing or to be initialized by Patroni.
+-  **PATRONI\_POSTGRESQL\_BIN_DIR**: Path to PostgreSQL binaries. (pg_ctl, pg_rewind, pg_basebackup, postgres) The  default value is an empty string meaning that PATH environment variable will be used to find the executables.
 -  **PATRONI\_POSTGRESQL\_PGPASS**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni.
 -  **PATRONI\_REPLICATION\_USERNAME**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication
 -  **PATRONI\_REPLICATION\_PASSWORD**: replication password; the user will be created during initialization.

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = Patroni
+SOURCEDIR     = .
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/README.rst

@@ -0,0 +1,90 @@
+.. _readme:
+
+=================
+How Patroni Works
+=================
+
+Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features.
+
+For an example of a Docker-based deployment with Patroni, see `Spilo <https://github.com/zalando/spilo>`__, currently in use at Zalando.
+
+For additional background info, see:
+
+* `PostgreSQL HA with Kubernetes and Patroni <https://www.youtube.com/watch?v=iruaCgeG7qs>`__, talk by Josh Berkus at KubeCon 2016 (video)
+* `Feb. 2016 Zalando Tech blog post <https://tech.zalando.de/blog/zalandos-patroni-a-template-for-high-availability-postgresql/>`__
+
+==================
+Development Status
+==================
+
+Patroni is in active development and accepts contributions. See our :ref:`Contributing <contributing>` section below for more details.
+
+We report new releases information :ref:`here <releases>`.
+
+===================================
+Technical Requirements/Installation
+===================================
+
+**For Mac**
+
+To install requirements on a Mac, run the following:
+
+::
+
+    brew install postgresql etcd haproxy libyaml python
+    pip install psycopg2 pyyaml
+
+=======================
+Running and Configuring
+=======================
+
+To get started, do the following from different terminals:
+::
+
+    > etcd --data-dir=data/etcd
+    > ./patroni.py postgres0.yml
+    > ./patroni.py postgres1.yml
+
+You will then see a high-availability cluster start up. Test different settings in the YAML files to see how the cluster's behavior changes. Kill some of the components to see how the system behaves.
+
+Add more ``postgres*.yml`` files to create an even larger cluster.
+
+Patroni provides an `HAProxy <http://www.haproxy.org/>`__ configuration, which will give your application a single endpoint for connecting to the cluster's leader. To configure,
+run:
+
+::
+
+    > haproxy -f haproxy.cfg
+
+::
+
+    > psql --host 127.0.0.1 --port 5000 postgres
+
+==================
+YAML Configuration
+==================
+
+Go :ref:`here <settings>` for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
+
+=========================
+Environment Configuration
+=========================
+
+Go :ref:`here <environment>` for comprehensive information about configuring(overriding) settings via environment variables.
+
+===================
+Replication Choices
+===================
+
+Patroni uses Postgres' streaming replication, which is asynchronous by default. Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the leader. This setting should be increased or decreased based on business requirements. It's also possible to use synchronous replication for better durability guarantees. See :ref:`replication modes documentation <replication_modes>` for details.
+
+======================================
+Applications Should Not Use Superusers
+======================================
+
+When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
+
+.. |Build Status| image:: https://travis-ci.org/zalando/patroni.svg?branch=master
+   :target: https://travis-ci.org/zalando/patroni
+.. |Coverage Status| image:: https://coveralls.io/repos/zalando/patroni/badge.svg?branch=master
+   :target: https://coveralls.io/r/zalando/patroni?branch=master

docs/SETTINGS.rst

@@ -1,3 +1,5 @@
+.. _settings:
+
 ===========================
 YAML Configuration Settings
 ===========================
@@ -14,6 +16,8 @@ Bootstrap configuration
     -  **loop\_wait**: the number of seconds the loop will sleep. Default value: 10
     -  **ttl**: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process. Default value: 30
     -  **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election.
+    -  **master\_start\_timeout**: the amount of time a master is allowed to recover from failures before failover is triggered. Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Best worst case failover time for master failure is: loop\_wait + master\_start\_timeout + loop\_wait, unless master\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff.
+    -  **synchronous\_mode**: turns on synchronous replication mode. In this mode a replica will be chosen as synchronous and only the latest leader and synchronous replica are able to participate in leader election. Synchronous mode makes sure that succesfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See `replication modes documentation <https://github.com/zalando/patroni/blob/master/docs/replication_modes.rst>`__ for details.
     -  **postgresql**:
         -  **use\_pg\_rewind**:whether or not to use pg_rewind
         -  **use\_slots**: whether or not to use replication_slots. Must be False for PostgreSQL 9.3. You should comment out max_replication_slots before it becomes ineligible for leader status.
@@ -32,6 +36,7 @@ Bootstrap configuration
         -  **options**: list of options for CREATE USER statement
             -  **- createrole**
             -  **- createdb**
+-  **post_init**: An additional script that will be executed after initializing the cluster. The script receives a connection string URL (with the cluster superuser as a user name). The PGPASSFILE variable is set to the location of pgpass file.
 
 Consul
 ------
@@ -39,7 +44,17 @@ Consul
 
 Etcd
 ----
+Most of the parameters are optional, but you have to specify one of the **host**, **url**, **proxy** or **srv**
 -  **host**: the host:port for the etcd endpoint.
+-  **url**: url for the etcd
+-  **proxy**: proxy url for the etcd. If you are connecting to the etcd using proxy, use this parameter instead of **url**
+-  **srv**: Domain to search the SRV record(s) for cluster autodiscovery.
+-  **protocol**: (optional) http or https, if not specified http is used. If the **url** or **proxy** is specified - will take protocol from them.
+-  **username**: (optional) username for etcd authentication
+-  **password**: (optional) password for etcd authentication.
+-  **cacert**: (optional) The ca certificate. If pressent it will enable validation.
+-  **cert**: (optional) file with the client certificate
+-  **key**: (optional) file with the client key. Can be empty if the key is part of **cert**.
 
 Exhibitor
 ---------
@@ -47,6 +62,8 @@ Exhibitor
 -  **poll\_interval**: how often the list of ZooKeeper and Exhibitor nodes should be updated from Exhibitor
 -  **port**: Exhibitor port.
 
+.. _postgresql_settings:
+
 PostgreSQL
 ----------
 -  **authentication**:
@@ -65,9 +82,11 @@ PostgreSQL
 -  **connect\_address**: IP address + port through which Postgres is accessible from other nodes and applications.
 -  **create\_replica\_methods**: an ordered list of the create methods for turning a Patroni node into a new replica. "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured as its own config item.
 -  **data\_dir**: The location of the Postgres data directory, either existing or to be initialized by Patroni.
+-  **bin\_dir**: Path to PostgreSQL binaries. (pg_ctl, pg_rewind, pg_basebackup, postgres) The  default value is an empty string meaning that PATH environment variable will be used to find the executables.
 -  **listen**: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
--  **pgpass**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni.
+-  **pgpass**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup, the post_init script and under some other circumstances. The location must be writable by Patroni.
 -  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower.
+-  **custom_conf** : path to an optional custom ``postgresql.conf`` file, that will be used in place of ``postgresql.base.conf``. The file must exist on all cluster nodes, be readable by PostgreSQL and will be included from its location on the real ``postgresql.conf``. Note that Patroni will not monitor this file for changes, nor backup it. However, its settings can still be overriden by Patroni's own configuration facilities - see `dynamic configuration <https://github.com/zalando/patroni/blob/master/docs/dynamic_configuration.rst>`__ for details.
 -  **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work.
 -  **pg\_ctl\_timeout**: How long should pg_ctl wait when doing ``start``, ``stop`` or ``restart``. Default value is 60 seconds.
 -  **use\_pg\_rewind**: try to use pg\_rewind on the former leader when it joins cluster as a replica.

docs/_static/custom.css

@@ -0,0 +1,3 @@
+li {
+   margin-bottom: 0.5em
+}

docs/conf.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Patroni documentation build configuration file, created by
+# sphinx-quickstart on Mon Dec 19 16:54:09 2016.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
+    'sphinx.ext.mathjax',
+    'sphinx.ext.ifconfig',
+    'sphinx.ext.viewcode']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'Patroni'
+copyright = '2016, Zalando SE'
+author = 'Zalando SE'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '1.2'
+# The full version, including alpha/beta/rc tags.
+release = '1.2.2'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+if not on_rtd:  # only import and set the theme if we're building docs locally
+    import sphinx_rtd_theme
+    html_theme = 'sphinx_rtd_theme'
+    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Patronidoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'Patroni.tex', 'Patroni Documentation',
+     'Zalando SE', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'patroni', 'Patroni Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'Patroni', 'Patroni Documentation',
+     author, 'Patroni', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+
+# -- Options for Epub output ----------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+epub_author = author
+epub_publisher = author
+epub_copyright = copyright
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {'https://docs.python.org/': None}
+
+# A possibility to have an own stylesheet, to add new rules or override existing ones
+# For the latter case, the CSS specificity of the rules should be higher than the default ones
+def setup(app):
+   app.add_stylesheet("custom.css")

docs/dynamic_configuration.rst

@@ -1,3 +1,5 @@
+.. _dynamic_configuration:
+
 Patroni configuration
 =====================
 
@@ -10,11 +12,11 @@ Patroni configuration is stored in the DCS (Distributed Configuration Store). Th
 	have changed), a special flag, ``pending_restart`` indicating this, is set in the members.data JSON.
 	Additionally, the node status also indicates this, by showing ``"restart_pending": true``.
 
-- Local `configuration <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ (patroni.yml).
+- Local :ref:`configuration <settings>` (patroni.yml).
 	These options are defined in the configuration file and take precedence over dynamic configuration.
 	patroni.yml could be changed and reload in runtime (without restart of Patroni) by sending SIGHUP to the Patroni process or by performing ``POST /reload`` REST-API request.
 
-- Environment `configuration <https://github.com/zalando/patroni/blob/master/docs/ENVIRONMENT.rst>`__ .
+- Environment :ref:`configuration <environment>` .
 	It is possible to set/override some of the "Local" configuration parameters with environment variables.
 	Environment configuration is very useful when you are running in a dynamic environment and you don't know some of the parameters in advance (for example it's not possible to know you external IP address when you are running inside ``docker``).
 
@@ -48,23 +50,24 @@ To be on the safe side parameters from the above lists are not written into ``po
 
 When applying the local or dynamic configuration options, the following actions are taken:
 
-- The node first checks if there is a postgresql.base.conf.
-- If it exists, it contains the renamed "original" configuration.
-- If it doesn't, the original postgresql.conf is taken and renamed to postgresql.base.conf.
+- The node first checks if there is a postgresql.base.conf or if the ``custom_conf`` parameter is set.
+- If the `custom_conf` parameter is set, it will take the file specified on it as a base configuration, ignoring `postgresql.base.conf` and `postgresql.conf`.
+- If the `custom_conf` parameter is not set and `postgresql.base.conf` exists, it contains the renamed "original" configuration and it will be used as a base configuration.
+- If there is no `custom_conf` nor `postgresql.base.conf`, the original postgresql.conf is taken and renamed to postgresql.base.conf.
 - The dynamic options (with the exceptions above) are dumped into the postgresql.conf and an include is set in
-  postgresql.conf to postgresql.base.conf. Therefore, we would be able to apply new options without re-reading the configuration file to check if the include is present not.
+  postgresql.conf to the used base configuration (either postgresql.base.conf or what is on ``custom_conf``). Therefore, we would be able to apply new options without re-reading the configuration file to check if the include is present not.
 - Some parameters that are essential for Patroni to manage the cluster are overridden using the command line.
 - If some of the options that require restart are changed (we should look at the context in pg_settings and at the actual
   values of those options), a pending_restart flag of a given node is set. This flag is reset on any restart.
 
 The parameters would be applied in the following order (run-time are given the highest priority):
 
-1. load parameters from file `postgresql.base.conf`
+1. load parameters from file `postgresql.base.conf` (or from a `custom_conf` file, if set)
 2. load parameters from file `postgresql.conf`
 3. load parameters from file `postgresql.auto.conf`
 4. run-time parameter using `-o --name=value`
 
-This allows configuration for all the nodes (2), configuration for a specific node using `ALTER SYSTEM` (3) and ensures that parameters essential to the running of Patroni are enforced. (4)
+This allows configuration for all the nodes (2), configuration for a specific node using `ALTER SYSTEM` (3) and ensures that parameters essential to the running of Patroni are enforced (4), as well as leaves room for configuration tools that manage `postgresql.conf` directly without involving Patroni (1).
 
 
 Also, the following Patroni configuration options can be changed only dynamically:

docs/ha_loop_diagram.dot

@@ -0,0 +1,148 @@
+// Graphviz source for ha_loop_diagram.png
+// recompile with:
+// dot -Tpng ha_loop_diagram.dot -o ha_loop_diagram.png
+
+digraph G {
+  rankdir=TB;
+  fontname="sans-serif";
+  penwidth="0.3";
+  layout="dot";
+  newrank=true;
+  edge [fontname="sans-serif", 
+        fontsize=12, 
+        color=black, 
+        fontcolor=black];
+  node [fontname=serif, 
+        fontsize=12, 
+        fillcolor=white, 
+        color=black, 
+        fontcolor=black,
+        style=filled];
+  "start" [label=Start, shape="rectangle", fillcolor="green"]
+  "start" -> "load_cluster_from_dcs";
+  "update_member" [label="Persist node state in DCS"]
+  "update_member" -> "start"
+  
+  subgraph cluster_run_cycle {
+    label="run_cycle"
+    "load_cluster_from_dcs" [label="Load cluster from DCS"];
+    "touch_member" [label="Persist node in DCS"];
+    "cluster.has_member" [shape="diamond", label="Is node registered on DCS?"]
+    "cluster.has_member" -> "touch_member"  [label="no" color="red"]
+    "long_action_in_progress?" [shape="diamond" label="Is the PostgreSQL currently being\nstopping/starting/restarting/reinitializing?"]
+    "load_cluster_from_dcs" -> "cluster.has_member";
+    "touch_member" -> "long_action_in_progress?";
+    "cluster.has_member" -> "long_action_in_progress?"  [label="yes" color="green"];
+    "long_action_in_progress?" -> "recovering?" [label="no" color="red"]
+    "recovering?" [label="Was cluster recovering and failed?", shape="diamond"];
+    "recovering?" -> "post_recover" [label="yes" color="green"];
+    "recovering?" -> "data_directory_empty" [label="no" color="red"];
+    "post_recover" [label="Remove leader key (if I was the leader)"];
+    "data_directory_empty" [label="Is data folder empty?", shape="diamond"];
+    "data_directory_empty" -> "cluster_initialize" [label="no" color="red"];
+    "data_belongs_to_cluster" [label="Does data dir belong to cluster?", shape="diamond"];
+    "data_belongs_to_cluster" -> "exit" [label="no" color="red"];
+    "data_belongs_to_cluster" -> "is_healthy" [label="yes" color="green"]
+    "exit" [label="Fail and exit", fillcolor=red];
+    "cluster_initialize" [label="Is cluster initialized on DCS?" shape="diamond"]
+    "cluster_initialize" -> "cluster.has_leader" [label="no" color="red"]
+    "cluster.has_leader" [label="Does the cluster has leader?", shape="diamond"]
+    "cluster.has_leader" -> "dcs.initialize" [label="no", color="red"]
+    "cluster.has_leader" -> "is_healthy" [label="yes", color="green"]
+    "cluster_initialize" -> "data_belongs_to_cluster" [label="yes" color="green"]
+    "dcs.initialize" [label="Initialize new cluster"];
+    "dcs.initialize" -> "is_healthy"
+    "is_healthy" [label="Is node healthy?\n(running Postgres)", shape="diamond"];
+    "recover" [label="Start as read-only\nand set Recover flag"]
+    "is_healthy" -> "recover" [label="no" color="red"]; 
+    "is_healthy" -> "cluster.is_unlocked" [label="yes" color="green"]; 
+    "cluster.is_unlocked" [label="Does the cluster has a leader?", shape="diamond"]
+  }
+
+  "post_recover" -> "update_member"
+  "recover" -> "update_member"
+  "long_action_in_progress?" -> "async_has_lock?" [label="yes" color="green"];
+  "cluster.is_unlocked" -> "unhealthy_is_healthiest" [label="no" color="red"]
+  "cluster.is_unlocked" -> "healthy_has_lock" [label="yes" color="green"]
+  "data_directory_empty" -> "bootstrap.is_unlocked" [label="yes" color="green"]
+
+  subgraph cluster_async {
+    label = "Long action in progress\n(Start/Stop/Restart/Reinitialize)"
+    "async_has_lock?" [label="Do I have the leader lock?", shape="diamond"]
+    "async_update_lock" [label="Renew leader lock"]
+    "async_has_lock?" -> "async_update_lock" [label="yes" color="green"]
+  }
+  "async_update_lock" -> "update_member"
+  "async_has_lock?" -> "update_member" [label="no" color="red"]
+
+  subgraph cluster_bootstrap {
+    label = "Node bootstrap";
+    "bootstrap.is_unlocked" [label="Does the cluster has a leader?", shape="diamond"]
+    "bootstrap.is_initialized" [label="Does the cluster has an initialize key?", shape="diamond"]
+    "bootstrap.is_unlocked" -> "bootstrap.is_initialized" [label="no" color="red"]
+    "bootstrap.is_unlocked" -> "bootstrap.select_node" [label="yes" color="green"]
+    "bootstrap.select_node" [label="Select a node to take a backup from"]
+    "bootstrap.do_bootstrap" [label="Run pg_basebackup\n(async)"]
+    "bootstrap.select_node" -> "bootstrap.do_bootstrap"
+    "bootstrap.is_initialized" -> "bootstrap.initialization_race" [label="no" color="red"]
+    "bootstrap.is_initialized" -> "bootstrap.wait_for_leader" [label="yes" color="green"]
+    "bootstrap.initialization_race" [label="Race for initialize key"]
+    "bootstrap.initialization_race" -> "bootstrap.won_initialize_race?"
+    "bootstrap.won_initialize_race?" [label="Do I won initialize race?", shape="diamond"]
+    "bootstrap.won_initialize_race?" -> "bootstrap.initdb_and_start" [label="yes" color="green"]
+    "bootstrap.won_initialize_race?" -> "bootstrap.wait_for_leader" [label="no" color="red"]
+    "bootstrap.wait_for_leader" [label="Need to wait for leader key"]
+    "bootstrap.initdb_and_start" [label="Run initdb, start postgres and create roles"]
+    "bootstrap.initdb_and_start" -> "bootstrap.success?"
+    "bootstrap.success?" [label="Success", shape="diamond"]
+    "bootstrap.success?" -> "bootstrap.take_leader_key" [label="yes" color="green"]
+    "bootstrap.success?" -> "bootstrap.clean" [label="no" color="red"]
+    "bootstrap.clean" [label="Remove initialize key from DCS\nand data directory from filesystem"]
+    "bootstrap.take_leader_key" [label="Take a leader key in DCS"]
+  }
+
+  "bootstrap.do_bootstrap" -> "update_member"
+  "bootstrap.wait_for_leader" -> "update_member"
+  "bootstrap.clean" -> "update_member"
+  "bootstrap.take_leader_key" -> "update_member"
+
+  subgraph cluster_process_healthy_cluster {
+    label = "process_healthy_cluster"
+    "healthy_has_lock" [label="Am I the owner of the leader lock?", shape=diamond]
+    "healthy_is_leader" [label="Is Postgres running as master?", shape=diamond]
+    "healthy_no_lock" [label="Follow the leader (async,\ncreate/update recovery.conf and restart if necessary)"]
+    "healthy_has_lock" -> "healthy_no_lock" [label="no" color="red"]
+    "healthy_has_lock" -> "healthy_update_leader_lock" [label="yes" color="green"]
+    "healthy_update_leader_lock" [label="Try to update leader lock"]
+    "healthy_update_leader_lock" -> "healthy_update_success"
+    "healthy_update_success" [label="Success?", shape=diamond]
+    "healthy_update_success" -> "healthy_is_leader" [label="yes" color="green"]
+    "healthy_update_success" -> "healthy_demote" [label="no" color="red"]
+    "healthy_demote" [label="Demote (async,\nrestart in read-only)"]
+    "healthy_failover" [label="Promote Postgres to master"]
+    "healthy_is_leader" -> "healthy_failover" [label="no" color="red"]
+  }
+  "healthy_demote" -> "update_member"
+  "healthy_is_leader" -> "update_member" [label="yes" color="green"]
+  "healthy_failover" -> "update_member"
+  "healthy_no_lock" -> "update_member"
+
+  subgraph cluster_process_unhealthy_cluster {
+    label = "process_unhealthy_cluster"
+    "unhealthy_is_healthiest" [label="Am I the healthiest node?", shape="diamond"]
+    "unhealthy_is_healthiest" -> "unhealthy_leader_race" [label="yes", color="green"]
+    "unhealthy_leader_race" [label="Try to create leader key"]
+    "unhealthy_leader_race" -> "unhealthy_acquire_lock"
+    "unhealthy_acquire_lock" [label="Was I able to get the lock?", shape="diamond"]
+    "unhealthy_is_leader" [label="Is Postgres running as master?", shape=diamond]
+    "unhealthy_acquire_lock" -> "unhealthy_is_leader" [label="yes" color="green"]
+    "unhealthy_is_leader" -> "unhealthy_promote" [label="no" color="red"]
+    "unhealthy_promote" [label="Promote to master"]
+    "unhealthy_is_healthiest" -> "unhealthy_follow" [label="no" color="red"]
+    "unhealthy_follow" [label="try to follow somebody else()"]
+    "unhealthy_acquire_lock" -> "unhealthy_follow" [label="no" color="red"]
+  }
+  "unhealthy_follow" -> "update_member"
+  "unhealthy_promote" -> "update_member"
+  "unhealthy_is_leader" -> "update_member" [label="yes" color="green"]
+}

docs/index.rst

@@ -0,0 +1,36 @@
+.. Patroni documentation master file, created by
+   sphinx-quickstart on Mon Dec 19 16:54:09 2016.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Introduction
+============
+
+Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__ or `Consul <https://github.com/hashicorp/consul>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
+
+We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely. There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
+
+**Note to Kubernetes users**: We're currently developing Patroni to be as useful as possible for teams running Kubernetes on top of Google Compute Engine; Patroni can be the HA solution for Postgres in such an environment. To this end, we've created a `Helm Chart <https://github.com/kubernetes/charts/tree/master/incubator/patroni>`__ that enables you to deploy a five-node Patroni cluster using a Kubernetes PetSet.
+
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   README
+   dynamic_configuration
+   ENVIRONMENT
+   SETTINGS
+   replication_modes
+   pause
+   releases
+   CONTRIBUTING
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+
+

docs/pause.rst

@@ -0,0 +1,35 @@
+.. _pause:
+
+Pause/Resume mode for the cluster
+=================================
+
+The goal
+--------
+
+Under certain circumstances Patroni needs to temporary step down from managing the cluster, while still retaining the cluster state in DCS. Possible use cases are uncommon activities on the cluster, such as major version upgrades or corruption recovery. During those activities nodes are often started and stopped for the reason unknown to Patroni, some nodes can be even temporary promoted, violating the assumption of running only one master. Therefore, Patroni needs to be able to "detach" from the running cluster, implementing an equivalent of the maintenance mode in Pacemaker.
+
+
+
+The implementation
+------------------
+
+When Patroni runs in a paused mode, it does not change the state of PostgreSQL, except for the following cases:
+
+- For each node, the member key in DCS is updated with the current information about the cluster. This causes Patroni to run read-only queries on a member node if the member is running.
+
+- For the Postgres master with the leader lock Patroni updates the lock. If the node with the leader lock stops being the master (i.e. is demoted manually), Patroni will release the lock instead of promoting the node back.
+
+- Manual unscheduled restart, reinitialize and manual failover are allowed. Manual failover is only allowed if the node to failover to is specified. In the paused mode, manual failover does not require a running master node.
+
+- If 'parallel' masters are detected by Patroni, it emits a warning, but does not demote the masters without the leader lock.
+
+- If there is no leader lock in the cluster, the running master acquires the lock. If there is more than one master node, then the first master to acquire the lock wins. If there are no masters altogether, Patroni does not try to promote any replicas. There is an exception in this rule: if there is no leader lock because the old master has demoted itself due to the manual promotion, then only the candidate node mentioned in the promotion request may take the leader lock. When the new leader lock is granted (i.e. after promoting a replica manually), Patroni makes sure the replicas that were streaming from the previous leader will switch to the new one.
+
+- When Postgres is stopped, Patroni does not try to start it. When Patroni is stopped, it does not to stop Postgres instance it is managing.
+
+User guide
+----------
+
+``patronictl`` supports ``pause`` and ``resume`` commands.
+
+One can also issue a ``PATCH`` request to the ``{namespace}/{cluster}/config`` key with ``{"pause": true/false/null}``

docs/releases.rst

@@ -0,0 +1,394 @@
+.. _releases:
+
+Release notes
+=============
+
+Version 1.2
+-----------
+
+This version introduces significant improvements over the handling of synchronous replication, makes the startup process and failover more reliable, adds PostgreSQL 9.6 support and fixes plenty of bugs.
+In addition, the documentation, including these release notes, has been moved to https://patroni.readthedocs.io.
+
+**Synchronous replication**
+
+- Add synchronous replication support. (Ants Aasma)
+
+  Adds a new configuration variable ``synchronous_mode``. When enabled, Patroni will manage ``synchronous_standby_names`` to enable synchronous replication whenever there are healthy standbys available. When synchronous mode is enabled, Patroni will automatically fail over only to a standby that was synchronously replicating at the time of the master failure. This effectively means that no user visible transaction gets lost in such a case. See the
+  :ref:`feature documentation <synchronous_mode>` for the detailed description and implementation details.
+
+**Reliability improvements**
+
+- Do not try to update the leader position stored in the ``leader optime`` key when PostgreSQL is not 100% healthy. Demote immediately when the update of the leader key failed. (Alexander Kukushkin)
+
+- Exclude unhealthy nodes from the list of targets to clone the new replica from. (Alexander)
+
+- Implement retry and timeout strategy for Consul similar to how it is done for Etcd. (Alexander)
+
+- Make ``--dcs`` and ``--config-file`` apply to all options in ``patronictl``. (Alexander)
+
+- Write all postgres parameters into postgresql.conf. (Alexander)
+
+  It allows starting PostgreSQL configured by Patroni with just ``pg_ctl``.
+
+- Avoid exceptions when there are no users in the config. (Kirill Pushkin)
+
+- Allow pausing an unhealthy cluster. Before this fix, ``patronictl`` would bail out if the node it tries to execute pause on is unhealthy. (Alexander)
+
+- Improve the leader watch functionality. (Alexander)
+
+  Previously the replicas were always watching the leader key (sleeping until the timeout or the leader key changes). With this change, they only watch
+  when the replica's PostgreSQL is in the ``running`` state and not when it is stopped/starting or restarting PostgreSQL.
+
+- Avoid running into race conditions when handling SIGCHILD as a PID 1. (Alexander)
+
+  Previously a race condition could occur when running inside the Docker containers, since the same process inside Patroni both spawned new processes
+  and handled SIGCHILD from them. This change uses fork/execs for Patroni and leaves the original PID 1 process responsible for handling signals from children.
+
+- Fix WAL-E restore. (Oleksii Kliukin)
+
+  Previously WAL-E restore used the ``no_master`` flag to avoid consulting with the master altogether, making Patroni always choose restoring
+  from WAL over the ``pg_basebackup``. This change reverts it to the original meaning of ``no_master``, namely Patroni WAL-E restore may be selected as a replication method if the master is not running.
+  The latter is checked by examining the connection string passed to the method. In addition, it makes the retry mechanism more robust and handles other minutia.
+
+- Implement asynchronous DNS resolver cache. (Alexander)
+
+  Avoid failing when DNS is temporary unavailable (for instance, due to an excessive traffic received by the node).
+
+- Implement starting state and master start timeout. (Ants, Alexander)
+
+  Previously ``pg_ctl`` waited for a timeout and then happily trodded on considering PostgreSQL to be running. This caused PostgreSQL to show up in listings as running when it was actually not and caused a race condition that   resulted in either a failover, or a crash recovery, or a crash recovery interrupted by failover and a missed rewind.
+  This change adds a ``master_start_timeout`` parameter and introduces a new state for the main HA loop: ``starting``. When ``master_start_timeout`` is 0 we will failover immediately when the master crashes as soon as there is a failover candidate. Otherwise, Patroni will wait after attempting to start PostgreSQL on the master for the duration of the timeout; when it expires, it will failover if possible. Manual failover requests will be honored during the crash of the master even before the timeout expiration.
+
+  Introduce the ``timeout`` parameter to the ``restart`` API endpoint and ``patronictl``. When it is set and restart takes longer than the timeout, PostgreSQL is considered unhealthy and the other nodes becomes eligible to take the leader lock.
+
+- Fix ``pg_rewind`` behavior in a pause mode. (Ants)
+
+  Avoid unnecessary restart in a pause mode when Patroni thinks it needs to rewind but rewind is not possible (i.e. ``pg_rewind`` is not present). Fallback to default ``libpq`` values for the ``superuser`` (default OS user) if ``superuser`` authentication is missing from the ``pg_rewind`` related Patroni configuration section.
+
+- Serialize callback execution. Kill the previous callback of the same type when the new one is about to run. Fix the issue of spawning zombie processes when running callbacks. (Alexander)
+
+- Avoid promoting a former master when the leader key is set in DCS but update to this leader key fails. (Alexander)
+
+  This avoids the issue of a current master continuing to keep its role when it is partitioned together with the minority of nodes in Etcd and other DCSs that allow "inconsistent reads".
+
+**Miscellaneous**
+
+- Add ``post_init`` configuration option on bootstrap. (Alejandro Martínez)
+
+  Patroni will call the script argument of this option right after running ``initdb`` and starting up PostgreSQL for a new cluster. The script receives a connection URL with ``superuser``
+  and sets ``PGPASSFILE`` to point to the ``.pgpass`` file containing the password. If the script fails, Patroni initialization fails as well. It is useful for adding
+  new users or creating extensions in the new cluster.
+
+- Implement PostgreSQL 9.6 support. (Alexander)
+
+  Use ``wal_level = replica`` as a synonym for ``hot_standby``, avoiding pending_restart flag when it changes from one to another. (Alexander)
+
+**Documentation improvements**
+
+- Add a Patroni main `loop workflow diagram <https://raw.githubusercontent.com/zalando/patroni/master/docs/ha_loop_diagram.png>`__. (Alejandro, Alexander)
+
+- Improve README, adding the Helm chart and links to release notes. (Lauri Apple)
+
+- Move Patroni documentation to ``Read the Docs``. The up-to-date documentation is available at https://patroni.readthedocs.io. (Oleksii)
+
+  Makes the documentation easily viewable from different devices (including smartphones) and searchable.
+
+- Move the package to the semantic versioning. (Oleksii)
+
+  Patroni will follow the major.minor.patch version schema to avoid releasing the new minor version on small but critical bugfixes. We will only publish the release notes for the minor version, which will include all patches.
+
+
+Version 1.1
+-----------
+
+This release improves management of Patroni cluster by bring in pause mode, improves maintenance with scheduled and conditional restarts, makes Patroni interaction with Etcd or Zookeeper more resilient and greatly enhances patronictl.
+
+**Upgrade notice**
+
+When upgrading from releases below 1.0 read about changing of credentials and configuration format at 1.0 release notes.
+
+**Pause mode**
+
+- Introduce pause mode to temporary detach Patroni from managing PostgreSQL instance (Murat Kabilov, Alexander Kukushkin, Oleksii Kliukin).
+
+  Previously, one had to send SIGKILL signal to Patroni to stop it without terminating PostgreSQL. The new pause mode detaches Patroni from PostgreSQL cluster-wide without terminating Patroni. It is similar to the maintenance mode in Pacemaker. Patroni is still responsible for updating member and leader keys in DCS, but it will not start, stop or restart PostgreSQL server in the process. There are a few exceptions, for instance, manual failovers, reinitializes and restarts are still allowed. You can read :ref:`a detailed description of this feature <pause>`.
+
+In addition, patronictl supports new ``pause`` and ``resume`` commands to toggle the pause mode.
+
+**Scheduled and conditional restarts**
+
+- Add conditions to the restart API command (Oleksii)
+
+  This change enhances Patroni restarts by adding a couple of conditions that can be verified in order to do the restart. Among the conditions are restarting when PostgreSQL role is either a master or a replica, checking the PostgreSQL version number or restarting only when restart is necessary in order to apply configuration changes.
+
+- Add scheduled restarts (Oleksii)
+
+  It is now possible to schedule a restart in the future. Only one scheduled restart per node is supported. It is possible to clear the scheduled restart if it is not needed anymore. A combination of scheduled and conditional restarts is supported, making it possible, for instance, to scheduled minor PostgreSQL upgrades in the night, restarting only the instances that are running the outdated minor version without adding postgres-specific logic to administration scripts.
+
+- Add support for conditional and scheduled restarts to patronictl (Murat).
+
+  patronictl restart supports several new options. There is also patronictl flush command to clean the scheduled actions.
+
+**Robust DCS interaction**
+
+- Set Kazoo timeouts depending on the loop_wait (Alexander)
+
+  Originally, ping_timeout and connect_timeout values were calculated from the negotiated session timeout. Patroni loop_wait was not taken into account. As
+  a result, a single retry could take more time than the session timeout, forcing Patroni to release the lock and demote.
+
+  This change set ping and connect timeout to half of the value of loop_wait, speeding up detection of connection issues and  leaving enough time to retry the connection attempt before loosing the lock.
+
+- Update Etcd topology only after original request succeed (Alexander)
+
+  Postpone updating the Etcd topology known to the client until after the original request. When retrieving the cluster topology, implement the retry timeouts depending on the known number of nodes in the Etcd cluster. This makes our client prefer to get the results of the request to having the up-to-date list of nodes.
+
+  Both changes make Patroni connections to DCS more robust in the face of network issues.
+
+**Patronictl, monitoring and configuration**
+
+- Return information about streaming replicas via the API (Feike Steenbergen)
+
+Previously, there was no reliable way to query Patroni about PostgreSQL instances that fail to stream changes (for instance, due to connection issues). This change exposes the contents of pg_stat_replication via the /patroni endpoint.
+
+- Add patronictl scaffold command (Oleksii)
+
+  Add a command to create cluster structure in Etcd. The cluster is created with user-specified sysid and leader, and both leader and member keys are made persistent. This command is useful to create so-called master-less configurations, where Patroni cluster consisting of only replicas replicate  from the external master node that is unaware of Patroni. Subsequently, one
+  may remove the leader key, promoting one of the Patroni nodes and replacing
+  the original master with the Patroni-based HA cluster.
+
+- Add configuration option ``bin_dir`` to locate PostgreSQL binaries (Ants Aasma)
+
+  It is useful to be able to specify the location of PostgreSQL binaries explicitly when Linux distros that support installing multiple PostgreSQL versions at the same time.
+
+- Allow configuration file path to be overridden using ``custom_conf`` of (Alejandro Martínez)
+
+  Allows for custom configuration file paths, which will be unmanaged by Patroni, :ref:`details <postgresql_settings>`.
+
+**Bug fixes and code improvements**
+
+- Make Patroni compatible with new version schema in PostgreSQL 10 and above (Feike)
+
+  Make sure that Patroni understand 2-digits version numbers when doing conditional restarts based on the PostgreSQL version.
+
+- Use pkgutil to find DCS modules (Alexander)
+
+  Use the dedicated python module instead of traversing directories manually in order to find DCS modules.
+
+- Always call on_start callback when starting Patroni (Alexander)
+
+  Previously, Patroni did not call any callbacks when attaching to the already running node with the correct role. Since callbacks are often used to route
+  client connections that could result in the failure to register the running
+  node in the connection routing scheme. With this fix, Patroni calls on_start
+  callback even when attaching to the already running node.
+
+- Do not drop active replication slots (Murat, Oleksii)
+
+  Avoid dropping active physical replication slots on master. PostgreSQL cannot
+  drop such slots anyway. This change makes possible to run non-Patroni managed
+  replicas/consumers on the master.
+
+- Close Patroni connections during start of the PostgreSQL instance (Alexander)
+
+  Forces Patroni to close all former connections when PostgreSQL node is started. Avoids the trap of reusing former connections if postmaster was killed with SIGKILL.
+
+- Replace invalid characters when constructing slot names from member names (Ants)
+
+  Make sure that standby names that do not comply with the slot naming rules don't cause the slot creation and standby startup to fail. Replace the dashes in the slot names with underscores and all other characters not allowed in slot names with their unicode codepoints.
+
+Version 1.0
+-----------
+
+This release introduces the global dynamic configuration that allows dynamic changes of the PostgreSQL and Patroni configuration parameters for the entire HA cluster. It also delivers numerous bugfixes.
+
+**Upgrade notice**
+
+When upgrading from v0.90 or below, always upgrade all replicas before the master. Since we don't store replication credentials in DCS anymore, an old replica won't be able to connect to the new master.
+
+**Dynamic Configuration**
+
+- Implement the dynamic global configuration (Alexander Kukushkin)
+
+  Introduce new REST API endpoint /config to provide PostgreSQL and Patroni configuration parameters that should be set globally for the entire HA cluster (master and all the replicas). Those parameters are set in DCS and in many cases can be applied without disrupting PostgreSQL or Patroni. Patroni sets a special flag called "pending restart" visible via the API when some of the values require the PostgreSQL restart. In that case, restart should be issued manually via the API.
+
+  Patroni SIGHUP or POST to /reload will make it re-read the configuration file.
+
+  See the :ref:`dynamic configuration <dynamic_configuration>`  for the details on which parameters can be changed and the order of processing difference configuration sources.
+
+  The configuration file format *has changed* since the v0.90. Patroni is still compatible with the old configuration files, but in order to take advantage of the bootstrap parameters one needs to change it. Users are encourage to update them by referring to the :ref:`dynamic configuraton documentation page <dynamic_configuration>`.
+
+**More flexible configuration***
+
+- Make postgresql configuration and database name Patroni connects to configurable (Misja Hoebe)
+
+  Introduce `database` and `config_base_name` configuration parameters. Among others, it makes possible to run Patroni with PipelineDB and other PostgreSQL forks.
+
+- Implement possibility to configure some Patroni configuration parameters via environment (Alexander)
+
+  Those include the scope, the node name and the namespace, as well as the secrets and makes it easier to run Patroni in a dynamic environment, i.e. Kubernetes  Please, refer to the :ref:`supported environment variables <environment>` for further details.
+
+- Update the built-in Patroni docker container  to take advantage of environment-based configuration (Feike Steenbergen).
+
+- Add Zookeeper support to Patroni docker image (Alexander)
+
+- Split the Zookeeper and Exhibitor configuration options (Alexander)
+
+- Make patronictl reuse the code from Patroni to read configuration (Alexander)
+
+  This allows patronictl to take advantage of environment-based configuration.
+
+- Set application name to node name in primary_conninfo (Alexander)
+
+  This simplifies identification and configuration of synchronous replication for a given node.
+
+**Stability, security and usability improvements**
+
+- Reset sysid and do not call pg_controldata when restore of backup in progress (Alexander)
+
+  This change reduces the amount of noise generated by Patroni API health checks during the lengthy initialization of this node from the backup.
+
+- Fix a bunch of pg_rewind corner-cases (Alexander)
+
+  Avoid running pg_rewind if the source cluster is not the master.
+
+  In addition, avoid removing the data directory on an unsuccessful rewind, unless the new parameter *remove_data_directory_on_rewind_failure* is set to true. By default it is false.
+
+- Remove passwords from the replication connection string in DCS (Alexander)
+
+  Previously, Patroni always used the replication credentials from the Postgres URL in DCS. That is now changed to take the credentials from the patroni configuration. The secrets (replication username and password) and no longer exposed in DCS.
+
+- Fix the asynchronous machinery around the demote call (Alexander)
+
+  Demote now runs totally asynchronously without blocking the DCS interactions.
+
+- Make patronictl always send the authorization header if it is configured (Alexander)
+
+  This allows patronictl to issue "protected" requests, i.e. restart or reinitialize, when Patroni is configured to require authorization on those.
+
+- Handle the SystemExit exception correctly (Alexander)
+
+  Avoids the issues of Patroni not stopping properly when receiving the SIGTERM
+
+- Sample haproxy templates for confd (Alexander)
+
+  Generates and dynamically changes haproxy configuration from the patroni state in the DCS using confide
+
+- Improve and restructure the documentation to make it more friendly to the new users (Lauri Apple)
+
+- API must report role=master during pg_ctl stop (Alexander)
+
+  Makes the callback calls more reliable, particularly in the cluster stop case. In addition, introduce the `pg_ctl_timeout` option to set the timeout for the start, stop and restart calls via the `pg_ctl`.
+
+- Fix the retry logic in etcd (Alexander)
+
+  Make retries more predictable and robust.
+
+- Make Zookeeper code more resilient against short network hiccups (Alexander)
+
+  Reduce the connection timeouts to make Zookeeper connection attempts more frequent.
+
+Version 0.90
+------------
+
+This releases adds support for Consul, includes a new *noloadbalance* tag, changes the behavior of the *clonefrom* tag, improves *pg_rewind* handling and improves *patronictl* control program.
+
+**Consul support**
+
+- Implement Consul support (Alexander Kukushkin)
+
+  Patroni runs against Consul, in addition to Etcd and Zookeeper. the connection parameters can be configured in the YAML file.
+
+**New and improved tags**
+
+- Implement *noloadbalance* tag (Alexander)
+
+  This tag makes Patroni always return that the replica is not available to the load balancer.
+
+- Change the implementation of the *clonefrom* tag (Alexander)
+
+  Previously, a node name had to be supplied to the *clonefrom*, forcing a tagged replica to clone from the specific node. The new implementation makes *clonefrom* a boolean tag: if it is set to true, the replica becomes a candidate for other replicas to clone from it. When multiple candidates are present, the replicas picks one randomly.
+
+**Stability and security improvements**
+
+- Numerous reliability improvements (Alexander)
+
+  Removes some spurious error messages, improves the stability of the failover, addresses some corner cases with reading data from DCS, shutdown, demote and reattaching of the former leader.
+
+- Improve systems script to avoid killing Patroni children on stop (Jan Keirse, Alexander Kukushkin)
+
+  Previously, when stopping Patroni, *systemd* also sent a signal to PostgreSQL. Since Patroni also tried to stop PostgreSQL by itself, it resulted in sending to different shutdown requests (the smart shutdown, followed by the fast shutdown). That resulted in replicas disconnecting too early and a former master not being able to rejoin after demote. Fix by Jan with prior research by Alexander.
+
+- Eliminate some cases where the former master was unable to call pg_rewind before rejoining as a replica (Oleksii Kliukin)
+
+  Previously, we only called *pg_rewind* if the former master had crashed. Change this to always run pg_rewind for the former master as long as pg_rewind is present in the system. This fixes the case when the master is shut down before the replicas managed to get the latest changes (i.e. during the "smart" shutdown).
+
+- Numerous improvements to unit- and acceptance- tests, in particular, enable support for Zookeeper and Consul (Alexander).
+
+- Make Travis CI faster and implement support for running tests against Zookeeper (Exhibitor) and Consul (Alexander)
+
+  Both unit and acceptance tests run automatically against Etcd, Zookeeper and Consul on each commit or pull-request.
+
+- Clear environment variables before calling PostgreSQL commands from Patroni (Feike Steenbergen)
+
+  This prevents  a possibility of reading system environment variables by connecting to the PostgreSQL cluster managed by Patroni.
+
+**Configuration and control changes**
+
+- Unify patronictl and Patroni configuration (Feike)
+
+  patronictl can use the same configuration file as Patroni itself.
+
+- Enable Patroni to read the configuration from the environment variables (Oleksii)
+
+  This simplifies generating configuration for Patroni automatically, or merging a single configuration from different sources.
+
+- Include database system identifier in the information returned by the API (Feike)
+
+- Implement *delete_cluster* for all available DCSs (Alexander)
+
+  Enables support for DCSs other than Etcd in patronictl.
+
+
+Version 0.80
+------------
+
+This release adds support for *cascading replication* and simplifies Patroni management by providing *scheduled failovers*. One may use older versions of Patroni (in particular, 0.78) combined with this one in order to migrate to the new release. Note that the scheduled failover and cascading replication related features will only work with Patroni 0.80 and above.
+
+**Cascading replication**
+
+ - Add support for the *replicatefrom* and *clonefrom* tags for the patroni node (Oleksii Kliukin).
+
+ The tag *replicatefrom*  allows a replica to use an arbitrary node a source, not necessary the master. The *clonefrom* does the same for the initial backup. Together, they enable Patroni to fully support cascading replication.
+
+- Add support for running replication methods to initialize the replica even without a running replication connection (Oleksii).
+
+ This is useful in order to create replicas from the snapshots stored on S3 or FTP.  A replication method that does not require a running replication connection should supply *no_master: true* in the yaml configuration. Those scripts will still be called in order if the replication connection is present.
+
+**Patronictl, API and DCS improvements**
+
+- Implement scheduled failovers (Feike Steenbergen).
+
+  Failovers can be scheduled to happen at a certain time in the future, using either patronictl, or API calls.
+
+- Add support for *dbuser* and *password* parameters in patronictl (Feike).
+
+- Add PostgreSQL version to the health check output (Feike).
+
+- Improve Zookeeper support in patronictl (Oleksandr Shulgin)
+
+- Migrate to python-etcd 0.43 (Alexander Kukushkin)
+
+**Configuration**
+
+- Add a sample systems configuration script for Patroni (Jan Keirse).
+
+- Fix the problem of Patroni ignoring the superuser name specified in the configuration file for DB connections  (Alexander).
+
+- Fix the handling of CTRL-C by creating a separate session ID and process group for the postmaster launched by Patroni (Alexander).
+
+**Tests**
+
+- Add acceptance tests with *behave* in order to check real-world scenarios of running Patroni (Alexander, Oleksii).
+
+  The tests can be launched manually using the *behave* command. They are also launched automatically for pull requests and after commits.
+
+  Releases notes for some older versions can be found on `project's github page <https://github.com/zalando/patroni/releases>`__.

docs/replication_modes.rst

@@ -0,0 +1,68 @@
+.. _replication_modes:
+
+=================
+Replication modes
+=================
+
+Patroni uses PostgreSQL streaming replication. For more information about streaming replication, see the `Postgres documentation <http://www.postgresql.org/docs/current/static/warm-standby.html#STREAMING-REPLICATION>`__. By default Patroni configures PostgreSQL for asynchronous replication. Choosing your replication schema is dependent on your business considerations. Investigate both async and sync replication, as well as other HA solutions, to determine which solution is best for you.
+
+Asynchronous mode durability
+----------------------------
+
+In asynchronous mode the cluster is allowed to lose some committed transactions to ensure availability. When master server fails or becomes unavailable for any other reason Patroni will automatically promote a sufficiently healthy standby to master. Any transactions that have not been replicated to that standby remain in a "forked timeline" on the master, and are effectively unrecoverable [1]_.
+
+The amount of transactions that can be lost is controlled via ``maximum_lag_on_failover`` parameter. Because master transaction log position is not sampled in real time, in reality the amount of lost data on failover is worst case bounded by  ``maximum_lag_on_failover`` bytes of transaction log plus the amount that is written in the last ``ttl`` seconds (``loop_wait``/2 seconds in the average case). However typical steady state replication delay is well under a second.
+
+PostgreSQL synchronous replication
+----------------------------------
+
+You can use Postgres's `synchronous replication <http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION>`__ with Patroni. Synchronous replication ensures consistency across a cluster by confirming that writes are written to a secondary before returning to the connecting client with a success. The cost of synchronous replication: reduced throughput on writes. This throughput will be entirely based on network performance.
+
+In hosted datacenter environments (like AWS, Rackspace, or any network you do not control), synchronous replication significantly increases the variability of write performance. If followers become inaccessible from the leader, the leader effectively becomes read-only.
+
+To enable a simple synchronous replication test, add the follow lines to the ``parameters`` section of your YAML configuration files:
+
+.. code:: YAML
+
+        synchronous_commit: "on"
+        synchronous_standby_names: "*"
+
+When using PostgreSQL synchronous replication, use at least three Postgres data nodes to ensure write availability if one host fails.
+
+Using PostgreSQL synchronous replication does not guarantee zero lost transactions under all circumstances. When master and standby that is currently acting as synchronous fail simultaneously a third node that might not contain all transactions will be promoted.
+
+.. _synchronous_mode:
+
+Synchronous mode
+----------------
+
+For use cases where losing committed transactions is not permissible you can turn on Patronis ``synchronous_mode``. When ``synchronous_mode`` is turned on Patroni will not promote a standby unless it is certain that the standby contains all transactions that may have returned a successful commit status to client [2]_. This means that the system may be unavailable for writes even though some servers are available. System administrators can still use manual failover commmands to promote a standby even if it results in transaction loss.
+
+Turning on ``synchronous_mode`` does not guarantee multi node durability of commits under all circumstances. When no suitable standby is available, master server will still accept writes, but does not guarantee their replication. When the master fails in this mode no standby will be promote.  When the host that used to be master comes back it will get promoted automatically, unless system administrator performed a manual failover. This behavior makes synchronous mode usable with 2 node clusters.
+
+When ``synchronous_mode`` is on and a standby crashes, commits will block until next iteration of Patroni runs and switches master to standalone mode (worst case delay for writes ``ttl`` seconds, average case ``loop_wait``/2 seconds). Manually shutting down or restarting a standby will not cause a commit service interruption. Standby will signal the master to release itself from synchronous standby duties before PostgreSQL shutdown is initiated.
+
+You can ensure that a standby never becomes the synchronous standby by setting ``nosync`` tag to true. This is recommended to set for standbys that are behind slow network connections and would cause performance degradation when becoming a synchronous standby.
+
+Synchronous mode can be switched on and off via Patroni REST interface. See `dynamic configuration <https://github.com/zalando/patroni/blob/master/docs/dynamic_configuration.rst>`__ for instructions.
+
+
+Synchronous mode implementation
+-------------------------------
+
+When in synchronous mode Patroni maintains synchronization state in the DCS, containing the latest master and current synchronous standby. This state is updated with strict ordering constraints to ensure the following invariants:
+
+- A node must be marked as the latest leader whenever it can accept write transactions. Patroni crashing or PostgreSQL not shutting down can cause violations of this invariant.
+
+- A node must be set as the synchronous standby in PostgreSQL as long as it is published as the synchronous standby.
+
+- A node that is not the leader or current synchronous standby is not allowed to promote itself automatically.
+
+Patroni will only ever assign one standby to ``synchronous_standby_names`` because with multiple candidates it is not possible to know which node was acting as synchronous during the failure.
+
+On each HA loop iteration Patroni re-evaluates synchronous standby choice. If the current synchronous standby is connected and has not requested its synchronous status to be removed it remains picked. Otherwise the cluster member avaiable for sync that is furthest ahead in replication is picked.
+
+
+.. [1] The data is still there, but recovering it requires a manual recovery effort by data recovery specialists. When Patroni is allowed to rewind with ``use_pg_rewind`` the forked timeline will be automatically erased to rejoin the failed master with the cluster.
+
+.. [2] Clients can change the behavior per transaction using PostgreSQL's ``synchronous_commit`` setting. Transactions with ``synchronous_commit`` values of ``off`` and ``local`` may be lost on fail over, but will not be blocked by replication delays.

features/basic_replication.feature

@@ -3,15 +3,38 @@ Feature: basic replication
 
   Scenario: check replication of a single table
     Given I start postgres0
-    And postgres0 is a leader after 10 seconds
-    And I start postgres1
-    When I add the table foo to postgres0
+    Then postgres0 is a leader after 10 seconds
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "loop_wait": 2, "synchronous_mode": true}
+    Then I receive a response code 200
+    When I start postgres1
+    And I configure and start postgres2 with a tag replicatefrom postgres0
+    And "sync" key in DCS has leader=postgres0 after 20 seconds
+    And I add the table foo to postgres0
     Then table foo is present on postgres1 after 20 seconds
+    Then table foo is present on postgres2 after 20 seconds
 
-  Scenario: check the basic failover
+  Scenario: check restart of sync replica
+    Given I run patronictl.py restart batman postgres2 --force
+    And "sync" key in DCS has sync_standby=postgres1 after 2 seconds
+    And I run patronictl.py restart batman postgres1 --force
+    Then I receive a response returncode 0
+    And "sync" key in DCS has sync_standby=postgres2 after 10 seconds
+
+  Scenario: check the basic failover in synchronous mode
     When I kill postgres0
-    Then postgres1 role is the primary after 32 seconds
-    When I start postgres0
+    Then postgres2 role is the primary after 22 seconds
+    When I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_mode": null, "master_start_timeout": 0}
+    Then I receive a response code 200
+    When I add the table bar to postgres2
+    Then table bar is present on postgres1 after 20 seconds
+
+  Scenario: check immediate failover when master_start_timeout=0
+    Given I kill postmaster on postgres2
+    Then postgres1 is a leader after 10 seconds
+    And postgres1 role is the primary after 10 seconds
+
+  Scenario: check rejoin of the former master with pg_rewind
+    Given I start postgres0
     Then postgres0 role is the secondary after 20 seconds
-    When I add the table bar to postgres1
-    Then table bar is present on postgres0 after 20 seconds
+    When I add the table buz to postgres1
+    Then table buz is present on postgres0 after 20 seconds

features/cascading_replication.feature

@@ -8,6 +8,7 @@ Scenario: check a base backup and streaming replication from a replica
 	And replication works from postgres0 to postgres1 after 20 seconds
 	And I create label with "postgres0" in postgres0 data directory
 	And I create label with "postgres1" in postgres1 data directory
+	And "members/postgres1" key in DCS has state=running after 12 seconds
 	And I configure and start postgres2 with a tag replicatefrom postgres1
 	Then replication works from postgres0 to postgres2 after 30 seconds
 	And there is a label with "postgres1" in postgres2 data directory

features/environment.py

@@ -16,7 +16,8 @@ import yaml
 @six.add_metaclass(abc.ABCMeta)
 class AbstractController(object):
 
-    def __init__(self, name, work_directory, output_dir):
+    def __init__(self, context, name, work_directory, output_dir):
+        self._context = context
         self._name = name
         self._work_directory = work_directory
         self._output_dir = output_dir
@@ -46,6 +47,7 @@ class AbstractController(object):
 
         assert self._has_started(), "Process {0} is not running after being started".format(self._name)
 
+        max_wait_limit *= self._context.timeout_multiplier
         for _ in range(max_wait_limit):
             if self._is_accessible():
                 break
@@ -54,10 +56,11 @@ class AbstractController(object):
             assert False,\
                 "{0} instance is not available for queries after {1} seconds".format(self._name, max_wait_limit)
 
-    def stop(self, kill=False, timeout=15):
+    def stop(self, kill=False, timeout=15, _=False):
         term = False
         start_time = time.time()
 
+        timeout *= self._context.timeout_multiplier
         while self._handle and self._is_running():
             if kill:
                 self._handle.kill()
@@ -77,12 +80,12 @@ class PatroniController(AbstractController):
     PATRONI_CONFIG = '{}.yml'
     """ starts and stops individual patronis"""
 
-    def __init__(self, dcs, name, work_directory, output_dir, tags=None):
-        super(PatroniController, self).__init__('patroni_' + name, work_directory, output_dir)
+    def __init__(self, context, name, work_directory, output_dir, tags=None):
+        super(PatroniController, self).__init__(context, 'patroni_' + name, work_directory, output_dir)
         PatroniController.__PORT += 1
         self._data_dir = os.path.join(work_directory, 'data', name)
         self._connstring = None
-        self._config = self._make_patroni_test_config(name, dcs, tags)
+        self._config = self._make_patroni_test_config(name, tags)
 
         self._conn = None
         self._curs = None
@@ -109,10 +112,15 @@ class PatroniController(AbstractController):
         return subprocess.Popen(['coverage', 'run', '--source=patroni', '-p', 'patroni.py', self._config],
                                 stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory)
 
+    def stop(self, kill=False, timeout=15, postgres=False):
+        if postgres:
+            return subprocess.call(['pg_ctl', '-D', self._data_dir, 'stop', '-mi', '-w'])
+        super(PatroniController, self).stop(kill, timeout)
+
     def _is_accessible(self):
         return self.query("SELECT 1", fail_ok=True) is not None
 
-    def _make_patroni_test_config(self, name, dcs, tags):
+    def _make_patroni_test_config(self, name, tags):
         patroni_config_name = self.PATRONI_CONFIG.format(name)
         patroni_config_path = os.path.join(self._output_dir, patroni_config_name)
 
@@ -182,12 +190,16 @@ class AbstractDcsController(AbstractController):
 
     _CLUSTER_NODE = '/service/batman'
 
+    def __init__(self, context, mktemp=True):
+        work_directory = mktemp and tempfile.mkdtemp() or None
+        super(AbstractDcsController, self).__init__(context, self.name(), work_directory, context.pctl.output_dir)
+
     def _is_accessible(self):
         return self._is_running()
 
-    def stop_and_remove_work_directory(self, timeout=15):
+    def stop(self, kill=False, timeout=15):
         """ terminate process and wipe out the temp work directory, but only if we actually started it"""
-        self.stop(timeout=timeout)
+        super(AbstractDcsController, self).stop(kill=kill, timeout=timeout)
         if self._work_directory:
             shutil.rmtree(self._work_directory)
 
@@ -206,17 +218,36 @@ class AbstractDcsController(AbstractController):
     def cleanup_service_tree(self):
         """ clean all contents stored in the tree used for the tests """
 
+    @classmethod
+    def get_subclasses(cls):
+        for subclass in cls.__subclasses__():
+            for subsubclass in subclass.get_subclasses():
+                yield subsubclass
+            yield subclass
+
+    @classmethod
+    def name(cls):
+        return cls.__name__[:-10].lower()
+
 
 class ConsulController(AbstractDcsController):
 
-    def __init__(self, output_dir):
-        super(ConsulController, self).__init__('consul', tempfile.mkdtemp(), output_dir)
+    def __init__(self, context):
+        super(ConsulController, self).__init__(context)
         os.environ['PATRONI_CONSUL_HOST'] = 'localhost:8500'
         self._client = consul.Consul()
 
     def _start(self):
-        return subprocess.Popen(['consul', 'agent', '-server', '-bootstrap', '-advertise=127.0.0.1',
-                                 '-data-dir', self._work_directory], stdout=self._log, stderr=subprocess.STDOUT)
+        config_file = self._work_directory + '.json'
+        with open(config_file, 'wb') as f:
+            f.write(b'{"session_ttl_min":"5s","server":true,"bootstrap":true,"advertise_addr":"127.0.0.1"}')
+        return subprocess.Popen(['consul', 'agent', '-config-file', config_file, '-data-dir', self._work_directory],
+                                stdout=self._log, stderr=subprocess.STDOUT)
+
+    def stop(self, kill=False, timeout=15):
+        super(ConsulController, self).stop(kill=kill, timeout=timeout)
+        if self._work_directory:
+            os.unlink(self._work_directory + '.json')
 
     def _is_running(self):
         try:
@@ -237,15 +268,18 @@ class ConsulController(AbstractDcsController):
     def cleanup_service_tree(self):
         self._client.kv.delete(self.path(), recurse=True)
 
+    def start(self, max_wait_limit=15):
+        super(ConsulController, self).start(max_wait_limit)
+
 
 class EtcdController(AbstractDcsController):
 
     """ handles all etcd related tasks, used for the tests setup and cleanup """
 
-    def __init__(self, output_dir):
-        super(EtcdController, self).__init__('etcd', tempfile.mkdtemp(), output_dir)
-        os.environ['PATRONI_ETCD_HOST'] = 'localhost:4001'
-        self._client = etcd.Client()
+    def __init__(self, context):
+        super(EtcdController, self).__init__(context)
+        os.environ['PATRONI_ETCD_HOST'] = 'localhost:2379'
+        self._client = etcd.Client(port=2379)
 
     def _start(self):
         return subprocess.Popen(["etcd", "--debug", "--data-dir", self._work_directory],
@@ -280,8 +314,8 @@ class ZooKeeperController(AbstractDcsController):
 
     """ handles all zookeeper related tasks, used for the tests setup and cleanup """
 
-    def __init__(self, output_dir, export_env=True):
-        super(ZooKeeperController, self).__init__('zookeeper', None, output_dir)
+    def __init__(self, context, export_env=True):
+        super(ZooKeeperController, self).__init__(context, False)
         if export_env:
             os.environ['PATRONI_ZOOKEEPER_HOSTS'] = "'localhost:2181'"
         self._client = kazoo.client.KazooClient()
@@ -318,22 +352,21 @@ class ZooKeeperController(AbstractDcsController):
 
 class ExhibitorController(ZooKeeperController):
 
-    def __init__(self, output_dir):
-        super(ExhibitorController, self).__init__(output_dir, False)
+    def __init__(self, context):
+        super(ExhibitorController, self).__init__(context, False)
         os.environ.update({'PATRONI_EXHIBITOR_HOSTS': 'localhost', 'PATRONI_EXHIBITOR_PORT': '8181'})
 
 
 class PatroniPoolController(object):
 
-    KNOWN_DCS = {'consul': ConsulController, 'etcd': EtcdController,
-                 'zookeeper': ZooKeeperController, 'exhibitor': ExhibitorController}
-
-    def __init__(self):
+    def __init__(self, context):
+        self._context = context
         self._dcs = None
         self._output_dir = None
         self._patroni_path = None
         self._processes = {}
         self.create_and_set_output_directory('')
+        self.known_dcs = {subclass.name(): subclass for subclass in AbstractDcsController.get_subclasses()}
 
     @property
     def patroni_path(self):
@@ -350,17 +383,17 @@ class PatroniPoolController(object):
     def output_dir(self):
         return self._output_dir
 
-    def start(self, pg_name, max_wait_limit=20, tags=None):
-        if pg_name not in self._processes:
-            self._processes[pg_name] = PatroniController(self.dcs, pg_name, self.patroni_path, self._output_dir, tags)
-        self._processes[pg_name].start(max_wait_limit)
+    def start(self, name, max_wait_limit=20, tags=None):
+        if name not in self._processes:
+            self._processes[name] = PatroniController(self._context, name, self.patroni_path, self._output_dir, tags)
+        self._processes[name].start(max_wait_limit)
 
     def __getattr__(self, func):
         if func not in ['stop', 'query', 'write_label', 'read_label', 'check_role_has_changed_to', 'add_tag_to_config']:
             raise AttributeError("PatroniPoolController instance has no attribute '{0}'".format(func))
 
-        def wrapper(pg_name, *args, **kwargs):
-            return getattr(self._processes[pg_name], func)(*args, **kwargs)
+        def wrapper(name, *args, **kwargs):
+            return getattr(self._processes[name], func)(*args, **kwargs)
         return wrapper
 
     def stop_all(self):
@@ -379,24 +412,26 @@ class PatroniPoolController(object):
     def dcs(self):
         if self._dcs is None:
             self._dcs = os.environ.pop('DCS', 'etcd')
-            assert self._dcs in self.KNOWN_DCS, 'Unsupported dcs: ' + self._dcs
+            assert self._dcs in self.known_dcs, 'Unsupported dcs: ' + self._dcs
         return self._dcs
 
 
 # actions to execute on start/stop of the tests and before running invidual features
 def before_all(context):
-    context.pctl = PatroniPoolController()
-    context.dcs_ctl = context.pctl.KNOWN_DCS[context.pctl.dcs](context.pctl.output_dir)
+    context.ci = 'TRAVIS_BUILD_NUMBER' in os.environ or 'BUILD_NUMBER' in os.environ
+    context.timeout_multiplier = 2 if context.ci else 1
+    context.pctl = PatroniPoolController(context)
+    context.dcs_ctl = context.pctl.known_dcs[context.pctl.dcs](context)
     context.dcs_ctl.start()
     try:
         context.dcs_ctl.cleanup_service_tree()
     except AssertionError:  # after_all handlers won't be executed in before_all
-        context.dcs_ctl.stop_and_remove_work_directory()
+        context.dcs_ctl.stop()
         raise
 
 
 def after_all(context):
-    context.dcs_ctl.stop_and_remove_work_directory()
+    context.dcs_ctl.stop()
     subprocess.call(['coverage', 'combine'])
     subprocess.call(['coverage', 'report'])
 

features/patroni_api.feature

@@ -12,7 +12,7 @@ Scenario: check API requests on a stand-alone server
 	Then I receive a response code 503
 	When I run patronictl.py reinit batman postgres0 --force
 	Then I receive a response returncode 0
-	And I receive a response output "reinitialize failed for member postgres0, status code=503, (I am the leader, can not reinitialize)"
+	And I receive a response output "Failed: reinitialize for member postgres0, status code=503, (I am the leader, can not reinitialize)"
 	When I run patronictl.py failover batman --master postgres0 --force
 	Then I receive a response returncode 1
 	And I receive a response output "Error: No candidates found to failover to"
@@ -34,46 +34,66 @@ Scenario: check local configuration reload
 	Then I receive a response code 202
 
 Scenario: check dynamic configuration change via DCS
-	Given I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "loop_wait": 1, "postgresql": {"parameters": {"max_connections": 101}}}
+	Given I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 10, "loop_wait": 2, "postgresql": {"parameters": {"max_connections": 101}}}
 	Then I receive a response code 200
-	And I receive a response loop_wait 1
+	And I receive a response loop_wait 2
 	And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 11 seconds
 	When I issue a GET request to http://127.0.0.1:8008/config
 	Then I receive a response code 200
-	And I receive a response loop_wait 1
+	And I receive a response loop_wait 2
 	When I issue a GET request to http://127.0.0.1:8008/patroni
 	Then I receive a response code 200
 	And I receive a response tags {'tag': 'new_value'}
 
-Scenario: check API requests for the primary-replica pair
-	Given I start postgres1
-	And replication works from postgres0 to postgres1 after 20 seconds
+Scenario: check API requests for the primary-replica pair in the pause mode
+	Given I run patronictl.py pause batman
+	Then I receive a response returncode 0
+	When I start postgres1
+	Then replication works from postgres0 to postgres1 after 20 seconds
 	When I issue a GET request to http://127.0.0.1:8009/replica
 	Then I receive a response code 200
 	And I receive a response state running
 	And I receive a response role replica
 	When I run patronictl.py reinit batman postgres1 --force
 	Then I receive a response returncode 0
-	And I receive a response output "Succesful reinitialize on member postgres1"
+	And I receive a response output "Success: reinitialize for member postgres1"
 	When I run patronictl.py restart batman postgres0 --force
 	Then I receive a response returncode 0
-	And I receive a response output "Succesful restart on member postgres0"
+	And I receive a response output "Success: restart on member postgres0"
 	And postgres0 role is the primary after 5 seconds
 	When I sleep for 10 seconds
 	Then postgres1 role is the secondary after 15 seconds
 
-Scenario: check the failover via the API
+Scenario: check the failover via the API in the pause mode
 	Given I run patronictl.py failover batman --master postgres0 --candidate postgres1 --force
 	Then I receive a response returncode 0
 	And postgres1 is a leader after 5 seconds
-	And postgres1 role is the primary after 5 seconds
+	And postgres1 role is the primary after 10 seconds
 	And postgres0 role is the secondary after 10 seconds
 	And replication works from postgres1 to postgres0 after 20 seconds
 
 Scenario: check the scheduled failover
-	Given I issue a scheduled failover from postgres1 to postgres0 in 1 seconds
+	Given I issue a scheduled failover from postgres1 to postgres0 in 3 seconds
+	Then I receive a response returncode 1
+	And I receive a response output "Can't schedule failover in the paused state"
+	When I run patronictl.py resume batman
+	Then I receive a response returncode 0
+	Given I issue a scheduled failover from postgres1 to postgres0 in 3 seconds
 	Then I receive a response returncode 0
 	And postgres0 is a leader after 20 seconds
-	And postgres0 role is the primary after 5 seconds
+	And postgres0 role is the primary after 10 seconds
 	And postgres1 role is the secondary after 10 seconds
 	And replication works from postgres0 to postgres1 after 25 seconds
+
+Scenario: check the scheduled restart
+	Given I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"superuser_reserved_connections": "6"}}}
+	Then I receive a response code 200
+		And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 5 seconds
+	Given I issue a scheduled restart at http://127.0.0.1:8008 in 3 seconds with {"role": "replica"}
+	Then I receive a response code 202
+		And I sleep for 4 seconds
+		And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 10 seconds
+	Given I issue a scheduled restart at http://127.0.0.1:8008 in 3 seconds with {"restart_pending": "True"}
+	Then I receive a response code 202
+		And Response on GET http://127.0.0.1:8008/patroni does not contain pending_restart after 10 seconds
+

features/steps/basic_replication.py

@@ -19,6 +19,11 @@ def kill_patroni(context, name):
     return context.pctl.stop(name, kill=True)
 
 
+@step('I kill postmaster on {name:w}')
+def stop_postgres(context, name):
+    return context.pctl.stop(name, postgres=True)
+
+
 @step('I add the table {table_name:w} to {pg_name:w}')
 def add_table(context, table_name, pg_name):
     # parse the configuration file and get the port
@@ -30,6 +35,7 @@ def add_table(context, table_name, pg_name):
 
 @then('Table {table_name:w} is present on {pg_name:w} after {max_replication_delay:d} seconds')
 def table_is_present_on(context, table_name, pg_name, max_replication_delay):
+    max_replication_delay *= context.timeout_multiplier
     for _ in range(int(max_replication_delay)):
         if context.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
             break
@@ -41,6 +47,7 @@ def table_is_present_on(context, table_name, pg_name, max_replication_delay):
 
 @then('{pg_name:w} role is the {pg_role:w} after {max_promotion_timeout:d} seconds')
 def check_role(context, pg_name, pg_role, max_promotion_timeout):
+    max_promotion_timeout *= context.timeout_multiplier
     assert context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)),\
         "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
 

features/steps/cascading_replication.py

@@ -1,3 +1,6 @@
+import json
+import time
+
 from behave import step, then
 
 
@@ -15,3 +18,18 @@ def check_label(context, content, name):
 @step('I create label with "{content:w}" in {name:w} data directory')
 def write_label(context, content, name):
     context.pctl.write_label(name, content)
+
+
+@step('"{name}" key in DCS has {key:w}={value:w} after {time_limit:d} seconds')
+def check_member(context, name, key, value, time_limit):
+    time_limit *= context.timeout_multiplier
+    max_time = time.time() + int(time_limit)
+    while time.time() < max_time:
+        try:
+            response = json.loads(context.dcs_ctl.query(name))
+            if response.get(key) == value:
+                return
+        except Exception:
+            pass
+        time.sleep(1)
+    assert False, "{0} does not have {1}={2} in dcs after {3} seconds".format(name, key, value, time_limit)

features/steps/patroni_api.py

@@ -1,6 +1,6 @@
 import json
+import os
 import parse
-import pytz
 import requests
 import shlex
 import subprocess
@@ -8,8 +8,11 @@ import time
 import yaml
 
 from behave import register_type, step, then
+from dateutil import tz
 from datetime import datetime, timedelta
 
+tzutc = tz.tzutc()
+
 
 @parse.with_pattern(r'https?://(?:\w|\.|:|/)+')
 def parse_url(text):
@@ -27,6 +30,7 @@ register_type(url=parse_url)
 @step('{name:w} is a leader after {time_limit:d} seconds')
 @then('{name:w} is a leader after {time_limit:d} seconds')
 def is_a_leader(context, name, time_limit):
+    time_limit *= context.timeout_multiplier
     max_time = time.time() + int(time_limit)
     while (context.dcs_ctl.query("leader") != name):
         time.sleep(1)
@@ -90,7 +94,10 @@ def do_request(context, request_method, url, data):
 def do_run(context, cmd):
     cmd = ['coverage', 'run', '--source=patroni', '-p'] + shlex.split(cmd)
     try:
-        response = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+        # XXX: Dirty hack! We need to take name/passwd from the config!
+        env = os.environ.copy()
+        env.update({'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password'})
+        response = subprocess.check_output(cmd, stderr=subprocess.STDOUT, env=env)
         context.status_code = 0
     except subprocess.CalledProcessError as e:
         response = e.output
@@ -118,7 +125,14 @@ def check_response(context, component, data):
 def scheduled_failover(context, from_host, to_host, in_seconds):
     context.execute_steps(u"""
         Given I run patronictl.py failover batman --master {0} --candidate {1} --scheduled "{2}" --force
-    """.format(from_host, to_host, datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds))))
+    """.format(from_host, to_host, datetime.now(tzutc) + timedelta(seconds=int(in_seconds))))
+
+
+@step('I issue a scheduled restart at {url:url} in {in_seconds:d} seconds with {data}')
+def scheduled_restart(context, url, in_seconds, data):
+    data = data and json.loads(data) or {}
+    data.update(schedule='{0}'.format((datetime.now(tzutc) + timedelta(seconds=int(in_seconds))).isoformat()))
+    context.execute_steps(u"""Given I issue a POST request to {0}/restart with {1}""".format(url, json.dumps(data)))
 
 
 @step('I add tag {tag:w} {value:w} to {pg_name:w} config')
@@ -127,12 +141,18 @@ def add_tag_to_config(context, tag, value, pg_name):
 
 
 @then('Response on GET {url} contains {value} after {timeout:d} seconds')
-def check_http_response(context, url, value, timeout):
+def check_http_response(context, url, value, timeout, negate=False):
+    timeout *= context.timeout_multiplier
     for _ in range(int(timeout)):
         r = requests.get(url)
-        if value in r.content.decode('utf-8'):
+        if (value in r.content.decode('utf-8')) != negate:
             break
         time.sleep(1)
     else:
         assert False,\
-            "Value {0} is not present in response after {1} seconds".format(value, timeout)
+            "Value {0} is {1} present in response after {2} seconds".format(value, "not" if not negate else "", timeout)
+
+
+@then('Response on GET {url} does not contain {value} after {timeout:d} seconds')
+def check_not_in_http_response(context, url, value, timeout):
+    check_http_response(context, url, value, timeout, negate=True)

mkbinary.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -e
+
+pip install --ignore-installed setuptools==19.2 pyinstaller
+pyinstaller --clean --onefile patroni.spec

patroni.spec

@@ -0,0 +1,39 @@
+# -*- mode: python -*-
+
+block_cipher = None
+
+
+def hiddenimports():
+    import sys
+    sys.path.insert(0, '.')
+    try:
+        import patroni.dcs
+        return patroni.dcs.dcs_modules()
+    finally:
+        sys.path.pop(0)
+
+
+a = Analysis(['patroni/__main__.py'],
+             pathex=[],
+             binaries=None,
+             datas=None,
+             hiddenimports=hiddenimports(),
+             hookspath=[],
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher)
+
+pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
+
+exe = EXE(pyz,
+    a.scripts,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    name='patroni',
+    debug=False,
+    strip=False,
+    upx=True,
+    console=True)

patroni/__init__.py

@@ -1,23 +1,22 @@
 import logging
+import os
 import signal
 import sys
 import time
 
-from patroni.api import RestApiServer
-from patroni.config import Config
-from patroni.dcs import get_dcs
-from patroni.exceptions import DCSError
-from patroni.ha import Ha
-from patroni.postgresql import Postgresql
-from patroni.utils import reap_children, sigchld_handler
-from patroni.version import __version__
-
 logger = logging.getLogger(__name__)
 
 
 class Patroni(object):
 
     def __init__(self):
+        from patroni.api import RestApiServer
+        from patroni.config import Config
+        from patroni.dcs import get_dcs
+        from patroni.ha import Ha
+        from patroni.postgresql import Postgresql
+        from patroni.version import __version__
+
         self.setup_signal_handlers()
 
         self.version = __version__
@@ -30,35 +29,40 @@ class Patroni(object):
         self.ha = Ha(self)
 
         self.tags = self.get_tags()
-        self.nap_time = self.config['loop_wait']
         self.next_run = time.time()
+        self.scheduled_restart = {}
 
     def load_dynamic_configuration(self):
+        from patroni.exceptions import DCSError
         while True:
             try:
                 cluster = self.dcs.get_cluster()
                 if cluster and cluster.config:
-                    self.config.set_dynamic_configuration(cluster.config)
+                    if self.config.set_dynamic_configuration(cluster.config):
+                        self.dcs.reload_config(self.config)
                 elif not self.config.dynamic_configuration and 'bootstrap' in self.config:
-                    self.config.set_dynamic_configuration(self.config['bootstrap']['dcs'])
+                    if self.config.set_dynamic_configuration(self.config['bootstrap']['dcs']):
+                        self.dcs.reload_config(self.config)
                 break
             except DCSError:
                 logger.warning('Can not get cluster from dcs')
 
     def get_tags(self):
         return {tag: value for tag, value in self.config.get('tags', {}).items()
-                if tag not in ('clonefrom', 'nofailover', 'noloadbalance') or value}
+                if tag not in ('clonefrom', 'nofailover', 'noloadbalance', 'nosync') or value}
 
     @property
     def nofailover(self):
-        return self.tags.get('nofailover', False)
+        return bool(self.tags.get('nofailover', False))
+
+    @property
+    def nosync(self):
+        return bool(self.tags.get('nosync', False))
 
     def reload_config(self):
         try:
             self.tags = self.get_tags()
-            self.nap_time = self.config['loop_wait']
-            self.dcs.set_ttl(self.config.get('ttl') or 30)
-            self.dcs.set_retry_timeout(self.config.get('retry_timeout') or self.nap_time)
+            self.dcs.reload_config(self.config)
             self.api.reload_config(self.config['restapi'])
             self.postgresql.reload_config(self.config['postgresql'])
         except Exception:
@@ -78,15 +82,19 @@ class Patroni(object):
 
     @property
     def noloadbalance(self):
-        return self.tags.get('noloadbalance', False)
+        return bool(self.tags.get('noloadbalance', False))
 
     def schedule_next_run(self):
-        self.next_run += self.nap_time
+        self.next_run += self.dcs.loop_wait
         current_time = time.time()
         nap_time = self.next_run - current_time
         if nap_time <= 0:
             self.next_run = current_time
-        elif self.dcs.watch(nap_time):
+            # Release the GIL so we don't starve anyone waiting on async_executor lock
+            time.sleep(0.001)
+            # Warn user that Patroni is not keeping up
+            logger.warning("Loop time exceeded, rescheduling immediately.")
+        elif self.ha.watch(nap_time):
             self.next_run = time.time()
 
     def run(self):
@@ -108,7 +116,6 @@ class Patroni(object):
             if not self.postgresql.data_directory_empty():
                 self.config.save_cache()
 
-            reap_children()
             self.schedule_next_run()
 
     def setup_signal_handlers(self):
@@ -116,10 +123,9 @@ class Patroni(object):
         self._received_sigterm = False
         signal.signal(signal.SIGHUP, self.sighup_handler)
         signal.signal(signal.SIGTERM, self.sigterm_handler)
-        signal.signal(signal.SIGCHLD, sigchld_handler)
 
 
-def main():
+def patroni_main():
     logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
     logging.getLogger('requests').setLevel(logging.WARNING)
 
@@ -130,5 +136,65 @@ def main():
         pass
     finally:
         patroni.api.shutdown()
-        patroni.postgresql.stop(checkpoint=False)
-        patroni.dcs.delete_leader()
+        if patroni.ha.is_paused():
+            logger.info('Leader key is not deleted and Postgresql is not stopped due paused state')
+        else:
+            patroni.ha.while_not_sync_standby(lambda: patroni.postgresql.stop(checkpoint=False))
+            patroni.dcs.delete_leader()
+
+
+def pg_ctl_start(args):
+    import subprocess
+    postmaster = subprocess.Popen(args)
+    print(postmaster.pid)
+
+
+def call_self(args, **kwargs):
+    """This function executes Patroni once again with provided arguments.
+
+    :args: list of arguments to call Patroni with.
+    :returns: `Popen` object"""
+
+    exe = [sys.executable]
+    if not getattr(sys, 'frozen', False):  # Binary distribution?
+        exe.append(sys.argv[0])
+
+    import subprocess
+    return subprocess.Popen(exe + args, **kwargs)
+
+
+def main():
+    if os.getpid() != 1:
+        if len(sys.argv) > 5 and sys.argv[1] == 'pg_ctl_start':
+            return pg_ctl_start(sys.argv[2:])
+        return patroni_main()
+
+    pid = 0
+
+    # Looks like we are in a docker, so we will act like init
+    def sigchld_handler(signo, stack_frame):
+        try:
+            while True:
+                ret = os.waitpid(-1, os.WNOHANG)
+                if ret == (0, 0):
+                    break
+                elif ret[0] != pid:
+                    logging.info('Reaped pid=%s, exit status=%s', *ret)
+        except OSError:
+            pass
+
+    def passtochild(signo, stack_frame):
+        if pid:
+            os.kill(pid, signo)
+
+    signal.signal(signal.SIGCHLD, sigchld_handler)
+    signal.signal(signal.SIGHUP, passtochild)
+    signal.signal(signal.SIGINT, passtochild)
+    signal.signal(signal.SIGUSR1, passtochild)
+    signal.signal(signal.SIGUSR2, passtochild)
+    signal.signal(signal.SIGQUIT, passtochild)
+    signal.signal(signal.SIGTERM, passtochild)
+
+    patroni = call_self(sys.argv[1:])
+    pid = patroni.pid
+    patroni.wait()

patroni/api.py

@@ -6,10 +6,9 @@ import psycopg2
 import time
 import dateutil.parser
 import datetime
-import pytz
 
 from patroni.exceptions import PostgresConnectionException
-from patroni.utils import deep_compare, patch_config, Retry, RetryFailedError
+from patroni.utils import deep_compare, patch_config, Retry, RetryFailedError, is_valid_pg_version, parse_int, tzutc
 from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 from six.moves.socketserver import ThreadingMixIn
 from threading import Thread
@@ -57,12 +56,18 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
     def _write_status_response(self, status_code, response):
         patroni = self.server.patroni
-        response.update({'tags': patroni.tags} if patroni.tags else {})
+        tags = patroni.ha.get_effective_tags()
+        if tags:
+            response['tags'] = tags
         if patroni.postgresql.sysid:
             response['database_system_identifier'] = patroni.postgresql.sysid
         if patroni.postgresql.pending_restart:
             response['pending_restart'] = True
         response['patroni'] = {'version': patroni.version, 'scope': patroni.postgresql.scope}
+        if patroni.scheduled_restart and isinstance(patroni.scheduled_restart, dict):
+            response['scheduled_restart'] = patroni.scheduled_restart.copy()
+            del response['scheduled_restart']['postmaster_start_time']
+            response['scheduled_restart']['schedule'] = (response['scheduled_restart']['schedule']).isoformat()
         self._write_json_response(status_code, response)
 
     def do_GET(self, write_status_code_only=False):
@@ -106,19 +111,21 @@ class RestApiHandler(BaseHTTPRequestHandler):
         self._write_status_response(200, response)
 
     def do_GET_config(self):
-        cluster = self.server.patroni.ha.dcs.cluster or self.server.patroni.ha.dcs.get_cluster()
+        cluster = self.server.patroni.dcs.cluster or self.server.patroni.dcs.get_cluster()
         if cluster.config:
             self._write_json_response(200, cluster.config.data)
         else:
             self.send_error(502)
 
-    def _read_json_content(self):
+    def _read_json_content(self, body_is_optional=False):
         if 'content-length' not in self.headers:
-            return self.send_error(411)
+            return self.send_error(411) if not body_is_optional else {}
         try:
             content_length = int(self.headers.get('content-length'))
+            if content_length == 0 and body_is_optional:
+                return {}
             request = json.loads(self.rfile.read(content_length).decode('utf-8'))
-            if isinstance(request, dict) and request:
+            if isinstance(request, dict) and (request or body_is_optional):
                 return request
         except Exception:
             logger.exception('Bad request')
@@ -128,22 +135,23 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_PATCH_config(self):
         request = self._read_json_content()
         if request:
-            cluster = self.server.patroni.ha.dcs.get_cluster()
+            cluster = self.server.patroni.dcs.get_cluster()
             data = cluster.config.data.copy()
             if patch_config(data, request):
                 value = json.dumps(data, separators=(',', ':'))
-                if not self.server.patroni.ha.dcs.set_config_value(value, cluster.config.index):
+                if not self.server.patroni.dcs.set_config_value(value, cluster.config.index):
                     return self.send_error(409)
+            self.server.patroni.ha.wakeup()
             self._write_json_response(200, data)
 
     @check_auth
     def do_PUT_config(self):
         request = self._read_json_content()
         if request:
-            cluster = self.server.patroni.ha.dcs.get_cluster()
+            cluster = self.server.patroni.dcs.get_cluster()
             if not deep_compare(request, cluster.config.data):
                 value = json.dumps(request, separators=(',', ':'))
-                if not self.server.patroni.ha.dcs.set_config_value(value):
+                if not self.server.patroni.dcs.set_config_value(value):
                     return self.send_error(502)
             self._write_json_response(200, request)
 
@@ -162,39 +170,108 @@ class RestApiHandler(BaseHTTPRequestHandler):
             response = str(e)
         self._write_response(status_code, response)
 
+    @staticmethod
+    def parse_schedule(schedule, action):
+        """ parses the given schedule and validates at """
+        error = None
+        scheduled_at = None
+        try:
+            scheduled_at = dateutil.parser.parse(schedule)
+            if scheduled_at.tzinfo is None:
+                error = 'Timezone information is mandatory for the scheduled {0}'.format(action)
+                status_code = 400
+            elif scheduled_at < datetime.datetime.now(tzutc):
+                error = 'Cannot schedule {0} in the past'.format(action)
+                status_code = 422
+            else:
+                status_code = None
+        except (ValueError, TypeError):
+            logger.exception('Invalid scheduled %s time: %s', action, schedule)
+            error = 'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601'
+            status_code = 422
+        return (status_code, error, scheduled_at)
+
     @check_auth
     def do_POST_restart(self):
         status_code = 500
         data = 'restart failed'
-        try:
-            status, data = self.server.patroni.ha.restart()
-            status_code = 200 if status else 503
-        except Exception:
-            logger.exception('Exception during restart')
+        request = self._read_json_content(body_is_optional=True)
+        cluster = self.server.patroni.dcs.get_cluster()
+        if request is None:
+            # failed to parse the json
+            return
+        if request:
+            logger.debug("received restart request: {0}".format(request))
+
+        if cluster.is_paused() and 'schedule' in request:
+            self._write_response(status_code, "Can't schedule restart in the paused state")
+            return
+
+        for k in request:
+            if k == 'schedule':
+                (_, data, request[k]) = self.parse_schedule(request[k], "restart")
+                if _:
+                    status_code = _
+                    break
+            elif k == 'role':
+                if request[k] not in ('master', 'replica'):
+                    status_code = 400
+                    data = "PostgreSQL role should be either master or replica"
+                    break
+            elif k == 'postgres_version':
+                if not is_valid_pg_version(request[k]):
+                    status_code = 400
+                    data = "PostgreSQL version should be in the first.major.minor format"
+                    break
+            elif k == 'timeout':
+                request[k] = parse_int(request[k], 's')
+                if request[k] is None or request[k] <= 0:
+                    status_code = 400
+                    data = "Timeout should be a positive number of seconds"
+                    break
+            elif k != 'restart_pending':
+                status_code = 400
+                data = "Unknown filter for the scheduled restart: {0}".format(k)
+                break
+        else:
+            if 'schedule' not in request:
+                try:
+                    status, data = self.server.patroni.ha.restart(request)
+                    status_code = 200 if status else 503
+                except Exception:
+                    logger.exception('Exception during restart')
+                    status_code = 400
+            else:
+                if self.server.patroni.ha.schedule_future_restart(request):
+                    data = "Restart scheduled"
+                    status_code = 202
+                else:
+                    data = "Another restart is already scheduled"
+                    status_code = 409
         self._write_response(status_code, data)
 
     @check_auth
-    def do_POST_reinitialize(self):
-        ha = self.server.patroni.ha
-        cluster = ha.dcs.get_cluster()
-        if cluster.is_unlocked():
-            status_code = 503
-            data = 'Cluster has no leader, can not reinitialize'
-        elif cluster.leader.name == ha.state_handler.name:
-            status_code = 503
-            data = 'I am the leader, can not reinitialize'
+    def do_DELETE_restart(self):
+        if self.server.patroni.ha.delete_future_restart():
+            data = "scheduled restart deleted"
+            code = 200
         else:
-            action = ha.schedule_reinitialize()
-            if action is not None:
-                status_code = 503
-                data = action + ' already in progress'
-            else:
-                status_code = 200
-                data = 'reinitialize scheduled'
+            data = "no restarts are scheduled"
+            code = 404
+        self._write_response(code, data)
+
+    @check_auth
+    def do_POST_reinitialize(self):
+        data = self.server.patroni.ha.reinitialize()
+        if data is None:
+            status_code = 200
+            data = 'reinitialize started'
+        else:
+            status_code = 503
         self._write_response(status_code, data)
 
     def poll_failover_result(self, leader, candidate):
-        timeout = 10 if self.server.patroni.nap_time < 10 else self.server.patroni.nap_time
+        timeout = max(10, self.server.patroni.dcs.loop_wait)
         for _ in range(0, timeout*2):
             time.sleep(1)
             try:
@@ -211,7 +288,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         return 503, 'Failover status unknown'
 
     def is_failover_possible(self, cluster, leader, candidate):
-        if leader and not cluster.leader or cluster.leader.name != leader:
+        if leader and (not cluster.leader or cluster.leader.name != leader):
             return 'leader name does not match'
         if candidate:
             members = [m for m in cluster.members if m.name == candidate]
@@ -229,14 +306,17 @@ class RestApiHandler(BaseHTTPRequestHandler):
     @check_auth
     def do_POST_failover(self):
         request = self._read_json_content()
+        status_code = 500
         if not request:
             return
 
         leader = request.get('leader')
         candidate = request.get('candidate') or request.get('member')
         scheduled_at = request.get('scheduled_at')
-        cluster = self.server.patroni.ha.dcs.get_cluster()
-        status_code = 500
+        cluster = self.server.patroni.dcs.get_cluster()
+
+        if scheduled_at and cluster.is_paused():
+            self._write_response(status_code, "Can't schedule failover in the paused state")
 
         logger.info("received failover request with leader=%s candidate=%s scheduled_at=%s",
                     leader, candidate, scheduled_at)
@@ -244,30 +324,21 @@ class RestApiHandler(BaseHTTPRequestHandler):
         data = ''
         if leader or candidate:
             if scheduled_at:
-                try:
-                    scheduled_at = dateutil.parser.parse(scheduled_at)
-                    if scheduled_at.tzinfo is None:
-                        data = 'Timezone information is mandatory for scheduled_at'
-                        status_code = 400
-                    elif scheduled_at < datetime.datetime.now(pytz.utc):
-                        data = 'Cannot schedule failover in the past'
-                        status_code = 422
-                    elif self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
-                        self.server.patroni.dcs.event.set()
-                        data = 'Failover scheduled'
-                        status_code = 202
-                    else:
-                        data = 'failed to write failover key into DCS'
-                        status_code = 503
-                except (ValueError, TypeError):
-                    logger.exception('Invalid scheduled failover time: %s', request['scheduled_at'])
-                    data = 'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601'
-                    status_code = 422
+                (_, data, scheduled_at) = self.parse_schedule(scheduled_at, "failover")
+                if _:
+                    status_code = _
+                elif self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
+                    self.server.patroni.ha.wakeup()
+                    data = 'Failover scheduled'
+                    status_code = 202
+                else:
+                    data = 'failed to write failover key into DCS'
+                    status_code = 503
             else:
                 data = self.is_failover_possible(cluster, leader, candidate)
                 if not data:
                     if self.server.patroni.dcs.manual_failover(leader, candidate):
-                        self.server.patroni.dcs.event.set()
+                        self.server.patroni.ha.wakeup()
                         status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, candidate)
                     else:
                         data = 'failed to write failover key into DCS'
@@ -303,17 +374,25 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
     def get_postgresql_status(self, retry=False):
         try:
-            row = self.query("""SELECT to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
+            row = self.query("""WITH replication_info AS (
+                                    SELECT usename, application_name, client_addr, state, sync_state, sync_priority
+                                      FROM pg_stat_replication
+                                )
+                                SELECT to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
                                        pg_is_in_recovery(),
                                        CASE WHEN pg_is_in_recovery()
                                             THEN 0
                                             ELSE pg_xlog_location_diff(pg_current_xlog_location(), '0/0')::bigint
                                        END,
-                                       pg_xlog_location_diff(pg_last_xlog_receive_location(), '0/0')::bigint,
+                                       pg_xlog_location_diff(COALESCE(pg_last_xlog_receive_location(),
+                                                                      pg_last_xlog_replay_location()), '0/0')::bigint,
                                        pg_xlog_location_diff(pg_last_xlog_replay_location(), '0/0')::bigint,
                                        to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
-                                       pg_is_in_recovery() AND pg_is_xlog_replay_paused()""", retry=retry)[0]
-            return {
+                                       pg_is_in_recovery() AND pg_is_xlog_replay_paused(),
+                                       (SELECT array_to_json(array_agg(row_to_json(ri))) FROM replication_info ri)""",
+                             retry=retry)[0]
+
+            result = {
                 'state': self.server.patroni.postgresql.state,
                 'postmaster_start_time': row[0],
                 'role': 'replica' if row[1] else 'master',
@@ -326,6 +405,11 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     'location': row[2]
                 })
             }
+
+            if row[7]:
+                result['replication'] = row[7]
+
+            return result
         except (psycopg2.Error, RetryFailedError, PostgresConnectionException):
             state = self.server.patroni.postgresql.state
             if state == 'running':

patroni/async_executor.py

@@ -1,27 +1,26 @@
 import logging
-from threading import Lock, Thread
+from threading import RLock, Thread
 
 logger = logging.getLogger(__name__)
 
 
 class AsyncExecutor(object):
 
-    def __init__(self):
-        self._busy = False
-        self._thread_lock = Lock()
+    def __init__(self, ha_wakeup):
+        self._ha_wakeup = ha_wakeup
+        self._thread_lock = RLock()
         self._scheduled_action = None
-        self._scheduled_action_lock = Lock()
+        self._scheduled_action_lock = RLock()
 
     @property
     def busy(self):
-        return self._busy
+        return self.scheduled_action is not None
 
     def schedule(self, action, immediately=False):
         with self._scheduled_action_lock:
             if self._scheduled_action is not None:
                 return self._scheduled_action
             self._scheduled_action = action
-            self._busy = immediately
         return None
 
     @property
@@ -34,17 +33,20 @@ class AsyncExecutor(object):
             self._scheduled_action = None
 
     def run(self, func, args=()):
+        wakeup = False
         try:
-            return func(*args) if args else func()
+            # if the func returned something (not None) - wake up main HA loop
+            wakeup = func(*args) if args else func()
+            return wakeup
         except:
             logger.exception('Exception during execution of long running task %s', self.scheduled_action)
         finally:
             with self:
-                self._busy = False
                 self.reset_scheduled_action()
+            if wakeup is not None:
+                self._ha_wakeup()
 
     def run_async(self, func, args=()):
-        self._busy = True
         Thread(target=self.run, args=(func, args)).start()
 
     def __enter__(self):

patroni/callback_executor.py

@@ -0,0 +1,37 @@
+import logging
+import subprocess
+from threading import Event, Lock, Thread
+
+logger = logging.getLogger(__name__)
+
+
+class CallbackExecutor(Thread):
+
+    def __init__(self):
+        super(CallbackExecutor, self).__init__()
+        self.daemon = True
+        self._lock = Lock()
+        self._cmd = None
+        self._process = None
+        self._callback_event = Event()
+        self.start()
+
+    def call(self, cmd):
+        with self._lock:
+            if self._process and self._process.poll() is None:
+                self._process.kill()
+                logger.warning('Killed the old callback process because it was still running: %s', self._cmd)
+        self._cmd = cmd
+        self._callback_event.set()
+
+    def run(self):
+        while True:
+            self._callback_event.wait()
+            self._callback_event.clear()
+            with self._lock:
+                try:
+                    self._process = subprocess.Popen(self._cmd, close_fds=True)
+                except Exception:
+                    logger.exception('Failed to execute %s',  self._cmd)
+                    continue
+            self._process.wait()

patroni/config.py

@@ -41,7 +41,10 @@ class Config(object):
     __DEFAULT_CONFIG = {
         'ttl': 30, 'loop_wait': 10, 'retry_timeout': 10,
         'maximum_lag_on_failover': 1048576,
+        'master_start_timeout': 300,
+        'synchronous_mode': False,
         'postgresql': {
+            'bin_dir': '',
             'use_slots': True,
             'parameters': {p: v[0] for p, v in Postgresql.CMDLINE_OPTIONS.items()}
         }
@@ -64,7 +67,7 @@ class Config(object):
                 print('Usage: {0} config.yml'.format(sys.argv[0]))
                 print('\tPatroni may also read the configuration from the {0} environment variable'.
                       format(self.PATRONI_CONFIG_VARIABLE))
-                exit(1)
+                sys.exit(1)
 
         self.__effective_configuration = self._build_effective_configuration({}, self._local_configuration)
         self._data_dir = self.__effective_configuration['postgresql']['data_dir']
@@ -171,7 +174,10 @@ class Config(object):
                     elif name not in ('connect_address', 'listen', 'data_dir', 'pgpass', 'authentication'):
                         config['postgresql'][name] = deepcopy(value)
             elif name in config:  # only variables present in __DEFAULT_CONFIG allowed to be overriden from DCS
-                config[name] = int(value)
+                if name == 'synchronous_mode':
+                    config[name] = value
+                else:
+                    config[name] = int(value)
         return config
 
     @staticmethod
@@ -193,7 +199,7 @@ class Config(object):
                     ret[section][param] = value
 
         _set_section_values('restapi', ['listen', 'connect_address', 'certfile', 'keyfile'])
-        _set_section_values('postgresql', ['listen', 'connect_address', 'data_dir', 'pgpass'])
+        _set_section_values('postgresql', ['listen', 'connect_address', 'data_dir', 'pgpass', 'bin_dir'])
 
         def _get_auth(name):
             ret = {}
@@ -231,8 +237,9 @@ class Config(object):
             if param.startswith(Config.PATRONI_ENV_PREFIX):
                 name, suffix = (param[8:].rsplit('_', 1) + [''])[:2]
                 if name and suffix:
-                    # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT)
-                    if suffix in ('HOST', 'HOSTS', 'PORT') and '_' not in name:
+                    # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..)
+                    if suffix in ('HOST', 'HOSTS', 'PORT', 'SRV', 'URL', 'PROXY', 'CACERT', 'CERT', 'KEY') \
+                            and '_' not in name:
                         value = os.environ.pop(param)
                         if suffix == 'PORT':
                             value = value and parse_int(value)
@@ -269,9 +276,7 @@ class Config(object):
 
         # restapi server expects to get restapi.auth = 'username:password'
         if 'authentication' in config['restapi']:
-            restapi = config['restapi']
-            auth = restapi['authentication']
-            restapi['auth'] = '{0}:{1}'.format(auth['username'], auth['password'])
+            config['restapi']['auth'] = '{username}:{password}'.format(**config['restapi']['authentication'])
 
         # special treatment for old config
 
@@ -294,7 +299,7 @@ class Config(object):
             config['name'] = pg_config['name']
 
         pg_config.update({p: config[p] for p in ('name', 'scope', 'retry_timeout',
-                          'maximum_lag_on_failover') if p in config})
+                          'synchronous_mode', 'maximum_lag_on_failover') if p in config})
 
         return config
 

patroni/ctl.py

@@ -21,17 +21,17 @@ from click import ClickException
 from patroni.config import Config
 from patroni.dcs import get_dcs as _get_dcs
 from patroni.exceptions import PatroniException
-from patroni.postgresql import get_conn_kwargs
+from patroni.postgresql import Postgresql
+from patroni.utils import is_valid_pg_version
 from prettytable import PrettyTable
 from six.moves.urllib_parse import urlparse
 
 CONFIG_DIR_PATH = click.get_app_dir('patroni')
 CONFIG_FILE_PATH = os.path.join(CONFIG_DIR_PATH, 'patronictl.yaml')
-LOGLEVEL = 'WARNING'
 DCS_DEFAULTS = {'zookeeper': {'port': 2181, 'template': "zookeeper:\n hosts: ['{host}:{port}']"},
                 'exhibitor': {'port': 8181, 'template': "exhibitor:\n hosts: [{host}]\n port: {port}"},
                 'consul': {'port': 8500, 'template': "consul:\n host: '{host}:{port}'"},
-                'etcd': {'port': 4001, 'template': "etcd:\n host: '{host}:{port}'"}}
+                'etcd': {'port': 2379, 'template': "etcd:\n host: '{host}:{port}'"}}
 
 
 class PatroniCtlException(ClickException):
@@ -76,7 +76,6 @@ def load_config(path, dcs):
         for d in DCS_DEFAULTS:
             config.pop(d, None)
         config.update(dcs)
-
     return config
 
 
@@ -88,25 +87,23 @@ def store_config(config, path):
         yaml.dump(config, fd)
 
 
-option_config_file = click.option('--config-file', '-c', help='Configuration file', default=CONFIG_FILE_PATH)
 option_format = click.option('--format', '-f', 'fmt', help='Output format (pretty, json)', default='pretty')
-option_dcs = click.option('--dcs', '-d', help='Use this DCS', envvar='DCS')
 option_watchrefresh = click.option('-w', '--watch', type=float, help='Auto update the screen every X seconds')
 option_watch = click.option('-W', is_flag=True, help='Auto update the screen every 2 seconds')
 option_force = click.option('--force', is_flag=True, help='Do not ask for confirmation at any point')
 
 
 @click.group()
+@click.option('--config-file', '-c', help='Configuration file', default=CONFIG_FILE_PATH)
+@click.option('--dcs', '-d', help='Use this DCS', envvar='DCS')
 @click.pass_context
-def ctl(ctx):
-    global LOGLEVEL
-    LOGLEVEL = os.environ.get('LOGLEVEL', LOGLEVEL)
-
-    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=LOGLEVEL)
+def ctl(ctx, config_file, dcs):
+    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=os.environ.get('LOGLEVEL', 'WARNING'))
+    ctx.obj = load_config(config_file, dcs)
 
 
 def get_dcs(config, scope):
-    config.setdefault('scope', scope)
+    config.update({'scope': scope, 'patronictl': True})
     config.setdefault('name', scope)
     try:
         return _get_dcs(config)
@@ -119,15 +116,17 @@ def auth_header(config):
         return {'Authorization': 'Basic ' + base64.b64encode(config['restapi']['auth'].encode('utf-8')).decode('utf-8')}
 
 
-def post_patroni(member, endpoint, content, headers=None):
+def request_patroni(member, request_type, endpoint, content=None, headers=None):
     headers = headers or {}
-    url = urlparse(member.api_url)
-    logging.debug(url)
+    url_parts = urlparse(member.api_url)
+    logging.debug(url_parts)
     if 'Content-Type' not in headers:
         headers['Content-Type'] = 'application/json'
-    return requests.post('{0}://{1}/{2}'.format(url.scheme, url.netloc, endpoint),
-                         headers=headers,
-                         data=json.dumps(content), timeout=60)
+
+    url = '{0}://{1}/{2}'.format(url_parts.scheme, url_parts.netloc, endpoint)
+
+    return getattr(requests, request_type)(url, headers=headers,
+                                           data=json.dumps(content) if content else None, timeout=60)
 
 
 def print_output(columns, rows=None, alignment=None, fmt='pretty', header=True, delimiter='\t'):
@@ -182,16 +181,6 @@ def watching(w, watch, max_count=None, clear=True):
         yield 0
 
 
-def build_connect_parameters(conn_url, connect_parameters):
-    params = get_conn_kwargs(conn_url, connect_parameters)
-    params.update({'fallback_application_name': 'Patroni ctl', 'connect_timeout': '5'})
-    if 'database' in connect_parameters:
-        params['database'] = connect_parameters['database']
-    else:
-        params.pop('database')
-    return params
-
-
 def get_all_members(cluster, role='master'):
     if role == 'master':
         if cluster.leader is not None:
@@ -216,7 +205,12 @@ def get_cursor(cluster, connect_parameters, role='master', member=None):
     if member is None:
         return None
 
-    params = build_connect_parameters(member.conn_url, connect_parameters)
+    params = member.conn_kwargs(connect_parameters)
+    params.update({'fallback_application_name': 'Patroni ctl', 'connect_timeout': '5'})
+    if 'database' in connect_parameters:
+        params['database'] = connect_parameters['database']
+    else:
+        params.pop('database')
 
     conn = psycopg2.connect(**params)
     conn.autocommit = True
@@ -235,37 +229,65 @@ def get_cursor(cluster, connect_parameters, role='master', member=None):
     return None
 
 
+def get_members(cluster, cluster_name, member_names, role, force, action):
+    candidates = {m.name: m for m in cluster.members}
+
+    if not force or role:
+        output_members(cluster, cluster_name)
+
+    if role:
+        role_names = [m.name for m in get_all_members(cluster, role)]
+        if member_names:
+            member_names = list(set(member_names) & set(role_names))
+            if not member_names:
+                raise PatroniCtlException('No {0} among provided members'.format(role))
+        else:
+            member_names = role_names
+
+    if not member_names and not force:
+        member_names = [click.prompt('Which member do you want to {0} [{1}]?'.format(action,
+                        ', '.join(candidates.keys())), type=str, default='')]
+
+    for mn in member_names:
+        if mn not in candidates:
+            raise PatroniCtlException('{0} is not a member of cluster'.format(mn))
+
+    if not force:
+        confirm = click.confirm('Are you sure you want to {0} members {1}?'.format(action, ', '.join(member_names)))
+        if not confirm:
+            raise PatroniCtlException('Aborted {0}'.format(action))
+
+    return [candidates[n] for n in member_names]
+
+
 @ctl.command('dsn', help='Generate a dsn for the provided member, defaults to a dsn of the master')
 @click.option('--role', '-r', help='Give a dsn of any member with this role', type=click.Choice(['master', 'replica',
               'any']), default=None)
 @click.option('--member', '-m', help='Generate a dsn for this member', type=str)
-@option_dcs
-@option_config_file
 @click.argument('cluster_name')
-def dsn(cluster_name, config_file, dcs, role, member):
+@click.pass_obj
+def dsn(obj, cluster_name, role, member):
     if role is not None and member is not None:
         raise PatroniCtlException('--role and --member are mutually exclusive options')
     if member is None and role is None:
         role = 'master'
 
-    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    cluster = get_dcs(obj, cluster_name).get_cluster()
     m = get_any_member(cluster, role=role, member=member)
     if m is None:
         raise PatroniCtlException('Can not find a suitable member')
 
-    params = get_conn_kwargs(m.conn_url)
+    params = m.conn_kwargs()
     click.echo('host={host} port={port}'.format(**params))
 
 
 @ctl.command('query', help='Query a Patroni PostgreSQL member')
 @click.argument('cluster_name')
-@option_config_file
 @option_format
 @click.option('--format', 'fmt', help='Output format (pretty, json)', default='tsv')
 @click.option('--file', '-f', 'p_file', help='Execute the SQL commands from this file', type=click.File('rb'))
 @click.option('--password', help='force password prompt', is_flag=True)
 @click.option('-U', '--username', help='database user name', type=str)
-@option_dcs
 @option_watch
 @option_watchrefresh
 @click.option('--role', '-r', help='The role of the query', type=click.Choice(['master', 'replica', 'any']),
@@ -274,10 +296,10 @@ def dsn(cluster_name, config_file, dcs, role, member):
 @click.option('--delimiter', help='The column delimiter', default='\t')
 @click.option('--command', '-c', help='The SQL commands to execute')
 @click.option('-d', '--dbname', help='database name to connect to', type=str)
+@click.pass_obj
 def query(
+    obj,
     cluster_name,
-    config_file,
-    dcs,
     role,
     member,
     w,
@@ -301,7 +323,7 @@ def query(
     if p_file is None and command is None:
         raise PatroniCtlException('You need to specify either --command or --file')
 
-    connect_parameters = dict()
+    connect_parameters = {}
     if username:
         connect_parameters['username'] = username
     if password:
@@ -312,17 +334,16 @@ def query(
     if p_file is not None:
         command = p_file.read()
 
-    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    dcs = get_dcs(obj, cluster_name)
 
     cursor = None
     for _ in watching(w, watch, clear=False):
+        if cursor is None:
+            cluster = dcs.get_cluster()
 
         output, cursor = query_member(cluster, cursor, member, role, command, connect_parameters)
         print_output(None, output, fmt=fmt, delimiter=delimiter)
 
-        if cursor is None:
-            cluster = dcs.get_cluster()
-
 
 def query_member(cluster, cursor, member, role, command, connect_parameters):
     try:
@@ -357,13 +378,13 @@ def query_member(cluster, cursor, member, role, command, connect_parameters):
 
 @ctl.command('remove', help='Remove cluster from DCS')
 @click.argument('cluster_name')
-@option_config_file
 @option_format
-@option_dcs
-def remove(config_file, cluster_name, fmt, dcs):
-    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+@click.pass_obj
+def remove(obj, cluster_name, fmt):
+    dcs = get_dcs(obj, cluster_name)
+    cluster = dcs.get_cluster()
 
-    output_members(cluster, cluster_name, fmt)
+    output_members(cluster, cluster_name, fmt=fmt)
 
     confirm = click.prompt('Please confirm the cluster name to remove', type=str)
     if confirm != cluster_name:
@@ -384,50 +405,27 @@ def remove(config_file, cluster_name, fmt, dcs):
     dcs.delete_cluster()
 
 
-def wait_for_leader(dcs, timeout=30):
-    t_stop = time.time() + timeout
-    timeout /= 2
-
-    while time.time() < t_stop:
-        dcs.watch(timeout)
-        cluster = dcs.get_cluster()
-
-        if cluster.leader:
-            return cluster
-
-    raise PatroniCtlException('Timeout occured')
+def check_response(response, member_name, action_name, silent_success=False):
+    if response.status_code >= 400:
+        click.echo('Failed: {0} for member {1}, status code={2}, ({3})'.format(
+            action_name, member_name, response.status_code, response.text
+        ))
+    elif not silent_success:
+        click.echo('Success: {0} for member {1}'.format(action_name, member_name))
 
 
-def empty_post_to_members(cluster, member_names, force, endpoint, headers=None):
-    candidates = {m.name: m for m in cluster.members}
+def parse_scheduled(scheduled):
+    if (scheduled or 'now') != 'now':
+        try:
+            scheduled_at = dateutil.parser.parse(scheduled)
+            if scheduled_at.tzinfo is None:
+                scheduled_at = tzlocal.get_localzone().localize(scheduled_at)
+        except (ValueError, TypeError):
+            message = 'Unable to parse scheduled timestamp ({0}). It should be in an unambiguous format (e.g. ISO 8601)'
+            raise PatroniCtlException(message.format(scheduled))
+        return scheduled_at
 
-    if not member_names:
-        member_names = [click.prompt('Which member do you want to {0} [{1}]?'.format(endpoint,
-                        ', '.join(candidates.keys())), type=str, default='')]
-
-    for mn in member_names:
-        if mn not in candidates:
-            raise PatroniCtlException('{0} is not a member of cluster'.format(mn))
-
-    if not force:
-        confirm = click.confirm('Are you sure you want to {0} members {1}?'.format(endpoint, ', '.join(member_names)))
-        if not confirm:
-            raise PatroniCtlException('Aborted {0}'.format(endpoint))
-
-    for mn in member_names:
-        r = post_patroni(candidates[mn], endpoint, '', headers)
-        if r.status_code != 200:
-            click.echo('{0} failed for member {1}, status code={2}, ({3})'.format(endpoint, mn, r.status_code, r.text))
-        else:
-            click.echo('Succesful {0} on member {1}'.format(endpoint, mn))
-
-
-def ctl_load_config(cluster_name, config_file, dcs):
-    config = load_config(config_file, dcs)
-    dcs = get_dcs(config, cluster_name)
-    cluster = dcs.get_cluster()
-
-    return config, dcs, cluster
+    return None
 
 
 @ctl.command('restart', help='Restart cluster member')
@@ -436,36 +434,81 @@ def ctl_load_config(cluster_name, config_file, dcs):
 @click.option('--role', '-r', help='Restart only members with this role', default='any',
               type=click.Choice(['master', 'replica', 'any']))
 @click.option('--any', 'p_any', help='Restart a single member only', is_flag=True)
-@option_config_file
+@click.option('--scheduled', help='Timestamp of a scheduled restart in unambiguous format (e.g. ISO 8601)',
+              default=None)
+@click.option('--pg-version', 'version', help='Restart if the PostgreSQL version is less than provided (e.g. 9.5.2)',
+              default=None)
+@click.option('--pending', help='Restart if pending', is_flag=True)
+@click.option('--timeout',
+              help='Return error and fail over if necessary when restarting takes longer than this.')
 @option_force
-@option_dcs
-def restart(cluster_name, member_names, config_file, dcs, force, role, p_any):
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
-
-    role_names = [m.name for m in get_all_members(cluster, role)]
-
-    if member_names:
-        member_names = list(set(member_names) & set(role_names))
-    else:
-        member_names = role_names
+@click.pass_obj
+def restart(obj, cluster_name, member_names, force, role, p_any, scheduled, version, pending, timeout):
+    cluster = get_dcs(obj, cluster_name).get_cluster()
 
+    members = get_members(cluster, cluster_name, member_names, role, force, 'restart')
     if p_any:
-        random.shuffle(member_names)
-        member_names = member_names[:1]
+        random.shuffle(members)
+        members = members[:1]
 
-    output_members(cluster, cluster_name)
-    empty_post_to_members(cluster, member_names, force, 'restart', auth_header(config))
+    if version is None and not force:
+        version = click.prompt('Restart if the PostgreSQL version is less than provided (e.g. 9.5.2) ',
+                               type=str, default='')
+
+    content = {}
+    if pending:
+        content['restart_pending'] = True
+
+    if version:
+        if not is_valid_pg_version(version):
+            message = 'PostgreSQL version should be in the first.major.minor format'
+            raise PatroniCtlException(message)
+        else:
+            content['postgres_version'] = version
+
+    if scheduled is None and not force:
+        scheduled = click.prompt('When should the restart take place (e.g. 2015-10-01T14:30) ', type=str, default='now')
+
+    scheduled_at = parse_scheduled(scheduled)
+    if scheduled_at:
+        if cluster.is_paused():
+            raise PatroniCtlException("Can't schedule restart in the paused state")
+        content['schedule'] = scheduled_at.isoformat()
+
+    if timeout is not None:
+        content['timeout'] = timeout
+
+    for member in members:
+        if 'schedule' in content:
+            if force and member.data.get('scheduled_restart'):
+                r = request_patroni(member, 'delete', 'restart', headers=auth_header(obj))
+                check_response(r, member.name, 'flush scheduled restart', True)
+
+        r = request_patroni(member, 'post', 'restart', content, auth_header(obj))
+        if r.status_code == 200:
+            click.echo('Success: restart on member {0}'.format(member.name))
+        elif r.status_code == 202:
+            click.echo('Success: restart scheduled on member {0}'.format(member.name))
+        elif r.status_code == 409:
+            click.echo('Failed: another restart is already scheduled on member {0}'.format(member.name))
+        else:
+            click.echo('Failed: restart for member {0}, status code={1}, ({2})'.format(
+                member.name, r.status_code, r.text)
+            )
 
 
 @ctl.command('reinit', help='Reinitialize cluster member')
 @click.argument('cluster_name')
 @click.argument('member_names', nargs=-1)
-@option_config_file
 @option_force
-@option_dcs
-def reinit(cluster_name, member_names, config_file, dcs, force):
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
-    empty_post_to_members(cluster, member_names, force, 'reinitialize', auth_header(config))
+@click.pass_obj
+def reinit(obj, cluster_name, member_names, force):
+    cluster = get_dcs(obj, cluster_name).get_cluster()
+    members = get_members(cluster, cluster_name, member_names, None, force, 'reinitialize')
+
+    for member in members:
+        r = request_patroni(member, 'post', 'reinitialize', headers=auth_header(obj))
+        check_response(r, member.name, 'reinitialize')
 
 
 @ctl.command('failover', help='Failover to a replica')
@@ -474,10 +517,9 @@ def reinit(cluster_name, member_names, config_file, dcs, force):
 @click.option('--candidate', help='The name of the candidate', default=None)
 @click.option('--scheduled', help='Timestamp of a scheduled failover in unambiguous format (e.g. ISO 8601)',
               default=None)
-@click.option('--force', is_flag=True)
-@option_config_file
-@option_dcs
-def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled):
+@option_force
+@click.pass_obj
+def failover(obj, cluster_name, master, candidate, force, scheduled):
     """
         We want to trigger a failover for the specified cluster name.
 
@@ -485,18 +527,19 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
         If so, we trigger a failover and keep the client up to date.
     """
 
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    dcs = get_dcs(obj, cluster_name)
+    cluster = dcs.get_cluster()
 
-    if cluster.leader is None:
+    if cluster.leader is None and not cluster.is_paused():
         raise PatroniCtlException('This cluster has no master')
 
-    if master is None:
+    if master is None and (not cluster.is_paused() or cluster.leader):
         if force:
             master = cluster.leader.member.name
         else:
             master = click.prompt('Master', type=str, default=cluster.leader.member.name)
 
-    if cluster.leader.member.name != master:
+    if master is not None and cluster.leader and cluster.leader.member.name != master:
         raise PatroniCtlException('Member {0} is not the leader of cluster {1}'.format(master, cluster_name))
 
     candidate_names = [str(m.name) for m in cluster.members if m.name != master]
@@ -519,19 +562,16 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
         scheduled = click.prompt('When should the failover take place (e.g. 2015-10-01T14:30) ', type=str,
                                  default='now')
 
-    if (scheduled or 'now') == 'now':
-        scheduled_at = None
-    else:
-        try:
-            scheduled_at = dateutil.parser.parse(scheduled)
-            if scheduled_at.tzinfo is None:
-                scheduled_at = tzlocal.get_localzone().localize(scheduled_at)
-        except (ValueError, TypeError):
-            message = 'Unable to parse scheduled timestamp ({0}). It should be in an unambiguous format (e.g. ISO 8601)'
-            raise PatroniCtlException(message.format(scheduled))
-        scheduled_at = scheduled_at.isoformat()
+    scheduled_at = parse_scheduled(scheduled)
+
+    scheduled_at_str = None
+    if scheduled_at:
+        if cluster.is_paused():
+            raise PatroniCtlException("Can't schedule failover in the paused state")
+        scheduled_at_str = scheduled_at.isoformat()
+
+    failover_value = {'leader': master, 'candidate': candidate, 'scheduled_at': scheduled_at_str}
 
-    failover_value = {'leader': master, 'candidate': candidate, 'scheduled_at': scheduled_at}
     logging.debug(failover_value)
 
     # By now we have established that the leader exists and the candidate exists
@@ -547,7 +587,9 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
 
     r = None
     try:
-        r = post_patroni(cluster.leader.member, 'failover', failover_value, auth_header(config))
+        member = cluster.leader.member if cluster.leader else [m for m in cluster.members if m.name == candidate][0]
+
+        r = request_patroni(member, 'post', 'failover', failover_value, auth_header(obj))
         if r.status_code in (200, 202):
             logging.debug(r)
             cluster = dcs.get_cluster()
@@ -561,12 +603,12 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
         logging.warning('Failing over to DCS')
         click.echo(timestamp() + ' Could not failover using Patroni api, falling back to DCS')
         click.echo(timestamp() + ' Initializing failover from master {0}'.format(master))
-        dcs.manual_failover(master, candidate, scheduled_at=failover_value)
+        dcs.manual_failover(master, candidate, scheduled_at=scheduled_at)
 
     output_members(cluster, cluster_name)
 
 
-def output_members(cluster, name, fmt='pretty'):
+def output_members(cluster, name, extended=False, fmt='pretty'):
     rows = []
     logging.debug(cluster)
     leader_name = None
@@ -580,57 +622,74 @@ def output_members(cluster, name, fmt='pretty'):
     for m in cluster.members:
         logging.debug(m)
 
-        leader = ''
+        role = ''
         if m.name == leader_name:
-            leader = '*'
+            role = 'Leader'
+        elif m.name == cluster.sync.sync_standby:
+            role = 'Sync standby'
 
-        host = get_conn_kwargs(m.conn_url)['host']
+        host = m.conn_kwargs()['host']
 
         xlog_location = m.data.get('xlog_location') or 0
         lag = ''
-        if (xlog_location_cluster >= xlog_location):
+        if xlog_location_cluster >= xlog_location:
             lag = round((xlog_location_cluster - xlog_location)/1024/1024)
 
-        rows.append([
+        row = [
             name,
             m.name,
             host,
-            leader,
+            role,
             m.data.get('state', ''),
-            lag
-        ])
+            lag,
+        ]
+        if extended:
+            value = ''
+            scheduled_restart = m.data.get('scheduled_restart')
+            if scheduled_restart:
+                value = scheduled_restart['schedule']
+                if 'postgres_version' in scheduled_restart:
+                    value += ' if version < {0}'.format(scheduled_restart['postgres_version'])
+
+            row.append(value)
+
+        rows.append(row)
 
     columns = [
         'Cluster',
         'Member',
         'Host',
-        'Leader',
+        'Role',
         'State',
         'Lag in MB',
     ]
     alignment = {'Cluster': 'l', 'Member': 'l', 'Host': 'l', 'Lag in MB': 'r'}
 
+    if extended:
+        columns.append('Scheduled restart')
+        alignment['Scheduled restart'] = 'l'
+
     print_output(columns, rows, alignment, fmt)
 
 
 @ctl.command('list', help='List the Patroni members for a given Patroni')
 @click.argument('cluster_names', nargs=-1)
-@option_config_file
+@click.option('--extended', '-e', help='Show some extra information', is_flag=True)
 @option_format
 @option_watch
 @option_watchrefresh
-@option_dcs
-def members(config_file, cluster_names, fmt, watch, w, dcs):
+@click.pass_obj
+def members(obj, cluster_names, fmt, watch, w, extended):
     if not cluster_names:
         logging.warning('Listing members: No cluster names were provided')
         return
 
-    config = load_config(config_file, dcs)
     for cluster_name in cluster_names:
-        dcs = get_dcs(config, cluster_name)
+        dcs = get_dcs(obj, cluster_name)
 
         for _ in watching(w, watch):
-            output_members(dcs.get_cluster(), cluster_name, fmt)
+            cluster = dcs.get_cluster()
+            output_members(cluster, cluster_name, extended, fmt)
 
 
 def timestamp(precision=6):
@@ -639,10 +698,124 @@ def timestamp(precision=6):
 
 @ctl.command('configure', help='Create configuration file')
 @click.option('--config-file', '-c', help='Configuration file', prompt='Configuration file', default=CONFIG_FILE_PATH)
-@click.option('--dcs', '-d', help='The DCS connect url', prompt='DCS connect url', default='etcd://localhost:4001')
+@click.option('--dcs', '-d', help='The DCS connect url', prompt='DCS connect url', default='etcd://localhost:2379')
 @click.option('--namespace', '-n', help='The namespace', prompt='Namespace', default='/service/')
 def configure(config_file, dcs, namespace):
-    config = dict()
-    config['dcs_api'] = str(dcs)
-    config['namespace'] = str(namespace)
-    store_config(config, config_file)
+    store_config({'dcs_api': str(dcs), 'namespace': str(namespace)}, config_file)
+
+
+def touch_member(config, dcs):
+    ''' Rip-off of the ha.touch_member without inter-class dependencies '''
+    p = Postgresql(config['postgresql'])
+    p.set_state('running')
+    p.set_role('master')
+
+    def restapi_connection_string(config):
+        protocol = 'https' if config.get('certfile') else 'http'
+        connect_address = config.get('connect_address')
+        listen = config['listen']
+        return '{0}://{1}/patroni'.format(protocol, connect_address or listen)
+
+    data = {
+        'conn_url': p.connection_string,
+        'api_url': restapi_connection_string(config['restapi']),
+        'state': p.state,
+        'role': p.role
+    }
+
+    return dcs.touch_member(json.dumps(data, separators=(',', ':')), permanent=True)
+
+
+def set_defaults(config, cluster_name):
+    """fill-in some basic configuration parameters if config file is not set """
+    config['postgresql'].setdefault('name', cluster_name)
+    config['postgresql'].setdefault('scope', cluster_name)
+    config['postgresql'].setdefault('listen', '127.0.0.1')
+    config['postgresql']['authentication'] = {'replication': None}
+    config['restapi']['listen'] = ':' in config['restapi']['listen'] and config['restapi']['listen'] or '127.0.0.1:8008'
+
+
+@ctl.command('scaffold', help='Create a structure for the cluster in DCS')
+@click.argument('cluster_name')
+@click.option('--sysid', '-s', help='System ID of the cluster to put into the initialize key', default="")
+@click.pass_obj
+def scaffold(obj, cluster_name, sysid):
+    dcs = get_dcs(obj, cluster_name)
+    cluster = dcs.get_cluster()
+    if cluster and cluster.initialize is not None:
+        raise PatroniCtlException("This cluster is already initialized")
+
+    if not dcs.initialize(create_new=True, sysid=sysid):
+        # initialize key already exists, don't touch this cluster
+        raise PatroniCtlException("Initialize key for cluster {0} already exists".format(cluster_name))
+
+    set_defaults(obj, cluster_name)
+
+    # make sure the leader keys will never expire
+    if not (touch_member(obj, dcs) and dcs.attempt_to_acquire_leader(permanent=True)):
+        # we did initialize this cluster, but failed to write the leader or member keys, wipe it down completely.
+        dcs.delete_cluster()
+        raise PatroniCtlException("Unable to install permanent leader for cluster {0}".format(cluster_name))
+    click.echo("Cluster {0} has been created successfully".format(cluster_name))
+
+
+@ctl.command('flush', help='Flush scheduled events')
+@click.argument('cluster_name')
+@click.argument('member_names', nargs=-1)
+@click.argument('target', type=click.Choice(['restart']))
+@click.option('--role', '-r', help='Flush only members with this role', default='any',
+              type=click.Choice(['master', 'replica', 'any']))
+@option_force
+@click.pass_obj
+def flush(obj, cluster_name, member_names, force, role, target):
+    cluster = get_dcs(obj, cluster_name).get_cluster()
+
+    members = get_members(cluster, cluster_name, member_names, role, force, 'flush')
+    for member in members:
+        if target == 'restart':
+            if member.data.get('scheduled_restart'):
+                r = request_patroni(member, 'delete', 'restart', None, auth_header(obj))
+                check_response(r, member.name, 'flush scheduled restart')
+            else:
+                click.echo('No scheduled restart for member {0}'.format(member.name))
+
+
+def toggle_pause(config, cluster_name, paused):
+    cluster = get_dcs(config, cluster_name).get_cluster()
+    if cluster.is_paused() == paused:
+        raise PatroniCtlException('Cluster is {0} paused'.format(paused and 'already' or 'not'))
+
+    members = []
+    if cluster.leader:
+        members.append(cluster.leader.member)
+    members.extend([m for m in cluster.members if m.api_url and (not members or members[0].name != m.name)])
+
+    for member in members:
+        try:
+            r = request_patroni(member, 'patch', 'config', {'pause': paused or None}, auth_header(config))
+        except Exception:
+            logging.warning('Member %s is not accessible', member.name)
+            continue
+
+        if r.status_code == 200:
+            click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
+        else:
+            click.echo('Failed: {0} cluster management status code={1}, ({2})'.format(
+                       paused and 'pause' or 'resume', r.status_code, r.text))
+        break
+    else:
+        raise PatroniCtlException('Can not find accessible cluster member')
+
+
+@ctl.command('pause', help='Disable auto failover')
+@click.argument('cluster_name')
+@click.pass_obj
+def pause(obj, cluster_name):
+    return toggle_pause(obj, cluster_name, True)
+
+
+@ctl.command('resume', help='Resume auto failover')
+@click.argument('cluster_name')
+@click.pass_obj
+def resume(obj, cluster_name):
+    return toggle_pause(obj, cluster_name, False)

patroni/dcs/__init__.py

@@ -4,7 +4,9 @@ import importlib
 import inspect
 import json
 import os
+import pkgutil
 import six
+import sys
 
 from collections import namedtuple
 from patroni.exceptions import PatroniException
@@ -30,23 +32,37 @@ def parse_connection_string(value):
     return conn_url, api_url
 
 
+def dcs_modules():
+    """Get names of DCS modules, depending on execution environment. If being packaged with PyInstaller,
+    modules aren't discoverable dynamically by scanning source directory because `FrozenImporter` doesn't
+    implement `iter_modules` method. But it is still possible to find all potential DCS modules by
+    iterating through `toc`, which contains list of all "frozen" resources."""
+
+    dcs_dirname = os.path.dirname(__file__)
+    module_prefix = __package__ + '.'
+
+    if getattr(sys, 'frozen', False):
+        importer = pkgutil.get_importer(dcs_dirname)
+        return [module for module in list(importer.toc) if module.startswith(module_prefix) and module.count('.') == 2]
+    else:
+        return [module_prefix + name for _, name, is_pkg in pkgutil.iter_modules([dcs_dirname]) if not is_pkg]
+
+
 def get_dcs(config):
     available_implementations = set()
-    for module in os.listdir(os.path.dirname(__file__)):
-        if module.endswith('.py') and not module.startswith('__'):  # find module
-            module_name = module[:-3].lower()
-            module = importlib.import_module(__package__ + '.' + module[:-3])
-            for name in filter(lambda name: not name.startswith('__'), dir(module)):  # iterate through module content
-                value = getattr(module, name)
-                name = name.lower()
-                # try to find implementation of AbstractDCS interface, class name must match with module_name
-                if inspect.isclass(value) and issubclass(value, AbstractDCS) and name == module_name:
-                    available_implementations.add(name)
-                    if name in config:  # which has configuration section in the config file
-                        # propagate some parameters
-                        config[name].update({p: config[p] for p in ('namespace', 'name',
-                                             'scope', 'ttl', 'retry_timeout') if p in config})
-                        return value(config[name])
+    for module_name in dcs_modules():
+        module = importlib.import_module(module_name)
+        for name in filter(lambda name: not name.startswith('__'), dir(module)):  # iterate through module content
+            value = getattr(module, name)
+            name = name.lower()
+            # try to find implementation of AbstractDCS interface, class name must match with module_name
+            if inspect.isclass(value) and issubclass(value, AbstractDCS) and __package__ + '.' + name == module_name:
+                available_implementations.add(name)
+                if name in config:  # which has configuration section in the config file
+                    # propagate some parameters
+                    config[name].update({p: config[p] for p in ('namespace', 'name', 'scope', 'loop_wait',
+                                         'patronictl', 'ttl', 'retry_timeout') if p in config})
+                    return value(config[name])
     raise PatroniException("""Can not find suitable configuration of distributed configuration store
 Available implementations: """ + ', '.join(available_implementations))
 
@@ -86,6 +102,26 @@ class Member(namedtuple('Member', 'index,name,session,data')):
     def conn_url(self):
         return self.data.get('conn_url')
 
+    def conn_kwargs(self, auth=None):
+        ret = self.data.get('conn_kwargs')
+        if ret:
+            ret = ret.copy()
+        else:
+            r = urlparse(self.conn_url)
+            ret = {
+                'host': r.hostname,
+                'port': r.port or 5432,
+                'database': r.path[1:]
+            }
+            self.data['conn_kwargs'] = ret.copy()
+
+        if auth and isinstance(auth, dict):
+            if 'username' in auth:
+                ret['user'] = auth['username']
+            if 'password' in auth:
+                ret['password'] = auth['password']
+        return ret
+
     @property
     def api_url(self):
         return self.data.get('api_url')
@@ -104,7 +140,15 @@ class Member(namedtuple('Member', 'index,name,session,data')):
 
     @property
     def clonefrom(self):
-        return self.tags.get('clonefrom', False)
+        return self.tags.get('clonefrom', False) and bool(self.conn_url)
+
+    @property
+    def state(self):
+        return self.data.get('state', 'unknown')
+
+    @property
+    def is_running(self):
+        return self.state == 'running'
 
 
 class Leader(namedtuple('Leader', 'index,session,member')):
@@ -119,6 +163,9 @@ class Leader(namedtuple('Leader', 'index,session,member')):
     def name(self):
         return self.member.name
 
+    def conn_kwargs(self, auth=None):
+        return self.member.conn_kwargs(auth)
+
     @property
     def conn_url(self):
         return self.member.conn_url
@@ -163,6 +210,9 @@ class Failover(namedtuple('Failover', 'index,leader,candidate,scheduled_at')):
 
         return Failover(index, data.get('leader'), data.get('member'), data.get('scheduled_at'))
 
+    def __len__(self):
+        return int(bool(self.leader)) + int(bool(self.candidate))
+
 
 class ClusterConfig(namedtuple('ClusterConfig', 'index,data,modify_index')):
 
@@ -180,7 +230,59 @@ class ClusterConfig(namedtuple('ClusterConfig', 'index,data,modify_index')):
         return ClusterConfig(index, data, modify_index or index)
 
 
-class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operation,members,failover')):
+class SyncState(namedtuple('SyncState', 'index,leader,sync_standby')):
+    """Immutable object (namedtuple) which represents last observed synhcronous replication state
+
+    :param index: modification index of a synchronization key in a Configuration Store
+    :param leader: reference to member that was leader
+    :param sync_standby: standby that was last synchronized to leader
+    """
+
+    @staticmethod
+    def from_node(index, value):
+        """
+        >>> SyncState.from_node(1, None).leader is None
+        True
+        >>> SyncState.from_node(1, '{}').leader is None
+        True
+        >>> SyncState.from_node(1, '{').leader is None
+        True
+        >>> SyncState.from_node(1, '[]').leader is None
+        True
+        >>> SyncState.from_node(1, '{"leader": "leader"}').leader == "leader"
+        True
+        """
+        if value:
+            try:
+                data = json.loads(value)
+                if not isinstance(data, dict):
+                    data = {}
+            except (TypeError, ValueError):
+                data = {}
+        else:
+            data = {}
+        return SyncState(index, data.get('leader'), data.get('sync_standby'))
+
+    def matches(self, name):
+        """
+        Returns if a node name matches one of the nodes in the sync state
+
+        >>> s = SyncState(1, 'foo', 'bar')
+        >>> s.matches('foo')
+        True
+        >>> s.matches('bar')
+        True
+        >>> s.matches('baz')
+        False
+        >>> s.matches(None)
+        False
+        >>> SyncState(1, None, None).matches('foo')
+        False
+        """
+        return name is not None and name in (self.leader, self.sync_standby)
+
+
+class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operation,members,failover,sync')):
 
     """Immutable object (namedtuple) which represents PostgreSQL cluster.
     Consists of the following fields:
@@ -190,7 +292,9 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operat
     :param last_leader_operation: int or long object containing position of last known leader operation.
         This value is stored in `/optime/leader` key
     :param members: list of Member object, all PostgreSQL cluster members including leader
-    :param failover: reference to `Failover` object"""
+    :param failover: reference to `Failover` object
+    :param sync: reference to `SyncState` object, last observed synchronous replication state.
+    """
 
     def is_unlocked(self):
         return not (self.leader and self.leader.name)
@@ -201,10 +305,14 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operat
     def get_member(self, member_name, fallback_to_leader=True):
         return ([m for m in self.members if m.name == member_name] or [self.leader if fallback_to_leader else None])[0]
 
-    def get_clone_member(self):
-        candidates = [m for m in self.members if m.clonefrom and (not self.leader or m.name != self.leader.name)]
+    def get_clone_member(self, exclude):
+        exclude = [exclude] + [self.leader.name] if self.leader else []
+        candidates = [m for m in self.members if m.clonefrom and m.is_running and m.name not in exclude]
         return candidates[randint(0, len(candidates) - 1)] if candidates else self.leader
 
+    def is_paused(self):
+        return self.config and self.config.data.get('pause', False) or False
+
 
 @six.add_metaclass(abc.ABCMeta)
 class AbstractDCS(object):
@@ -216,6 +324,7 @@ class AbstractDCS(object):
     _MEMBERS = 'members/'
     _OPTIME = 'optime'
     _LEADER_OPTIME = _OPTIME + '/' + _LEADER
+    _SYNC = 'sync'
 
     def __init__(self, config):
         """
@@ -225,9 +334,12 @@ class AbstractDCS(object):
         self._name = config['name']
         self._namespace = '/{0}'.format(config.get('namespace', '/service/').strip('/'))
         self._base_path = '/'.join([self._namespace, config['scope']])
+        self._set_loop_wait(config.get('loop_wait', 10))
 
+        self._ctl = bool(config.get('patronictl', False))
         self._cluster = None
         self._cluster_thread_lock = Lock()
+        self._last_leader_operation = ''
         self.event = Event()
 
     def client_path(self, path):
@@ -261,6 +373,10 @@ class AbstractDCS(object):
     def leader_optime_path(self):
         return self.client_path(self._LEADER_OPTIME)
 
+    @property
+    def sync_path(self):
+        return self.client_path(self._SYNC)
+
     @abc.abstractmethod
     def set_ttl(self, ttl):
         """Set the new ttl value for leader key"""
@@ -269,6 +385,18 @@ class AbstractDCS(object):
     def set_retry_timeout(self, retry_timeout):
         """Set the new value for retry_timeout"""
 
+    def _set_loop_wait(self, loop_wait):
+        self._loop_wait = loop_wait
+
+    def reload_config(self, config):
+        self._set_loop_wait(config['loop_wait'])
+        self.set_ttl(config['ttl'])
+        self.set_retry_timeout(config['retry_timeout'])
+
+    @property
+    def loop_wait(self):
+        return self._loop_wait
+
     @abc.abstractmethod
     def _load_cluster(self):
         """Internally this method should build  `Cluster` object which
@@ -298,9 +426,14 @@ class AbstractDCS(object):
             self._cluster = None
 
     @abc.abstractmethod
-    def write_leader_optime(self, last_operation):
+    def _write_leader_optime(self, last_operation):
         """write current xlog location into `/optime/leader` key in DCS
-        :param last_operation: absolute xlog location in bytes"""
+        :param last_operation: absolute xlog location in bytes
+        :returns: `!True` on success."""
+
+    def write_leader_optime(self, last_operation):
+        if self._last_leader_operation != last_operation and self._write_leader_optime(last_operation):
+            self._last_leader_operation = last_operation
 
     @abc.abstractmethod
     def update_leader(self):
@@ -313,9 +446,11 @@ class AbstractDCS(object):
         for example for etcd `prevValue` parameter must be used."""
 
     @abc.abstractmethod
-    def attempt_to_acquire_leader(self):
+    def attempt_to_acquire_leader(self, permanent=False):
         """Attempt to acquire leader lock
         This method should create `/leader` key with value=`~self._name`
+        :param permanent: if set to `!True`, the leader key will never expire.
+         Used in patronictl for the external master
         :returns: `!True` if key has been created successfully.
 
         Key must be created atomically. In case if key already exists it should not be
@@ -343,13 +478,15 @@ class AbstractDCS(object):
         """Create or update `/config` key"""
 
     @abc.abstractmethod
-    def touch_member(self, data, ttl=None):
+    def touch_member(self, data, ttl=None, permanent=False):
         """Update member key in DCS.
         This method should create or update key with the name = '/members/' + `~self._name`
         and value = data in a given DCS.
 
         :param data: json serialized information about instance (including connection strings)
         :param ttl: ttl for member key, optional parameter. If it is None `~self.member_ttl will be used`
+        :param permanent: if set to `!True`, the member key will never expire.
+         Used in patronictl for the external master.
         :returns: `!True` on success otherwise `!False`
         """
 
@@ -383,10 +520,22 @@ class AbstractDCS(object):
     def delete_cluster(self):
         """Delete cluster from DCS"""
 
-    def watch(self, timeout):
+    def write_sync_state(self, leader, sync_standby, index=None):
+        return self.set_sync_state_value(json.dumps({'leader': leader, 'sync_standby': sync_standby}), index=index)
+
+    @abc.abstractmethod
+    def set_sync_state_value(self, value, index=None):
+        """"""
+
+    @abc.abstractmethod
+    def delete_sync_state(self, index=None):
+        """"""
+
+    def watch(self, leader_index, timeout):
         """If the current node is a master it should just sleep.
         Any other node should watch for changes of leader key with a given timeout
 
+        :param leader_index: index of a leader key
         :param timeout: timeout in seconds
         :returns: `!True` if you would like to reschedule the next run of ha cycle"""
 

patroni/dcs/consul.py

@@ -1,14 +1,17 @@
 from __future__ import absolute_import
 import logging
 import os
+import socket
 import time
-import six
+import urllib3
 
-from consul import ConsulException, NotFound, base, std
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member
+from consul import ConsulException, NotFound, base
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
 from patroni.exceptions import DCSError
-from patroni.utils import sleep
-from requests.exceptions import RequestException
+from patroni.utils import Retry, RetryFailedError
+from urllib3.exceptions import HTTPError
+from six.moves.urllib.parse import urlencode
+from six.moves.http_client import HTTPException
 
 logger = logging.getLogger(__name__)
 
@@ -17,38 +20,60 @@ class ConsulError(DCSError):
     pass
 
 
-class HTTPClient(std.HTTPClient):
+class ConsulInternalError(ConsulException):
+    """An internal Consul server error occurred"""
 
-    def __init__(self, *args, **kwargs):
-        super(HTTPClient, self).__init__(*args, **kwargs)
 
-    def patch_default_timeout(self, timeout):
-        # Set a default timeout for the `request.session.request` method, that is used
-        # internally  by the methods request.session.get, request.session.post and
-        # others. We monkey-patch here to avoid reimplementing each individual method from
-        # `std.HTTPClient`. By default, the timeout is not set. It means that a new
-        # session may hang almost indefinitely waiting for the server to respond,
-        # which is not what we want in Patroni.
+class HTTPClient(object):
 
-        request_func = getattr(self.session.request, '__func__' if six.PY3 else 'im_func')
-        defaults_attr_name = '__defaults__' if six.PY3 else 'func_defaults'
-        defaults = list(getattr(request_func, defaults_attr_name))
-        code = request_func.__code__ if six.PY3 else request_func.func_code
-        defaults[code.co_varnames[code.co_argcount - len(defaults):code.co_argcount].index('timeout')] = timeout
-        setattr(request_func, defaults_attr_name, tuple(defaults))  # monkeypatching
+    def __init__(self, host='127.0.0.1', port=8500, scheme='http', verify=True, timeout=10):
+        self.host = host
+        self.port = port
+        self.scheme = scheme
+        self.verify = verify
+        self.set_read_timeout(timeout)
+        self.base_uri = '{0}://{1}:{2}'.format(self.scheme, self.host, self.port)
+        self.http = urllib3.PoolManager(num_pools=10)
+        self._ttl = None
 
-    def get(self, callback, path, params=None):
-        # The get function is overridden to handle a special case of it being called
-        # with an index and wait parameters. That form indicates that a user needs to
-        # wait for the given key to change its value, with a wait timeout supplied. We
-        # don't want our monkey-patched timeout to be less than the value of the wait
-        # parameter, therefore, we set it to either the value of wait or a default of 5 minutes.
+    def set_read_timeout(self, timeout):
+        self._read_timeout = timeout/3.0
 
-        if isinstance(params, dict) and 'index' in params:
-            timeout = (float(params['wait'][:-1]) if 'wait' in params else 300) + 1
-        else:
-            timeout = None
-        return callback(self.response(self.session.get(self.uri(path, params), verify=self.verify, timeout=timeout)))
+    def set_ttl(self, ttl):
+        ret = self._ttl != ttl
+        self._ttl = ttl
+        return ret
+
+    @staticmethod
+    def response(response):
+        data = response.data.decode('utf-8')
+        if response.status == 500:
+            raise ConsulInternalError('{0} {1}'.format(response.status, data))
+        return base.Response(response.status, response.headers, data)
+
+    def uri(self, path, params=None):
+        return '{0}{1}{2}'.format(self.base_uri, path, params and '?' + urlencode(params) or '')
+
+    def __getattr__(self, method):
+        if method not in ('get', 'post', 'put', 'delete'):
+            raise AttributeError("HTTPClient instance has no attribute '{0}'".format(method))
+
+        def wrapper(callback, path, params=None, data=''):
+            # python-consul doesn't allow to specify ttl smaller then 10 seconds
+            # because session_ttl_min defaults to 10s, so we have to do this ugly dirty hack...
+            if method == 'put' and path == '/v1/session/create':
+                ttl = '"ttl": "{0}s"'.format(self._ttl)
+                if not data or data == '{}':
+                    data = '{' + ttl + '}'
+                else:
+                    data = data[:-1] + ', ' + ttl + '}'
+            kwargs = {'retries': 0, 'preload_content': False, 'body': data}
+            if method == 'get' and isinstance(params, dict) and 'index' in params:
+                kwargs['timeout'] = (float(params['wait'][:-1]) if 'wait' in params else 300) + 1
+            else:
+                kwargs['timeout'] = self._read_timeout
+            return callback(self.response(self.http.request(method.upper(), self.uri(path, params), **kwargs)))
+        return wrapper
 
 
 class ConsulClient(base.Consul):
@@ -62,7 +87,7 @@ def catch_consul_errors(func):
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
-        except (ConsulException, RequestException):
+        except (RetryFailedError, ConsulException, HTTPException, HTTPError, socket.error, socket.timeout):
             return False
     return wrapper
 
@@ -71,16 +96,24 @@ class Consul(AbstractDCS):
 
     def __init__(self, config):
         super(Consul, self).__init__(config)
-        self._ttl = None
+        self._scope = config['scope']
         self._session = None
+        self.__do_not_watch = False
+        self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
+                            retry_exceptions=(ConsulInternalError, HTTPException,
+                                              HTTPError, socket.error, socket.timeout))
+
         self._my_member_data = None
-        self.set_ttl(config.get('ttl') or 30)
         host, port = config.get('host', '127.0.0.1:8500').split(':')
         self._client = ConsulClient(host=host, port=port)
-        self._client.http.patch_default_timeout(config['retry_timeout']/2.0)
-        self._scope = config['scope']
-        self.create_session()
-        self.__do_not_watch = False
+        self.set_retry_timeout(config['retry_timeout'])
+        self.set_ttl(config.get('ttl') or 30)
+        self._last_session_refresh = 0
+        if not self._ctl:
+            self.create_session()
+
+    def retry(self, *args, **kwargs):
+        return self._retry.copy()(*args, **kwargs)
 
     def create_session(self):
         while not self._session:
@@ -88,34 +121,40 @@ class Consul(AbstractDCS):
                 self.refresh_session()
             except ConsulError:
                 logger.info('waiting on consul')
-                sleep(5)
+                time.sleep(5)
 
     def set_ttl(self, ttl):
-        ttl = ttl/2.0  # My experiments have shown that session expires after 2*ttl time
-        if self._ttl != ttl:
+        if self._client.http.set_ttl(ttl/2.0):  # Consul multiplies the TTL by 2x
             self._session = None
             self.__do_not_watch = True
-        self._ttl = ttl
 
     def set_retry_timeout(self, retry_timeout):
-        self._client.http.patch_default_timeout(retry_timeout/2.0)
+        self._retry.deadline = retry_timeout
+        self._client.http.set_read_timeout(retry_timeout)
 
-    def refresh_session(self):
+    def _do_refresh_session(self):
         """:returns: `!True` if it had to create new session"""
+        if self._session and self._last_session_refresh + self._loop_wait > time.time():
+            return False
+
         if self._session:
             try:
-                return self._client.session.renew(self._session) is None
+                self._client.session.renew(self._session)
             except NotFound:
                 self._session = None
-        if not self._session:
-            name = self._scope + '-' + self._name
-            try:
-                self._session = self._client.session.create(name=name, lock_delay=0, behavior='delete', ttl=self._ttl)
-            except (ConsulException, RequestException):
-                logger.exception('session.create')
-        if not self._session:
-            raise ConsulError('Failed to renew/create session')
-        return True
+        ret = not self._session
+        if ret:
+            self._session = self._client.session.create(name=self._scope + '-' + self._name,
+                                                        lock_delay=0.001, behavior='delete')
+        self._last_session_refresh = time.time()
+        return ret
+
+    def refresh_session(self):
+        try:
+            return self.retry(self._do_refresh_session)
+        except (ConsulException, RetryFailedError):
+            logger.exception('refresh_session')
+        raise ConsulError('Failed to renew/create session')
 
     def client_path(self, path):
         return super(Consul, self).client_path(path)[1:]
@@ -127,7 +166,7 @@ class Consul(AbstractDCS):
     def _load_cluster(self):
         try:
             path = self.client_path('/')
-            _, results = self._client.kv.get(path, recurse=True)
+            _, results = self.retry(self._client.kv.get, path, recurse=True)
 
             if results is None:
                 raise NotFound
@@ -154,7 +193,8 @@ class Consul(AbstractDCS):
 
             # get leader
             leader = nodes.get(self._LEADER)
-            if leader and leader['Value'] == self._name and self._session != leader.get('Session', 'x'):
+            if not self._ctl and leader and leader['Value'] == self._name \
+                    and self._session != leader.get('Session', 'x'):
                 logger.info('I am leader but not owner of the session. Removing leader node')
                 self._client.kv.delete(self.leader_path, cas=leader['ModifyIndex'])
                 leader = None
@@ -169,17 +209,22 @@ class Consul(AbstractDCS):
             if failover:
                 failover = Failover.from_node(failover['ModifyIndex'], failover['Value'])
 
-            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover)
+            # get synchronization state
+            sync = nodes.get(self._SYNC)
+            sync = SyncState.from_node(sync and sync['ModifyIndex'], sync and sync['Value'])
+
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
         except NotFound:
-            self._cluster = Cluster(False, None, None, None, [], None)
+            self._cluster = Cluster(None, None, None, None, [], None, None)
         except:
             logger.exception('get_cluster')
             raise ConsulError('Consul is not responding properly')
 
     def touch_member(self, data, **kwargs):
         cluster = self.cluster
-        member = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
+        member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
         create_member = self.refresh_session()
+
         if member and (create_member or member.session != self._session):
             try:
                 self._client.kv.delete(self.member_path)
@@ -191,7 +236,8 @@ class Consul(AbstractDCS):
             return True
 
         try:
-            self._client.kv.put(self.member_path, data, acquire=self._session)
+            args = {} if kwargs.get('permanent', False) else {'acquire': self._session}
+            self._client.kv.put(self.member_path, data, **args)
             self._my_member_data = data
             return True
         except Exception:
@@ -199,8 +245,12 @@ class Consul(AbstractDCS):
         return False
 
     @catch_consul_errors
-    def attempt_to_acquire_leader(self):
-        ret = self._client.kv.put(self.leader_path, self._name, acquire=self._session)
+    def attempt_to_acquire_leader(self, permanent=False):
+        if not self._session and not permanent:
+            self.refresh_session()
+
+        args = {} if permanent else {'acquire': self._session}
+        ret = self.retry(self._client.kv.put, self.leader_path, self._name, **args)
         if not ret:
             logger.info('Could not take out TTL lock')
         return ret
@@ -217,25 +267,28 @@ class Consul(AbstractDCS):
         return self._client.kv.put(self.config_path, value, cas=index)
 
     @catch_consul_errors
-    def write_leader_optime(self, last_operation):
+    def _write_leader_optime(self, last_operation):
         return self._client.kv.put(self.leader_optime_path, last_operation)
 
-    @staticmethod
-    def update_leader():
-        return True
+    @catch_consul_errors
+    def update_leader(self):
+        if self._session:
+            self.retry(self._client.session.renew, self._session)
+            self._last_session_refresh = time.time()
+        return bool(self._session)
 
     @catch_consul_errors
     def initialize(self, create_new=True, sysid=''):
         kwargs = {'cas': 0} if create_new else {}
-        return self._client.kv.put(self.initialize_path, sysid, **kwargs)
+        return self.retry(self._client.kv.put, self.initialize_path, sysid, **kwargs)
 
     @catch_consul_errors
     def cancel_initialization(self):
-        return self._client.kv.delete(self.initialize_path)
+        return self.retry(self._client.kv.delete, self.initialize_path)
 
     @catch_consul_errors
     def delete_cluster(self):
-        return self._client.kv.delete(self.client_path(''), recurse=True)
+        return self.retry(self._client.kv.delete, self.client_path(''), recurse=True)
 
     @catch_consul_errors
     def delete_leader(self):
@@ -243,24 +296,31 @@ class Consul(AbstractDCS):
         if cluster and isinstance(cluster.leader, Leader) and cluster.leader.name == self._name:
             return self._client.kv.delete(self.leader_path, cas=cluster.leader.index)
 
-    def watch(self, timeout):
+    @catch_consul_errors
+    def set_sync_state_value(self, value, index=None):
+        return self._client.kv.put(self.sync_path, value, cas=index)
+
+    @catch_consul_errors
+    def delete_sync_state(self, index=None):
+        return self._client.kv.delete(self.sync_path, cas=index)
+
+    def watch(self, leader_index, timeout):
         if self.__do_not_watch:
             self.__do_not_watch = False
             return True
 
-        cluster = self.cluster
-        if cluster and cluster.leader and cluster.leader.name != self._name and cluster.leader.index:
+        if leader_index:
             end_time = time.time() + timeout
             while timeout >= 1:
                 try:
-                    idx, _ = self._client.kv.get(self.leader_path, index=cluster.leader.index, wait=str(timeout) + 's')
-                    return str(idx) != str(cluster.leader.index)
-                except (ConsulException, RequestException):
+                    idx, _ = self._client.kv.get(self.leader_path, index=leader_index, wait=str(timeout) + 's')
+                    return str(idx) != str(leader_index)
+                except (ConsulException, HTTPException, HTTPError, socket.error, socket.timeout):
                     logging.exception('watch')
 
                 timeout = end_time - time.time()
 
         try:
-            return super(Consul, self).watch(timeout)
+            return super(Consul, self).watch(None, timeout)
         finally:
             self.event.clear()

patroni/dcs/etcd.py

@@ -2,6 +2,7 @@ from __future__ import absolute_import
 import etcd
 import logging
 import os
+import urllib3.util.connection
 import random
 import requests
 import socket
@@ -9,12 +10,15 @@ import time
 
 from dns.exception import DNSException
 from dns import resolver
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
 from patroni.exceptions import DCSError
-from patroni.utils import Retry, RetryFailedError, sleep
+from patroni.utils import Retry, RetryFailedError
 from urllib3.exceptions import HTTPError, ReadTimeoutError
 from requests.exceptions import RequestException
+from six.moves.queue import Queue
 from six.moves.http_client import HTTPException
+from six.moves.urllib_parse import urlparse
+from threading import Thread
 
 logger = logging.getLogger(__name__)
 
@@ -23,32 +27,122 @@ class EtcdError(DCSError):
     pass
 
 
+class DnsCachingResolver(Thread):
+
+    def __init__(self, cache_time=600.0, cache_fail_time=30.0):
+        super(DnsCachingResolver, self).__init__()
+        self._cache = {}
+        self._cache_time = cache_time
+        self._cache_fail_time = cache_fail_time
+        self._resolve_queue = Queue()
+        self.daemon = True
+        self.start()
+
+    def run(self):
+        while True:
+            (host, port), attempt = self._resolve_queue.get()
+            response = self._do_resolve(host, port)
+            if response:
+                self._cache[(host, port)] = (time.time(), response)
+            else:
+                if attempt < 10:
+                    self.resolve_async(host, port, attempt + 1)
+                    time.sleep(1)
+
+    def resolve(self, host, port):
+        current_time = time.time()
+        cached_time, response = self._cache.get((host, port), (0, []))
+        time_passed = current_time - cached_time
+        if time_passed > self._cache_time or (not response and time_passed > self._cache_fail_time):
+            new_response = self._do_resolve(host, port)
+            if new_response:
+                self._cache[(host, port)] = (current_time, new_response)
+                response = new_response
+        return response
+
+    def resolve_async(self, host, port, attempt=0):
+        self._resolve_queue.put(((host, port), attempt))
+
+    @staticmethod
+    def _do_resolve(host, port):
+        try:
+            return socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, socket.IPPROTO_TCP)
+        except socket.gaierror:
+            logger.warning('failed to resolve host %s', host)
+            return []
+
+
 class Client(etcd.Client):
 
-    def __init__(self, config):
-        super(Client, self).__init__(read_timeout=config['retry_timeout'])
+    def __init__(self, config, dns_resolver, cache_ttl=300):
+        self._dns_resolver = dns_resolver
+        self.set_machines_cache_ttl(cache_ttl)
+        self._machines_cache_updated = 0
+        args = {p: config.get(p) for p in ('host', 'port', 'protocol', 'use_proxies', 'username', 'password',
+                                           'cert', 'ca_cert') if config.get(p)}
+        super(Client, self).__init__(read_timeout=config['retry_timeout'], **args)
         self._config = config
         self._load_machines_cache()
-        self._allow_reconnect = True
+        self._allow_reconnect = not self._use_proxies
+
+    def _build_request_parameters(self):
+        kwargs = {'headers': self._get_headers(), 'redirect': self.allow_redirect}
+
+        # calculate the number of retries and timeout *per node*
+        # actual number of retries depends on the number of nodes
+        etcd_nodes = len(self._machines_cache) + 1
+        kwargs['retries'] = 0 if etcd_nodes > 3 else (1 if etcd_nodes > 1 else 2)
+
+        # if etcd_nodes > 3:
+        #     kwargs.update({'retries': 0, 'timeout': float(self.read_timeout)/etcd_nodes})
+        # elif etcd_nodes > 1:
+        #     kwargs.update({'retries': 1, 'timeout': self.read_timeout/2.0/etcd_nodes})
+        # else:
+        #     kwargs.update({'retries': 2, 'timeout': self.read_timeout/3.0})
+        kwargs['timeout'] = self.read_timeout/float(kwargs['retries'] + 1)/etcd_nodes
+        return kwargs
+
+    def set_machines_cache_ttl(self, cache_ttl):
+        self._machines_cache_ttl = cache_ttl
 
     @property
     def machines(self):
         """Original `machines` method(property) of `etcd.Client` class raise exception
         when it failed to get list of etcd cluster members. This method is being called
         only when request failed on one of the etcd members during `api_execute` call.
-        For us it's more important to execute original request rather then get new
-        topology of etcd cluster. So we will catch this exception and return valid list
-        of machines with setting flag `self._update_machines_cache` to `!True`.
-        Later, during next `api_execute` call we will forcefully update machines_cache"""
-        try:
-            ret = super(Client, self).machines
-            random.shuffle(ret)
-            return ret
-        except etcd.EtcdException:
-            if self._update_machines_cache:  # We are updating machines_cache
-                raise  # This exception is fatal, we should re-raise it.
-            self._update_machines_cache = True
-            return [self._base_uri]
+        For us it's more important to execute original request rather then get new topology
+        of etcd cluster. So we will catch this exception and return empty list of machines.
+        Later, during next `api_execute` call we will forcefully update machines_cache.
+
+        Also this method implements the same timeout-retry logic as `api_execute`, because
+        the original method was retrying 2 times with the `read_timeout` on each node."""
+
+        kwargs = self._build_request_parameters()
+
+        while True:
+            try:
+                response = self.http.request(self._MGET, self._base_uri + self.version_prefix + '/machines', **kwargs)
+                machines = [n.strip() for n in self._handle_server_response(response).data.decode('utf-8').split(',')]
+                logger.debug("Retrieved list of machines: %s", machines)
+                random.shuffle(machines)
+                for url in machines:
+                    r = urlparse(url)
+                    port = r.port or (443 if r.scheme == 'https' else 80)
+                    self._dns_resolver.resolve_async(r.hostname, port)
+                return machines
+            except Exception as e:
+                # We can't get the list of machines, if one server is in the
+                # machines cache, try on it
+                logger.error("Failed to get list of machines from %s%s: %r", self._base_uri, self.version_prefix, e)
+                if self._machines_cache:
+                    self._base_uri = self._machines_cache.pop(0)
+                    logger.info("Retrying on %s", self._base_uri)
+                elif self._update_machines_cache:
+                    raise etcd.EtcdException("Could not get the list of servers, "
+                                             "maybe you provided the wrong "
+                                             "host(s) to connect to?")
+                else:
+                    return []
 
     def set_read_timeout(self, timeout):
         self._read_timeout = timeout
@@ -73,8 +167,7 @@ class Client(etcd.Client):
         if not path.startswith('/'):
             raise ValueError('Path does not start with /')
 
-        kwargs = {'fields': params, 'redirect': self.allow_redirect,
-                  'headers': self._get_headers(), 'preload_content': False}
+        kwargs = {'fields': params, 'preload_content': False}
 
         if method in [self._MGET, self._MDELETE]:
             request_executor = self.http.request
@@ -87,74 +180,84 @@ class Client(etcd.Client):
         # Update machines_cache if previous attempt of update has failed
         if self._update_machines_cache:
             self._load_machines_cache()
+        elif time.time() - self._machines_cache_updated > self._machines_cache_ttl:
+            self._machines_cache = self.machines
+            if self._base_uri in self._machines_cache:
+                self._machines_cache.remove(self._base_uri)
+            self._machines_cache_updated = time.time()
 
-        if timeout is None:
-            # calculate the number of retries and timeout *per node*
-            # actual number of retries depends on the number of nodes
-            etcd_nodes = len(self._machines_cache) + 1
-            kwargs['retries'] = 0 if etcd_nodes > 3 else (1 if etcd_nodes > 1 else 2)
+        kwargs.update(self._build_request_parameters())
 
-            # if etcd_nodes > 3:
-            #     kwargs.update({'retries': 0, 'timeout': float(self.read_timeout)/etcd_nodes})
-            # elif etcd_nodes > 1:
-            #     kwargs.update({'retries': 1, 'timeout': self.read_timeout/2.0/etcd_nodes})
-            # else:
-            #     kwargs.update({'retries': 2, 'timeout': self.read_timeout/3.0})
-            kwargs['timeout'] = self.read_timeout/float(kwargs['retries'] + 1)/etcd_nodes
-        else:
+        if timeout is not None:
             kwargs.update({'retries': 0, 'timeout': timeout})
 
         response = False
 
         try:
+            some_request_failed = False
             while not response:
                 response = self._do_http_request(request_executor, method, self._base_uri + path, **kwargs)
 
-                if response is False and not self._use_proxies:
-                    self._machines_cache = self.machines
+                if response is False:
+                    some_request_failed = True
+            if some_request_failed and not self._use_proxies:
+                self._machines_cache = self.machines
+                if self._base_uri in self._machines_cache:
                     self._machines_cache.remove(self._base_uri)
-            return self._handle_server_response(response)
         except etcd.EtcdConnectionFailed:
             self._update_machines_cache = True
-            raise
+            if not response:
+                raise
+        return self._handle_server_response(response)
 
     @staticmethod
     def get_srv_record(host):
         try:
-            return [(str(r.target).rstrip('.'), r.port) for r in resolver.query('_etcd-server._tcp.' + host, 'SRV')]
+            return [(r.target.to_text(True), r.port) for r in resolver.query(host, 'SRV')]
         except DNSException:
-            logger.exception('Can not resolve SRV for %s', host)
-        return []
+            return []
 
-    def _get_machines_cache_from_srv(self, discovery_srv):
+    def _get_machines_cache_from_srv(self, srv):
         """Fetch list of etcd-cluster member by resolving _etcd-server._tcp. SRV record.
         This record should contain list of host and peer ports which could be used to run
         'GET http://{host}:{port}/members' request (peer protocol)"""
 
         ret = []
-        for host, port in self.get_srv_record(discovery_srv):
-            url = '{0}://{1}:{2}/members'.format(self._protocol, host, port)
-            try:
-                response = requests.get(url, timeout=self.read_timeout)
-                if response.ok:
-                    for member in response.json():
-                        ret.extend(member['clientURLs'])
-                    break
-            except RequestException:
-                logger.exception('GET %s', url)
+        for r in ['-client-ssl', '-client', '-ssl', '', '-server-ssl', '-server']:
+            protocol = 'https' if '-ssl' in r else 'http'
+            endpoint = '/members' if '-server' in r else ''
+            for host, port in self.get_srv_record('_etcd{0}._tcp.{1}'.format(r, srv)):
+                url = '{0}://{1}:{2}{3}'.format(protocol, host, port, endpoint)
+                if endpoint:
+                    try:
+                        response = requests.get(url, timeout=self.read_timeout, verify=False)
+                        if response.ok:
+                            for member in response.json():
+                                ret.extend(member['clientURLs'])
+                            break
+                    except RequestException:
+                        logger.exception('GET %s', url)
+                else:
+                    ret.append(url)
+            if ret:
+                self._protocol = protocol
+                break
+        else:
+            logger.warning('Can not resolve SRV for %s', srv)
         return list(set(ret))
 
-    def _get_machines_cache_from_dns(self, addr):
+    def _get_machines_cache_from_dns(self, host, port):
         """One host might be resolved into multiple ip addresses. We will make list out of it"""
-
-        ret = []
-        host, port = addr.split(':')
-        try:
-            for r in set(socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)):
-                ret.append('{0}://{1}:{2}'.format(self._protocol, r[4][0], r[4][1]))
-        except socket.error:
-            logger.exception('Can not resolve %s', host)
-        return list(set(ret)) if ret else ['{0}://{1}:{2}'.format(self._protocol, host, port)]
+        if self.protocol == 'http':
+            ret = []
+            for af, _, _, _, sa in self._dns_resolver.resolve(host, port):
+                host, port = sa[:2]
+                if af == socket.AF_INET6:
+                    host = '[{0}]'.format(host)
+                ret.append('{0}://{1}:{2}'.format(self.protocol, host, port))
+            if ret:
+                return list(set(ret))
+        return ['{0}://{1}:{2}'.format(self.protocol, host, port)]
 
     def _load_machines_cache(self):
         """This method should fill up `_machines_cache` from scratch.
@@ -164,29 +267,33 @@ class Client(etcd.Client):
 
         self._update_machines_cache = True
 
-        if 'discovery_srv' not in self._config and 'host' not in self._config:
-            raise Exception('Neither discovery_srv nor host are defined in etcd section of config')
+        if 'srv' not in self._config and 'host' not in self._config:
+            raise Exception('Neither srv nor host url are defined in etcd section of config')
 
-        self._machines_cache = []
+        if self._use_proxies:
+            self._machines_cache = ['{0}://{1}:{2}'.format(self.protocol, self._config['host'], self._config['port'])]
+        else:
+            self._machines_cache = []
 
-        if 'discovery_srv' in self._config:
-            self._machines_cache = self._get_machines_cache_from_srv(self._config['discovery_srv'])
+            if 'srv' in self._config:
+                self._machines_cache = self._get_machines_cache_from_srv(self._config['srv'])
 
-        if not self._machines_cache and 'host' in self._config:
-            self._machines_cache = self._get_machines_cache_from_dns(self._config['host'])
+            if not self._machines_cache and 'host' in self._config:
+                self._machines_cache = self._get_machines_cache_from_dns(self._config['host'], self._config['port'])
 
         # Can not bootstrap list of etcd-cluster members, giving up
         if not self._machines_cache:
             raise etcd.EtcdException
 
         # After filling up initial list of machines_cache we should ask etcd-cluster about actual list
-        self._base_uri = self._machines_cache.pop(0)
+        self._base_uri = self._next_server()
         self._machines_cache = self.machines
 
         if self._base_uri in self._machines_cache:
             self._machines_cache.remove(self._base_uri)
 
         self._update_machines_cache = False
+        self._machines_cache_updated = time.time()
 
 
 def catch_etcd_errors(func):
@@ -219,19 +326,79 @@ class Etcd(AbstractDCS):
 
     @staticmethod
     def get_etcd_client(config):
+        if 'proxy' in config:
+            config['use_proxies'] = True
+            config['url'] = config['proxy']
+
+        if 'url' in config:
+            r = urlparse(config['url'])
+            config.update({'protocol': r.scheme, 'host': r.hostname, 'port': r.port or 2379,
+                           'username': r.username, 'password': r.password})
+        elif 'host' in config:
+            host, port = (config['host'] + ':2379').split(':')[:2]
+            config['host'] = host
+            if 'port' not in config:
+                config['port'] = int(port)
+
+        if config.get('cacert'):
+            config['ca_cert'] = config.pop('cacert')
+
+        if config.get('key') and config.get('cert'):
+            config['cert'] = (config['cert'], config['key'])
+
+        for p in ('discovery_srv', 'srv_domain'):
+            if p in config:
+                config['srv'] = config.pop(p)
+
+        dns_resolver = DnsCachingResolver()
+
+        def create_connection_patched(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                                      source_address=None, socket_options=None):
+            host, port = address
+            if host.startswith('['):
+                host = host.strip('[]')
+            err = None
+            for af, socktype, proto, _, sa in dns_resolver.resolve(host, port):
+                sock = None
+                try:
+                    sock = socket.socket(af, socktype, proto)
+                    if socket_options:
+                        for opt in socket_options:
+                            sock.setsockopt(*opt)
+                    if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
+                        sock.settimeout(timeout)
+                    if source_address:
+                        sock.bind(source_address)
+                    sock.connect(sa)
+                    return sock
+
+                except socket.error as e:
+                    err = e
+                    if sock is not None:
+                        sock.close()
+                        sock = None
+
+            if err is not None:
+                raise err
+
+            raise socket.error("getaddrinfo returns an empty list")
+
+        urllib3.util.connection.create_connection = create_connection_patched
+
         client = None
         while not client:
             try:
-                client = Client(config)
+                client = Client(config, dns_resolver)
             except etcd.EtcdException:
                 logger.info('waiting on etcd')
-                sleep(5)
+                time.sleep(5)
         return client
 
     def set_ttl(self, ttl):
         ttl = int(ttl)
         self.__do_not_watch = self._ttl != ttl
         self._ttl = ttl
+        self._client.set_machines_cache_ttl(ttl*10)
 
     def set_retry_timeout(self, retry_timeout):
         self._retry.deadline = retry_timeout
@@ -266,31 +433,40 @@ class Etcd(AbstractDCS):
             if leader:
                 member = Member(-1, leader.value, None, {})
                 member = ([m for m in members if m.name == leader.value] or [member])[0]
-                leader = Leader(leader.modifiedIndex, leader.ttl, member)
+                index = result.etcd_index if result.etcd_index > leader.modifiedIndex else leader.modifiedIndex + 1
+                leader = Leader(index, leader.ttl, member)
 
             # failover key
             failover = nodes.get(self._FAILOVER)
             if failover:
                 failover = Failover.from_node(failover.modifiedIndex, failover.value)
 
-            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover)
+            # get synchronization state
+            sync = nodes.get(self._SYNC)
+            sync = SyncState.from_node(sync and sync.modifiedIndex, sync and sync.value)
+
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
         except etcd.EtcdKeyNotFound:
-            self._cluster = Cluster(False, None, None, None, [], None)
+            self._cluster = Cluster(None, None, None, None, [], None, None)
         except:
             logger.exception('get_cluster')
             raise EtcdError('Etcd is not responding properly')
 
     @catch_etcd_errors
-    def touch_member(self, data, ttl=None):
-        return self.retry(self._client.set, self.member_path, data, ttl or self._ttl)
+    def touch_member(self, data, ttl=None, permanent=False):
+        return self.retry(self._client.set, self.member_path, data, None if permanent else ttl or self._ttl)
 
     @catch_etcd_errors
     def take_leader(self):
         return self.retry(self._client.set, self.leader_path, self._name, self._ttl)
 
-    def attempt_to_acquire_leader(self):
+    def attempt_to_acquire_leader(self, permanent=False):
         try:
-            return bool(self.retry(self._client.write, self.leader_path, self._name, ttl=self._ttl, prevExist=False))
+            return bool(self.retry(self._client.write,
+                                   self.leader_path,
+                                   self._name,
+                                   ttl=None if permanent else self._ttl,
+                                   prevExist=False))
         except etcd.EtcdAlreadyExist:
             logger.info('Could not take out TTL lock')
         except (RetryFailedError, etcd.EtcdException):
@@ -306,7 +482,7 @@ class Etcd(AbstractDCS):
         return self._client.write(self.config_path, value, prevIndex=index or 0)
 
     @catch_etcd_errors
-    def write_leader_optime(self, last_operation):
+    def _write_leader_optime(self, last_operation):
         return self._client.set(self.leader_optime_path, last_operation)
 
     @catch_etcd_errors
@@ -329,19 +505,25 @@ class Etcd(AbstractDCS):
     def delete_cluster(self):
         return self.retry(self._client.delete, self.client_path(''), recursive=True)
 
-    def watch(self, timeout):
+    @catch_etcd_errors
+    def set_sync_state_value(self, value, index=None):
+        return self._client.write(self.sync_path, value, prevIndex=index or 0)
+
+    @catch_etcd_errors
+    def delete_sync_state(self, index=None):
+        return self.retry(self._client.delete, self.sync_path, prevIndex=index or 0)
+
+    def watch(self, leader_index, timeout):
         if self.__do_not_watch:
             self.__do_not_watch = False
             return True
 
-        cluster = self.cluster
-        # watch on leader key changes if it is defined and current node is not lock owner
-        if cluster and cluster.leader and cluster.leader.name != self._name and cluster.leader.index:
+        if leader_index:
             end_time = time.time() + timeout
 
             while timeout >= 1:  # when timeout is too small urllib3 doesn't have enough time to connect
                 try:
-                    self._client.watch(self.leader_path, index=cluster.leader.index + 1, timeout=timeout + 0.5)
+                    self._client.watch(self.leader_path, index=leader_index, timeout=timeout + 0.5)
                     # Synchronous work of all cluster members with etcd is less expensive
                     # than reestablishing http connection every time from every replica.
                     return True
@@ -349,11 +531,11 @@ class Etcd(AbstractDCS):
                     self._client.http.clear()
                     return False
                 except etcd.EtcdException:
-                    logging.exception('watch')
+                    logger.exception('watch')
 
                 timeout = end_time - time.time()
 
         try:
-            return super(Etcd, self).watch(timeout)
+            return super(Etcd, self).watch(None, timeout)
         finally:
             self.event.clear()

patroni/dcs/exhibitor.py

@@ -4,7 +4,6 @@ import requests
 import time
 
 from patroni.dcs.zookeeper import ZooKeeper
-from patroni.utils import sleep
 from requests.exceptions import RequestException
 
 logger = logging.getLogger(__name__)
@@ -24,7 +23,7 @@ class ExhibitorEnsembleProvider(object):
         self._next_poll = None
         while not self.poll():
             logger.info('waiting on exhibitor')
-            sleep(5)
+            time.sleep(5)
 
     def poll(self):
         if self._next_poll and self._next_poll > time.time():

patroni/dcs/zookeeper.py

@@ -3,7 +3,7 @@ import logging
 from kazoo.client import KazooClient, KazooState
 from kazoo.exceptions import NoNodeError, NodeExistsError
 from kazoo.handlers.threading import SequentialThreadingHandler
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
 from patroni.exceptions import DCSError
 
 logger = logging.getLogger(__name__)
@@ -20,7 +20,7 @@ class PatroniSequentialThreadingHandler(SequentialThreadingHandler):
         self.set_connect_timeout(connect_timeout)
 
     def set_connect_timeout(self, connect_timeout):
-        self._connect_timeout = max(1.0, connect_timeout/4.0)
+        self._connect_timeout = max(1.0, connect_timeout/2.0)  # try to connect to zookeeper node during loop_wait/2
 
     def create_connection(self, *args, **kwargs):
         """This method is trying to establish connection with one of the zookeeper nodes.
@@ -57,10 +57,27 @@ class ZooKeeper(AbstractDCS):
 
         self._my_member_data = None
         self._fetch_cluster = True
-        self._last_leader_operation = 0
+
+        self._orig_kazoo_connect = self._client._connection._connect
+        self._client._connection._connect = self._kazoo_connect
 
         self._client.start()
 
+    def _kazoo_connect(self, host, port):
+        """Kazoo is using Ping's to determine health of connection to zookeeper. If there is no
+        response on Ping after Ping interval (1/2 from read_timeout) it will consider current
+        connection dead and try to connect to another node. Without this "magic" it was taking
+        up to 2/3 from session timeout (ttl) to figure out that connection was dead and we had
+        only small time for reconnect and retry.
+
+        This method is needed to return different value of read_timeout, which is not calculated
+        from negotiated session timeout but from value of `loop_wait`. And it is 2 sec smaller
+        than loop_wait, because we can spend up to 2 seconds when calling `touch_member()` and
+        `write_leader_optime()` methods, which also may hang..."""
+
+        ret = self._orig_kazoo_connect(host, port)
+        return max(self.loop_wait - 2, 2)*1000, ret[1]
+
     def session_listener(self, state):
         if state in [KazooState.SUSPENDED, KazooState.LOST]:
             self.cluster_watcher(None)
@@ -69,15 +86,34 @@ class ZooKeeper(AbstractDCS):
         self._fetch_cluster = True
         self.event.set()
 
+    def reload_config(self, config):
+        self.set_retry_timeout(config['retry_timeout'])
+
+        loop_wait = config['loop_wait']
+
+        loop_wait_changed = self._loop_wait != loop_wait
+        self._loop_wait = loop_wait
+        self._client.handler.set_connect_timeout(loop_wait)
+
+        # We need to reestablish connection to zookeeper if we want to change
+        # read_timeout (and Ping interval respectively), because read_timeout
+        # is calculated in `_kazoo_connect` method. If we are changing ttl at
+        # the same time, set_ttl method will reestablish connection and return
+        # `!True`, otherwise we will close existing connection and let kazoo
+        # open the new one.
+        if not self.set_ttl(int(config['ttl'] * 1000)) and loop_wait_changed:
+            self._client._connection._socket.close()
+
     def set_ttl(self, ttl):
-        ttl = int(ttl * 1000)
-        # I know, it's weird to access private attributes
+        """It is not possible to change ttl (session_timeout) in zookeeper without
+        destroying old session and creating the new one. This method returns `!True`
+        if session_timeout has been changed (`restart()` has been called)."""
         if self._client._session_timeout != ttl:
             self._client._session_timeout = ttl
             self._client.restart()
+            return True
 
     def set_retry_timeout(self, retry_timeout):
-        self._client.handler.set_connect_timeout(retry_timeout)
         self._client._retry.deadline = retry_timeout
 
     def get_node(self, key, watch=None):
@@ -97,10 +133,11 @@ class ZooKeeper(AbstractDCS):
         except NoNodeError:
             return []
 
-    def load_members(self):
+    def load_members(self, sync_standby):
         members = []
         for member in self.get_children(self.members_path, self.cluster_watcher):
-            data = self.get_node(self.members_path + member)
+            watch = member == sync_standby and self.cluster_watcher or None
+            data = self.get_node(self.members_path + member, watch)
             if data is not None:
                 members.append(self.member(member, *data))
         return members
@@ -119,14 +156,24 @@ class ZooKeeper(AbstractDCS):
         config = self.get_node(self.config_path, watch=self.cluster_watcher) if self._CONFIG in nodes else None
         config = config and ClusterConfig.from_node(config[1].version, config[0], config[1].mzxid)
 
+        # get last leader operation
+        last_leader_operation = self._OPTIME in nodes and self._fetch_cluster and self.get_node(self.leader_optime_path)
+        last_leader_operation = last_leader_operation and int(last_leader_operation[0]) or 0
+
+        # get synchronization state
+        sync = self.get_node(self.sync_path, watch=self.cluster_watcher) if self._SYNC in nodes else None
+        sync = SyncState.from_node(sync and sync[1].version, sync and sync[0])
+
         # get list of members
-        members = self.load_members() if self._MEMBERS[:-1] in nodes else []
+        sync_standby = sync.leader == self._name and sync.sync_standby or None
+        members = self.load_members(sync_standby) if self._MEMBERS[:-1] in nodes else []
 
         # get leader
         leader = self.get_node(self.leader_path) if self._LEADER in nodes else None
         if leader:
             client_id = self._client.client_id
-            if leader[0] == self._name and client_id is not None and client_id[0] != leader[1].ephemeralOwner:
+            if not self._ctl and leader[0] == self._name and client_id is not None \
+                    and client_id[0] != leader[1].ephemeralOwner:
                 logger.info('I am leader but not owner of the session. Removing leader node')
                 self._client.delete(self.leader_path)
                 leader = None
@@ -141,16 +188,13 @@ class ZooKeeper(AbstractDCS):
         failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
         failover = failover and Failover.from_node(failover[1].version, failover[0])
 
-        # get last leader operation
-        optime = self.get_node(self.leader_optime_path) if self._OPTIME in nodes and self._fetch_cluster else None
-        self._last_leader_operation = 0 if optime is None else int(optime[0])
-        self._cluster = Cluster(initialize, config, leader, self._last_leader_operation, members, failover)
+        self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
 
     def _load_cluster(self):
         if self._fetch_cluster or self._cluster is None:
             try:
                 self._client.retry(self._inner_load_cluster)
-            except:
+            except Exception:
                 logger.exception('get_cluster')
                 self.cluster_watcher(None)
                 raise ZooKeeperError('ZooKeeper in not responding properly')
@@ -162,8 +206,8 @@ class ZooKeeper(AbstractDCS):
         except:
             return False
 
-    def attempt_to_acquire_leader(self):
-        ret = self._create(self.leader_path, self._name, makepath=True, ephemeral=True)
+    def attempt_to_acquire_leader(self, permanent=False):
+        ret = self._create(self.leader_path, self._name, makepath=True, ephemeral=not permanent)
         if not ret:
             logger.info('Could not take out TTL lock')
         return ret
@@ -190,60 +234,59 @@ class ZooKeeper(AbstractDCS):
 
     def initialize(self, create_new=True, sysid=""):
         return self._create(self.initialize_path, sysid, makepath=True) if create_new \
-            else self._client.retry(self._client.set, self.initialize_path,  sysid.encode("utf-8"))
+            else self._client.retry(self._client.set, self.initialize_path, sysid.encode("utf-8"))
 
-    def touch_member(self, data, ttl=None):
+    def touch_member(self, data, ttl=None, permanent=False):
         cluster = self.cluster
-        member = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
-        path = self.member_path
+        member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
         data = data.encode('utf-8')
         if member and self._client.client_id is not None and member.session != self._client.client_id[0]:
             try:
-                self._client.retry(self._client.delete, path)
+                self._client.delete_async(self.member_path).get(timeout=1)
             except NoNodeError:
                 pass
             except:
                 return False
             member = None
 
-        if member and data == self._my_member_data:
-            return True
-
-        try:
-            if member:
-                self._client.retry(self._client.set, path, data)
-            else:
-                self._client.retry(self._client.create, path, data, makepath=True, ephemeral=True)
-            self._my_member_data = data
-            return True
-        except NodeExistsError:
+        if member:
+            if data == self._my_member_data:
+                return True
+        else:
             try:
-                self._client.retry(self._client.set, path, data)
+                self._client.create_async(self.member_path, data, makepath=True, ephemeral=not permanent).get(timeout=1)
                 self._my_member_data = data
                 return True
-            except:
-                logger.exception('touch_member')
+            except Exception as e:
+                if not isinstance(e, NodeExistsError):
+                    logger.exception('touch_member')
+                    return False
+        try:
+            self._client.set_async(self.member_path, data).get(timeout=1)
+            self._my_member_data = data
+            return True
         except:
             logger.exception('touch_member')
+
         return False
 
     def take_leader(self):
         return self.attempt_to_acquire_leader()
 
-    def write_leader_optime(self, last_operation):
+    def _write_leader_optime(self, last_operation):
         last_operation = last_operation.encode('utf-8')
-        if last_operation != self._last_leader_operation:
-            self._last_leader_operation = last_operation
-            path = self.leader_optime_path
+        try:
+            self._client.set_async(self.leader_optime_path, last_operation).get(timeout=1)
+            return True
+        except NoNodeError:
             try:
-                self._client.retry(self._client.set, path, last_operation)
-            except NoNodeError:
-                try:
-                    self._client.retry(self._client.create, path, last_operation, makepath=True)
-                except:
-                    logger.exception('Failed to create %s', path)
+                self._client.create_async(self.leader_optime_path, last_operation, makepath=True).get(timeout=1)
+                return True
             except:
-                logger.exception('Failed to update %s', path)
+                logger.exception('Failed to create %s', self.leader_optime_path)
+        except:
+            logger.exception('Failed to update %s', self.leader_optime_path)
+        return False
 
     def update_leader(self):
         return True
@@ -270,7 +313,20 @@ class ZooKeeper(AbstractDCS):
         except NoNodeError:
             return True
 
-    def watch(self, timeout):
-        if super(ZooKeeper, self).watch(timeout):
+    def set_sync_state_value(self, value, index=None):
+        try:
+            self._client.retry(self._client.set, self.sync_path, value.encode('utf-8'), version=index or -1)
+            return True
+        except NoNodeError:
+            return value == '' or (index is None and self._create(self.sync_path, value))
+        except:
+            logging.exception('set_sync_state_value')
+            return False
+
+    def delete_sync_state(self, index=None):
+        return self.set_sync_state_value("{}", index)
+
+    def watch(self, leader_index, timeout):
+        if super(ZooKeeper, self).watch(leader_index, timeout):
             self._fetch_cluster = True
         return self._fetch_cluster

patroni/scripts/aws.py

@@ -72,5 +72,6 @@ def main():
     else:
         sys.exit("Usage: {0} action role name".format(sys.argv[0]))
 
+
 if __name__ == '__main__':
     main()

patroni/scripts/wale_restore.py

@@ -30,18 +30,28 @@ import os
 import psycopg2
 import subprocess
 import sys
+import time
 import argparse
 
-
-if sys.hexversion >= 0x0300000:
-    long = int
-
 logger = logging.getLogger(__name__)
 
+RETRY_SLEEP_INTERVAL = 1
+
+
+# We need to know the current PG version in order to figure out the correct WAL directory name
+def get_major_version(data_dir):
+    version_file = os.path.join(data_dir, 'PG_VERSION')
+    if os.path.isfile(version_file):  # version file exists
+        try:
+            with open(version_file) as f:
+                return float(f.read())
+        except Exception:
+            logger.exception('Failed to read PG_VERSION from %s', data_dir)
+    return 0.0
+
 
 class WALERestore(object):
-
-    def __init__(self, scope, datadir, connstring, env_dir, threshold_mb, threshold_pct, use_iam, no_master):
+    def __init__(self, scope, datadir, connstring, env_dir, threshold_mb, threshold_pct, use_iam, no_master, retries):
         self.scope = scope
         self.master_connection = connstring
         self.data_dir = datadir
@@ -53,11 +63,19 @@ class WALERestore(object):
         self.no_master = no_master
         self.wal_e.cmd = 'envdir {0} wal-e {1} '.format(self.wal_e.dir, self.wal_e.iam_string)
         self.init_error = (not os.path.exists(self.wal_e.dir))
+        self.retries = retries
 
     def run(self):
         """ creates a new replica using WAL-E """
-        if not self.init_error and self.should_use_s3_to_create_replica():
-            return self.create_replica_with_s3()
+        if not self.init_error:
+            try:
+                ret = self.should_use_s3_to_create_replica()
+                if ret:
+                    return self.create_replica_with_s3()
+                elif ret is None:  # caught an exception, need to retry
+                    return 1
+            except Exception:
+                logger.exception("Exception when running WAL-E restore")
         return 2
 
     def should_use_s3_to_create_replica(self):
@@ -73,7 +91,7 @@ class WALERestore(object):
             # base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z
             # 20310671    00000001000000000000007F    00000040
             # 00000001000000000000007F    00000240
-            backup_strings = latest_backup.splitlines() if latest_backup else ()
+            backup_strings = latest_backup.decode('utf-8').splitlines() if latest_backup else ()
             if len(backup_strings) != 2:
                 return False
 
@@ -83,17 +101,17 @@ class WALERestore(object):
                 return False
 
             backup_info = dict(zip(names, vals))
-        except subprocess.CalledProcessError as e:
-            logger.error("could not query wal-e latest backup: {}".format(e))
-            return False
+        except subprocess.CalledProcessError:
+            logger.exception("could not query wal-e latest backup")
+            return None
 
         try:
             backup_size = backup_info['expanded_size_bytes']
             backup_start_segment = backup_info['wal_segment_backup_start']
             backup_start_offset = backup_info['wal_segment_offset_backup_start']
-        except Exception as e:
-            logger.error("unable to get some of WALE backup parameters: {}".format(e))
-            return False
+        except Exception:
+            logger.exception("unable to get some of WALE backup parameters")
+            return None
 
         # WAL filename is XXXXXXXXYYYYYYYY000000ZZ, where X - timeline, Y - LSN logical log file,
         # ZZ - 2 high digits of LSN offset. The rest of the offset is the provided decimal offset,
@@ -101,31 +119,62 @@ class WALERestore(object):
 
         lsn_segment = backup_start_segment[8:16]
         # first 2 characters of the result are 0x and the last one is L
-        lsn_offset = hex((long(backup_start_segment[16:32], 16) << 24) + long(backup_start_offset))[2:-1]
+        lsn_offset = hex((int(backup_start_segment[16:32], 16) << 24) + int(backup_start_offset))[2:-1]
 
         # construct the LSN from the segment and offset
         backup_start_lsn = '{0}/{1}'.format(lsn_segment, lsn_offset)
 
-        diff_in_bytes = long(backup_size)
-        if not self.no_master:
-            try:
-                # get the difference in bytes between the current WAL location and the backup start offset
-                with psycopg2.connect(self.master_connection) as con:
-                    con.autocommit = True
-                    with con.cursor() as cur:
-                        cur.execute("SELECT pg_xlog_location_diff(pg_current_xlog_location(), %s)", (backup_start_lsn,))
-                        diff_in_bytes = long(cur.fetchone()[0])
-            except psycopg2.Error as e:
-                logger.error('could not determine difference with the master location: %s', e)
-                return False
-        else:
-            # always try to use WAL-E if base backup is available
-            diff_in_bytes = 0
+        diff_in_bytes = int(backup_size)
+        attempts_no = 0
+        while True:
+            if self.master_connection:
+                try:
+                    # get the difference in bytes between the current WAL location and the backup start offset
+                    with psycopg2.connect(self.master_connection) as con:
+                        con.autocommit = True
+                        with con.cursor() as cur:
+                            cur.execute("SELECT pg_xlog_location_diff(pg_current_xlog_location(), %s)",
+                                        (backup_start_lsn,))
+                            diff_in_bytes = int(cur.fetchone()[0])
+                except psycopg2.Error:
+                    logger.exception('could not determine difference with the master location')
+                    if attempts_no < self.retries:  # retry in case of a temporarily connection issue
+                        attempts_no = attempts_no + 1
+                        time.sleep(RETRY_SLEEP_INTERVAL)
+                        continue
+                    else:
+                        if not self.no_master:
+                            return False  # do no more retries on the outer level
+                        logger.info("continue with base backup from S3 since master is not available")
+                        diff_in_bytes = 0
+                        break
+            else:
+                # always try to use WAL-E if master connection string is not available
+                diff_in_bytes = 0
+            break
 
         # if the size of the accumulated WAL segments is more than a certan percentage of the backup size
         # or exceeds the pre-determined size - pg_basebackup is chosen instead.
-        return (diff_in_bytes < long(threshold_megabytes) * 1048576) and\
-            (diff_in_bytes < long(backup_size) * float(threshold_backup_size_percentage) / 100)
+        return (diff_in_bytes < int(threshold_megabytes) * 1048576) and\
+            (diff_in_bytes < int(backup_size) * float(threshold_backup_size_percentage) / 100)
+
+    def fix_subdirectory_path_if_broken(self, dirname):
+        # in case it is a symlink pointing to a non-existing location, remove it and create the actual directory
+        path = os.path.join(self.data_dir, dirname)
+        if not os.path.exists(path):
+            if os.path.islink(path):  # broken xlog symlink, to remove
+                try:
+                    os.remove(path)
+                except OSError:
+                    logger.exception("could not remove broken %s symlink pointing to %s",
+                                     dirname, os.readlink(path))
+                    return False
+            try:
+                os.mkdir(path)
+            except OSError:
+                logger.exception("coud not create missing %s directory path", dirname)
+                return False
+        return True
 
     def create_replica_with_s3(self):
         # if we're set up, restore the replica using fetch latest
@@ -135,6 +184,9 @@ class WALERestore(object):
             logger.error('Error when fetching backup with WAL-E: {0}'.format(e))
             return 1
 
+        if (ret == 0 and not
+           self.fix_subdirectory_path_if_broken('pg_xlog' if get_major_version(self.data_dir) < 10.0 else 'pg_wal')):
+            return 2
         return ret
 
 
@@ -153,17 +205,23 @@ def main():
     parser.add_argument('--no_master', type=int, default=0)
     args = parser.parse_args()
 
-    # retry cloning in a loop
+    # Retry cloning in a loop. We do separate retries for the master
+    # connection attempt inside should_use_s3_to_create_replica,
+    # because we need to differentiate between the last attempt and
+    # the rest and make a decision when the last attempt fails on
+    # whether to use WAL-E or not depending on the no_master flag.
     for _ in range(0, args.retries + 1):
         restore = WALERestore(scope=args.scope, datadir=args.datadir, connstring=args.connstring,
                               env_dir=args.envdir, threshold_mb=args.threshold_megabytes,
                               threshold_pct=args.threshold_backup_size_percentage, use_iam=args.use_iam,
-                              no_master=args.no_master)
+                              no_master=args.no_master, retries=args.retries)
         ret = restore.run()
-        if ret == 0:
+        if ret != 1:  # only WAL-E failures lead to the retry
             break
+        time.sleep(RETRY_SLEEP_INTERVAL)
+
+    return ret
 
-    sys.exit(ret)
 
 if __name__ == '__main__':
-    main()
+    sys.exit(main())

patroni/utils.py

@@ -1,15 +1,11 @@
-import os
 import random
-import sys
 import time
+import re
 
+from dateutil import tz
 from patroni.exceptions import PatroniException
 
-if sys.hexversion >= 0x0300000:
-    long = int
-
-__interrupted_sleep = False
-__reap_children = False
+tzutc = tz.tzutc()
 
 
 def deep_compare(obj1, obj2):
@@ -121,7 +117,7 @@ def strtol(value, strict=True):
         while i <= l:
             try:  # try to find maximally long number
                 i += 1  # by giving to `int` longer and longer strings
-                ret = long(value[:i], base)
+                ret = int(value[:i], base)
             except ValueError:  # until we will not get an exception or end of the string
                 i -= 1
                 break
@@ -194,36 +190,12 @@ def compare_values(vartype, unit, old_value, new_value):
     return old_value is not None and new_value is not None and old_value == new_value
 
 
-def sigchld_handler(signo, stack_frame):
-    global __interrupted_sleep, __reap_children
-    __reap_children = __interrupted_sleep = True
+def _sleep(interval):
+    time.sleep(interval)
 
 
-def sleep(interval):
-    global __interrupted_sleep
-    current_time = time.time()
-    end_time = current_time + interval
-    while current_time < end_time:
-        __interrupted_sleep = False
-        time.sleep(end_time - current_time)
-        if not __interrupted_sleep:  # we will ignore only sigchld
-            break
-        current_time = time.time()
-    __interrupted_sleep = False
-
-
-def reap_children():
-    global __reap_children
-    if __reap_children:
-        try:
-            while True:
-                ret = os.waitpid(-1, os.WNOHANG)
-                if ret == (0, 0):
-                    break
-        except OSError:
-            pass
-        finally:
-            __reap_children = False
+def is_valid_pg_version(version):
+    return re.match(r'[1-9][0-9]?(\.(0|([1-9][0-9]?))){2}$', version)
 
 
 class RetryFailedError(PatroniException):
@@ -236,7 +208,7 @@ class Retry(object):
     """Helper for retrying a method in the face of retry-able exceptions"""
 
     def __init__(self, max_tries=1, delay=0.1, backoff=2, max_jitter=0.8, max_delay=3600,
-                 sleep_func=sleep, deadline=None, retry_exceptions=PatroniException):
+                 sleep_func=_sleep, deadline=None, retry_exceptions=PatroniException):
         """Create a :class:`Retry` instance for retrying function calls
 
         :param max_tries: How many times to retry the command. -1 means infinite tries.
@@ -297,3 +269,14 @@ class Retry(object):
                 else:
                     self.sleep_func(sleeptime)
                 self._cur_delay = min(self._cur_delay * self.backoff, self.max_delay)
+
+
+def polling_loop(timeout, interval=1):
+    """Returns an iterator that returns values until timeout has passed. Timeout is measured from start of iteration."""
+    start_time = time.time()
+    iteration = 0
+    end_time = start_time + timeout
+    while time.time() < end_time:
+        yield iteration
+        iteration += 1
+        time.sleep(interval)

patroni/version.py

@@ -1 +1 @@
-__version__ = '1.0'
+__version__ = '1.2.3'

postgres0.yml

@@ -4,13 +4,15 @@ name: postgresql0
 
 restapi:
   listen: 127.0.0.1:8008
+  connect_address: 127.0.0.1:8008
+#  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
+#  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
 #  authentication:
 #    username: username
 #    password: password
-  connect_address: 127.0.0.1:8008
 
 etcd:
-  host: 127.0.0.1:4001
+  host: 127.0.0.1:2379
 
 bootstrap:
   # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
@@ -20,6 +22,8 @@ bootstrap:
     loop_wait: 10
     retry_timeout: 10
     maximum_lag_on_failover: 1048576
+#    master_start_timeout: 300
+#    synchronous_mode: false
     postgresql:
       use_pg_rewind: true
 #      use_slots: true
@@ -30,11 +34,11 @@ bootstrap:
 #        max_wal_senders: 5
 #        max_replication_slots: 5
 #        wal_log_hints: "on"
-        archive_mode: "on"
-        archive_timeout: 1800s
-        archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
-      recovery_conf:
-        restore_command: cp ../wal_archive/%f %p
+#        archive_mode: "on"
+#        archive_timeout: 1800s
+#        archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
+#      recovery_conf:
+#        restore_command: cp ../wal_archive/%f %p
 
   # some desired options for 'initdb'
   initdb:  # Note: It needs to be a list (some options need values, others are switches)
@@ -46,6 +50,9 @@ bootstrap:
   - host all all 0.0.0.0/0 md5
 #  - hostssl all all 0.0.0.0/0 md5
 
+  # Additional script to be launched after initial cluster creation (will be passed the connection URL as parameter)
+# post_init: /usr/local/bin/setup_cluster.sh
+
   # Some additional users users which needs to be created after initializing new cluster
   users:
     admin:
@@ -58,6 +65,7 @@ postgresql:
   listen: 127.0.0.1:5432
   connect_address: 127.0.0.1:5432
   data_dir: data/postgresql0
+#  bin_dir:
   pgpass: /tmp/pgpass0
   authentication:
     replication:
@@ -72,3 +80,4 @@ tags:
     nofailover: false
     noloadbalance: false
     clonefrom: false
+    nosync: false

postgres1.yml

@@ -1,105 +1,80 @@
-ttl: &ttl 30
-loop_wait: &loop_wait 10
-scope: &scope batman
+scope: batman
+#namespace: /service/
+name: postgresql1
+
 restapi:
   listen: 127.0.0.1:8009
   connect_address: 127.0.0.1:8009
-#  auth: 'username:password'
 #  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
 #  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
+#  authentication:
+#    username: username
+#    password: password
+
 etcd:
-  scope: *scope
-  ttl: *ttl
-  host: 127.0.0.1:4001
-  #discovery_srv: my-etcd.domain
-#consul:
-#  scope: *scope
-#  ttl: *ttl
-#  host: 127.0.0.1:8500
-#zookeeper:
-#  scope: *scope
-#  session_timeout: *ttl
-#  reconnect_timeout: *loop_wait
-#  hosts:
-#    - 127.0.0.1:2181
-#    - 127.0.0.2:2181
-#exhibitor:
-#  poll_interval: 300
-#  port: 8181
-#  hosts:
-#    - host1
-#    - host2
-#    - host3
+  host: 127.0.0.1:2379
+
+bootstrap:
+  # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
+  # and all other cluster members will use it as a `global configuration`
+  dcs:
+    ttl: 30
+    loop_wait: 10
+    retry_timeout: 10
+    maximum_lag_on_failover: 1048576
+    postgresql:
+      use_pg_rewind: true
+#      use_slots: true
+      parameters:
+#        wal_level: hot_standby
+#        hot_standby: "on"
+#        wal_keep_segments: 8
+#        max_wal_senders: 5
+#        max_replication_slots: 5
+#        wal_log_hints: "on"
+#        archive_mode: "on"
+#        archive_timeout: 1800s
+#        archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
+#      recovery_conf:
+#        restore_command: cp ../wal_archive/%f %p
+
+  # some desired options for 'initdb'
+  initdb:  # Note: It needs to be a list (some options need values, others are switches)
+  - encoding: UTF8
+  - data-checksums
+
+  pg_hba:  # Add following lines to pg_hba.conf after running 'initdb'
+  - host replication replicator 127.0.0.1/32 md5
+  - host all all 0.0.0.0/0 md5
+#  - hostssl all all 0.0.0.0/0 md5
+
+  # Additional script to be launched after initial cluster creation (will be passed the connection URL as parameter)
+# post_init: /usr/local/bin/setup_cluster.sh
+
+  # Some additional users users which needs to be created after initializing new cluster
+  users:
+    admin:
+      password: admin
+      options:
+        - createrole
+        - createdb
+
 postgresql:
-  name: postgresql1
-  scope: *scope
   listen: 127.0.0.1:5433
   connect_address: 127.0.0.1:5433
   data_dir: data/postgresql1
-  maximum_lag_on_failover: 1048576 # 1 megabyte in bytes
-  use_slots: True
+#  bin_dir:
   pgpass: /tmp/pgpass1
-  initdb:  ## We allow the following options to be passed on to initdb
-  # - auth: authmethod
-  # - auth-host: authmethod
-  # - auth-local: authmethod
-  - encoding: UTF8
-  # - data-checksums  # When pg_rewind is needed on 9.3, this needs to be enabled
-  # - locale: locale
-  # - lc-collate: locale
-  # - lc-ctype: locale
-  # - lc-messages: locale
-  # - lc-monetary: locale
-  # - lc-numeric: locale
-  # - lc-time: locale
-  # - text-search-config: CFG
-  # - xlogdir: directory
-  # - debug
-  # - noclean
-  pg_rewind:
-    username: postgres
-    password: zalando
-  pg_hba:
-  - host replication replicator 127.0.0.1/32 md5
-  - host all all 0.0.0.0/0 md5
-  # - hostssl all all 0.0.0.0/0 md5
-  replication:
-    username: replicator
-    password: rep-pass
-  superuser:
-    username: postgres
-    password: zalando
-  admin:
-    username: admin
-    password: admin
-# commented-out example for wal-e provisioning
-  create_replica_method:
-    - basebackup
-#   - wal_e
-# commented-out example for wal-e provisioning
-  #wal_e:
-    #command: /patroni/scripts/wale_restore.py
-    #env_dir: /home/postgres/etc/wal-e.d/env
-    #threshold_megabytes: 10240
-    #threshold_backup_size_percentage: 30
-    #retries: 2
-    #use_iam: 1
-  #recovery_conf:
-    #restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1
-  recovery_conf:
-    restore_command: cp ../wal_archive/%f %p
+  authentication:
+    replication:
+      username: replicator
+      password: rep-pass
+    superuser:
+      username: postgres
+      password: zalando
   parameters:
-    archive_mode: "on"
-    wal_level: hot_standby
-    archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
-    max_wal_senders: 10
-    wal_keep_segments: 8
-    archive_timeout: 1800s
-    max_replication_slots: 10
-    hot_standby: "on"
-    wal_log_hints: "on"
     unix_socket_directories: '.'
 tags:
-    nofailover: False
-    noloadbalance: False
-    clonefrom: False
+    nofailover: false
+    noloadbalance: false
+    clonefrom: false

postgres2.yml

@@ -1,106 +1,78 @@
-ttl: &ttl 30
-loop_wait: &loop_wait 10
-scope: &scope batman
+scope: batman
+#namespace: /service/
+name: postgresql2
+
 restapi:
   listen: 127.0.0.1:8010
   connect_address: 127.0.0.1:8010
-  auth: 'username:password'
 #  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
 #  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
+  authentication:
+    username: username
+    password: password
+
 etcd:
-  scope: *scope
-  ttl: *ttl
-  host: 127.0.0.1:4001
-  #discovery_srv: my-etcd.domain
-#consul:
-#  scope: *scope
-#  ttl: *ttl
-#  host: 127.0.0.1:8500
-#zookeeper:
-#  scope: *scope
-#  session_timeout: *ttl
-#  reconnect_timeout: *loop_wait
-#  hosts:
-#    - 127.0.0.1:2181
-#    - 127.0.0.2:2181
-#exhibitor:
-#  poll_interval: 300
-#  port: 8181
-#  hosts:
-#    - host1
-#    - host2
-#    - host3
+  host: 127.0.0.1:2379
+
+bootstrap:
+  # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
+  # and all other cluster members will use it as a `global configuration`
+  dcs:
+    ttl: 30
+    loop_wait: 10
+    retry_timeout: 10
+    maximum_lag_on_failover: 1048576
+    postgresql:
+      use_pg_rewind: true
+#      use_slots: true
+      parameters:
+#        wal_level: hot_standby
+#        hot_standby: "on"
+#        wal_keep_segments: 8
+#        max_wal_senders: 5
+#        max_replication_slots: 5
+#        wal_log_hints: "on"
+#        archive_mode: "on"
+#        archive_timeout: 1800s
+#        archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
+#      recovery_conf:
+#        restore_command: cp ../wal_archive/%f %p
+
+  # some desired options for 'initdb'
+  initdb:  # Note: It needs to be a list (some options need values, others are switches)
+  - encoding: UTF8
+  - data-checksums
+
+  pg_hba:  # Add following lines to pg_hba.conf after running 'initdb'
+  - host replication replicator 127.0.0.1/32 md5
+  - host all all 0.0.0.0/0 md5
+#  - hostssl all all 0.0.0.0/0 md5
+
+  # Some additional users users which needs to be created after initializing new cluster
+  users:
+    admin:
+      password: admin
+      options:
+        - createrole
+        - createdb
+
 postgresql:
-  name: postgresql2
-  scope: *scope
   listen: 127.0.0.1:5434
   connect_address: 127.0.0.1:5434
   data_dir: data/postgresql2
-  maximum_lag_on_failover: 1048576 # 1 megabyte in bytes
-  use_slots: True
+#  bin_dir:
   pgpass: /tmp/pgpass2
-  initdb:  ## We allow the following options to be passed on to initdb
-  # - auth: authmethod
-  # - auth-host: authmethod
-  # - auth-local: authmethod
-  - encoding: UTF8
-  # - data-checksums  # When pg_rewind is needed on 9.3, this needs to be enabled
-  # - locale: locale
-  # - lc-collate: locale
-  # - lc-ctype: locale
-  # - lc-messages: locale
-  # - lc-monetary: locale
-  # - lc-numeric: locale
-  # - lc-time: locale
-  # - text-search-config: CFG
-  # - xlogdir: directory
-  # - debug
-  # - noclean
-  pg_rewind:
-    username: postgres
-    password: zalando
-  pg_hba:
-  - host replication replicator 127.0.0.1/32 md5
-  - host all all 0.0.0.0/0 md5
-  # - hostssl all all 0.0.0.0/0 md5
-  replication:
-    username: replicator
-    password: rep-pass
-  superuser:
-    username: postgres
-    password: zalando
-  admin:
-    username: admin
-    password: admin
-# commented-out example for wal-e provisioning
-  create_replica_method:
-    - basebackup
-#   - wal_e
-# commented-out example for wal-e provisioning
-  #wal_e:
-    #command: /patroni/scripts/wale_restore.py
-    #env_dir: /home/postgres/etc/wal-e.d/env
-    #threshold_megabytes: 10240
-    #threshold_backup_size_percentage: 30
-    #retries: 2
-    #use_iam: 1
-  #recovery_conf:
-    #restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1
-  recovery_conf:
-    restore_command: cp ../wal_archive/%f %p
+  authentication:
+    replication:
+      username: replicator
+      password: rep-pass
+    superuser:
+      username: postgres
+      password: zalando
   parameters:
-    archive_mode: "on"
-    wal_level: hot_standby
-    archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
-    max_wal_senders: 10
-    wal_keep_segments: 8
-    archive_timeout: 1800s
-    max_replication_slots: 10
-    hot_standby: "on"
-    wal_log_hints: "on"
     unix_socket_directories: '.'
 tags:
-    nofailover: False
-    noloadbalance: False
-    clonefrom: False
-    replicatefrom: postgresql1
+    nofailover: false
+    noloadbalance: false
+    clonefrom: false
+    replicatefrom: postgres1

requirements.txt

@@ -1,3 +1,4 @@
+urllib3>=1.9
 boto
 psycopg2>=2.6.1
 PyYAML
@@ -5,7 +6,7 @@ requests
 six >= 1.7
 kazoo==2.2.1
 python-etcd==0.4.3
-python-consul==0.6.0
+python-consul==0.7.0
 click>=4.1
 prettytable>=0.7
 tzlocal

setup.py

@@ -24,6 +24,7 @@ def read_version(package):
         exec(fd.read(), data)
     return data['__version__']
 
+
 NAME = 'patroni'
 MAIN_PACKAGE = NAME
 SCRIPTS = 'scripts'

tests/test_api.py

@@ -1,3 +1,4 @@
+import datetime
 import json
 import psycopg2
 import unittest
@@ -5,11 +6,16 @@ import unittest
 from mock import Mock, patch
 from patroni.api import RestApiHandler, RestApiServer
 from patroni.dcs import ClusterConfig, Member
+from patroni.utils import tzutc
 from six import BytesIO as IO
 from six.moves import BaseHTTPServer
 from test_postgresql import psycopg2_connect, MockCursor
 
 
+future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5)
+postmaster_start_time = datetime.datetime.now(tzutc)
+
+
 class MockPostgresql(object):
 
     name = 'test'
@@ -24,14 +30,17 @@ class MockPostgresql(object):
     def connection():
         return psycopg2_connect()
 
+    @staticmethod
+    def postmaster_start_time():
+        return str(postmaster_start_time)
+
 
 class MockHa(object):
 
-    dcs = Mock()
     state_handler = MockPostgresql()
 
     @staticmethod
-    def schedule_reinitialize():
+    def reinitialize():
         return 'reinitialize'
 
     @staticmethod
@@ -42,21 +51,42 @@ class MockHa(object):
     def restart_scheduled():
         return False
 
+    @staticmethod
+    def delete_future_restart():
+        return True
+
     @staticmethod
     def fetch_nodes_statuses(members):
         return [[None, True, None, None, {}]]
 
+    @staticmethod
+    def schedule_future_restart(data):
+        return True
+
+    @staticmethod
+    def is_lagging(xlog):
+        return False
+
+    @staticmethod
+    def get_effective_tags():
+        return {'nosync': True}
+
+    @staticmethod
+    def wakeup():
+        pass
+
 
 class MockPatroni(object):
 
-    nap_time = 10
-    config = Mock()
-    postgresql = MockPostgresql()
     ha = MockHa()
+    config = Mock()
+    postgresql = ha.state_handler
     dcs = Mock()
     tags = {}
     version = '0.00'
     noloadbalance = Mock(return_value=False)
+    scheduled_restart = {'schedule': future_restart_time,
+                         'postmaster_start_time': postgresql.postmaster_start_time()}
 
     @staticmethod
     def sighup_handler():
@@ -118,14 +148,14 @@ class TestRestApiHandler(unittest.TestCase):
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0'))
         MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0\nAuthorization:')
 
-    @patch.object(MockHa, 'dcs')
+    @patch.object(MockPatroni, 'dcs')
     def test_do_GET_config(self, mock_dcs):
         mock_dcs.cluster.config.data = {}
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /config'))
         mock_dcs.cluster.config = None
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /config'))
 
-    @patch.object(MockHa, 'dcs')
+    @patch.object(MockPatroni, 'dcs')
     def test_do_PATCH_config(self, mock_dcs):
         config = {'postgresql': {'use_slots': False, 'use_pg_rewind': True, 'parameters': {'wal_level': 'logical'}}}
         mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, json.dumps(config))
@@ -141,7 +171,7 @@ class TestRestApiHandler(unittest.TestCase):
         mock_dcs.set_config_value.return_value = False
         MockRestApiServer(RestApiHandler, request)
 
-    @patch.object(MockHa, 'dcs')
+    @patch.object(MockPatroni, 'dcs')
     def test_do_PUT_config(self, mock_dcs):
         mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, '{}')
         request = 'PUT /config HTTP/1.0' + self._authorization + '\nContent-Length: '
@@ -161,23 +191,71 @@ class TestRestApiHandler(unittest.TestCase):
             MockRestApiServer(RestApiHandler, 'POST /reload HTTP/1.0' + self._authorization)
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /reload HTTP/1.0' + self._authorization))
 
-    def test_do_POST_restart(self):
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_POST_restart(self, mock_dcs):
+        mock_dcs.get_cluster.return_value.is_paused.return_value = False
         request = 'POST /restart HTTP/1.0' + self._authorization
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
+
         with patch.object(MockHa, 'restart', Mock(side_effect=Exception)):
             MockRestApiServer(RestApiHandler, request)
 
-    @patch.object(MockHa, 'dcs')
-    def test_do_POST_reinitialize(self, dcs):
-        cluster = dcs.get_cluster.return_value
+        post = request + '\nContent-Length: '
+
+        def make_request(request=None, **kwargs):
+            request = json.dumps(kwargs) if request is None else request
+            return '{0}{1}\n\n{2}'.format(post, len(request), request)
+
+        # empty request
+        request = make_request('')
+        MockRestApiServer(RestApiHandler, request)
+        # invalid request
+        request = make_request('foobar=baz')
+        MockRestApiServer(RestApiHandler, request)
+        # wrong role
+        request = make_request(schedule=future_restart_time.isoformat(), role='unknown', postgres_version='9.5.3')
+        MockRestApiServer(RestApiHandler, request)
+        # wrong version
+        request = make_request(schedule=future_restart_time.isoformat(), role='master', postgres_version='9.5.3.1')
+        MockRestApiServer(RestApiHandler, request)
+        # unknown filter
+        request = make_request(schedule=future_restart_time.isoformat(), batman='lives')
+        MockRestApiServer(RestApiHandler, request)
+        # incorrect schedule
+        request = make_request(schedule='2016-08-42 12:45TZ+1', role='master')
+        MockRestApiServer(RestApiHandler, request)
+        # everything fine, but the schedule is missing
+        request = make_request(role='master', postgres_version='9.5.2')
+        MockRestApiServer(RestApiHandler, request)
+        for retval in (True, False):
+            with patch.object(MockHa, 'schedule_future_restart', Mock(return_value=retval)):
+                request = make_request(schedule=future_restart_time.isoformat())
+                MockRestApiServer(RestApiHandler, request)
+            with patch.object(MockHa, 'restart', Mock(return_value=(retval, "foo"))):
+                request = make_request(role='master', postgres_version='9.5.2')
+                MockRestApiServer(RestApiHandler, request)
+
+        mock_dcs.get_cluster.return_value.is_paused.return_value = True
+        MockRestApiServer(RestApiHandler, make_request(schedule='2016-08-42 12:45TZ+1', role='master'))
+        # Valid timeout
+        MockRestApiServer(RestApiHandler, make_request(timeout='60s'))
+        # Invalid timeout
+        MockRestApiServer(RestApiHandler, make_request(timeout='42towels'))
+
+    def test_do_DELETE_restart(self):
+        for retval in (True, False):
+            with patch.object(MockHa, 'delete_future_restart', Mock(return_value=retval)):
+                request = 'DELETE /restart HTTP/1.0' + self._authorization
+                self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
+
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_POST_reinitialize(self, mock_dcs):
+        cluster = mock_dcs.get_cluster.return_value
+        cluster.is_paused.return_value = False
         request = 'POST /reinitialize HTTP/1.0' + self._authorization
         MockRestApiServer(RestApiHandler, request)
-        cluster.is_unlocked.return_value = False
-        MockRestApiServer(RestApiHandler, request)
-        with patch.object(MockHa, 'schedule_reinitialize', Mock(return_value=None)):
+        with patch.object(MockHa, 'reinitialize', Mock(return_value=None)):
             MockRestApiServer(RestApiHandler, request)
-        cluster.leader.name = 'test'
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
 
     @patch('time.sleep', Mock())
     def test_RestApiServer_query(self):
@@ -187,8 +265,9 @@ class TestRestApiHandler(unittest.TestCase):
             self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
 
     @patch('time.sleep', Mock())
-    @patch.object(MockHa, 'dcs')
+    @patch.object(MockPatroni, 'dcs')
     def test_do_POST_failover(self, dcs):
+        dcs.loop_wait = 10
         cluster = dcs.get_cluster.return_value
 
         post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
@@ -213,19 +292,27 @@ class TestRestApiHandler(unittest.TestCase):
         cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
                            Member(0, 'postgresql2', 30, {'api_url': 'http'})]
         MockRestApiServer(RestApiHandler, request)
-        with patch.object(MockPatroni, 'dcs') as d:
-            cluster = d.get_cluster.return_value
-            cluster.leader.name = 'postgresql0'
-            MockRestApiServer(RestApiHandler, request)
-            cluster.leader.name = 'postgresql2'
-            MockRestApiServer(RestApiHandler, request)
-            cluster.leader.name = 'postgresql1'
-            cluster.failover = None
-            MockRestApiServer(RestApiHandler, request)
-            d.get_cluster = Mock(side_effect=Exception)
-            MockRestApiServer(RestApiHandler, request)
-            d.manual_failover.return_value = False
-            MockRestApiServer(RestApiHandler, request)
+
+        cluster.failover = None
+        MockRestApiServer(RestApiHandler, request)
+
+        dcs.get_cluster.side_effect = [cluster]
+        MockRestApiServer(RestApiHandler, request)
+
+        cluster2 = cluster.copy()
+        cluster2.leader.name = 'postgresql0'
+        dcs.get_cluster.side_effect = [cluster, cluster2]
+        MockRestApiServer(RestApiHandler, request)
+
+        cluster2.leader.name = 'postgresql2'
+        dcs.get_cluster.side_effect = [cluster, cluster2]
+        MockRestApiServer(RestApiHandler, request)
+
+        dcs.get_cluster.side_effect = None
+        dcs.manual_failover.return_value = False
+        MockRestApiServer(RestApiHandler, request)
+        dcs.manual_failover.return_value = True
+
         with patch.object(MockHa, 'fetch_nodes_statuses', Mock(return_value=[])):
             MockRestApiServer(RestApiHandler, request)
 

tests/test_async_executor.py

@@ -8,7 +8,7 @@ from threading import Thread
 class TestAsyncExecutor(unittest.TestCase):
 
     def setUp(self):
-        self.a = AsyncExecutor()
+        self.a = AsyncExecutor(Mock())
 
     @patch.object(Thread, 'start', Mock())
     def test_run_async(self):

tests/test_callback_executor.py

@@ -0,0 +1,24 @@
+import unittest
+
+from mock import Mock, patch
+from patroni.callback_executor import CallbackExecutor
+
+
+class TestCallbackExecutor(unittest.TestCase):
+
+    @patch('subprocess.Popen')
+    def test_callback_executor(self, mock_popen):
+        mock_popen.return_value.wait.side_effect = Exception
+        mock_popen.return_value.poll.return_value = None
+
+        ce = CallbackExecutor()
+        self.assertIsNone(ce.call([]))
+        ce.join()
+
+        self.assertIsNone(ce.call([]))
+
+        mock_popen.side_effect = Exception
+        ce = CallbackExecutor()
+        ce._callback_event.wait = Mock(side_effect=[None, Exception])
+        self.assertIsNone(ce.call([]))
+        ce.join()

tests/test_config.py

@@ -20,9 +20,10 @@ class TestConfig(unittest.TestCase):
     def test_no_config(self):
         self.assertRaises(SystemExit, Config)
 
-    @patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception))
     def test_set_dynamic_configuration(self):
-        self.assertIsNone(self.config.set_dynamic_configuration({'foo': 'bar'}))
+        with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
+            self.assertIsNone(self.config.set_dynamic_configuration({'foo': 'bar'}))
+        self.assertTrue(self.config.set_dynamic_configuration({'synchronous_mode': True}))
 
     def test_reload_local_configuration(self):
         os.environ.update({
@@ -40,6 +41,12 @@ class TestConfig(unittest.TestCase):
             'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0',
             'PATRONI_POSTGRESQL_PGPASS': '/tmp/pgpass0',
             'PATRONI_ETCD_HOST': '127.0.0.1:2379',
+            'PATRONI_ETCD_URL': 'https://127.0.0.1:2379',
+            'PATRONI_ETCD_PROXY': 'http://127.0.0.1:2379',
+            'PATRONI_ETCD_SRV': 'test',
+            'PATRONI_ETCD_CACERT': '/cacert',
+            'PATRONI_ETCD_CERT': '/cert',
+            'PATRONI_ETCD_KEY': '/key',
             'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
             'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
             'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',

tests/test_consul.py

@@ -1,8 +1,9 @@
 import consul
 import unittest
 
+from consul import ConsulException, NotFound
 from mock import Mock, patch
-from patroni.dcs.consul import AbstractDCS, Cluster, Consul, ConsulError, ConsulException, HTTPClient, NotFound
+from patroni.dcs.consul import AbstractDCS, Cluster, Consul, ConsulInternalError, ConsulError, HTTPClient
 from test_etcd import SleepException
 
 
@@ -30,17 +31,34 @@ def kv_get(self, key, **kwargs):
                   'Value': ('postgres://replicator:[email protected]:5433/postgres' +
                             '?application_name=http://127.0.0.1:8009/patroni').encode('utf-8')},
                  {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'optime/leader', 'LockIndex': 0,
-                  'ModifyIndex': 6429, 'Value': b'4496294792'}])
+                  'ModifyIndex': 6429, 'Value': b'4496294792'},
+                 {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'sync', 'LockIndex': 0,
+                  'ModifyIndex': 6429, 'Value': b'{"leader": "leader", "sync_standby": null}'}])
     raise ConsulException
 
 
 class TestHTTPClient(unittest.TestCase):
 
-    def test_get(self):
+    def setUp(self):
         self.client = HTTPClient('127.0.0.1', '8500', 'http', False)
-        self.client.session.get = Mock()
+        self.client.http.request = Mock()
+
+    def test_get(self):
         self.client.get(Mock(), '')
         self.client.get(Mock(), '', {'wait': '1s', 'index': 1})
+        self.client.http.request.return_value.status = 500
+        self.assertRaises(ConsulInternalError, self.client.get, Mock(), '')
+
+    def test_unknown_method(self):
+        try:
+            self.client.bla(Mock(), '')
+            self.assertFail()
+        except Exception as e:
+            self.assertTrue(isinstance(e, AttributeError))
+
+    def test_put(self):
+        self.client.put(Mock(), '/v1/session/create')
+        self.client.put(Mock(), '/v1/session/create', data='{"foo": "bar"}')
 
 
 @patch.object(consul.Consul.KV, 'get', kv_get)
@@ -65,7 +83,8 @@ class TestConsul(unittest.TestCase):
     @patch.object(consul.Consul.Session, 'create', Mock(side_effect=ConsulException))
     def test_referesh_session(self):
         self.c._session = '1'
-        self.c._name = ''
+        self.assertFalse(self.c.refresh_session())
+        self.c._last_session_refresh = 0
         self.assertRaises(ConsulError, self.c.refresh_session)
 
     @patch.object(consul.Consul.KV, 'delete', Mock())
@@ -91,6 +110,8 @@ class TestConsul(unittest.TestCase):
 
     @patch.object(consul.Consul.KV, 'put', Mock(return_value=False))
     def test_take_leader(self):
+        self.c.set_ttl(20)
+        self.c.refresh_session = Mock()
         self.c.take_leader()
 
     @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
@@ -103,8 +124,9 @@ class TestConsul(unittest.TestCase):
 
     @patch.object(consul.Consul.KV, 'put', Mock(side_effect=ConsulException))
     def test_write_leader_optime(self):
-        self.c.write_leader_optime('')
+        self.c.write_leader_optime('1')
 
+    @patch.object(consul.Consul.Session, 'renew', Mock())
     def test_update_leader(self):
         self.c.update_leader()
 
@@ -126,15 +148,17 @@ class TestConsul(unittest.TestCase):
 
     @patch.object(AbstractDCS, 'watch', Mock())
     def test_watch(self):
+        self.c.watch(None, 1)
         self.c._name = ''
-        self.c.watch(1)
+        self.c.watch(6429, 1)
         with patch.object(consul.Consul.KV, 'get', Mock(side_effect=ConsulException)):
-            self.c.watch(1)
-
-    @patch.object(consul.Consul.Session, 'destroy', Mock(side_effect=ConsulException))
-    def test_set_ttl(self):
-        self.c.set_ttl(20)
-        self.assertTrue(self.c.watch(1))
+            self.c.watch(6429, 1)
 
     def test_set_retry_timeout(self):
         self.c.set_retry_timeout(10)
+
+    @patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
+    @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
+    def test_sync_state(self):
+        self.assertTrue(self.c.set_sync_state_value('{}'))
+        self.assertTrue(self.c.delete_sync_state())

tests/test_ctl.py

@@ -6,12 +6,13 @@ import unittest
 
 from click.testing import CliRunner
 from mock import patch, Mock
-from patroni.ctl import ctl, members, store_config, load_config, output_members, post_patroni, get_dcs, parse_dcs, \
-    wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException
+from patroni.ctl import ctl, members, store_config, load_config, output_members, request_patroni, get_dcs, parse_dcs, \
+    get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException
+from patroni.dcs.etcd import Client
 from psycopg2 import OperationalError
 from test_etcd import etcd_read, requests_get, socket_getaddrinfo, MockResponse
 from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \
-    get_cluster_initialized_with_only_leader
+    get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader
 from test_postgresql import MockConnect, psycopg2_connect
 
 CONFIG_FILE_PATH = './test-ctl.yaml'
@@ -28,14 +29,16 @@ def test_rw_config():
         os.rmdir(CONFIG_FILE_PATH)
 
 
-@patch('patroni.ctl.load_config', Mock(return_value={'restapi': {'auth': 'u:p'}, 'etcd': {'host': 'localhost:4001'}}))
+@patch('patroni.ctl.load_config',
+       Mock(return_value={'postgresql': {'data_dir': '.', 'parameters': {}, 'retry_timeout': 5},
+                          'restapi': {'auth': 'u:p', 'listen': ''}, 'etcd': {'host': 'localhost:2379'}}))
 class TestCtl(unittest.TestCase):
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
     def setUp(self):
-        self.runner = CliRunner()
-        with patch.object(etcd.Client, 'machines') as mock_machines:
+        with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
+            self.runner = CliRunner()
             self.e = get_dcs({'etcd': {'ttl': 30, 'host': 'ok:2379', 'retry_timeout': 10}}, 'foo')
 
     @patch('psycopg2.connect', psycopg2_connect)
@@ -51,8 +54,8 @@ class TestCtl(unittest.TestCase):
 
     def test_parse_dcs(self):
         assert parse_dcs(None) is None
-        assert parse_dcs('localhost') == {'etcd': {'host': 'localhost:4001'}}
-        assert parse_dcs('') == {'etcd': {'host': 'localhost:4001'}}
+        assert parse_dcs('localhost') == {'etcd': {'host': 'localhost:2379'}}
+        assert parse_dcs('') == {'etcd': {'host': 'localhost:2379'}}
         assert parse_dcs('localhost:8500') == {'consul': {'host': 'localhost:8500'}}
         assert parse_dcs('zookeeper://localhost') == {'zookeeper': {'hosts': ['localhost:2181']}}
         assert parse_dcs('exhibitor://dummy') == {'exhibitor': {'hosts': ['dummy'], 'port': 8181}}
@@ -66,18 +69,20 @@ class TestCtl(unittest.TestCase):
         self.assertIsNone(output_members(cluster, name='abc', fmt='tsv'))
 
     @patch('patroni.ctl.get_dcs')
-    @patch('patroni.ctl.post_patroni', Mock(return_value=MockResponse()))
+    @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
     def test_failover(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        mock_get_dcs.return_value.set_failover_value = Mock()
         result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
         assert 'leader' in result.output
 
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2100-01-01T12:23:00\ny')
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
         assert result.exit_code == 0
 
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2030-01-01T12:23:00\ny')
-        assert result.exit_code == 0
+        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+            result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00'])
+            assert result.exit_code == 1
 
         # Aborting failover,as we anser NO to the confirmation
         result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\nN')
@@ -109,12 +114,12 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['failover', 'dummy'], input='dummy')
         assert result.exit_code == 1
 
-        with patch('patroni.ctl.post_patroni', Mock(side_effect=Exception)):
+        with patch('patroni.ctl.request_patroni', Mock(side_effect=Exception)):
             # Non-responding patroni
-            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
+            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
             assert 'falling back to DCS' in result.output
 
-        with patch('patroni.ctl.post_patroni') as mocked:
+        with patch('patroni.ctl.request_patroni') as mocked:
             mocked.return_value.status_code = 500
             result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
             assert 'Failover failed' in result.output
@@ -205,23 +210,82 @@ class TestCtl(unittest.TestCase):
     @patch('patroni.ctl.get_dcs')
     def test_restart_reinit(self, mock_get_dcs):
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
-        result = self.runner.invoke(ctl, ['restart', 'alpha'], input='y')
-        assert 'restart failed for' in result.output
+        result = self.runner.invoke(ctl, ['restart', 'alpha'], input='y\n\nnow')
+        assert 'Failed: restart for' in result.output
         assert result.exit_code == 0
 
         result = self.runner.invoke(ctl, ['reinit', 'alpha'], input='y')
         assert result.exit_code == 1
 
+        # successful reinit
+        result = self.runner.invoke(ctl, ['reinit', 'alpha', 'other'], input='y')
+        assert result.exit_code == 0
+
         # Aborted restart
         result = self.runner.invoke(ctl, ['restart', 'alpha'], input='N')
         assert result.exit_code == 1
 
+        result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force'])
+        assert result.exit_code == 0
+
         # Not a member
         result = self.runner.invoke(ctl, ['restart', 'alpha', 'dummy', '--any'], input='y')
         assert result.exit_code == 1
 
+        # Wrong pg version
+        result = self.runner.invoke(ctl, ['restart', 'alpha', '--any', '--pg-version', '9.1'], input='y')
+        assert 'Error: PostgreSQL version' in result.output
+        assert result.exit_code == 1
+
+        result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force', '--timeout', '10min'])
+        assert result.exit_code == 0
+
+        with patch('requests.delete', Mock(return_value=MockResponse(500))):
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force',
+                                              '--scheduled', '2300-10-01T14:30'])
+            assert 'Failed: flush scheduled restart' in result.output
+
+        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+            result = self.runner.invoke(ctl,
+                                        ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
+            assert result.exit_code == 1
+
         with patch('requests.post', Mock(return_value=MockResponse())):
-            result = self.runner.invoke(ctl, ['restart', 'alpha'], input='y')
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0.0',
+                                        '--scheduled', '2300-10-01T14:30'], input='y')
+            assert result.exit_code == 0
+
+        with patch('requests.post', Mock(return_value=MockResponse(204))):
+            # get restart with the non-200 return code
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0.0',
+                                        '--scheduled', '2300-10-01T14:30'], input='y')
+            assert result.exit_code == 0
+
+        # force restart with restart already present
+        with patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse(204))):
+            result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force',
+                                              '--scheduled', '2300-10-01T14:30'])
+            assert result.exit_code == 0
+
+        with patch('requests.post', Mock(return_value=MockResponse(202))):
+            # get restart with the non-200 return code
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(
+                ctl, ['restart', 'alpha', '--pg-version', '99.0.0', '--scheduled', '2300-10-01T14:30'], input='y'
+            )
+            assert 'Success: restart scheduled' in result.output
+            assert result.exit_code == 0
+
+        with patch('requests.post', Mock(return_value=MockResponse(409))):
+            # get restart with the non-200 return code
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(
+                ctl, ['restart', 'alpha', '--pg-version', '99.0.0', '--scheduled', '2300-10-01T14:30'], input='y'
+            )
+            assert 'Failed: another restart is already' in result.output
             assert result.exit_code == 0
 
     @patch('patroni.ctl.get_dcs')
@@ -244,18 +308,10 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['remove', 'alpha'], input='alpha\nYes I am aware\nleader')
         assert result.exit_code == 0
 
-    @patch('patroni.dcs.AbstractDCS.watch', Mock(return_value=None))
-    @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
-    def test_wait_for_leader(self):
-        self.assertRaises(PatroniCtlException, wait_for_leader, self.e, 0)
-
-        cluster = wait_for_leader(self.e, timeout=2)
-        assert cluster.leader.member.name == 'leader'
-
     @patch('requests.post', Mock(side_effect=requests.exceptions.ConnectionError('foo')))
-    def test_post_patroni(self):
+    def test_request_patroni(self):
         member = get_cluster_initialized_with_leader().leader.member
-        self.assertRaises(requests.exceptions.ConnectionError, post_patroni, member, 'dummy', {})
+        self.assertRaises(requests.exceptions.ConnectionError, request_patroni, member, 'post', 'dummy', {})
 
     def test_ctl(self):
         self.runner.invoke(ctl, ['list'])
@@ -292,3 +348,93 @@ class TestCtl(unittest.TestCase):
     def test_configure(self):
         result = self.runner.invoke(configure, ['--dcs', 'abc', '-c', 'dummy', '-n', 'bla'])
         assert result.exit_code == 0
+
+    @patch('patroni.ctl.get_dcs')
+    def test_scaffold(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_not_initialized_without_leader
+        mock_get_dcs.return_value.initialize = Mock(return_value=True)
+        mock_get_dcs.return_value.touch_member = Mock(return_value=True)
+        mock_get_dcs.return_value.attempt_to_acquire_leader = Mock(return_value=True)
+        mock_get_dcs.return_value.delete_cluster = Mock()
+
+        with patch.object(self.e, 'initialize', return_value=False):
+            result = self.runner.invoke(ctl, ['scaffold', 'alpha'])
+            assert result.exception
+
+        with patch.object(mock_get_dcs.return_value, 'touch_member', Mock(return_value=False)):
+            result = self.runner.invoke(ctl, ['scaffold', 'alpha'])
+            assert result.exception
+
+        result = self.runner.invoke(ctl, ['scaffold', 'alpha'])
+        assert result.exit_code == 0
+
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        result = self.runner.invoke(ctl, ['scaffold', 'alpha'])
+        assert result.exception
+
+    @patch('patroni.ctl.get_dcs')
+    def test_list_extended(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
+        mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster)
+
+        result = self.runner.invoke(ctl, ['list', 'dummy', '--extended'])
+        assert '2100' in result.output
+        assert 'Scheduled restart' in result.output
+
+    @patch('patroni.ctl.get_dcs')
+    @patch('requests.delete', Mock(return_value=MockResponse()))
+    def test_flush(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+
+        result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '-r', 'master'], input='y')
+        assert 'No scheduled restart' in result.output
+
+        result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '--force'])
+        assert 'Success: flush scheduled restart' in result.output
+        with patch.object(requests, 'delete', return_value=MockResponse(404)):
+            result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '--force'])
+            assert 'Failed: flush scheduled restart' in result.output
+
+    @patch('patroni.ctl.get_dcs')
+    def test_pause_cluster(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+
+        with patch('requests.patch', Mock(return_value=MockResponse(200))):
+            result = self.runner.invoke(ctl, ['pause', 'dummy'])
+            assert 'Success' in result.output
+
+        with patch('requests.patch', Mock(return_value=MockResponse(500))):
+            result = self.runner.invoke(ctl, ['pause', 'dummy'])
+            assert 'Failed' in result.output
+
+        with patch('requests.patch', Mock(return_value=MockResponse(200))),\
+                patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+            result = self.runner.invoke(ctl, ['pause', 'dummy'])
+            assert 'Cluster is already paused' in result.output
+
+    @patch('patroni.ctl.get_dcs')
+    def test_resume_cluster(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+
+        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+            with patch('requests.patch', Mock(return_value=MockResponse(200))):
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Success' in result.output
+
+            with patch('requests.patch', Mock(return_value=MockResponse(500))):
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Failed' in result.output
+
+            with patch('requests.patch', Mock(return_value=MockResponse(200))),\
+                    patch('patroni.dcs.Cluster.is_paused', Mock(return_value=False)):
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Cluster is not paused' in result.output
+
+            with patch('requests.patch', Mock(side_effect=Exception)):
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Can not find accessible cluster member' in result.output

tests/test_etcd.py

@@ -1,20 +1,21 @@
 import etcd
 import json
+import urllib3.util.connection
 import requests
 import socket
 import unittest
 
 from dns.exception import DNSException
 from mock import Mock, patch
-from patroni.dcs.etcd import AbstractDCS, Client, Cluster, Etcd, EtcdError
+from patroni.dcs.etcd import AbstractDCS, Client, Cluster, Etcd, EtcdError, DnsCachingResolver
 from patroni.exceptions import DCSError
 from urllib3.exceptions import ReadTimeoutError
 
 
 class MockResponse(object):
 
-    def __init__(self):
-        self.status_code = 200
+    def __init__(self, status_code=200):
+        self.status_code = status_code
         self.content = '{}'
         self.ok = True
         self.text = ''
@@ -42,7 +43,7 @@ def requests_get(url, **kwargs):
     if url.startswith('http://local'):
         raise requests.exceptions.RequestException()
     elif ':8011/patroni' in url:
-        response.content = '{"role": "replica", "xlog": {"replayed_location": 0}, "tags": {}}'
+        response.content = '{"role": "replica", "xlog": {"received_location": 0}, "tags": {}}'
     elif url.endswith('/members'):
         response.content = '[{}]' if url.startswith('http://error') else members
     elif url.startswith('http://exhibitor'):
@@ -91,6 +92,8 @@ def etcd_read(self, key, **kwargs):
                     {"key": "/service/batman5/optime/leader", "value": "2164261704",
                      "modifiedIndex": 20729, "createdIndex": 20729}],
                  "modifiedIndex": 20437, "createdIndex": 20437},
+                {"key": "/service/batman5/sync", "value": '{"leader": "leader"}',
+                 "modifiedIndex": 1582, "createdIndex": 1582},
                 {"key": "/service/batman5/members", "dir": True, "nodes": [
                     {"key": "/service/batman5/members/postgresql1",
                      "value": "postgres://replicator:[email protected]:5434/postgres" +
@@ -103,93 +106,140 @@ def etcd_read(self, key, **kwargs):
                      "expiration": "2015-05-15T09:11:09.611860899Z", "ttl": 30,
                      "modifiedIndex": 20730, "createdIndex": 20730}],
                  "modifiedIndex": 1581, "createdIndex": 1581}], "modifiedIndex": 1581, "createdIndex": 1581}}
-    return etcd.EtcdResult(**response)
+    result = etcd.EtcdResult(**response)
+    result.etcd_index = 0
+    return result
 
 
 class SleepException(Exception):
     pass
 
 
-class MockSRV(object):
-    port = 2380
-    target = '127.0.0.1'
-
-
 def dns_query(name, _):
+    if '-server' not in name or '-ssl' in name:
+        return []
     if name == '_etcd-server._tcp.blabla':
         return []
     elif name == '_etcd-server._tcp.exception':
         raise DNSException()
-    return [MockSRV()]
+    srv = Mock()
+    srv.port = 2380
+    srv.target.to_text.return_value = 'localhost' if name == '_etcd-server._tcp.foobar' else '127.0.0.1'
+    return [srv]
 
 
 def socket_getaddrinfo(*args):
-    if args[0] == 'ok':
-        return [(2, 1, 6, '', ('127.0.0.1', 2379)), (2, 1, 6, '', ('127.0.0.1', 2379))]
-    raise socket.error
+    if args[0] in ('ok', 'localhost', '127.0.0.1'):
+        return [(socket.AF_INET, 1, 6, '', ('127.0.0.1', 0)), (socket.AF_INET6, 1, 6, '', ('::1', 0))]
+    raise socket.gaierror
 
 
 def http_request(method, url, **kwargs):
     if url == 'http://localhost:2379/timeout':
         raise ReadTimeoutError(None, None, None)
+    if url == 'http://localhost:2379/v2/machines':
+        ret = MockResponse()
+        ret.content = 'http://localhost:2379,http://localhost:4001'
+        return ret
     if url == 'http://localhost:2379/':
         return MockResponse()
     raise socket.error
 
 
+class TestDnsCachingResolver(unittest.TestCase):
+
+    @patch('time.sleep', Mock(side_effect=SleepException))
+    @patch('socket.getaddrinfo', Mock(side_effect=socket.gaierror))
+    def test_run(self):
+        r = DnsCachingResolver()
+        self.assertIsNone(r.resolve_async('', 0))
+        r.join()
+
+
 @patch('dns.resolver.query', dns_query)
 @patch('socket.getaddrinfo', socket_getaddrinfo)
 @patch('requests.get', requests_get)
 class TestClient(unittest.TestCase):
 
     @patch('dns.resolver.query', dns_query)
+    @patch('socket.getaddrinfo', socket_getaddrinfo)
     @patch('requests.get', requests_get)
     def setUp(self):
-        with patch.object(etcd.Client, 'machines') as mock_machines:
+        with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
-            self.client = Client({'discovery_srv': 'test', 'retry_timeout': 3})
+            self.client = Client({'srv': 'test', 'retry_timeout': 3}, DnsCachingResolver())
             self.client.http.request = http_request
             self.client.http.request_encode_body = http_request
 
-    def test_api_execute(self):
+    def test_machines(self):
         self.client._base_uri = 'http://localhost:4001'
         self.client._machines_cache = ['http://localhost:2379']
-        self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
-        self.client._update_machines_cache = False
-        self.client.api_execute('/', 'POST', timeout=0)
-        self.client._update_machines_cache = False
+        self.assertIsNotNone(self.client.machines)
         self.client._base_uri = 'http://localhost:4001'
         self.client._machines_cache = []
-        self.assertRaises(etcd.EtcdConnectionFailed, self.client.api_execute, '/', 'GET')
-        self.assertTrue(self.client._update_machines_cache)
-        self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'GET')
-        self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '')
+        self.assertIsNotNone(self.client.machines)
+        self.client._update_machines_cache = True
+        machines = None
+        try:
+            machines = self.client.machines
+            self.assertFail()
+        except Exception:
+            self.assertIsNone(machines)
+
+    @patch.object(Client, 'machines')
+    def test_api_execute(self, mock_machines):
+        mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
         self.assertRaises(ValueError, self.client.api_execute, '', '')
+        self.client._base_uri = 'http://localhost:4001'
+        self.client._machines_cache = ['http://localhost:2379']
+        self.client.api_execute('/', 'POST', timeout=0)
+        mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
+        self.client._machines_cache_updated = 0
+        self.client.api_execute('/', 'POST', timeout=0)
+        self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
+        self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '')
+        self.client._update_machines_cache = True
+        with patch.object(Client, '_load_machines_cache', Mock(side_effect=etcd.EtcdException)):
+            self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'GET')
 
     def test_get_srv_record(self):
-        self.assertEquals(self.client.get_srv_record('blabla'), [])
-        self.assertEquals(self.client.get_srv_record('exception'), [])
+        self.assertEquals(self.client.get_srv_record('_etcd-server._tcp.blabla'), [])
+        self.assertEquals(self.client.get_srv_record('_etcd-server._tcp.exception'), [])
 
     def test__get_machines_cache_from_srv(self):
+        self.client._get_machines_cache_from_srv('foobar')
         self.client.get_srv_record = Mock(return_value=[('localhost', 2380)])
         self.client._get_machines_cache_from_srv('blabla')
 
     def test__get_machines_cache_from_dns(self):
-        self.client._get_machines_cache_from_dns('error:2379')
+        self.client._get_machines_cache_from_dns('error', 2379)
 
-    def test__load_machines_cache(self):
+    @patch.object(Client, 'machines')
+    def test__load_machines_cache(self, mock_machines):
+        mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
         self.client._config = {}
         self.assertRaises(Exception, self.client._load_machines_cache)
-        self.client._config = {'discovery_srv': 'blabla'}
+        self.client._config = {'srv': 'blabla'}
         self.assertRaises(etcd.EtcdException, self.client._load_machines_cache)
 
+    @patch.object(socket.socket, 'connect')
+    def test_create_connection_patched(self, mock_connect):
+        self.assertRaises(socket.error, urllib3.util.connection.create_connection, ('fail', 2379))
+        urllib3.util.connection.create_connection(('[localhost]', 2379))
+        mock_connect.side_effect = socket.error
+        self.assertRaises(socket.error, urllib3.util.connection.create_connection, ('[localhost]', 2379),
+                          timeout=1, source_address=('localhost', 53333),
+                          socket_options=[(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)])
+
 
 @patch('requests.get', requests_get)
+@patch('socket.getaddrinfo', socket_getaddrinfo)
 @patch.object(etcd.Client, 'write', etcd_write)
 @patch.object(etcd.Client, 'read', etcd_read)
 @patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
 class TestEtcd(unittest.TestCase):
 
+    @patch('socket.getaddrinfo', socket_getaddrinfo)
     def setUp(self):
         with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
@@ -201,11 +251,15 @@ class TestEtcd(unittest.TestCase):
 
     @patch('dns.resolver.query', dns_query)
     def test_get_etcd_client(self):
-        with patch.object(etcd.Client, 'machines') as mock_machines:
+        with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(side_effect=etcd.EtcdException)
-            with patch('time.sleep', Mock(side_effect=SleepException())):
+            with patch('time.sleep', Mock(side_effect=SleepException)):
                 self.assertRaises(SleepException, self.etcd.get_etcd_client,
-                                  {'discovery_srv': 'test', 'retry_timeout': 10})
+                                  {'discovery_srv': 'test', 'retry_timeout': 10, 'cacert': '1', 'key': '1', 'cert': 1})
+                self.assertRaises(SleepException, self.etcd.get_etcd_client,
+                                  {'url': 'https://test:2379', 'retry_timeout': 10})
+                self.assertRaises(SleepException, self.etcd.get_etcd_client,
+                                  {'proxy': 'https://user:password@test:2379', 'retry_timeout': 10})
 
     def test_get_cluster(self):
         self.assertIsInstance(self.etcd.get_cluster(), Cluster)
@@ -248,12 +302,12 @@ class TestEtcd(unittest.TestCase):
 
     @patch.object(etcd.Client, 'watch', etcd_watch)
     def test_watch(self):
-        self.etcd.watch(0)
+        self.etcd.watch(None, 0)
         self.etcd.get_cluster()
-        self.etcd.watch(1.5)
-        self.etcd.watch(4.5)
+        self.etcd.watch(20729, 1.5)
+        self.etcd.watch(20729, 4.5)
         with patch.object(AbstractDCS, 'watch', Mock()):
-            self.etcd.watch(9.5)
+            self.etcd.watch(20729, 9.5)
 
     def test_other_exceptions(self):
         self.etcd.retry = Mock(side_effect=AttributeError('foo'))
@@ -261,4 +315,8 @@ class TestEtcd(unittest.TestCase):
 
     def test_set_ttl(self):
         self.etcd.set_ttl(20)
-        self.assertTrue(self.etcd.watch(1))
+        self.assertTrue(self.etcd.watch(None, 1))
+
+    def test_sync_state(self):
+        self.assertFalse(self.etcd.write_sync_state('leader', None))
+        self.assertFalse(self.etcd.delete_sync_state())

tests/test_ha.py

@@ -1,16 +1,18 @@
 import datetime
 import etcd
 import os
-import pytz
 import unittest
 
-from mock import Mock, MagicMock, patch
+from mock import Mock, MagicMock, PropertyMock, patch
 from patroni.config import Config
-from patroni.dcs import Cluster, Failover, Leader, Member, get_dcs
+from patroni.dcs import Cluster, ClusterConfig, Failover, Leader, Member, get_dcs, SyncState
+from patroni.dcs.etcd import Client
 from patroni.exceptions import DCSError, PostgresException
-from patroni.ha import Ha
+from patroni.ha import Ha, _MemberStatus
 from patroni.postgresql import Postgresql
+from patroni.utils import tzutc
 from test_etcd import socket_getaddrinfo, etcd_read, etcd_write, requests_get
+from test_postgresql import psycopg2_connect
 
 
 def true(*args, **kwargs):
@@ -21,30 +23,47 @@ def false(*args, **kwargs):
     return False
 
 
-def get_cluster(initialize, leader, members, failover):
-    return Cluster(initialize, None, leader, 10, members, failover)
+def get_cluster(initialize, leader, members, failover, sync):
+    return Cluster(initialize, ClusterConfig(1, {1: 2}, 1), leader, 10, members, failover, sync)
 
 
 def get_cluster_not_initialized_without_leader():
-    return get_cluster(None, None, [], None)
+    return get_cluster(None, None, [], None, SyncState(None, None, None))
 
 
-def get_cluster_initialized_without_leader(leader=False, failover=None):
+def get_cluster_initialized_without_leader(leader=False, failover=None, sync=None):
     m1 = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres',
                                   'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location': 4})
     l = Leader(0, 0, m1) if leader else None
     m2 = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
-                                 'api_url': 'http://127.0.0.1:8011/patroni', 'tags': {'clonefrom': True}})
-    return get_cluster(True, l, [m1, m2], failover)
+                                 'api_url': 'http://127.0.0.1:8011/patroni',
+                                 'state': 'running',
+                                 'tags': {'clonefrom': True},
+                                 'scheduled_restart': {'schedule': "2100-01-01 10:53:07.560445+00:00",
+                                                       'postgres_version': '99.0.0'}})
+    syncstate = SyncState(0 if sync else None, sync and sync[0], sync and sync[1])
+    return get_cluster(True, l, [m1, m2], failover, syncstate)
 
 
-def get_cluster_initialized_with_leader(failover=None):
-    return get_cluster_initialized_without_leader(leader=True, failover=failover)
+def get_cluster_initialized_with_leader(failover=None, sync=None):
+    return get_cluster_initialized_without_leader(leader=True, failover=failover, sync=sync)
 
 
 def get_cluster_initialized_with_only_leader(failover=None):
     l = get_cluster_initialized_without_leader(leader=True, failover=failover).leader
-    return get_cluster(True, l, [l], failover)
+    return get_cluster(True, l, [l], failover, None)
+
+
+def get_node_status(reachable=True, in_recovery=True, xlog_location=10, nofailover=False):
+    def fetch_node_status(e):
+        tags = {}
+        if nofailover:
+            tags['nofailover'] = True
+        return _MemberStatus(e, reachable, in_recovery, xlog_location, tags)
+    return fetch_node_status
+
+future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5)
+postmaster_start_time = datetime.datetime.now(tzutc)
 
 
 class MockPatroni(object):
@@ -76,19 +95,21 @@ zookeeper:
         self.api = Mock()
         self.tags = {'foo': 'bar'}
         self.nofailover = None
-        self.nap_time = 10
         self.replicatefrom = None
         self.api.connection_string = 'http://127.0.0.1:8008'
         self.clonefrom = None
+        self.nosync = False
+        self.scheduled_restart = {'schedule': future_restart_time,
+                                  'postmaster_start_time': str(postmaster_start_time)}
 
 
-def run_async(func, args=()):
+def run_async(self, func, args=()):
     return func(*args) if args else func()
 
 
 @patch.object(Postgresql, 'is_running', Mock(return_value=True))
 @patch.object(Postgresql, 'is_leader', Mock(return_value=True))
-@patch.object(Postgresql, 'xlog_position', Mock(return_value=0))
+@patch.object(Postgresql, 'xlog_position', Mock(return_value=10))
 @patch.object(Postgresql, 'call_nowait', Mock(return_value=True))
 @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
 @patch.object(Postgresql, 'controldata', Mock(return_value={'Database system identifier': '1234567890'}))
@@ -98,38 +119,43 @@ def run_async(func, args=()):
 @patch.object(Postgresql, 'write_recovery_conf', Mock())
 @patch.object(Postgresql, 'query', Mock())
 @patch.object(Postgresql, 'checkpoint', Mock())
+@patch.object(Postgresql, 'call_nowait', Mock())
 @patch.object(etcd.Client, 'write', etcd_write)
 @patch.object(etcd.Client, 'read', etcd_read)
 @patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
+@patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=False))
+@patch('patroni.async_executor.AsyncExecutor.run_async', run_async)
 @patch('subprocess.call', Mock(return_value=0))
 class TestHa(unittest.TestCase):
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
+    @patch('psycopg2.connect', psycopg2_connect)
     @patch.object(etcd.Client, 'read', etcd_read)
     def setUp(self):
-        with patch.object(etcd.Client, 'machines') as mock_machines:
+        with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
             self.p = Postgresql({'name': 'postgresql0', 'scope': 'dummy', 'listen': '127.0.0.1:5432',
                                  'data_dir': 'data/postgresql0', 'retry_timeout': 10,
+                                 'maximum_lag_on_failover': 5,
                                  'authentication': {'superuser': {'username': 'foo', 'password': 'bar'},
                                                     'replication': {'username': '', 'password': ''}},
                                  'parameters': {'wal_level': 'hot_standby', 'max_replication_slots': 5, 'foo': 'bar',
                                                 'hot_standby': 'on', 'max_wal_senders': 5, 'wal_keep_segments': 8}})
             self.p.set_state('running')
             self.p.set_role('replica')
-            self.p.check_replication_lag = true
+            self.p.postmaster_start_time = MagicMock(return_value=str(postmaster_start_time))
             self.p.can_create_replica_without_replication_connection = MagicMock(return_value=False)
             self.e = get_dcs({'etcd': {'ttl': 30, 'host': 'ok:2379', 'scope': 'test',
                                        'name': 'foo', 'retry_timeout': 10}})
             self.ha = Ha(MockPatroni(self.p, self.e))
-            self.ha._async_executor.run_async = run_async
             self.ha.old_cluster = self.e.get_cluster()
             self.ha.cluster = get_cluster_not_initialized_without_leader()
             self.ha.load_cluster_from_dcs = Mock()
+            self.ha.is_synchronous_mode = false
 
     def test_update_lock(self):
         self.p.last_operation = Mock(side_effect=PostgresException(''))
-        self.assertTrue(self.ha.update_lock())
+        self.assertTrue(self.ha.update_lock(True))
 
     def test_touch_member(self):
         self.p.xlog_position = Mock(side_effect=Exception)
@@ -216,7 +242,9 @@ class TestHa(unittest.TestCase):
         self.ha.cluster.is_unlocked = false
         self.ha.has_lock = true
         self.ha.update_lock = false
-        self.assertEquals(self.ha.run_cycle(), 'demoting self because i do not have the lock and i was a leader')
+        self.assertEquals(self.ha.run_cycle(), 'demoted self because failed to update leader lock in DCS')
+        self.p.is_leader = false
+        self.assertEquals(self.ha.run_cycle(), 'not promoting because failed to update leader lock in DCS')
 
     def test_follow(self):
         self.ha.cluster.is_unlocked = false
@@ -225,6 +253,13 @@ class TestHa(unittest.TestCase):
         self.ha.patroni.replicatefrom = "foo"
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am a secondary and i am following a leader')
 
+    def test_follow_in_pause(self):
+        self.ha.cluster.is_unlocked = false
+        self.ha.is_paused = true
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: continue to run as master without lock')
+        self.p.is_leader = false
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: no action')
+
     def test_no_etcd_connection_master_demote(self):
         self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
         self.assertEquals(self.ha.run_cycle(), 'demoted self because DCS is not accessible and i was a leader')
@@ -258,45 +293,47 @@ class TestHa(unittest.TestCase):
         self.assertRaises(PostgresException, self.ha.bootstrap)
 
     def test_reinitialize(self):
-        self.ha.schedule_reinitialize()
-        self.ha.schedule_reinitialize()
-        self.ha.run_cycle()
-        self.assertIsNone(self.ha._async_executor.scheduled_action)
+        self.assertIsNotNone(self.ha.reinitialize())
 
         self.ha.cluster = get_cluster_initialized_with_leader()
-        self.ha.has_lock = true
-        self.ha.schedule_reinitialize()
-        self.ha.run_cycle()
-        self.assertIsNone(self.ha._async_executor.scheduled_action)
+        self.assertIsNone(self.ha.reinitialize())
 
-        self.ha.has_lock = false
-        self.ha.schedule_reinitialize()
-        self.ha.run_cycle()
+        self.assertIsNotNone(self.ha.reinitialize())
+
+        self.ha.state_handler.name = self.ha.cluster.leader.name
+        self.assertIsNotNone(self.ha.reinitialize())
 
     def test_restart(self):
-        self.assertEquals(self.ha.restart(), (True, 'restarted successfully'))
+        self.assertEquals(self.ha.restart({}), (True, 'restarted successfully'))
+        self.p.restart = Mock(return_value=None)
+        self.assertEquals(self.ha.restart({}), (False, 'postgres is still starting'))
         self.p.restart = false
-        self.assertEquals(self.ha.restart(), (False, 'restart failed'))
-        self.ha.schedule_reinitialize()
-        self.assertEquals(self.ha.restart(), (False, 'reinitialize already in progress'))
+        self.assertEquals(self.ha.restart({}), (False, 'restart failed'))
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.ha.reinitialize()
+        self.assertEquals(self.ha.restart({}), (False, 'reinitialize already in progress'))
+        with patch.object(self.ha, "restart_matches", return_value=False):
+            self.assertEquals(self.ha.restart({'foo': 'bar'}), (False, "restart conditions are not satisfied"))
 
     def test_restart_in_progress(self):
-        self.ha._async_executor.schedule('restart', True)
-        self.assertTrue(self.ha.restart_scheduled())
-        self.assertEquals(self.ha.run_cycle(), 'not healthy enough for leader race')
+        with patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=True)):
+            self.ha.restart({}, run_async=True)
+            self.assertTrue(self.ha.restart_scheduled())
+            self.assertEquals(self.ha.run_cycle(), 'not healthy enough for leader race')
 
-        self.ha.cluster = get_cluster_initialized_with_leader()
-        self.assertEquals(self.ha.run_cycle(), 'restart in progress')
+            self.ha.cluster = get_cluster_initialized_with_leader()
+            self.assertEquals(self.ha.run_cycle(), 'restart in progress')
 
-        self.ha.has_lock = true
-        self.assertEquals(self.ha.run_cycle(), 'updated leader lock during restart')
+            self.ha.has_lock = true
+            self.assertEquals(self.ha.run_cycle(), 'updated leader lock during restart')
 
-        self.ha.update_lock = false
-        self.assertEquals(self.ha.run_cycle(), 'failed to update leader lock during restart')
+            self.ha.update_lock = false
+            self.assertEquals(self.ha.run_cycle(), 'failed to update leader lock during restart')
 
     @patch('requests.get', requests_get)
     @patch('time.sleep', Mock())
     def test_manual_failover_from_leader(self):
+        self.ha.fetch_node_status = get_node_status()
         self.ha.has_lock = true
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None))
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
@@ -307,7 +344,9 @@ class TestHa(unittest.TestCase):
         f = Failover(0, self.p.name, '', None)
         self.ha.cluster = get_cluster_initialized_with_leader(f)
         self.assertEquals(self.ha.run_cycle(), 'manual failover: demoting myself')
-        self.ha.fetch_node_status = lambda e: (e, True, True, 0, {'nofailover': 'True'})
+        self.ha.fetch_node_status = get_node_status(nofailover=True)
+        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
+        self.ha.fetch_node_status = get_node_status(xlog_location=1)
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
         # manual failover from the previous leader to us won't happen if we hold the nofailover flag
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, None))
@@ -318,7 +357,7 @@ class TestHa(unittest.TestCase):
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
         self.ha.run_cycle()
 
-        scheduled = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
+        scheduled = datetime.datetime.utcnow().replace(tzinfo=tzutc)
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
         self.assertEquals('no action.  i am the leader with the lock', self.ha.run_cycle())
 
@@ -335,6 +374,17 @@ class TestHa(unittest.TestCase):
         self.assertEquals('no action.  i am the leader with the lock', self.ha.run_cycle())
 
     @patch('requests.get', requests_get)
+    def test_manual_failover_from_leader_in_pause(self):
+        self.ha.has_lock = true
+        self.ha.is_paused = true
+        scheduled = datetime.datetime.now()
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
+        self.assertEquals('PAUSE: no action.  i am the leader with the lock', self.ha.run_cycle())
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, '', None))
+        self.assertEquals('PAUSE: no action.  i am the leader with the lock', self.ha.run_cycle())
+
+    @patch('requests.get', requests_get)
+    @patch('time.sleep', Mock())
     def test_manual_failover_process_no_leader(self):
         self.p.is_leader = false
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', self.p.name, None))
@@ -342,40 +392,58 @@ class TestHa(unittest.TestCase):
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader', None))
         self.p.set_role('replica')
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
-        self.ha.fetch_node_status = lambda e: (e, True, True, 0, {})  # accessible, in_recovery
+        self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
         self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, self.p.name, '', None))
         self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
-        self.ha.fetch_node_status = lambda e: (e, False, True, 0, {})  # inaccessible, in_recovery
+        self.ha.fetch_node_status = get_node_status(reachable=False)  # inaccessible, in_recovery
         self.p.set_role('replica')
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
         # set failover flag to True for all members of the cluster
         # this should elect the current member, as we are not going to call the API for it.
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
-        self.ha.fetch_node_status = lambda e: (e, True, True, 0, {'nofailover': 'True'})  # accessible, in_recovery
+        self.ha.fetch_node_status = get_node_status(nofailover=True)  # accessible, in_recovery
         self.p.set_role('replica')
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
         # same as previous, but set the current member to nofailover. In no case it should be elected as a leader
         self.ha.patroni.nofailover = True
         self.assertEquals(self.ha.run_cycle(), 'following a different leader because I am not allowed to promote')
 
+    @patch('time.sleep', Mock())
+    def test_manual_failover_process_no_leader_in_pause(self):
+        self.ha.is_paused = true
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: continue to run as master without lock')
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', '', None))
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: continue to run as master without lock')
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'blabla', None))
+        self.assertEquals('PAUSE: acquired session lock as a leader', self.ha.run_cycle())
+        self.p.is_leader = false
+        self.p.set_role('replica')
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', self.p.name, None))
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: promoted self to leader by acquiring session lock')
+
     def test_is_healthiest_node(self):
         self.ha.state_handler.is_leader = false
         self.ha.patroni.nofailover = False
-        self.ha.fetch_node_status = lambda e: (e, True, True, 0, {})
+        self.ha.fetch_node_status = get_node_status()
         self.assertTrue(self.ha.is_healthiest_node())
+        with patch('patroni.postgresql.Postgresql.is_starting', return_value=True):
+            self.assertFalse(self.ha.is_healthiest_node())
+        self.ha.is_paused = true
+        self.assertFalse(self.ha.is_healthiest_node())
 
     def test__is_healthiest_node(self):
         self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.p.is_leader = false
-        self.ha.fetch_node_status = lambda e: (e, True, True, 0, {})  # accessible, in_recovery
+        self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
         self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
-        self.ha.fetch_node_status = lambda e: (e, True, False, 0, {})  # accessible, not in_recovery
+        self.ha.fetch_node_status = get_node_status(in_recovery=False)  # accessible, not in_recovery
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
-        self.ha.fetch_node_status = lambda e: (e, True, True, 1, {})  # accessible, in_recovery, xlog location ahead
-        self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
-        self.p.check_replication_lag = false
+        self.ha.fetch_node_status = get_node_status(xlog_location=11)  # accessible, in_recovery, xlog location ahead
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
+        with patch('patroni.postgresql.Postgresql.xlog_position', return_value=1):
+            self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.patroni.nofailover = True
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.patroni.nofailover = False
@@ -395,3 +463,311 @@ class TestHa(unittest.TestCase):
         self.assertEqual(self.ha.post_recover(), 'failed to start postgres')
         self.p.is_running = true
         self.assertIsNone(self.ha.post_recover())
+
+    def test_schedule_future_restart(self):
+        self.ha.patroni.scheduled_restart = {}
+        # do the restart 2 times. The first one should succeed, the second one should fail
+        self.assertTrue(self.ha.schedule_future_restart({'schedule': future_restart_time}))
+        self.assertFalse(self.ha.schedule_future_restart({'schedule': future_restart_time}))
+
+    def test_delete_future_restarts(self):
+        self.ha.delete_future_restart()
+
+    def test_evaluate_scheduled_restart(self):
+        self.p.postmaster_start_time = Mock(return_value=str(postmaster_start_time))
+        # restart already in progres
+        with patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=True)):
+            self.assertIsNone(self.ha.evaluate_scheduled_restart())
+        # restart while the postmaster has been already restarted, fails
+        with patch.object(self.ha,
+                          'future_restart_scheduled',
+                          Mock(return_value={'postmaster_start_time':
+                                             str(postmaster_start_time - datetime.timedelta(days=1)),
+                                             'schedule': str(future_restart_time)})):
+            self.assertIsNone(self.ha.evaluate_scheduled_restart())
+        with patch.object(self.ha,
+                          'future_restart_scheduled',
+                          Mock(return_value={'postmaster_start_time': str(postmaster_start_time),
+                                             'schedule': str(future_restart_time)})):
+            with patch.object(self.ha,
+                              'should_run_scheduled_action', Mock(return_value=True)):
+                # restart in the future, ok
+                self.assertIsNotNone(self.ha.evaluate_scheduled_restart())
+                with patch.object(self.ha, 'restart', Mock(return_value=(False, "Test"))):
+                    # restart in the future, bit the actual restart failed
+                    self.assertIsNone(self.ha.evaluate_scheduled_restart())
+
+    def test_scheduled_restart(self):
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        with patch.object(self.ha, "evaluate_scheduled_restart", Mock(return_value="restart scheduled")):
+            self.assertEquals(self.ha.run_cycle(), "restart scheduled")
+
+    def test_restart_matches(self):
+        self.p._role = 'replica'
+        self.p.server_version = 90500
+        self.p._pending_restart = True
+        self.assertFalse(self.ha.restart_matches("master", "9.5.0", True))
+        self.assertFalse(self.ha.restart_matches("replica", "9.4.3", True))
+        self.p._pending_restart = False
+        self.assertFalse(self.ha.restart_matches("replica", "9.5.2", True))
+        self.assertTrue(self.ha.restart_matches("replica", "9.5.2", False))
+
+    def test_process_healthy_cluster_in_pause(self):
+        self.p.is_leader = false
+        self.ha.is_paused = true
+        self.p.name = 'leader'
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: removed leader lock because postgres is not running as master')
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', self.p.name, None))
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: waiting to become master after promote...')
+
+    def test_postgres_unhealthy_in_pause(self):
+        self.ha.is_paused = true
+        self.p.is_healthy = false
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: postgres is not running')
+        self.ha.has_lock = true
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: removed leader lock because postgres is not running')
+
+    def test_no_etcd_connection_in_pause(self):
+        self.ha.is_paused = true
+        self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: DCS is not accessible')
+
+    @patch('patroni.ha.Ha.update_lock', return_value=True)
+    @patch('patroni.ha.Ha.demote')
+    def test_starting_timeout(self, demote, update_lock):
+        def check_calls(seq):
+            for mock, called in seq:
+                if called:
+                    mock.assert_called_once()
+                else:
+                    mock.assert_not_called()
+                mock.reset_mock()
+        self.ha.has_lock = true
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.p.check_for_startup = true
+        self.p.time_in_state = lambda: 30
+        self.assertEquals(self.ha.run_cycle(), 'PostgreSQL is still starting up, 270 seconds until timeout')
+        check_calls([(update_lock, True), (demote, False)])
+
+        self.p.time_in_state = lambda: 350
+        self.ha.fetch_node_status = get_node_status(reachable=False)  # inaccessible, in_recovery
+        self.assertEquals(self.ha.run_cycle(),
+                          'master start has timed out, but continuing to wait because failover is not possible')
+        check_calls([(update_lock, True), (demote, False)])
+
+        self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
+        self.assertEquals(self.ha.run_cycle(), 'stopped PostgreSQL because of startup timeout')
+        check_calls([(update_lock, True), (demote, True)])
+
+        update_lock.return_value = False
+        self.assertEquals(self.ha.run_cycle(), 'stopped PostgreSQL while starting up because leader key was lost')
+        check_calls([(update_lock, True), (demote, True)])
+
+        self.ha.has_lock = false
+        self.p.is_leader = false
+        self.assertEquals(self.ha.run_cycle(), 'no action.  i am a secondary and i am following a leader')
+        check_calls([(update_lock, False), (demote, False)])
+
+    @patch('time.sleep', Mock())
+    def test_manual_failover_while_starting(self):
+        self.ha.has_lock = true
+        self.p.check_for_startup = true
+        f = Failover(0, self.p.name, '', None)
+        self.ha.cluster = get_cluster_initialized_with_leader(f)
+        self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
+        self.assertEquals(self.ha.run_cycle(), 'manual failover: demoting myself')
+
+    @patch('patroni.ha.Ha.demote')
+    def test_failover_immediately_on_zero_master_start_timeout(self, demote):
+        self.p.is_running = false
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.ha.patroni.config.set_dynamic_configuration({'master_start_timeout': 0})
+        self.ha.has_lock = true
+        self.ha.update_lock = true
+        self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
+        self.assertEquals(self.ha.run_cycle(), 'stopped PostgreSQL to fail over after a crash')
+        demote.assert_called_once()
+
+    @patch('time.sleep', Mock())
+    @patch('patroni.postgresql.Postgresql.follow')
+    def test_demote_immediate(self, follow):
+        self.ha.has_lock = true
+        self.e.get_cluster = Mock(return_value=get_cluster_initialized_without_leader())
+        self.ha.demote('immediate')
+        follow.assert_called_once_with(None, None, True, None, True)
+
+    @patch('time.sleep', Mock())
+    def test_process_sync_replication(self):
+        self.ha.has_lock = true
+        mock_set_sync = self.p.set_synchronous_standby = Mock()
+        self.p.name = 'leader'
+
+        # Test sync key removed when sync mode disabled
+        self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
+        with patch.object(self.ha.dcs, 'delete_sync_state') as mock_delete_sync:
+            self.ha.run_cycle()
+            mock_delete_sync.assert_called_once()
+            mock_set_sync.assert_called_once_with(None)
+
+        mock_set_sync.reset_mock()
+        # Test sync key not touched when not there
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        with patch.object(self.ha.dcs, 'delete_sync_state') as mock_delete_sync:
+            self.ha.run_cycle()
+            mock_delete_sync.assert_not_called()
+            mock_set_sync.assert_called_once_with(None)
+
+        mock_set_sync.reset_mock()
+
+        self.ha.is_synchronous_mode = true
+
+        # Test sync standby not touched when picking the same node
+        self.p.pick_synchronous_standby = Mock(return_value=('other', True))
+        self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
+        self.ha.run_cycle()
+        mock_set_sync.assert_not_called()
+
+        mock_set_sync.reset_mock()
+
+        # Test sync standby is replaced when switching standbys
+        self.p.pick_synchronous_standby = Mock(return_value=('other2', False))
+        self.ha.dcs.write_sync_state = Mock(return_value=True)
+        self.ha.run_cycle()
+        mock_set_sync.assert_called_once_with('other2')
+
+        mock_set_sync.reset_mock()
+        # Test sync standby is not disabled when updating dcs fails
+        self.ha.dcs.write_sync_state = Mock(return_value=False)
+        self.ha.run_cycle()
+        mock_set_sync.assert_not_called()
+
+        mock_set_sync.reset_mock()
+        # Test changing sync standby
+        self.ha.dcs.write_sync_state = Mock(return_value=True)
+        self.ha.dcs.get_cluster = Mock(return_value=get_cluster_initialized_with_leader(sync=('leader', 'other')))
+        # self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
+        self.p.pick_synchronous_standby = Mock(return_value=('other2', True))
+        self.ha.run_cycle()
+        self.ha.dcs.get_cluster.assert_called_once()
+        self.assertEquals(self.ha.dcs.write_sync_state.call_count, 2)
+
+        # Test updating sync standby key failed due to race
+        self.ha.dcs.write_sync_state = Mock(side_effect=[True, False])
+        self.ha.run_cycle()
+        self.assertEquals(self.ha.dcs.write_sync_state.call_count, 2)
+
+        # Test changing sync standby failed due to race
+        self.ha.dcs.write_sync_state = Mock(return_value=True)
+        self.ha.dcs.get_cluster = Mock(return_value=get_cluster_initialized_with_leader(sync=('somebodyelse', None)))
+        self.ha.run_cycle()
+        self.assertEquals(self.ha.dcs.write_sync_state.call_count, 1)
+
+    def test_sync_replication_become_master(self):
+        self.ha.is_synchronous_mode = true
+
+        mock_set_sync = self.p.set_synchronous_standby = Mock()
+        self.p.is_leader = false
+        self.p.set_role('replica')
+        self.ha.has_lock = true
+        mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=True)
+        self.p.name = 'leader'
+        self.ha.cluster = get_cluster_initialized_with_leader(sync=('other', None))
+
+        # When we just became master nobody is sync
+        self.assertEquals(self.ha.enforce_master_role('msg', 'promote msg'), 'promote msg')
+        mock_set_sync.assert_called_once_with(None)
+        mock_write_sync.assert_called_once_with('leader', None, index=0)
+
+        mock_set_sync.reset_mock()
+
+        # When we just became master nobody is sync
+        self.p.set_role('replica')
+        mock_write_sync.return_value = False
+        self.assertTrue(self.ha.enforce_master_role('msg', 'promote msg') != 'promote msg')
+        mock_set_sync.assert_not_called()
+
+    def test_unhealthy_sync_mode(self):
+        self.ha.is_synchronous_mode = true
+
+        self.p.is_leader = false
+        self.p.set_role('replica')
+        self.p.name = 'other'
+        self.ha.cluster = get_cluster_initialized_without_leader(sync=('leader', 'other2'))
+        mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=True)
+        mock_acquire = self.ha.acquire_lock = Mock(return_value=True)
+        mock_follow = self.p.follow = Mock()
+        mock_promote = self.p.promote = Mock()
+
+        # If we don't match the sync replica we are not allowed to acquire lock
+        self.ha.run_cycle()
+        mock_acquire.assert_not_called()
+        mock_follow.assert_called_once()
+        self.assertEquals(mock_follow.call_args[0][0], None)
+        mock_write_sync.assert_not_called()
+
+        mock_follow.reset_mock()
+        # If we do match we will try to promote
+        self.ha._is_healthiest_node = true
+
+        self.ha.cluster = get_cluster_initialized_without_leader(sync=('leader', 'other'))
+        self.ha.run_cycle()
+        mock_acquire.assert_called_once()
+        mock_follow.assert_not_called()
+        mock_promote.assert_called_once()
+        mock_write_sync.assert_called_once_with('other', None, index=0)
+
+    @patch('time.sleep')
+    def test_disable_sync_when_restarting(self, mock_sleep):
+        self.ha.is_synchronous_mode = true
+
+        self.p.name = 'other'
+        self.p.is_leader = false
+        self.p.set_role('replica')
+        mock_restart = self.p.restart = Mock(return_value=True)
+        self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
+        self.ha.touch_member = Mock(return_value=True)
+        self.ha.dcs.get_cluster = Mock(side_effect=[
+            get_cluster_initialized_with_leader(sync=('leader', syncstandby))
+            for syncstandby in ['other', None]])
+
+        self.ha.restart({})
+
+        mock_restart.assert_called_once()
+        mock_sleep.assert_called()
+
+        # Restart is still called when DCS connection fails
+        mock_restart.reset_mock()
+        self.ha.dcs.get_cluster = Mock(side_effect=DCSError("foo"))
+        self.ha.restart({})
+
+        mock_restart.assert_called_once()
+
+        # We don't try to fetch the cluster state when touch_member fails
+        mock_restart.reset_mock()
+        self.ha.dcs.get_cluster.reset_mock()
+        self.ha.touch_member = Mock(return_value=False)
+
+        self.ha.restart({})
+
+        mock_restart.assert_called_once()
+        self.ha.dcs.get_cluster.assert_not_called()
+
+    def test_effective_tags(self):
+        self.ha._disable_sync = True
+        self.assertEquals(self.ha.get_effective_tags(), {'foo': 'bar', 'nosync': True})
+        self.ha._disable_sync = False
+        self.assertEquals(self.ha.get_effective_tags(), {'foo': 'bar'})
+
+    def test_restore_cluster_config(self):
+        self.ha.cluster.config.data.clear()
+        self.ha.has_lock = true
+        self.ha.cluster.is_unlocked = false
+        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
+
+    def test_watch(self):
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.ha.watch(0)
+
+    def test_wakup(self):
+        self.ha.wakeup()

tests/test_patroni.py

@@ -1,18 +1,25 @@
 import etcd
+import signal
 import sys
 import time
 import unittest
 
-from mock import Mock, patch
+from mock import Mock, PropertyMock, patch
 from patroni.api import RestApiServer
 from patroni.async_executor import AsyncExecutor
+from patroni.dcs.etcd import Client
 from patroni.exceptions import DCSError
-from patroni import Patroni, main as _main
+from patroni import Patroni, main as _main, patroni_main
 from six.moves import BaseHTTPServer
 from test_etcd import SleepException, etcd_read, etcd_write
 from test_postgresql import Postgresql, psycopg2_connect
 
 
+class MockFrozenImporter(object):
+
+    toc = set(['patroni.dcs.etcd'])
+
+
 @patch('time.sleep', Mock())
 @patch('subprocess.call', Mock(return_value=0))
 @patch('psycopg2.connect', psycopg2_connect)
@@ -20,18 +27,21 @@ from test_postgresql import Postgresql, psycopg2_connect
 @patch.object(Postgresql, '_write_postgresql_conf', Mock())
 @patch.object(Postgresql, 'write_recovery_conf', Mock())
 @patch.object(Postgresql, 'is_running', Mock(return_value=True))
+@patch.object(Postgresql, 'call_nowait', Mock())
 @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
 @patch.object(AsyncExecutor, 'run', Mock())
 @patch.object(etcd.Client, 'write', etcd_write)
 @patch.object(etcd.Client, 'read', etcd_read)
 class TestPatroni(unittest.TestCase):
 
+    @patch('pkgutil.get_importer', Mock(return_value=MockFrozenImporter()))
+    @patch('sys.frozen', Mock(return_value=True), create=True)
     @patch.object(etcd.Client, 'read', etcd_read)
     def setUp(self):
         RestApiServer._BaseServer__is_shut_down = Mock()
         RestApiServer._BaseServer__shutdown_request = True
         RestApiServer.socket = 0
-        with patch.object(etcd.Client, 'machines') as mock_machines:
+        with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
             sys.argv = ['patroni.py', 'postgres0.yml']
             self.p = Patroni()
@@ -44,19 +54,56 @@ class TestPatroni(unittest.TestCase):
 
     @patch('time.sleep', Mock(side_effect=SleepException))
     @patch.object(etcd.Client, 'delete', Mock())
-    @patch.object(etcd.Client, 'machines')
-    def test_patroni_main(self, mock_machines):
+    @patch.object(Client, 'machines')
+    def test_patroni_patroni_main(self, mock_machines):
         with patch('subprocess.call', Mock(return_value=1)):
             sys.argv = ['patroni.py', 'postgres0.yml']
 
             mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
             with patch.object(Patroni, 'run', Mock(side_effect=SleepException)):
-                self.assertRaises(SleepException, _main)
+                self.assertRaises(SleepException, patroni_main)
             with patch.object(Patroni, 'run', Mock(side_effect=KeyboardInterrupt())):
+                with patch('patroni.ha.Ha.is_paused', Mock(return_value=True)):
+                    patroni_main()
+
+    @patch('os.getpid')
+    @patch('subprocess.Popen', )
+    @patch('patroni.patroni_main', Mock())
+    def test_patroni_main(self, mock_popen, mock_getpid):
+        mock_getpid.return_value = 2
+        _main()
+
+        with patch('sys.frozen', Mock(return_value=True), create=True):
+            sys.argv = ['/patroni', 'pg_ctl_start', 'postgres', '-D', '/data', '--max_connections=100']
+            _main()
+
+        mock_getpid.return_value = 1
+
+        def mock_signal(signo, handler):
+            handler(signo, None)
+
+        with patch('signal.signal', mock_signal):
+            with patch('os.waitpid', Mock(side_effect=[(1, 0), (0, 0)])):
                 _main()
+            with patch('os.waitpid', Mock(side_effect=OSError)):
+                _main()
+
+        ref = {'passtochild': lambda signo, stack_frame: 0}
+
+        def mock_sighup(signo, handler):
+            if signo == signal.SIGHUP:
+                ref['passtochild'] = handler
+
+        def mock_wait():
+            ref['passtochild'](0, None)
+
+        mock_popen.return_value.wait = mock_wait
+        with patch('signal.signal', mock_sighup), patch('os.kill', Mock()):
+            self.assertIsNone(_main())
 
     @patch('patroni.config.Config.save_cache', Mock())
     @patch('patroni.config.Config.reload_local_configuration', Mock(return_value=True))
+    @patch.object(Postgresql, 'state', PropertyMock(return_value='running'))
     def test_run(self):
         self.p.sighup_handler()
         self.p.ha.dcs.watch = Mock(side_effect=SleepException)
@@ -74,7 +121,7 @@ class TestPatroni(unittest.TestCase):
     def test_schedule_next_run(self):
         self.p.ha.dcs.watch = Mock(return_value=True)
         self.p.schedule_next_run()
-        self.p.next_run = time.time() - self.p.nap_time - 1
+        self.p.next_run = time.time() - self.p.dcs.loop_wait - 1
         self.p.schedule_next_run()
 
     def test_noloadbalance(self):
@@ -96,3 +143,9 @@ class TestPatroni(unittest.TestCase):
         self.p.reload_config()
         self.p.get_tags = Mock(side_effect=Exception)
         self.p.reload_config()
+
+    def test_nosync(self):
+        self.p.tags['nosync'] = True
+        self.assertTrue(self.p.nosync)
+        self.p.tags['nosync'] = None
+        self.assertFalse(self.p.nosync)

tests/test_postgresql.py

@@ -6,12 +6,12 @@ import subprocess
 import unittest
 
 from mock import Mock, MagicMock, PropertyMock, patch, mock_open
-from patroni.dcs import Cluster, Leader, Member
+from patroni.dcs import Cluster, Leader, Member, SyncState
 from patroni.exceptions import PostgresException, PostgresConnectionException
-from patroni.postgresql import Postgresql
+from patroni.postgresql import Postgresql, STATE_REJECT, STATE_NO_RESPONSE
 from patroni.utils import RetryFailedError
 from six.moves import builtins
-from test_ha import false
+from threading import Thread
 
 
 class MockCursor(object):
@@ -28,18 +28,21 @@ class MockCursor(object):
             raise RetryFailedError('retry')
         elif sql.startswith('SELECT slot_name'):
             self.results = [('blabla',), ('foobar',)]
-        elif sql.startswith('SELECT pg_xlog_location_diff'):
+        elif sql.startswith('SELECT CASE WHEN pg_is_in_recovery()'):
             self.results = [(0,)]
         elif sql == 'SELECT pg_is_in_recovery()':
             self.results = [(False, )]
-        elif sql.startswith('SELECT to_char(pg_postmaster_start_time'):
-            self.results = [('', True, '', '', '', '', False)]
+        elif sql.startswith('WITH replication_info AS ('):
+            replication_info = '[{"application_name":"walreceiver","client_addr":"1.2.3.4",' +\
+                               '"state":"streaming","sync_state":"async","sync_priority":0}]'
+            self.results = [('', True, '', '', '', '', False, replication_info)]
         elif sql.startswith('SELECT name, setting'):
             self.results = [('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
                             ('search_path', 'public', None, 'string', 'user'),
                             ('port', '5433', None, 'integer', 'postmaster'),
                             ('listen_addresses', '*', None, 'string', 'postmaster'),
-                            ('autovacuum', 'on', None, 'bool', 'sighup')]
+                            ('autovacuum', 'on', None, 'bool', 'sighup'),
+                            ('wal_level', 'replica', None, 'enum', 'postmaster')]
         else:
             self.results = [(None, None, None, None, None, None, None, None, None, None)]
 
@@ -156,12 +159,13 @@ class TestPostgresql(unittest.TestCase):
     _PARAMETERS = {'wal_level': 'hot_standby', 'max_replication_slots': 5, 'f.oo': 'bar',
                    'search_path': 'public', 'hot_standby': 'on', 'max_wal_senders': 5,
                    'wal_keep_segments': 8, 'wal_log_hints': 'on', 'max_locks_per_transaction': 64,
-                   'max_worker_processes': 8, 'max_connections': 100, 'max_prepared_transactions': 0}
+                   'max_worker_processes': 8, 'max_connections': 100, 'max_prepared_transactions': 0,
+                   'track_commit_timestamp': 'off'}
 
     @patch('subprocess.call', Mock(return_value=0))
     @patch('psycopg2.connect', psycopg2_connect)
     @patch('os.rename', Mock())
-    @patch.object(Postgresql, 'get_major_version', Mock(return_value=9.4))
+    @patch.object(Postgresql, 'get_major_version', Mock(return_value=9.6))
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def setUp(self):
         self.data_dir = 'data/test0'
@@ -180,9 +184,10 @@ class TestPostgresql(unittest.TestCase):
                                            'on_reload': 'true'
                                            },
                              'restore': 'true'})
+        self.p._callback_executor = Mock()
         self.leadermem = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres'})
         self.leader = Leader(-1, 28, self.leadermem)
-        self.other = Member(0, 'test1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres',
+        self.other = Member(0, 'test-1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres',
                             'tags': {'replicatefrom': 'leader'}})
         self.me = Member(0, 'test0', 28, {'conn_url': 'postgres://replicator:[email protected]:5434/postgres'})
 
@@ -201,19 +206,64 @@ class TestPostgresql(unittest.TestCase):
     def test_delete_trigger_file(self):
         self.p.delete_trigger_file()
 
+    @patch('subprocess.Popen')
+    @patch.object(Postgresql, 'wait_for_startup')
+    @patch.object(Postgresql, 'wait_for_port_open')
     @patch.object(Postgresql, 'is_running')
-    def test_start(self, mock_is_running):
+    def test_start(self, mock_is_running, mock_wait_for_port_open, mock_wait_for_startup, mock_popen):
         mock_is_running.return_value = True
+        mock_wait_for_port_open.return_value = True
+        mock_wait_for_startup.return_value = False
+        mock_popen.stdout.readline.return_value = '123'
         self.assertTrue(self.p.start())
         mock_is_running.return_value = False
         open(os.path.join(self.data_dir, 'postmaster.pid'), 'w').close()
         pg_conf = os.path.join(self.data_dir, 'postgresql.conf')
         open(pg_conf, 'w').close()
-        self.assertTrue(self.p.start())
+        self.assertFalse(self.p.start())
         with open(pg_conf) as f:
             lines = f.readlines()
             self.assertTrue("f.oo = 'bar'\n" in lines)
 
+        mock_wait_for_startup.return_value = None
+        self.assertFalse(self.p.start(10))
+        self.assertIsNone(self.p.start())
+
+        mock_wait_for_port_open.return_value = False
+        self.assertFalse(self.p.start())
+
+    @patch.object(Postgresql, 'pg_isready')
+    @patch.object(Postgresql, 'read_pid_file')
+    @patch.object(Postgresql, 'is_pid_running')
+    @patch('patroni.postgresql.polling_loop', Mock(return_value=range(1)))
+    def test_wait_for_port_open(self, mock_is_pid_running, mock_read_pid_file, mock_pg_isready):
+        mock_is_pid_running.return_value = False
+        mock_pg_isready.return_value = STATE_NO_RESPONSE
+
+        # No pid file and postmaster death
+        mock_read_pid_file.return_value = {}
+        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
+
+        mock_is_pid_running.return_value = True
+
+        # timeout
+        mock_read_pid_file.return_value = {'pid', 1}
+        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
+
+        # Garbage pid
+        mock_read_pid_file.return_value = {'pid': 'garbage', 'start_time': '101', 'data_dir': '',
+                                           'socket_dir': '', 'port': '', 'listen_addr': ''}
+        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
+
+        # Not ready
+        mock_read_pid_file.return_value = {'pid': '42', 'start_time': '101', 'data_dir': '',
+                                           'socket_dir': '', 'port': '', 'listen_addr': ''}
+        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
+
+        # pg_isready failure
+        mock_pg_isready.return_value = 'garbage'
+        self.assertTrue(self.p.wait_for_port_open(42, 100., 1))
+
     @patch.object(Postgresql, 'is_running')
     def test_stop(self, mock_is_running):
         mock_is_running.return_value = True
@@ -223,12 +273,13 @@ class TestPostgresql(unittest.TestCase):
             self.assertTrue(self.p.stop())
 
     def test_restart(self):
-        self.p.start = false
+        self.p.start = Mock(return_value=False)
         self.assertFalse(self.p.restart())
         self.assertEquals(self.p.state, 'restart failed (restarting)')
 
     @patch.object(builtins, 'open', MagicMock())
     def test_write_pgpass(self):
+        self.p.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo'})
         self.p.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo', 'password': 'bar'})
 
     def test_checkpoint(self):
@@ -263,22 +314,18 @@ class TestPostgresql(unittest.TestCase):
         with patch.object(Postgresql, 'restart', Mock(return_value=False)):
             self.p.set_role('replica')
             self.p.follow(None, None)  # restart without rewind
-            self.p.set_role('master')
 
         with patch.object(Postgresql, 'stop', Mock(return_value=False)):
-            self.p.follow(self.leader, self.leader)  # failed to stop postgres
-
-        self.p.follow(self.leader, None)  # Leader unknown, can not rewind
+            self.p.follow(self.leader, self.leader, need_rewind=True)  # failed to stop postgres
 
         self.p.follow(self.leader, self.leader)  # "leader" is not accessible or is_in_recovery
 
         with patch.object(Postgresql, 'checkpoint', Mock(return_value=None)):
             self.p.follow(self.leader, self.leader)
-            self.p.set_role('master')
             mock_pg_rewind.return_value = True
-            self.p.follow(self.leader, self.leader)
+            self.p.follow(self.leader, self.leader, need_rewind=True)
 
-        self.assertTrue(self.p.follow(None, None))  # check_recovery_conf...
+        self.p.follow(None, None)  # check_recovery_conf...
 
     @patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string))
     def test_can_rewind(self):
@@ -292,6 +339,7 @@ class TestPostgresql(unittest.TestCase):
         self.assertFalse(self.p.can_rewind)
 
     @patch('time.sleep', Mock())
+    @patch.object(Postgresql, 'remove_data_directory', Mock(return_value=True))
     def test_create_replica(self):
         self.p.delete_trigger_file = Mock(side_effect=OSError)
         with patch('subprocess.call', Mock(side_effect=[1, 0])):
@@ -309,17 +357,27 @@ class TestPostgresql(unittest.TestCase):
         with patch('subprocess.call', Mock(side_effect=Exception("foo"))):
             self.assertEquals(self.p.create_replica(self.leader), 1)
 
+        with patch('subprocess.call', Mock(return_value=1)):
+            self.assertEquals(self.p.create_replica(self.leader), 1)
+
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def test_sync_replication_slots(self):
         self.p.start()
-        cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None)
+        cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None, None)
+        with mock.patch('patroni.postgresql.Postgresql._query', Mock(side_effect=psycopg2.OperationalError)):
+            self.p.sync_replication_slots(cluster)
         self.p.sync_replication_slots(cluster)
-        self.p.query = Mock(side_effect=psycopg2.OperationalError)
-        self.p.schedule_load_slots = True
-        self.p.sync_replication_slots(cluster)
-        self.p.schedule_load_slots = False
         with mock.patch('patroni.postgresql.Postgresql.role', new_callable=PropertyMock(return_value='replica')):
             self.p.sync_replication_slots(cluster)
+        with mock.patch('patroni.postgresql.logger.error', new_callable=Mock()) as errorlog_mock:
+            self.p.query = Mock()
+            alias1 = Member(0, 'test-3', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres'})
+            alias2 = Member(0, 'test.3', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres'})
+            cluster.members.extend([alias1, alias2])
+            self.p.sync_replication_slots(cluster)
+            errorlog_mock.assert_called_once()
+            assert "test-3" in errorlog_mock.call_args[0][1]
+            assert "test.3" in errorlog_mock.call_args[0][1]
 
     @patch.object(MockConnect, 'closed', 2)
     def test__query(self):
@@ -332,8 +390,11 @@ class TestPostgresql(unittest.TestCase):
         self.assertRaises(PostgresConnectionException, self.p.query, 'RetryFailedError')
         self.assertRaises(psycopg2.OperationalError, self.p.query, 'blabla')
 
+    @patch.object(Postgresql, 'pg_isready', Mock(return_value=STATE_REJECT))
     def test_is_leader(self):
         self.assertTrue(self.p.is_leader())
+        with patch.object(Postgresql, '_query', Mock(side_effect=RetryFailedError(''))):
+            self.assertRaises(PostgresConnectionException, self.p.is_leader)
 
     def test_reload(self):
         self.assertTrue(self.p.reload())
@@ -352,6 +413,7 @@ class TestPostgresql(unittest.TestCase):
 
     def test_last_operation(self):
         self.assertEquals(self.p.last_operation(), '0')
+        Thread(target=self.p.last_operation).start()
 
     @patch('os.path.isfile', Mock(return_value=True))
     @patch('os.kill', Mock(side_effect=Exception))
@@ -362,9 +424,9 @@ class TestPostgresql(unittest.TestCase):
     def test_is_running(self):
         self.assertFalse(self.p.is_running())
 
-    @patch('subprocess.Popen', Mock(side_effect=OSError))
+    @patch('shlex.split', Mock(side_effect=OSError))
     def test_call_nowait(self):
-        self.assertFalse(self.p.call_nowait('on_start'))
+        self.assertIsNone(self.p.call_nowait('on_start'))
 
     def test_non_existing_callback(self):
         self.assertFalse(self.p.call_nowait('foobar'))
@@ -375,9 +437,6 @@ class TestPostgresql(unittest.TestCase):
         self.p.query = Mock(side_effect=psycopg2.OperationalError("not supported"))
         self.assertTrue(self.p.stop())
 
-    def test_check_replication_lag(self):
-        self.assertTrue(self.p.check_replication_lag(0))
-
     @patch('os.rename', Mock())
     @patch('os.path.isdir', Mock(return_value=True))
     def test_move_data_directory(self):
@@ -390,15 +449,35 @@ class TestPostgresql(unittest.TestCase):
         with patch('subprocess.call', Mock(return_value=1)):
             self.assertRaises(PostgresException, self.p.bootstrap, {})
 
+        with patch.object(Postgresql, 'run_bootstrap_post_init', Mock(return_value=False)):
+            self.assertRaises(PostgresException, self.p.bootstrap, {})
+
         self.p.bootstrap({'users': {'replicator': {'password': 'rep-pass', 'options': ['replication']}},
                           'pg_hba': ['host replication replicator 127.0.0.1/32 md5',
                                      'hostssl all all 0.0.0.0/0 md5',
-                                     'host all all 0.0.0.0/0 md5']})
+                                     'host all all 0.0.0.0/0 md5'],
+                          'post_init': '/bin/false'})
         with open(os.path.join(self.data_dir, 'pg_hba.conf')) as f:
             lines = f.readlines()
             assert 'host replication replicator 127.0.0.1/32 md5\n' in lines
             assert 'host all all 0.0.0.0/0 md5\n' in lines
 
+    def test_run_bootstrap_post_init(self):
+        with patch('subprocess.call', Mock(return_value=1)):
+            self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+
+        with patch('subprocess.call', Mock(side_effect=OSError)):
+            self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+
+        with patch('subprocess.call', Mock(return_value=0)) as mock_method:
+            self.p._superuser.pop('username')
+            self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+
+        mock_method.assert_called()
+        args, kwargs = mock_method.call_args
+        assert 'PGPASSFILE' in kwargs['env'].keys()
+        self.assertEquals(args[0], ['/bin/false', 'postgres://localhost:5432/postgres'])
+
     @patch('patroni.postgresql.Postgresql.create_replica', Mock(return_value=0))
     def test_clone(self):
         self.p.clone(self.leader)
@@ -537,3 +616,157 @@ class TestPostgresql(unittest.TestCase):
             self.assertEquals(self.p.get_major_version(), 9.4)
         with patch.object(builtins, 'open', Mock(side_effect=Exception)):
             self.assertEquals(self.p.get_major_version(), 0.0)
+
+    def test_postmaster_start_time(self):
+        with patch.object(MockCursor, "fetchone", Mock(return_value=('foo', True, '', '', '', '', False))):
+            self.assertEqual(self.p.postmaster_start_time(), 'foo')
+        with patch.object(MockCursor, "execute", side_effect=psycopg2.Error):
+            self.assertIsNone(self.p.postmaster_start_time())
+
+    def test_check_for_startup(self):
+        with patch('subprocess.call', return_value=0):
+            self.p._state = 'starting'
+            self.assertFalse(self.p.check_for_startup())
+            self.assertEquals(self.p.state, 'running')
+
+        with patch('subprocess.call', return_value=1):
+            self.p._state = 'starting'
+            self.assertTrue(self.p.check_for_startup())
+            self.assertEquals(self.p.state, 'starting')
+
+        with patch('subprocess.call', return_value=2):
+            self.p._state = 'starting'
+            self.assertFalse(self.p.check_for_startup())
+            self.assertEquals(self.p.state, 'start failed')
+
+        with patch('subprocess.call', return_value=0):
+            self.p._state = 'running'
+            self.assertFalse(self.p.check_for_startup())
+            self.assertEquals(self.p.state, 'running')
+
+        with patch('subprocess.call', return_value=127):
+            self.p._state = 'running'
+            self.assertFalse(self.p.check_for_startup())
+            self.assertEquals(self.p.state, 'running')
+
+            self.p._state = 'starting'
+            self.assertFalse(self.p.check_for_startup())
+            self.assertEquals(self.p.state, 'running')
+
+    def test_wait_for_startup(self):
+        state = {'sleeps': 0, 'num_rejects': 0, 'final_return': 0}
+
+        def increment_sleeps(*args):
+            print("Sleep")
+            state['sleeps'] += 1
+
+        def isready_return(*args):
+            ret = 1 if state['sleeps'] < state['num_rejects'] else state['final_return']
+            print("Isready {0} {1}".format(ret, state))
+            return ret
+
+        def time_in_state(*args):
+            return state['sleeps']
+
+        with patch('subprocess.call', side_effect=isready_return):
+            with patch('time.sleep', side_effect=increment_sleeps):
+                self.p.time_in_state = Mock(side_effect=time_in_state)
+
+                self.p._state = 'stopped'
+                self.assertTrue(self.p.wait_for_startup())
+                self.assertEquals(state['sleeps'], 0)
+
+                self.p._state = 'starting'
+                state['num_rejects'] = 5
+                self.assertTrue(self.p.wait_for_startup())
+                self.assertEquals(state['sleeps'], 5)
+
+                self.p._state = 'starting'
+                state['sleeps'] = 0
+                state['final_return'] = 2
+                self.assertFalse(self.p.wait_for_startup())
+
+                self.p._state = 'starting'
+                state['sleeps'] = 0
+                state['final_return'] = 0
+                self.assertFalse(self.p.wait_for_startup(timeout=2))
+                self.assertEquals(state['sleeps'], 3)
+
+    def test_read_pid_file(self):
+        pidfile = os.path.join(self.data_dir, 'postmaster.pid')
+        if os.path.exists(pidfile):
+            os.remove(pidfile)
+        self.assertEquals(self.p.read_pid_file(), {})
+
+    @patch('os.kill')
+    def test_is_pid_running(self, mock_kill):
+        mock_kill.return_value = True
+        self.assertTrue(self.p.is_pid_running(-100))
+        self.assertFalse(self.p.is_pid_running(0))
+        self.assertFalse(self.p.is_pid_running(None))
+
+    def test_pick_sync_standby(self):
+        cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None,
+                          SyncState(0, self.me.name, self.leadermem.name))
+
+        with patch.object(Postgresql, "query", return_value=[
+                    (self.leadermem.name, 'streaming', 'sync'),
+                    (self.me.name, 'streaming', 'async'),
+                    (self.other.name, 'streaming', 'async'),
+                ]):
+            self.assertEquals(self.p.pick_synchronous_standby(cluster), (self.leadermem.name, True))
+
+        with patch.object(Postgresql, "query", return_value=[
+                    (self.me.name, 'streaming', 'async'),
+                    (self.leadermem.name, 'streaming', 'potential'),
+                    (self.other.name, 'streaming', 'async'),
+                ]):
+            self.assertEquals(self.p.pick_synchronous_standby(cluster), (self.leadermem.name, False))
+
+        with patch.object(Postgresql, "query", return_value=[
+                    (self.me.name, 'streaming', 'async'),
+                    (self.other.name, 'streaming', 'async'),
+                ]):
+            self.assertEquals(self.p.pick_synchronous_standby(cluster), (self.me.name, False))
+
+        with patch.object(Postgresql, "query", return_value=[
+                    ('missing', 'streaming', 'sync'),
+                    (self.me.name, 'streaming', 'async'),
+                    (self.other.name, 'streaming', 'async'),
+                ]):
+            self.assertEquals(self.p.pick_synchronous_standby(cluster), (self.me.name, False))
+
+        with patch.object(Postgresql, "query", return_value=[]):
+            self.assertEquals(self.p.pick_synchronous_standby(cluster), (None, False))
+
+    def test_set_sync_standby(self):
+        def value_in_conf():
+            with open(os.path.join(self.data_dir, 'postgresql.conf')) as f:
+                for line in f:
+                    if line.startswith('synchronous_standby_names'):
+                        return line.strip()
+
+        mock_reload = self.p.reload = Mock()
+        self.p.set_synchronous_standby('n1')
+        self.assertEquals(value_in_conf(), "synchronous_standby_names = 'n1'")
+        mock_reload.assert_called()
+
+        mock_reload.reset_mock()
+        self.p.set_synchronous_standby('n1')
+        mock_reload.assert_not_called()
+        self.assertEquals(value_in_conf(), "synchronous_standby_names = 'n1'")
+
+        self.p.set_synchronous_standby('n2')
+        mock_reload.assert_called()
+        self.assertEquals(value_in_conf(), "synchronous_standby_names = 'n2'")
+
+        mock_reload.reset_mock()
+        self.p.set_synchronous_standby(None)
+        mock_reload.assert_called()
+        self.assertEquals(value_in_conf(), None)
+
+    def test_get_server_parameters(self):
+        config = {'synchronous_mode': True, 'parameters': {}, 'listen': '0'}
+        self.p.get_server_parameters(config)
+        self.p.set_synchronous_standby('foo')
+        self.p.get_server_parameters(config)

tests/test_utils.py

@@ -2,25 +2,13 @@ import unittest
 
 from mock import Mock, patch
 from patroni.exceptions import PatroniException
-from patroni.utils import reap_children, Retry, RetryFailedError, sigchld_handler, sleep
-
-
-def time_sleep(_):
-    sigchld_handler(None, None)
+from patroni.utils import Retry, RetryFailedError, polling_loop
 
 
 class TestUtils(unittest.TestCase):
 
-    @patch('time.sleep', Mock())
-    def test_reap_children(self):
-        self.assertIsNone(reap_children())
-        with patch('os.waitpid', Mock(return_value=(0, 0))):
-            sigchld_handler(None, None)
-            self.assertIsNone(reap_children())
-
-    @patch('time.sleep', time_sleep)
-    def test_sleep(self):
-        self.assertIsNone(sleep(0.01))
+    def test_polling_loop(self):
+        self.assertEquals(list(polling_loop(0.001, interval=0.001)), [0])
 
 
 @patch('time.sleep', Mock())

tests/test_wale_restore.py

@@ -2,79 +2,100 @@ import psycopg2
 import subprocess
 import unittest
 
-from mock import MagicMock, patch, PropertyMock
-from patroni.scripts.wale_restore import WALERestore, main as _main
+from mock import Mock, MagicMock, patch, mock_open
+from patroni.scripts.wale_restore import WALERestore, main as _main, get_major_version
+from six.moves import builtins
 
 
-def fake_backup_data(self, *args, **kwargs):
-    """ return the fake result of WAL-E backup-list"""
-    return """name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop
-base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z 167772160   00000001000000000000007F    00000040 00000001000000000000007F    00000240
-"""
+wale_output = b'name last_modified expanded_size_bytes wal_segment_backup_start ' +\
+              b'wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop\n' +\
+              b'base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 ' +\
+              b'00000001000000000000007F 00000040 00000001000000000000007F 00000240\n'
 
 
-def fake_backup_data_2(self, *args, **kwargs):
-    """ return the fake result of WAL-E backup-list"""
-    return """name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop """
-
-
-def fake_backup_data_3(self, *args, **kwargs):
-    """ return the fake result of WAL-E backup-list"""
-    return """name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop
-base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z 167772160   00000001000000000000007F    00000040 00000001000000000000007F    00000240
-"""
-
-
-def fake_backup_data_4(self, *args, **kwargs):
-    """ return the fake result of WAL-E backup-list"""
-    return """name    last_modified   expanded_size_foo wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop
-base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z 167772160   00000001000000000000007F    00000040 00000001000000000000007F    00000240
-"""
-
-
-@patch('os.access', MagicMock(return_value=True))
-@patch('os.makedirs', MagicMock(return_value=True))
-@patch('os.path.exists', MagicMock(return_value=True))
-@patch('os.path.isdir', MagicMock(return_value=True))
-@patch('psycopg2.extensions.cursor', MagicMock(autospec=True))
-@patch('psycopg2.extensions.connection', MagicMock(autospec=True))
+@patch('os.access', Mock(return_value=True))
+@patch('os.makedirs', Mock(return_value=True))
+@patch('os.path.exists', Mock(return_value=True))
+@patch('os.path.isdir', Mock(return_value=True))
+@patch('psycopg2.extensions.cursor', Mock(autospec=True))
+@patch('psycopg2.extensions.connection', Mock(autospec=True))
 @patch('psycopg2.connect', MagicMock(autospec=True))
-@patch('subprocess.check_output', MagicMock(side_effect=fake_backup_data))
+@patch('subprocess.check_output', Mock(return_value=wale_output))
 class TestWALERestore(unittest.TestCase):
 
     def setUp(self):
-        self.wale_restore = WALERestore("batman", "/data", "host=batman port=5432 user=batman", "/etc", 100, 100, 1, 0)
+        self.wale_restore = WALERestore("batman", "/data", "host=batman port=5432 user=batman",
+                                        "/etc", 100, 100, 1, 0, 1)
 
     def test_should_use_s3_to_create_replica(self):
-        with patch('psycopg2.connect', MagicMock(side_effect=psycopg2.Error("foo"))):
-            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output', MagicMock(side_effect=subprocess.CalledProcessError(1, "cmd", "foo"))):
-            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output', MagicMock(side_effect=fake_backup_data_2)):
-            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output', MagicMock(side_effect=fake_backup_data_3)):
-            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output', MagicMock(side_effect=fake_backup_data_4)):
-            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-
-        self.wale_restore.should_use_s3_to_create_replica()
-        self.wale_restore.no_master = 1
         self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
 
+        with patch('psycopg2.connect', Mock(side_effect=psycopg2.Error("foo"))):
+            save_no_master = self.wale_restore.no_master
+            save_master_connection = self.wale_restore.master_connection
+
+            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
+            self.wale_restore.no_master = 1
+            self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())  # this would do 2 retries 1 sec each
+            self.wale_restore.master_connection = ''
+            self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
+
+            self.wale_restore.no_master = save_no_master
+            self.wale_restore.master_connection = save_master_connection
+
+        with patch('subprocess.check_output', Mock(side_effect=subprocess.CalledProcessError(1, "cmd", "foo"))):
+            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
+        with patch('subprocess.check_output', Mock(return_value=wale_output.split(b'\n')[0])):
+            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
+        with patch('subprocess.check_output',
+                   Mock(return_value=wale_output.replace(b' wal_segment_offset_backup_stop', b''))):
+            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
+        with patch('subprocess.check_output',
+                   Mock(return_value=wale_output.replace(b'expanded_size_bytes', b'expanded_size_foo'))):
+            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
+
     def test_create_replica_with_s3(self):
-        with patch('subprocess.call', MagicMock(return_value=0)):
+        with patch('subprocess.call', Mock(return_value=0)):
             self.assertEqual(self.wale_restore.create_replica_with_s3(), 0)
-        with patch('subprocess.call', MagicMock(side_effect=Exception("foo"))):
+            with patch.object(self.wale_restore, 'fix_subdirectory_path_if_broken', Mock(return_value=False)):
+                self.assertEqual(self.wale_restore.create_replica_with_s3(), 2)
+
+        with patch('subprocess.call', Mock(side_effect=Exception("foo"))):
             self.assertEqual(self.wale_restore.create_replica_with_s3(), 1)
 
     def test_run(self):
-        with patch.object(self.wale_restore, 'init_error', PropertyMock(return_value=True)):
-            self.assertEqual(self.wale_restore.run(), 2)
-        with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', MagicMock(return_value=True)):
-            with patch.object(self.wale_restore, 'create_replica_with_s3', MagicMock(return_value=0)):
+        self.wale_restore.init_error = True
+        self.assertEqual(self.wale_restore.run(), 2)  # this would do 2 retries 1 sec each
+        self.wale_restore.init_error = False
+        with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(return_value=True)):
+            with patch.object(self.wale_restore, 'create_replica_with_s3', Mock(return_value=0)):
                 self.assertEqual(self.wale_restore.run(), 0)
+            with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(return_value=None)):
+                self.assertEqual(self.wale_restore.run(), 1)
+            with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(side_effect=Exception)):
+                self.assertEqual(self.wale_restore.run(), 2)
 
-    @patch('sys.exit', MagicMock())
-    @patch.object(WALERestore, 'run', MagicMock(return_value=0))
+    @patch('sys.exit', Mock())
     def test_main(self):
-        self.assertEqual(_main(), None)
+        with patch.object(WALERestore, 'run', Mock(return_value=0)):
+            self.assertEqual(_main(), 0)
+        with patch.object(WALERestore, 'run', Mock(return_value=1)):
+            self.assertEqual(_main(), 1)
+
+    @patch('os.path.isfile', Mock(return_value=True))
+    def test_get_major_version(self):
+        with patch.object(builtins, 'open', mock_open(read_data='9.4')):
+            self.assertEqual(get_major_version("data"), 9.4)
+        with patch.object(builtins, 'open', side_effect=OSError):
+            self.assertEqual(get_major_version("data"), 0.0)
+
+    @patch('os.path.islink', Mock(return_value=True))
+    @patch('os.readlink', Mock(return_value="foo"))
+    @patch('os.remove', Mock())
+    @patch('os.mkdir', Mock())
+    def test_fix_subdirectory_path_if_broken(self):
+        with patch('os.path.exists', Mock(return_value=False)):  # overriding the class-wide mock
+            self.assertTrue(self.wale_restore.fix_subdirectory_path_if_broken("data1"))
+            for fn in ('os.remove', 'os.mkdir'):
+                with patch(fn, side_effect=OSError):
+                    self.assertFalse(self.wale_restore.fix_subdirectory_path_if_broken("data3"))

tests/test_zookeeper.py

@@ -50,7 +50,7 @@ class MockKazooClient(Mock):
         if path.startswith('/no_node'):
             raise NoNodeError
         elif path in ['/service/bla/', '/service/test/']:
-            return ['initialize', 'leader', 'members', 'optime', 'failover']
+            return ['initialize', 'leader', 'members', 'optime', 'failover', 'sync']
         return ['foo', 'bar', 'buzz']
 
     def create(self, path, value=b"", acl=None, ephemeral=False, sequence=False, makepath=False):
@@ -58,11 +58,16 @@ class MockKazooClient(Mock):
             raise TypeError("Invalid type for 'path' (string expected)")
         if not isinstance(value, (six.binary_type,)):
             raise TypeError("Invalid type for 'value' (must be a byte string)")
+        if value == b'Exception':
+            raise Exception
         if path.endswith('/initialize') or path == '/service/test/optime/leader':
             raise Exception
         elif value == b'retry' or (value == b'exists' and self.exists):
             raise NodeExistsError
 
+    def create_async(self, path, value=b"", acl=None, ephemeral=False, sequence=False, makepath=False):
+        return self.create(path, value, acl, ephemeral, sequence, makepath) or Mock()
+
     @staticmethod
     def set(path, value, version=-1):
         if not isinstance(path, six.string_types):
@@ -73,13 +78,16 @@ class MockKazooClient(Mock):
             raise Exception
         if path == '/service/test/members/bar' and value == b'retry':
             return
-        if path in ('/service/test/failover', '/service/test/config'):
+        if path in ('/service/test/failover', '/service/test/config', '/service/test/sync'):
             if value == b'Exception':
                 raise Exception
             elif value == b'ok':
                 return
         raise NoNodeError
 
+    def set_async(self, path, value, version=-1):
+        return self.set(path, value, version) or Mock()
+
     def delete(self, path, version=-1, recursive=False):
         if not isinstance(path, six.string_types):
             raise TypeError("Invalid type for 'path' (string expected)")
@@ -92,6 +100,9 @@ class MockKazooClient(Mock):
         elif path.endswith('/') or path.endswith('/initialize') or path == '/service/test/members/bar':
             raise NoNodeError
 
+    def delete_async(self, path, version=-1, recursive=False):
+        return self.delete(path, version, recursive) or Mock()
+
 
 class TestPatroniSequentialThreadingHandler(unittest.TestCase):
 
@@ -109,16 +120,14 @@ class TestZooKeeper(unittest.TestCase):
     @patch('patroni.dcs.zookeeper.KazooClient', MockKazooClient)
     def setUp(self):
         self.zk = ZooKeeper({'hosts': ['localhost:2181'], 'scope': 'test',
-                             'name': 'foo', 'ttl': 30, 'retry_timeout': 10})
+                             'name': 'foo', 'ttl': 30, 'retry_timeout': 10, 'loop_wait': 10})
 
     def test_session_listener(self):
         self.zk.session_listener(KazooState.SUSPENDED)
 
-    def test_set_ttl(self):
-        self.zk.set_ttl(20)
-
-    def test_set_retry_timeout(self):
-        self.zk.set_retry_timeout(10)
+    def test_reload_config(self):
+        self.zk.reload_config({'ttl': 20, 'retry_timeout': 10, 'loop_wait': 10})
+        self.zk.reload_config({'ttl': 20, 'retry_timeout': 10, 'loop_wait': 5})
 
     def test_get_node(self):
         self.assertIsNone(self.zk.get_node('/no_node'))
@@ -165,7 +174,7 @@ class TestZooKeeper(unittest.TestCase):
         self.zk.touch_member('new')
         self.zk._name = 'na'
         self.zk._client.exists = 1
-        self.zk.touch_member('exists')
+        self.zk.touch_member('Exception')
         self.zk._name = 'bar'
         self.zk.touch_member('retry')
         self.zk._fetch_cluster = True
@@ -183,13 +192,28 @@ class TestZooKeeper(unittest.TestCase):
     def test_write_leader_optime(self):
         self.zk.last_leader_operation = '0'
         self.zk.write_leader_optime('1')
+        with patch.object(MockKazooClient, 'create_async', Mock()):
+            self.zk.write_leader_optime('1')
+        with patch.object(MockKazooClient, 'set_async', Mock()):
+            self.zk.write_leader_optime('2')
         self.zk._base_path = self.zk._base_path.replace('test', 'bla')
-        self.zk.write_leader_optime('2')
+        self.zk.write_leader_optime('3')
 
     def test_delete_cluster(self):
         self.assertTrue(self.zk.delete_cluster())
 
     def test_watch(self):
-        self.zk.watch(0)
-        self.zk.event.isSet = lambda: True
-        self.zk.watch(0)
+        self.zk.watch(None, 0)
+        self.zk.event.isSet = Mock(return_value=True)
+        self.zk.watch(None, 0)
+
+    def test__kazoo_connect(self):
+        self.zk._client._retry.deadline = 1
+        self.zk._orig_kazoo_connect = Mock(return_value=(0, 0))
+        self.zk._kazoo_connect(None, None)
+
+    def test_sync_state(self):
+        self.zk.set_sync_state_value('')
+        self.zk.set_sync_state_value('ok')
+        self.zk.set_sync_state_value('Exception')
+        self.zk.delete_sync_state()

tox.ini

@@ -1,8 +1,2 @@
 [flake8]
 max-line-length=120
-
-[testenv:py27]
-deps = -rrequirements-py2.txt
-
-[testenv:py33]
-deps = -rrequirements-py3.txt