Merge branch 'master' of https://github.com/zalando/patroni

- add release notes
- update the version

.gitignore

@@ -46,3 +46,8 @@ dummy
 
 pgpass
 scm-source.json
+
+# Sphinx-generated documentation
+docs/build/
+docs/source/_static/
+docs/source/_templates/

.travis.yml

@@ -1,15 +1,12 @@
 sudo: false
+dist: trusty
 language: python
 python:
-- "3.5"
-addons:
-  apt:
-    packages:
-    - postgresql-contrib-9.5
-  postgresql: "9.5"
+- "3.4"  # 2.7 and 3.5 are preinstalled by default
 env:
   global:
-  - ETCDVERSION=2.3.2 ZKVERSION=3.4.6 CONSULVERSION=0.6.4
+  - ETCDVERSION=3.0.17 ZKVERSION=3.4.9 CONSULVERSION=0.7.4
+  - PYVERSIONS="2.7 3.4 3.5"
   matrix:
   - TEST_SUITE="python setup.py"
   - DCS="etcd" TEST_SUITE="behave"
@@ -17,66 +14,105 @@ env:
   - DCS="consul" TEST_SUITE="behave"
 cache:
   directories:
-  - $HOME/virtualenv/python2.7.9
-  - $HOME/virtualenv/python3.4.2
-  - $HOME/virtualenv/python3.5.2
+  - $HOME/mycache
+before_cache:
+- |
+  rm -fr $HOME/mycache/python*
+  for pv in $PYVERSIONS; do
+      if [[ $TEST_SUITE != "behave" || $pv != "3.4" ]]; then
+          fpv=$(basename $(readlink $HOME/virtualenv/python${pv}))
+          mv $HOME/virtualenv/${fpv} $HOME/mycache/${fpv}
+      fi
+  done
 install:
   - |
     set -e
 
     if [[ $TEST_SUITE == "behave" ]]; then
-        if [[ $DCS == "consul" ]]; then
-            curl -L https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip \
-                | gunzip > consul
-            chmod +x consul
-        fi
+        function get_consul() {
+            CC=~/mycache/consul_${CONSULVERSION}
+            if [[ ! -x $CC ]]; then
+                curl -L https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip \
+                    | gunzip > $CC
+                [[ ${PIPESTATUS[0]} == 0 ]] || return 1
+                chmod +x $CC
+            fi
+            ln -s $CC consul
+        }
 
-        if [[ $DCS == "etcd" ]]; then
-            curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
-                | tar xz -C . --strip=1 --wildcards --no-anchored etcd
-        fi
+        function get_etcd() {
+            EC=~/mycache/etcd_${ETCDVERSION}
+            if [[ ! -x $EC ]]; then
+                curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
+                    | tar xz -C . --strip=1 --wildcards --no-anchored etcd
+                [[ ${PIPESTATUS[0]} == 0 ]] || return 1
+                mv etcd $EC
+            fi
+            ln -s $EC etcd
+        }
 
-        if [[ $DCS == "exhibitor" ]]; then
-            curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz
-            mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg
-            zookeeper-${ZKVERSION}/bin/zkServer.sh start
+        function get_exhibitor() {
+            ZC=~/mycache/zookeeper-${ZKVERSION}
+            if [[ ! -d $ZC ]]; then
+                curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz
+                [[ ${PIPESTATUS[0]} == 0 ]] || return 1
+                mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg
+                mv zookeeper-${ZKVERSION} $ZC
+            fi
+            $ZC/bin/zkServer.sh start
             # following lines are 'emulating' exhibitor REST API
             while true; do
                 echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \
                     | nc -l 8181 &> /dev/null
             done&
-        fi
+            ZK_PID=$!
+        }
+
+        attempt_num=1
+        until get_${DCS}; do
+            [[ $attempt_num -ge 3 ]] && exit 1
+            echo "Attempt $attempt_num failed! Trying again in $attempt_num seconds..."
+            sleep $(( attempt_num++ ))
+        done
     fi
 
-    for pv in "2.7" "3.4" "3.5"; do
-        source ~/virtualenv/python${pv}/bin/activate
-        # explicitly install all needed python modules to cache them
-        for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8==2.6.0 mock>=2.0.0 pytest-cov pytest'; do
-            pip install $p
-        done
+    for pv in $PYVERSIONS; do
+        if [[ $TEST_SUITE != "behave" || $pv != "3.4" ]]; then
+            fpv=$(basename $(readlink $HOME/virtualenv/python$pv))
+            if [[ -d ~/mycache/${fpv} ]]; then
+                mv ~/virtualenv/${fpv} ~/virtualenv/${fpv}.bckp
+                mv ~/mycache/${fpv} ~/virtualenv/${fpv}
+            fi
+            source ~/virtualenv/python${pv}/bin/activate
+            # explicitly install all needed python modules to cache them
+            for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8 mock pytest-cov pytest setuptools'; do
+                pip install $p --upgrade
+            done
+        fi
     done
 script:
   - |
-    for pv in "2.7" "3.4" "3.5"; do
+    for pv in $PYVERSIONS; do
         source ~/virtualenv/python${pv}/bin/activate
 
-        if [[ $TEST_SUITE == "behave" ]]; then
-            if [[ $pv != "3.4" ]]; then
-                echo Running acceptance tests using python${pv}
-                if ! PATH=.:$PATH $TEST_SUITE; then
-                    # output all log files when tests are failing
-                    grep . features/output/*/*postgres?.*
-                    exit 1
-                fi
-            fi
-        else
+        if [[ $TEST_SUITE != "behave" ]]; then
             echo Running unit tests using python${pv}
             $TEST_SUITE test
             $TEST_SUITE flake8
+        elif [[ $pv != "3.4" ]]; then
+            echo Running acceptance tests using python${pv}
+            if ! PATH=.:/usr/lib/postgresql/9.6/bin:$PATH $TEST_SUITE; then
+                # output all log files when tests are failing
+                grep . features/output/*/*postgres?.*
+                exit 1
+            fi
         fi
     done
 
     set +e
 after_success:
+  # before_cache is executed earlier than after_success, so we need to restore one of virtualenv directories
+  - fpv=$(basename $(readlink $HOME/virtualenv/python3.5)) && mv $HOME/mycache/${fpv} $HOME/virtualenv/${fpv}
   - coveralls
   - if [[ $TEST_SUITE != "behave" ]]; then python-codacy-coverage -r coverage.xml; fi
+  - if [[ $DCS == "exhibitor" ]]; then ~/mycache/zookeeper-${ZKVERSION}/bin/zkServer.sh stop; kill -9 $ZK_PID; fi

Dockerfile

@@ -1,26 +1,30 @@
 ## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine
 ## It has all the necessary components to play/debug with a single node appliance, running etcd
-FROM ubuntu:16.04
-MAINTAINER Feike Steenbergen <feike.steenberge[email protected]>
+FROM postgres:9.6
+MAINTAINER Alexander Kukushkin <alexander.kukushki[email protected]>
 
-RUN echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/01norecommend \
-    && echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/01norecommend
-
-ENV PGVERSION 9.5
-ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:$PATH
-RUN apt-get update -y \
+RUN export DEBIAN_FRONTEND=noninteractive \
+    && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
+    && apt-get update -y \
     && apt-get upgrade -y \
-    && apt-get install -y curl jq haproxy zookeeper postgresql-${PGVERSION} python-psycopg2 python-yaml \
-        python-requests python-six python-click python-dateutil python-tzlocal python-urllib3 \
-        python-dnspython python-pip python-setuptools python-kazoo python-prettytable python \
-    && pip install python-etcd==0.4.3 python-consul==0.6.0 --upgrade \
+    && apt-get install -y curl jq haproxy python-psycopg2 python-yaml python-requests python-six python-pysocks \
+        python-dateutil python-pip python-setuptools python-prettytable python-wheel python-psutil python locales \
+
+    ## Make sure we have a en_US.UTF-8 locale available
+    && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
+
+    && pip install 'python-etcd>=0.4.3,<0.5' click tzlocal cdiff \
+
+    && mkdir -p /home/postgres \
+    && chown postgres:postgres /home/postgres \
+
+    # Clean up
     && apt-get remove -y python-pip python-setuptools \
     && apt-get autoremove -y \
-        # Clean up
     && apt-get clean -y \
     && rm -rf /var/lib/apt/lists/* /root/.cache
 
-ENV ETCDVERSION 2.3.6
+ENV ETCDVERSION 3.2.3
 RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
     | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl
 
@@ -35,10 +39,10 @@ RUN ln -s /patronictl.py /usr/local/bin/patronictl
 
 ### Setting up a simple script that will serve as an entrypoint
 RUN mkdir /data/ && touch /pgpass /patroni.yml \
-    && chown postgres:postgres -R /patroni/ /data/ /pgpass /patroni.yml /etc/haproxy /var/run/ /var/lib/ /var/log/ \
-    && echo 1 > /etc/zookeeper/conf/myid
+    && chown postgres:postgres -R /patroni/ /data/ /pgpass /patroni.yml /etc/haproxy /var/run/ /var/lib/ /var/log/
 
 EXPOSE 2379 5432 8008
 
+ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
 ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
 USER postgres

README.rst

@@ -2,13 +2,17 @@
 
 Patroni: A Template for PostgreSQL HA with ZooKeeper, etcd or Consul
 ------------------------------------------------------------
+
+You can find a version of this documentation that is searchable and also easier to navigate at `patroni.readthedocs.io <https://patroni.readthedocs.io>`__.
+
+
 There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
 
 Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__ or `Consul <https://github.com/hashicorp/consul>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
 
 We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely.
 
-**Note to Kubernetes users**: We're currently developing Patroni to be as useful as possible for teams running Kubernetes on top of Google Compute Engine; Patroni can be the HA solution for Postgres in such an environment. Please contact us via our Issues Tracker if this describes your team's current setup, and we'll follow up.
+**Note to Kubernetes users**: We're currently developing Patroni to be as useful as possible for teams running Kubernetes on top of Google Compute Engine; Patroni can be the HA solution for Postgres in such an environment. To this end, there is a `Helm chart <https://github.com/kubernetes/charts/tree/master/incubator/patroni>`__ that uses Patroni and  `Spilo  <https://github.com/zalando/spilo/>`__ to provision a five-node PostgreSQL HA cluster in a Kubernetes+GCE environment. (The Helm chart deploys Spilo Docker images, not just "bare" Patroni.)
 
 .. contents::
     :local:
@@ -19,12 +23,13 @@ We call Patroni a "template" because it is far from being a one-size-fits-all or
 How Patroni Works
 ==============
 
-Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features. 
+Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features.
 
 For an example of a Docker-based deployment with Patroni, see `Spilo <https://github.com/zalando/spilo>`__, currently in use at Zalando.
 
 For additional background info, see:
 
+* `Elephants on Automatic: HA Clustered PostgreSQL with Helm <https://www.youtube.com/watch?v=CftcVhFMGSY>`_, talk by Josh Berkus and Oleksii Kliukin at KubeCon Berlin 2017 
 * `PostgreSQL HA with Kubernetes and Patroni <https://www.youtube.com/watch?v=iruaCgeG7qs>`__, talk by Josh Berkus at KubeCon 2016 (video)
 * `Feb. 2016 Zalando Tech blog post <https://tech.zalando.de/blog/zalandos-patroni-a-template-for-high-availability-postgresql/>`__
 
@@ -32,7 +37,9 @@ For additional background info, see:
 Development Status
 ================
 
-Patroni is in active development and accepts contributions. See our `Contributing <https://github.com/zalando/patroni/blob/master/README.rst#contributing>`__ section below for more details. 
+Patroni is in active development and accepts contributions. See our `Contributing <https://github.com/zalando/patroni/blob/master/docs/CONTRIBUTING.rst>`__ section below for more details.
+
+We report new releases information `here <https://github.com/zalando/patroni/releases>`__.
 
 ===========================
 Technical Requirements/Installation
@@ -77,7 +84,7 @@ run:
 YAML Configuration
 ===============
 
-Go `here <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__. 
+Go `here <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
 
 =========================
 Environment Configuration
@@ -89,24 +96,7 @@ Go `here <https://github.com/zalando/patroni/blob/master/docs/ENVIRONMENT.rst>`_
 Replication Choices
 ===============
 
-Patroni uses Postgres' streaming replication, which is asynchronous by default. For more information, see the `Postgres documentation on streaming replication <http://www.postgresql.org/docs/current/static/warm-standby.html#STREAMING-REPLICATION>`__.
-
-Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the follower. This setting should be increased or decreased based on business requirements.
-
-When asynchronous replication is not optimal for your use case, investigate Postgres's `synchronous replication <http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION>`__. Synchronous replication ensures consistency across a cluster by confirming that writes are written to a secondary before returning to the connecting client with a success. The cost of synchronous replication: reduced throughput on writes. This throughput will be entirely based on network performance. 
-
-In hosted datacenter environments (like AWS, Rackspace, or any network you do not control), synchronous replication significantly increases the variability of write performance. If followers become inaccessible from the leader, the leader effectively becomes read-only.
-
-To enable a simple synchronous replication test, add the follow lines to the ``parameters`` section of your YAML configuration files:
-
-.. code:: YAML
-
-        synchronous_commit: "on"
-        synchronous_standby_names: "*"
-
-When using synchronous replication, use at least three Postgres data nodes to ensure write availability if one host fails.
-
-Choosing your replication schema is dependent on your business considerations. Investigate both async and sync replication, as well as other HA solutions, to determine which solution is best for you.
+Patroni uses Postgres' streaming replication, which is asynchronous by default. Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the leader. This setting should be increased or decreased based on business requirements. It's also possible to use synchronous replication for better durability guarantees. See `replication modes documentation <https://github.com/zalando/patroni/blob/master/docs/replication_modes.rst>`__ for details.
 
 ===============================
 Applications Should Not Use Superusers
@@ -114,14 +104,6 @@ Applications Should Not Use Superusers
 
 When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
 
-================
-Contributing
-================
-Patroni accepts contributions from the open-source community; see the `Issues Tracker <https://github.com/zalando/patroni/issues>`__ for current needs. 
-
-Before making a contribution, please let us know by posting a comment to the relevant issue. 
-If you would like to propose a new feature, please first file a new issue explaining the feature you'd like to create.
-
 .. |Build Status| image:: https://travis-ci.org/zalando/patroni.svg?branch=master
    :target: https://travis-ci.org/zalando/patroni
 .. |Coverage Status| image:: https://coveralls.io/repos/zalando/patroni/badge.svg?branch=master

docker-compose.yml

@@ -50,8 +50,8 @@ haproxy:
     links:
         - patroni_etcd:patroni_etcd
     ports:
-        - "5000"
-        - "5001"
+        - "5000:5000"
+        - "5001:5001"
     environment:
         PATRONI_ETCD_HOST: patroni_etcd:2379
         PATRONI_SCOPE: testcluster

docker/entrypoint.sh

@@ -22,7 +22,7 @@ __EOF__
 
 DOCKER_IP=$(hostname --ip-address)
 PATRONI_SCOPE=${PATRONI_SCOPE:-batman}
-ETCD_ARGS="--data-dir /tmp/etcd.data -advertise-client-urls=http://${DOCKER_IP}:2379 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://0.0.0.0:2380"
+ETCD_ARGS="--data-dir /tmp/etcd.data -advertise-client-urls=http://${DOCKER_IP}:2379 -listen-client-urls=http://0.0.0.0:2379"
 
 optspec=":vh-:"
 while getopts "$optspec" optchar; do

docs/CONTRIBUTING.rst

@@ -0,0 +1,24 @@
+.. _contributing:
+
+Contributing guidelines
+=======================
+
+Wanna contribute to Patroni? Yay - here is how!
+
+Reporting issues
+----------------
+
+If you have a question about patroni or have a problem using it, please read the :ref:`README <readme>` before filing an issue.
+Also double check with the current issues on our `Issues Tracker <https://github.com/zalando/patroni/issues>`__.
+
+Contributing a pull request
+---------------------------
+
+1) Submit a comment to the relevant issue or create a new issue describing your proposed change.
+2) Do a fork, develop and test your code changes.
+3) Include documentation
+4) Submit a pull request.
+
+You'll get feedback about your pull request as soon as possible.
+
+Happy Patroni hacking ;-)

docs/ENVIRONMENT.rst

@@ -1,3 +1,5 @@
+.. _environment:
+
 ==================================
 Environment Configuration Settings
 ==================================
@@ -27,6 +29,12 @@ Consul
 Etcd
 ----
 -  **PATRONI\_ETCD\_HOST**: the host:port for the etcd endpoint.
+-  **PATRONI\_ETCD\_URL**: url for the etcd, in format: http(s)://(username:password@)host:port
+-  **PATRONI\_ETCD\_PROXY**: proxy url for the etcd. If you are connecting to the etcd using proxy, use this parameter instead of **PATRONI\_ETCD\_URL**
+-  **PATRONI\_ETCD\_SRV**: Domain to search the SRV record(s) for cluster autodiscovery.
+-  **PATRONI\_ETCD\_CACERT**: The ca certificate. If pressent it will enable validation.
+-  **PATRONI\_ETCD\_CERT**: File with the client certificate
+-  **PATRONI\_ETCD\_KEY**: File with the client key. Can be empty if the key is part of certificate.
 
 Exhibitor
 ---------
@@ -38,6 +46,7 @@ PostgreSQL
 -  **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
 -  **PATRONI\_POSTGRESQL\_CONNECT\_ADDRESS**: IP address + port through which Postgres is accessible from other nodes and applications.
 -  **PATRONI\_POSTGRESQL\_DATA\_DIR**: The location of the Postgres data directory, either existing or to be initialized by Patroni.
+-  **PATRONI\_POSTGRESQL\_CONFIG\_DIR**: The location of the Postgres configuration directory, defaults to the data directory. Must be writable by Patroni.
 -  **PATRONI\_POSTGRESQL\_BIN_DIR**: Path to PostgreSQL binaries. (pg_ctl, pg_rewind, pg_basebackup, postgres) The  default value is an empty string meaning that PATH environment variable will be used to find the executables.
 -  **PATRONI\_POSTGRESQL\_PGPASS**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni.
 -  **PATRONI\_REPLICATION\_USERNAME**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = Patroni
+SOURCEDIR     = .
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/README.rst

@@ -0,0 +1,90 @@
+.. _readme:
+
+=================
+How Patroni Works
+=================
+
+Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features.
+
+For an example of a Docker-based deployment with Patroni, see `Spilo <https://github.com/zalando/spilo>`__, currently in use at Zalando.
+
+For additional background info, see:
+
+* `PostgreSQL HA with Kubernetes and Patroni <https://www.youtube.com/watch?v=iruaCgeG7qs>`__, talk by Josh Berkus at KubeCon 2016 (video)
+* `Feb. 2016 Zalando Tech blog post <https://tech.zalando.de/blog/zalandos-patroni-a-template-for-high-availability-postgresql/>`__
+
+==================
+Development Status
+==================
+
+Patroni is in active development and accepts contributions. See our :ref:`Contributing <contributing>` section below for more details.
+
+We report new releases information :ref:`here <releases>`.
+
+===================================
+Technical Requirements/Installation
+===================================
+
+**For Mac**
+
+To install requirements on a Mac, run the following:
+
+::
+
+    brew install postgresql etcd haproxy libyaml python
+    pip install psycopg2 pyyaml
+
+=======================
+Running and Configuring
+=======================
+
+To get started, do the following from different terminals:
+::
+
+    > etcd --data-dir=data/etcd
+    > ./patroni.py postgres0.yml
+    > ./patroni.py postgres1.yml
+
+You will then see a high-availability cluster start up. Test different settings in the YAML files to see how the cluster's behavior changes. Kill some of the components to see how the system behaves.
+
+Add more ``postgres*.yml`` files to create an even larger cluster.
+
+Patroni provides an `HAProxy <http://www.haproxy.org/>`__ configuration, which will give your application a single endpoint for connecting to the cluster's leader. To configure,
+run:
+
+::
+
+    > haproxy -f haproxy.cfg
+
+::
+
+    > psql --host 127.0.0.1 --port 5000 postgres
+
+==================
+YAML Configuration
+==================
+
+Go :ref:`here <settings>` for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
+
+=========================
+Environment Configuration
+=========================
+
+Go :ref:`here <environment>` for comprehensive information about configuring(overriding) settings via environment variables.
+
+===================
+Replication Choices
+===================
+
+Patroni uses Postgres' streaming replication, which is asynchronous by default. Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the leader. This setting should be increased or decreased based on business requirements. It's also possible to use synchronous replication for better durability guarantees. See :ref:`replication modes documentation <replication_modes>` for details.
+
+======================================
+Applications Should Not Use Superusers
+======================================
+
+When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
+
+.. |Build Status| image:: https://travis-ci.org/zalando/patroni.svg?branch=master
+   :target: https://travis-ci.org/zalando/patroni
+.. |Coverage Status| image:: https://coveralls.io/repos/zalando/patroni/badge.svg?branch=master
+   :target: https://coveralls.io/r/zalando/patroni?branch=master

docs/SETTINGS.rst

@@ -1,3 +1,5 @@
+.. _settings:
+
 ===========================
 YAML Configuration Settings
 ===========================
@@ -13,12 +15,19 @@ Bootstrap configuration
 -  **dcs**: This section will be written into `/<namespace>/<scope>/config` of a given configuration store after initializing of new cluster. This is the global configuration for the cluster. If you want to change some parameters for all cluster nodes - just do it in DCS (or via Patroni API) and all nodes will apply this configuration.
     -  **loop\_wait**: the number of seconds the loop will sleep. Default value: 10
     -  **ttl**: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process. Default value: 30
+    -  **retry\_timeout**: timeout for DCS and PostgreSQL operation retries. DCS or network issues shorter than this will not cause Patroni to demote the leader. Default value: 10
     -  **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election.
+    -  **master\_start\_timeout**: the amount of time a master is allowed to recover from failures before failover is triggered. Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Best worst case failover time for master failure is: loop\_wait + master\_start\_timeout + loop\_wait, unless master\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff.
+    -  **synchronous\_mode**: turns on synchronous replication mode. In this mode a replica will be chosen as synchronous and only the latest leader and synchronous replica are able to participate in leader election. Synchronous mode makes sure that succesfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See `replication modes documentation <https://github.com/zalando/patroni/blob/master/docs/replication_modes.rst>`__ for details.
     -  **postgresql**:
         -  **use\_pg\_rewind**:whether or not to use pg_rewind
         -  **use\_slots**: whether or not to use replication_slots. Must be False for PostgreSQL 9.3. You should comment out max_replication_slots before it becomes ineligible for leader status.
         -  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower. 
         -  **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work.
+-  **method**: custom script to use for bootstrpapping this cluster.
+   See :ref:`custom bootstrap methods documentation <custom_bootstrap>` for details.
+   When ``initdb`` is specified revert to the default ``initdb`` command. ``initdb`` is also triggered when no ``method``
+   parameter is present in the configuration file.
 -  **initdb**: List options to be passed on to initdb.
         -  **- data-checksums**: Must be enabled when pg_rewind is needed on 9.3.
         -  **- encoding: UTF8**: default encoding for new databases.
@@ -32,6 +41,7 @@ Bootstrap configuration
         -  **options**: list of options for CREATE USER statement
             -  **- createrole**
             -  **- createdb**
+-  **post\_bootstrap** or **post\_init**: An additional script that will be executed after initializing the cluster. The script receives a connection string URL (with the cluster superuser as a user name). The PGPASSFILE variable is set to the location of pgpass file.
 
 Consul
 ------
@@ -39,7 +49,17 @@ Consul
 
 Etcd
 ----
+Most of the parameters are optional, but you have to specify one of the **host**, **url**, **proxy** or **srv**
 -  **host**: the host:port for the etcd endpoint.
+-  **url**: url for the etcd
+-  **proxy**: proxy url for the etcd. If you are connecting to the etcd using proxy, use this parameter instead of **url**
+-  **srv**: Domain to search the SRV record(s) for cluster autodiscovery.
+-  **protocol**: (optional) http or https, if not specified http is used. If the **url** or **proxy** is specified - will take protocol from them.
+-  **username**: (optional) username for etcd authentication
+-  **password**: (optional) password for etcd authentication.
+-  **cacert**: (optional) The ca certificate. If pressent it will enable validation.
+-  **cert**: (optional) file with the client certificate
+-  **key**: (optional) file with the client key. Can be empty if the key is part of **cert**.
 
 Exhibitor
 ---------
@@ -47,6 +67,8 @@ Exhibitor
 -  **poll\_interval**: how often the list of ZooKeeper and Exhibitor nodes should be updated from Exhibitor
 -  **port**: Exhibitor port.
 
+.. _postgresql_settings:
+
 PostgreSQL
 ----------
 -  **authentication**:
@@ -63,14 +85,21 @@ PostgreSQL
         -  **on\_start**: run this script when the cluster starts.
         -  **on\_stop**: run this script when the cluster stops.
 -  **connect\_address**: IP address + port through which Postgres is accessible from other nodes and applications.
--  **create\_replica\_methods**: an ordered list of the create methods for turning a Patroni node into a new replica. "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured as its own config item.
+-  **create\_replica\_method**: an ordered list of the create methods for turning a Patroni node into a new replica.
+   "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured as its
+   own config item. See :ref:`custom replica creation methods documentation <custom_replica_creation>` for further explanation.
 -  **data\_dir**: The location of the Postgres data directory, either existing or to be initialized by Patroni.
+-  **config\_dir**: The location of the Postgres configuration directory, defaults to the data directory. Must be writable by Patroni.
 -  **bin\_dir**: Path to PostgreSQL binaries. (pg_ctl, pg_rewind, pg_basebackup, postgres) The  default value is an empty string meaning that PATH environment variable will be used to find the executables.
 -  **listen**: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
--  **pgpass**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni.
+-  **use\_unix\_socket**: specifies that Patroni should prefer to use unix sockets to connect to the cluster. Default value is ``false``. If ``unix_socket_directories`` is definded, Patroni will use first suitable value from it to connect to the cluster and fallback to tcp if nothing is suitable. If ``unix_socket_directories`` is not specified in ``postgresql.parameters``, Patroni will assume that default value should be used and omit ``host`` from connection parameters.
+-  **pgpass**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup, the post_init script and under some other circumstances. The location must be writable by Patroni.
 -  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower.
--  **custom_conf** : path to an optional custom ``postgresql.conf`` file, that will be used in place of ``postgresql.base.conf``. The file must exist on all cluster nodes, be readable by PostgreSQL and will be included from its location on the real ``postgresql.conf``. Note that Patroni will not monitor this file for changes, nor backup it. However, its settings can still be overriden by Patroni's own configuration facilities - see `dynamic configuration <https://github.com/zalando/patroni/blob/master/docs/dynamic_configuration.rst>`__ for details.
+-  **custom\_conf** : path to an optional custom ``postgresql.conf`` file, that will be used in place of ``postgresql.base.conf``. The file must exist on all cluster nodes, be readable by PostgreSQL and will be included from its location on the real ``postgresql.conf``. Note that Patroni will not monitor this file for changes, nor backup it. However, its settings can still be overriden by Patroni's own configuration facilities - see `dynamic configuration <https://github.com/zalando/patroni/blob/master/docs/dynamic_configuration.rst>`__ for details.
 -  **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work.
+-  **pg\_hba**: list of lines that Patroni will use to generate ``pg_hba.conf``. This parameter has higher priority than ``bootstrap.pg_hba``. Together with :ref:`dynamic configuration <dynamic_configuration>` it simplifies management of ``pg_hba.conf``.
+        -  **- host all all 0.0.0.0/0 md5**.
+        -  **- host replication replicator 127.0.0.1/32 md5**: A line like this is required for replication.
 -  **pg\_ctl\_timeout**: How long should pg_ctl wait when doing ``start``, ``stop`` or ``restart``. Default value is 60 seconds.
 -  **use\_pg\_rewind**: try to use pg\_rewind on the former leader when it joins cluster as a replica.
 -  **remove\_data\_directory\_on\_rewind\_failure**: If this option is enabled, Patroni will remove postgres data directory and recreate replica. Otherwise it will try to follow the new leader. Default value is **false**.
@@ -91,3 +120,9 @@ REST API
 ZooKeeper
 ----------
 -  **hosts**: list of ZooKeeper cluster members in format: ['host1:port1', 'host2:port2', 'etc...'].
+
+Watchdog
+--------
+- **mode**: ``off``, ``automatic`` or ``required``. When ``off`` watchdog is disabled. When ``automatic`` watchdog will be used if available, but ignored if it is not. When ``required`` the node will not become a leader unless watchdog can be succesfully enabled.
+- **device**: Path to watchdog device. Defaults to ``/dev/watchdog``.
+- **safety_margin**: Number of seconds of safety margin between watchdog triggering and leader key expiration.

docs/_static/custom.css

@@ -0,0 +1,3 @@
+li {
+   margin-bottom: 0.5em
+}

docs/conf.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Patroni documentation build configuration file, created by
+# sphinx-quickstart on Mon Dec 19 16:54:09 2016.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
+    'sphinx.ext.mathjax',
+    'sphinx.ext.ifconfig',
+    'sphinx.ext.viewcode']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'Patroni'
+copyright = '2016, Zalando SE'
+author = 'Zalando SE'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '1.2'
+# The full version, including alpha/beta/rc tags.
+release = '1.2.2'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+if not on_rtd:  # only import and set the theme if we're building docs locally
+    import sphinx_rtd_theme
+    html_theme = 'sphinx_rtd_theme'
+    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Patronidoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'Patroni.tex', 'Patroni Documentation',
+     'Zalando SE', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'patroni', 'Patroni Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'Patroni', 'Patroni Documentation',
+     author, 'Patroni', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+
+# -- Options for Epub output ----------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+epub_author = author
+epub_publisher = author
+epub_copyright = copyright
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {'https://docs.python.org/': None}
+
+# A possibility to have an own stylesheet, to add new rules or override existing ones
+# For the latter case, the CSS specificity of the rules should be higher than the default ones
+def setup(app):
+   app.add_stylesheet("custom.css")

docs/dynamic_configuration.rst

@@ -1,3 +1,5 @@
+.. _dynamic_configuration:
+
 Patroni configuration
 =====================
 
@@ -10,11 +12,11 @@ Patroni configuration is stored in the DCS (Distributed Configuration Store). Th
 	have changed), a special flag, ``pending_restart`` indicating this, is set in the members.data JSON.
 	Additionally, the node status also indicates this, by showing ``"restart_pending": true``.
 
-- Local `configuration <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ (patroni.yml).
+- Local :ref:`configuration <settings>` (patroni.yml).
 	These options are defined in the configuration file and take precedence over dynamic configuration.
 	patroni.yml could be changed and reload in runtime (without restart of Patroni) by sending SIGHUP to the Patroni process or by performing ``POST /reload`` REST-API request.
 
-- Environment `configuration <https://github.com/zalando/patroni/blob/master/docs/ENVIRONMENT.rst>`__ .
+- Environment :ref:`configuration <environment>` .
 	It is possible to set/override some of the "Local" configuration parameters with environment variables.
 	Environment configuration is very useful when you are running in a dynamic environment and you don't know some of the parameters in advance (for example it's not possible to know you external IP address when you are running inside ``docker``).
 

docs/ha_loop_diagram.dot

@@ -0,0 +1,148 @@
+// Graphviz source for ha_loop_diagram.png
+// recompile with:
+// dot -Tpng ha_loop_diagram.dot -o ha_loop_diagram.png
+
+digraph G {
+  rankdir=TB;
+  fontname="sans-serif";
+  penwidth="0.3";
+  layout="dot";
+  newrank=true;
+  edge [fontname="sans-serif", 
+        fontsize=12, 
+        color=black, 
+        fontcolor=black];
+  node [fontname=serif, 
+        fontsize=12, 
+        fillcolor=white, 
+        color=black, 
+        fontcolor=black,
+        style=filled];
+  "start" [label=Start, shape="rectangle", fillcolor="green"]
+  "start" -> "load_cluster_from_dcs";
+  "update_member" [label="Persist node state in DCS"]
+  "update_member" -> "start"
+  
+  subgraph cluster_run_cycle {
+    label="run_cycle"
+    "load_cluster_from_dcs" [label="Load cluster from DCS"];
+    "touch_member" [label="Persist node in DCS"];
+    "cluster.has_member" [shape="diamond", label="Is node registered on DCS?"]
+    "cluster.has_member" -> "touch_member"  [label="no" color="red"]
+    "long_action_in_progress?" [shape="diamond" label="Is the PostgreSQL currently being\nstopping/starting/restarting/reinitializing?"]
+    "load_cluster_from_dcs" -> "cluster.has_member";
+    "touch_member" -> "long_action_in_progress?";
+    "cluster.has_member" -> "long_action_in_progress?"  [label="yes" color="green"];
+    "long_action_in_progress?" -> "recovering?" [label="no" color="red"]
+    "recovering?" [label="Was cluster recovering and failed?", shape="diamond"];
+    "recovering?" -> "post_recover" [label="yes" color="green"];
+    "recovering?" -> "data_directory_empty" [label="no" color="red"];
+    "post_recover" [label="Remove leader key (if I was the leader)"];
+    "data_directory_empty" [label="Is data folder empty?", shape="diamond"];
+    "data_directory_empty" -> "cluster_initialize" [label="no" color="red"];
+    "data_belongs_to_cluster" [label="Does data dir belong to cluster?", shape="diamond"];
+    "data_belongs_to_cluster" -> "exit" [label="no" color="red"];
+    "data_belongs_to_cluster" -> "is_healthy" [label="yes" color="green"]
+    "exit" [label="Fail and exit", fillcolor=red];
+    "cluster_initialize" [label="Is cluster initialized on DCS?" shape="diamond"]
+    "cluster_initialize" -> "cluster.has_leader" [label="no" color="red"]
+    "cluster.has_leader" [label="Does the cluster has leader?", shape="diamond"]
+    "cluster.has_leader" -> "dcs.initialize" [label="no", color="red"]
+    "cluster.has_leader" -> "is_healthy" [label="yes", color="green"]
+    "cluster_initialize" -> "data_belongs_to_cluster" [label="yes" color="green"]
+    "dcs.initialize" [label="Initialize new cluster"];
+    "dcs.initialize" -> "is_healthy"
+    "is_healthy" [label="Is node healthy?\n(running Postgres)", shape="diamond"];
+    "recover" [label="Start as read-only\nand set Recover flag"]
+    "is_healthy" -> "recover" [label="no" color="red"]; 
+    "is_healthy" -> "cluster.is_unlocked" [label="yes" color="green"]; 
+    "cluster.is_unlocked" [label="Does the cluster has a leader?", shape="diamond"]
+  }
+
+  "post_recover" -> "update_member"
+  "recover" -> "update_member"
+  "long_action_in_progress?" -> "async_has_lock?" [label="yes" color="green"];
+  "cluster.is_unlocked" -> "unhealthy_is_healthiest" [label="no" color="red"]
+  "cluster.is_unlocked" -> "healthy_has_lock" [label="yes" color="green"]
+  "data_directory_empty" -> "bootstrap.is_unlocked" [label="yes" color="green"]
+
+  subgraph cluster_async {
+    label = "Long action in progress\n(Start/Stop/Restart/Reinitialize)"
+    "async_has_lock?" [label="Do I have the leader lock?", shape="diamond"]
+    "async_update_lock" [label="Renew leader lock"]
+    "async_has_lock?" -> "async_update_lock" [label="yes" color="green"]
+  }
+  "async_update_lock" -> "update_member"
+  "async_has_lock?" -> "update_member" [label="no" color="red"]
+
+  subgraph cluster_bootstrap {
+    label = "Node bootstrap";
+    "bootstrap.is_unlocked" [label="Does the cluster has a leader?", shape="diamond"]
+    "bootstrap.is_initialized" [label="Does the cluster has an initialize key?", shape="diamond"]
+    "bootstrap.is_unlocked" -> "bootstrap.is_initialized" [label="no" color="red"]
+    "bootstrap.is_unlocked" -> "bootstrap.select_node" [label="yes" color="green"]
+    "bootstrap.select_node" [label="Select a node to take a backup from"]
+    "bootstrap.do_bootstrap" [label="Run pg_basebackup\n(async)"]
+    "bootstrap.select_node" -> "bootstrap.do_bootstrap"
+    "bootstrap.is_initialized" -> "bootstrap.initialization_race" [label="no" color="red"]
+    "bootstrap.is_initialized" -> "bootstrap.wait_for_leader" [label="yes" color="green"]
+    "bootstrap.initialization_race" [label="Race for initialize key"]
+    "bootstrap.initialization_race" -> "bootstrap.won_initialize_race?"
+    "bootstrap.won_initialize_race?" [label="Do I won initialize race?", shape="diamond"]
+    "bootstrap.won_initialize_race?" -> "bootstrap.initdb_and_start" [label="yes" color="green"]
+    "bootstrap.won_initialize_race?" -> "bootstrap.wait_for_leader" [label="no" color="red"]
+    "bootstrap.wait_for_leader" [label="Need to wait for leader key"]
+    "bootstrap.initdb_and_start" [label="Run initdb, start postgres and create roles"]
+    "bootstrap.initdb_and_start" -> "bootstrap.success?"
+    "bootstrap.success?" [label="Success", shape="diamond"]
+    "bootstrap.success?" -> "bootstrap.take_leader_key" [label="yes" color="green"]
+    "bootstrap.success?" -> "bootstrap.clean" [label="no" color="red"]
+    "bootstrap.clean" [label="Remove initialize key from DCS\nand data directory from filesystem"]
+    "bootstrap.take_leader_key" [label="Take a leader key in DCS"]
+  }
+
+  "bootstrap.do_bootstrap" -> "update_member"
+  "bootstrap.wait_for_leader" -> "update_member"
+  "bootstrap.clean" -> "update_member"
+  "bootstrap.take_leader_key" -> "update_member"
+
+  subgraph cluster_process_healthy_cluster {
+    label = "process_healthy_cluster"
+    "healthy_has_lock" [label="Am I the owner of the leader lock?", shape=diamond]
+    "healthy_is_leader" [label="Is Postgres running as master?", shape=diamond]
+    "healthy_no_lock" [label="Follow the leader (async,\ncreate/update recovery.conf and restart if necessary)"]
+    "healthy_has_lock" -> "healthy_no_lock" [label="no" color="red"]
+    "healthy_has_lock" -> "healthy_update_leader_lock" [label="yes" color="green"]
+    "healthy_update_leader_lock" [label="Try to update leader lock"]
+    "healthy_update_leader_lock" -> "healthy_update_success"
+    "healthy_update_success" [label="Success?", shape=diamond]
+    "healthy_update_success" -> "healthy_is_leader" [label="yes" color="green"]
+    "healthy_update_success" -> "healthy_demote" [label="no" color="red"]
+    "healthy_demote" [label="Demote (async,\nrestart in read-only)"]
+    "healthy_failover" [label="Promote Postgres to master"]
+    "healthy_is_leader" -> "healthy_failover" [label="no" color="red"]
+  }
+  "healthy_demote" -> "update_member"
+  "healthy_is_leader" -> "update_member" [label="yes" color="green"]
+  "healthy_failover" -> "update_member"
+  "healthy_no_lock" -> "update_member"
+
+  subgraph cluster_process_unhealthy_cluster {
+    label = "process_unhealthy_cluster"
+    "unhealthy_is_healthiest" [label="Am I the healthiest node?", shape="diamond"]
+    "unhealthy_is_healthiest" -> "unhealthy_leader_race" [label="yes", color="green"]
+    "unhealthy_leader_race" [label="Try to create leader key"]
+    "unhealthy_leader_race" -> "unhealthy_acquire_lock"
+    "unhealthy_acquire_lock" [label="Was I able to get the lock?", shape="diamond"]
+    "unhealthy_is_leader" [label="Is Postgres running as master?", shape=diamond]
+    "unhealthy_acquire_lock" -> "unhealthy_is_leader" [label="yes" color="green"]
+    "unhealthy_is_leader" -> "unhealthy_promote" [label="no" color="red"]
+    "unhealthy_promote" [label="Promote to master"]
+    "unhealthy_is_healthiest" -> "unhealthy_follow" [label="no" color="red"]
+    "unhealthy_follow" [label="try to follow somebody else()"]
+    "unhealthy_acquire_lock" -> "unhealthy_follow" [label="no" color="red"]
+  }
+  "unhealthy_follow" -> "update_member"
+  "unhealthy_promote" -> "update_member"
+  "unhealthy_is_leader" -> "update_member" [label="yes" color="green"]
+}

docs/index.rst

@@ -0,0 +1,37 @@
+.. Patroni documentation master file, created by
+   sphinx-quickstart on Mon Dec 19 16:54:09 2016.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Introduction
+============
+
+Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__ or `Consul <https://github.com/hashicorp/consul>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
+
+We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely. There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
+
+**Note to Kubernetes users**: We're currently developing Patroni to be as useful as possible for teams running Kubernetes on top of Google Compute Engine; Patroni can be the HA solution for Postgres in such an environment. To this end, we've created a `Helm Chart <https://github.com/kubernetes/charts/tree/master/incubator/patroni>`__ that enables you to deploy a five-node Patroni cluster using a Kubernetes PetSet.
+
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   README
+   dynamic_configuration
+   ENVIRONMENT
+   SETTINGS
+   replica_bootstrap
+   replication_modes
+   pause
+   releases
+   CONTRIBUTING
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+
+

docs/pause.rst

@@ -1,3 +1,5 @@
+.. _pause:
+
 Pause/Resume mode for the cluster
 =================================
 

docs/releases.rst

@@ -0,0 +1,522 @@
+.. _releases:
+
+Release notes
+=============
+
+Version 1.3
+-----------
+
+Version 1.3 adds custom bootstrap possibility, significantly improves support for pg_rewind, enhances the
+synchronous mode support, adds configuration editing to patronictl and implements watchdog support on Linux.
+In addition, this is the first version to work correctly with PostgreSQL 10.
+
+**Upgrade notice**
+
+There are no known compatibility issues with the new version of Patroni. Configuration from version 1.2 should work
+without any changes. It is possible to upgrade by installing new packages and either  restarting Patroni (will cause
+PostgreSQL restart), or by putting Patroni into a :ref:`pause mode <pause>` first and then restarting Patroni on all
+nodes in the cluster (Patroni in a pause mode will not attempt to stop/start PostgreSQL), resuming from the pause mode
+at the end.
+
+**Custom bootstrap**
+
+- Make the process of bootstrapping the cluster configurable (Alexander Kukushkin)
+
+  Allow custom bootstrap scripts instead of ``initdb`` when initializing the very first node in the cluster.
+  The bootstrap command receives the name of the cluster and the path to the data directory. The resulting cluster can
+  be configured to perform recovery, making it possible to bootstrap from a backup and do point in time recovery. Refer
+  to the :ref:`documentaton page <custom_bootstrap>` for more detailed description of this feature.
+
+**Smarter pg_rewind support**
+
+-  Decide on whether to run pg_rewind by looking at the timeline differences from the current master (Alexander)
+
+   Previously, Patroni had a fixed set of conditions to trigger pg_rewind, namely when starting a former master, when
+   doing a switchover to the designated node for every other node in the cluster or when there is a replica with the
+   nofailover tag. All those cases have in common a chance that some replica may be ahead of the new master. In some cases,
+   pg_rewind did nothing, in some other ones it was not running when necessary. Instead of relying on this limited list
+   of rules make Patroni compare the master and the replica WAL positions (using the streaming replication protocol)
+   in order to reliably decide if rewind is necessary for the replica.
+
+**Synchronous replication mode strict**
+
+-  Enhance synchronous replication support by adding the strict mode (James Sewell, Alexander)
+
+   Normally, when ``synchronous_mode`` is enabled and there are no replicas attached to the master, Patroni will disable
+   synchronous replication in order to keep the master available for writes. The ``synchronous_mode_strict`` option
+   changes that, when it is set Patroni will not disable the synchronous replication in a lack of replicas, effectively
+   blocking all clients writing data to the master. In addition to the synchronous mode guarantee of preventing any data
+   loss due to automatic failover, the strict mode ensures that each write is either durably stored on two nodes or not
+   happening altogether if there is only one node in the cluster.
+
+**Configuration editing with patronictl**
+
+- Add configuration editing to patronictl (Ants Aasma, Alexander)
+
+  Add the ability to patronictl of editing dynamic cluster configuration stored in DCS. Support either specifying the
+  parameter/values from the command-line, invoking the $EDITOR, or applying configuration from the yaml file.
+
+**Linux watchdog support**
+
+- Implement watchdog support for Linux (Ants)
+
+  Support Linux software watchdog in order to reboot the node where Patroni is not running or not responding (e.g because
+  of the high load) The Linux software watchdog reboots the non-responsive node. It is possible to configure the watchdog
+  device to use (`/dev/watchdog` by default) and the mode (on, automatic, off) from the watchdog section of the Patroni
+  configuration. You can get more information from the :ref:`watchdog documentation <watchdog>`.
+
+**Add support for PostgreSQL 10**
+
+- Patroni is compatible with all beta versions of PostgreSQL 10 released so far and we expect it to be compatible with
+  the PostgreSQL 10 when it will be released.
+
+**PostgreSQL-related minor improvements**
+
+- Define pg_hba.conf via the Patroni configuration file or the dynamic configuration in DCS (Alexander)
+
+  Allow to define the contents of ``pg_hba.conf`` in the ``pg_hba`` sub-section of the ``postgresql`` section of the
+  configuration. This simplifies managing ``pg_hba.conf`` on multiple nodes, as one needs to define it only ones in DCS
+  instead of logging to every node, changing it manually and reload the configuration.
+
+  When defined, the contents of this section will replace the current ``pg_hba.conf`` completely. Patroni ignores it
+  if ``hba_file`` PostgreSQL parameter is set.
+
+- Support connecting via a UNIX socket to the local PostgreSQL cluster (Alexander)
+
+  Add the ``use_unix_socket`` option to the ``postgresql`` section of Patroni configuration. When set to true and the
+  PostgreSQL ``unix_socket_directories`` option is not empty, enables Patroni to use the first value from it to connect
+  to the local PostgreSQL cluster. If ``unix_socket_directories`` is not defined, Patroni will assume its default value
+  and omit the ``host`` parameter in the PostgreSQL connection string altogether.
+
+- Support change of superuser and replication credentials on reload (Alexander)
+
+- Support storing of configuration files outside of PostgreSQL data directory (@jouir)
+
+  Add the new configuration ``postgresql`` configuration directive ``config_dir``.
+  It defaults to the data directory and must be writable by Patroni.
+
+**Bug fixes and stability improvements**
+
+- Handle EtcdEventIndexCleared and EtcdWatcherCleared exceptions (Alexander)
+
+  Faster recovery when the watch operation is ended by Etcd by avoiding useless retries.
+
+- Remove error spinning on Etcd failure and reduce log spam (Ants)
+
+  Avoid immediate retrying and emitting stack traces in the log on the second and subsequent Etcd connection failures.
+
+- Export locale variables when forking PostgreSQL processes (Oleksii Kliukin)
+
+  Avoid the `postmaster became multithreaded during startup` fatal error on non-English locales for PostgreSQL built with NLS.
+
+- Extra checks when dropping the replication slot (Alexander)
+
+  In some cases Patroni is prevented from dropping the replication slot by the WAL sender.
+
+- Truncate the replication slot name to 63  (NAMEDATALEN - 1) characters to comply with PostgreSQL naming rules (Nick Scott)
+
+- Fix a race condition resulting in extra connections being opened to the PostgreSQL cluster from Patroni (Alexander)
+
+- Release the leader key when the node restarts with an empty data directory (Alex Kerney)
+
+- Set asynchronous executor busy when running bootstrap without a leader (Alexander)
+
+  Failure to do so could have resulted in errors stating the node belonged to a different cluster, as Patroni proceeded with
+  the normal business while being bootstrapped by a bootstrap method that doesn't require a leader to be present in the
+  cluster.
+
+- Improve WAL-E replica creation method (Joar Wandborg, Alexander).
+
+  - Use csv.DictReader when parsing WAL-E base backup, accepting ISO dates with space-delimited date and time.
+  - Support fetching current WAL position from the replica to estimate the amount of WAL to restore. Previously, the code used to call system information functions that were available only on the master node.
+
+
+Version 1.2
+-----------
+
+This version introduces significant improvements over the handling of synchronous replication, makes the startup process and failover more reliable, adds PostgreSQL 9.6 support and fixes plenty of bugs.
+In addition, the documentation, including these release notes, has been moved to https://patroni.readthedocs.io.
+
+**Synchronous replication**
+
+- Add synchronous replication support. (Ants Aasma)
+
+  Adds a new configuration variable ``synchronous_mode``. When enabled, Patroni will manage ``synchronous_standby_names`` to enable synchronous replication whenever there are healthy standbys available. When synchronous mode is enabled, Patroni will automatically fail over only to a standby that was synchronously replicating at the time of the master failure. This effectively means that no user visible transaction gets lost in such a case. See the
+  :ref:`feature documentation <synchronous_mode>` for the detailed description and implementation details.
+
+**Reliability improvements**
+
+- Do not try to update the leader position stored in the ``leader optime`` key when PostgreSQL is not 100% healthy. Demote immediately when the update of the leader key failed. (Alexander Kukushkin)
+
+- Exclude unhealthy nodes from the list of targets to clone the new replica from. (Alexander)
+
+- Implement retry and timeout strategy for Consul similar to how it is done for Etcd. (Alexander)
+
+- Make ``--dcs`` and ``--config-file`` apply to all options in ``patronictl``. (Alexander)
+
+- Write all postgres parameters into postgresql.conf. (Alexander)
+
+  It allows starting PostgreSQL configured by Patroni with just ``pg_ctl``.
+
+- Avoid exceptions when there are no users in the config. (Kirill Pushkin)
+
+- Allow pausing an unhealthy cluster. Before this fix, ``patronictl`` would bail out if the node it tries to execute pause on is unhealthy. (Alexander)
+
+- Improve the leader watch functionality. (Alexander)
+
+  Previously the replicas were always watching the leader key (sleeping until the timeout or the leader key changes). With this change, they only watch
+  when the replica's PostgreSQL is in the ``running`` state and not when it is stopped/starting or restarting PostgreSQL.
+
+- Avoid running into race conditions when handling SIGCHILD as a PID 1. (Alexander)
+
+  Previously a race condition could occur when running inside the Docker containers, since the same process inside Patroni both spawned new processes
+  and handled SIGCHILD from them. This change uses fork/execs for Patroni and leaves the original PID 1 process responsible for handling signals from children.
+
+- Fix WAL-E restore. (Oleksii Kliukin)
+
+  Previously WAL-E restore used the ``no_master`` flag to avoid consulting with the master altogether, making Patroni always choose restoring
+  from WAL over the ``pg_basebackup``. This change reverts it to the original meaning of ``no_master``, namely Patroni WAL-E restore may be selected as a replication method if the master is not running.
+  The latter is checked by examining the connection string passed to the method. In addition, it makes the retry mechanism more robust and handles other minutia.
+
+- Implement asynchronous DNS resolver cache. (Alexander)
+
+  Avoid failing when DNS is temporary unavailable (for instance, due to an excessive traffic received by the node).
+
+- Implement starting state and master start timeout. (Ants, Alexander)
+
+  Previously ``pg_ctl`` waited for a timeout and then happily trodded on considering PostgreSQL to be running. This caused PostgreSQL to show up in listings as running when it was actually not and caused a race condition that   resulted in either a failover, or a crash recovery, or a crash recovery interrupted by failover and a missed rewind.
+  This change adds a ``master_start_timeout`` parameter and introduces a new state for the main HA loop: ``starting``. When ``master_start_timeout`` is 0 we will failover immediately when the master crashes as soon as there is a failover candidate. Otherwise, Patroni will wait after attempting to start PostgreSQL on the master for the duration of the timeout; when it expires, it will failover if possible. Manual failover requests will be honored during the crash of the master even before the timeout expiration.
+
+  Introduce the ``timeout`` parameter to the ``restart`` API endpoint and ``patronictl``. When it is set and restart takes longer than the timeout, PostgreSQL is considered unhealthy and the other nodes becomes eligible to take the leader lock.
+
+- Fix ``pg_rewind`` behavior in a pause mode. (Ants)
+
+  Avoid unnecessary restart in a pause mode when Patroni thinks it needs to rewind but rewind is not possible (i.e. ``pg_rewind`` is not present). Fallback to default ``libpq`` values for the ``superuser`` (default OS user) if ``superuser`` authentication is missing from the ``pg_rewind`` related Patroni configuration section.
+
+- Serialize callback execution. Kill the previous callback of the same type when the new one is about to run. Fix the issue of spawning zombie processes when running callbacks. (Alexander)
+
+- Avoid promoting a former master when the leader key is set in DCS but update to this leader key fails. (Alexander)
+
+  This avoids the issue of a current master continuing to keep its role when it is partitioned together with the minority of nodes in Etcd and other DCSs that allow "inconsistent reads".
+
+**Miscellaneous**
+
+- Add ``post_init`` configuration option on bootstrap. (Alejandro Martínez)
+
+  Patroni will call the script argument of this option right after running ``initdb`` and starting up PostgreSQL for a new cluster. The script receives a connection URL with ``superuser``
+  and sets ``PGPASSFILE`` to point to the ``.pgpass`` file containing the password. If the script fails, Patroni initialization fails as well. It is useful for adding
+  new users or creating extensions in the new cluster.
+
+- Implement PostgreSQL 9.6 support. (Alexander)
+
+  Use ``wal_level = replica`` as a synonym for ``hot_standby``, avoiding pending_restart flag when it changes from one to another. (Alexander)
+
+**Documentation improvements**
+
+- Add a Patroni main `loop workflow diagram <https://raw.githubusercontent.com/zalando/patroni/master/docs/ha_loop_diagram.png>`__. (Alejandro, Alexander)
+
+- Improve README, adding the Helm chart and links to release notes. (Lauri Apple)
+
+- Move Patroni documentation to ``Read the Docs``. The up-to-date documentation is available at https://patroni.readthedocs.io. (Oleksii)
+
+  Makes the documentation easily viewable from different devices (including smartphones) and searchable.
+
+- Move the package to the semantic versioning. (Oleksii)
+
+  Patroni will follow the major.minor.patch version schema to avoid releasing the new minor version on small but critical bugfixes. We will only publish the release notes for the minor version, which will include all patches.
+
+
+Version 1.1
+-----------
+
+This release improves management of Patroni cluster by bring in pause mode, improves maintenance with scheduled and conditional restarts, makes Patroni interaction with Etcd or Zookeeper more resilient and greatly enhances patronictl.
+
+**Upgrade notice**
+
+When upgrading from releases below 1.0 read about changing of credentials and configuration format at 1.0 release notes.
+
+**Pause mode**
+
+- Introduce pause mode to temporary detach Patroni from managing PostgreSQL instance (Murat Kabilov, Alexander Kukushkin, Oleksii Kliukin).
+
+  Previously, one had to send SIGKILL signal to Patroni to stop it without terminating PostgreSQL. The new pause mode detaches Patroni from PostgreSQL cluster-wide without terminating Patroni. It is similar to the maintenance mode in Pacemaker. Patroni is still responsible for updating member and leader keys in DCS, but it will not start, stop or restart PostgreSQL server in the process. There are a few exceptions, for instance, manual failovers, reinitializes and restarts are still allowed. You can read :ref:`a detailed description of this feature <pause>`.
+
+In addition, patronictl supports new ``pause`` and ``resume`` commands to toggle the pause mode.
+
+**Scheduled and conditional restarts**
+
+- Add conditions to the restart API command (Oleksii)
+
+  This change enhances Patroni restarts by adding a couple of conditions that can be verified in order to do the restart. Among the conditions are restarting when PostgreSQL role is either a master or a replica, checking the PostgreSQL version number or restarting only when restart is necessary in order to apply configuration changes.
+
+- Add scheduled restarts (Oleksii)
+
+  It is now possible to schedule a restart in the future. Only one scheduled restart per node is supported. It is possible to clear the scheduled restart if it is not needed anymore. A combination of scheduled and conditional restarts is supported, making it possible, for instance, to scheduled minor PostgreSQL upgrades in the night, restarting only the instances that are running the outdated minor version without adding postgres-specific logic to administration scripts.
+
+- Add support for conditional and scheduled restarts to patronictl (Murat).
+
+  patronictl restart supports several new options. There is also patronictl flush command to clean the scheduled actions.
+
+**Robust DCS interaction**
+
+- Set Kazoo timeouts depending on the loop_wait (Alexander)
+
+  Originally, ping_timeout and connect_timeout values were calculated from the negotiated session timeout. Patroni loop_wait was not taken into account. As
+  a result, a single retry could take more time than the session timeout, forcing Patroni to release the lock and demote.
+
+  This change set ping and connect timeout to half of the value of loop_wait, speeding up detection of connection issues and  leaving enough time to retry the connection attempt before loosing the lock.
+
+- Update Etcd topology only after original request succeed (Alexander)
+
+  Postpone updating the Etcd topology known to the client until after the original request. When retrieving the cluster topology, implement the retry timeouts depending on the known number of nodes in the Etcd cluster. This makes our client prefer to get the results of the request to having the up-to-date list of nodes.
+
+  Both changes make Patroni connections to DCS more robust in the face of network issues.
+
+**Patronictl, monitoring and configuration**
+
+- Return information about streaming replicas via the API (Feike Steenbergen)
+
+Previously, there was no reliable way to query Patroni about PostgreSQL instances that fail to stream changes (for instance, due to connection issues). This change exposes the contents of pg_stat_replication via the /patroni endpoint.
+
+- Add patronictl scaffold command (Oleksii)
+
+  Add a command to create cluster structure in Etcd. The cluster is created with user-specified sysid and leader, and both leader and member keys are made persistent. This command is useful to create so-called master-less configurations, where Patroni cluster consisting of only replicas replicate  from the external master node that is unaware of Patroni. Subsequently, one
+  may remove the leader key, promoting one of the Patroni nodes and replacing
+  the original master with the Patroni-based HA cluster.
+
+- Add configuration option ``bin_dir`` to locate PostgreSQL binaries (Ants Aasma)
+
+  It is useful to be able to specify the location of PostgreSQL binaries explicitly when Linux distros that support installing multiple PostgreSQL versions at the same time.
+
+- Allow configuration file path to be overridden using ``custom_conf`` of (Alejandro Martínez)
+
+  Allows for custom configuration file paths, which will be unmanaged by Patroni, :ref:`details <postgresql_settings>`.
+
+**Bug fixes and code improvements**
+
+- Make Patroni compatible with new version schema in PostgreSQL 10 and above (Feike)
+
+  Make sure that Patroni understand 2-digits version numbers when doing conditional restarts based on the PostgreSQL version.
+
+- Use pkgutil to find DCS modules (Alexander)
+
+  Use the dedicated python module instead of traversing directories manually in order to find DCS modules.
+
+- Always call on_start callback when starting Patroni (Alexander)
+
+  Previously, Patroni did not call any callbacks when attaching to the already running node with the correct role. Since callbacks are often used to route
+  client connections that could result in the failure to register the running
+  node in the connection routing scheme. With this fix, Patroni calls on_start
+  callback even when attaching to the already running node.
+
+- Do not drop active replication slots (Murat, Oleksii)
+
+  Avoid dropping active physical replication slots on master. PostgreSQL cannot
+  drop such slots anyway. This change makes possible to run non-Patroni managed
+  replicas/consumers on the master.
+
+- Close Patroni connections during start of the PostgreSQL instance (Alexander)
+
+  Forces Patroni to close all former connections when PostgreSQL node is started. Avoids the trap of reusing former connections if postmaster was killed with SIGKILL.
+
+- Replace invalid characters when constructing slot names from member names (Ants)
+
+  Make sure that standby names that do not comply with the slot naming rules don't cause the slot creation and standby startup to fail. Replace the dashes in the slot names with underscores and all other characters not allowed in slot names with their unicode codepoints.
+
+Version 1.0
+-----------
+
+This release introduces the global dynamic configuration that allows dynamic changes of the PostgreSQL and Patroni configuration parameters for the entire HA cluster. It also delivers numerous bugfixes.
+
+**Upgrade notice**
+
+When upgrading from v0.90 or below, always upgrade all replicas before the master. Since we don't store replication credentials in DCS anymore, an old replica won't be able to connect to the new master.
+
+**Dynamic Configuration**
+
+- Implement the dynamic global configuration (Alexander Kukushkin)
+
+  Introduce new REST API endpoint /config to provide PostgreSQL and Patroni configuration parameters that should be set globally for the entire HA cluster (master and all the replicas). Those parameters are set in DCS and in many cases can be applied without disrupting PostgreSQL or Patroni. Patroni sets a special flag called "pending restart" visible via the API when some of the values require the PostgreSQL restart. In that case, restart should be issued manually via the API.
+
+  Patroni SIGHUP or POST to /reload will make it re-read the configuration file.
+
+  See the :ref:`dynamic configuration <dynamic_configuration>`  for the details on which parameters can be changed and the order of processing difference configuration sources.
+
+  The configuration file format *has changed* since the v0.90. Patroni is still compatible with the old configuration files, but in order to take advantage of the bootstrap parameters one needs to change it. Users are encourage to update them by referring to the :ref:`dynamic configuraton documentation page <dynamic_configuration>`.
+
+**More flexible configuration***
+
+- Make postgresql configuration and database name Patroni connects to configurable (Misja Hoebe)
+
+  Introduce `database` and `config_base_name` configuration parameters. Among others, it makes possible to run Patroni with PipelineDB and other PostgreSQL forks.
+
+- Implement possibility to configure some Patroni configuration parameters via environment (Alexander)
+
+  Those include the scope, the node name and the namespace, as well as the secrets and makes it easier to run Patroni in a dynamic environment, i.e. Kubernetes  Please, refer to the :ref:`supported environment variables <environment>` for further details.
+
+- Update the built-in Patroni docker container  to take advantage of environment-based configuration (Feike Steenbergen).
+
+- Add Zookeeper support to Patroni docker image (Alexander)
+
+- Split the Zookeeper and Exhibitor configuration options (Alexander)
+
+- Make patronictl reuse the code from Patroni to read configuration (Alexander)
+
+  This allows patronictl to take advantage of environment-based configuration.
+
+- Set application name to node name in primary_conninfo (Alexander)
+
+  This simplifies identification and configuration of synchronous replication for a given node.
+
+**Stability, security and usability improvements**
+
+- Reset sysid and do not call pg_controldata when restore of backup in progress (Alexander)
+
+  This change reduces the amount of noise generated by Patroni API health checks during the lengthy initialization of this node from the backup.
+
+- Fix a bunch of pg_rewind corner-cases (Alexander)
+
+  Avoid running pg_rewind if the source cluster is not the master.
+
+  In addition, avoid removing the data directory on an unsuccessful rewind, unless the new parameter *remove_data_directory_on_rewind_failure* is set to true. By default it is false.
+
+- Remove passwords from the replication connection string in DCS (Alexander)
+
+  Previously, Patroni always used the replication credentials from the Postgres URL in DCS. That is now changed to take the credentials from the patroni configuration. The secrets (replication username and password) and no longer exposed in DCS.
+
+- Fix the asynchronous machinery around the demote call (Alexander)
+
+  Demote now runs totally asynchronously without blocking the DCS interactions.
+
+- Make patronictl always send the authorization header if it is configured (Alexander)
+
+  This allows patronictl to issue "protected" requests, i.e. restart or reinitialize, when Patroni is configured to require authorization on those.
+
+- Handle the SystemExit exception correctly (Alexander)
+
+  Avoids the issues of Patroni not stopping properly when receiving the SIGTERM
+
+- Sample haproxy templates for confd (Alexander)
+
+  Generates and dynamically changes haproxy configuration from the patroni state in the DCS using confide
+
+- Improve and restructure the documentation to make it more friendly to the new users (Lauri Apple)
+
+- API must report role=master during pg_ctl stop (Alexander)
+
+  Makes the callback calls more reliable, particularly in the cluster stop case. In addition, introduce the `pg_ctl_timeout` option to set the timeout for the start, stop and restart calls via the `pg_ctl`.
+
+- Fix the retry logic in etcd (Alexander)
+
+  Make retries more predictable and robust.
+
+- Make Zookeeper code more resilient against short network hiccups (Alexander)
+
+  Reduce the connection timeouts to make Zookeeper connection attempts more frequent.
+
+Version 0.90
+------------
+
+This releases adds support for Consul, includes a new *noloadbalance* tag, changes the behavior of the *clonefrom* tag, improves *pg_rewind* handling and improves *patronictl* control program.
+
+**Consul support**
+
+- Implement Consul support (Alexander Kukushkin)
+
+  Patroni runs against Consul, in addition to Etcd and Zookeeper. the connection parameters can be configured in the YAML file.
+
+**New and improved tags**
+
+- Implement *noloadbalance* tag (Alexander)
+
+  This tag makes Patroni always return that the replica is not available to the load balancer.
+
+- Change the implementation of the *clonefrom* tag (Alexander)
+
+  Previously, a node name had to be supplied to the *clonefrom*, forcing a tagged replica to clone from the specific node. The new implementation makes *clonefrom* a boolean tag: if it is set to true, the replica becomes a candidate for other replicas to clone from it. When multiple candidates are present, the replicas picks one randomly.
+
+**Stability and security improvements**
+
+- Numerous reliability improvements (Alexander)
+
+  Removes some spurious error messages, improves the stability of the failover, addresses some corner cases with reading data from DCS, shutdown, demote and reattaching of the former leader.
+
+- Improve systems script to avoid killing Patroni children on stop (Jan Keirse, Alexander Kukushkin)
+
+  Previously, when stopping Patroni, *systemd* also sent a signal to PostgreSQL. Since Patroni also tried to stop PostgreSQL by itself, it resulted in sending to different shutdown requests (the smart shutdown, followed by the fast shutdown). That resulted in replicas disconnecting too early and a former master not being able to rejoin after demote. Fix by Jan with prior research by Alexander.
+
+- Eliminate some cases where the former master was unable to call pg_rewind before rejoining as a replica (Oleksii Kliukin)
+
+  Previously, we only called *pg_rewind* if the former master had crashed. Change this to always run pg_rewind for the former master as long as pg_rewind is present in the system. This fixes the case when the master is shut down before the replicas managed to get the latest changes (i.e. during the "smart" shutdown).
+
+- Numerous improvements to unit- and acceptance- tests, in particular, enable support for Zookeeper and Consul (Alexander).
+
+- Make Travis CI faster and implement support for running tests against Zookeeper (Exhibitor) and Consul (Alexander)
+
+  Both unit and acceptance tests run automatically against Etcd, Zookeeper and Consul on each commit or pull-request.
+
+- Clear environment variables before calling PostgreSQL commands from Patroni (Feike Steenbergen)
+
+  This prevents  a possibility of reading system environment variables by connecting to the PostgreSQL cluster managed by Patroni.
+
+**Configuration and control changes**
+
+- Unify patronictl and Patroni configuration (Feike)
+
+  patronictl can use the same configuration file as Patroni itself.
+
+- Enable Patroni to read the configuration from the environment variables (Oleksii)
+
+  This simplifies generating configuration for Patroni automatically, or merging a single configuration from different sources.
+
+- Include database system identifier in the information returned by the API (Feike)
+
+- Implement *delete_cluster* for all available DCSs (Alexander)
+
+  Enables support for DCSs other than Etcd in patronictl.
+
+
+Version 0.80
+------------
+
+This release adds support for *cascading replication* and simplifies Patroni management by providing *scheduled failovers*. One may use older versions of Patroni (in particular, 0.78) combined with this one in order to migrate to the new release. Note that the scheduled failover and cascading replication related features will only work with Patroni 0.80 and above.
+
+**Cascading replication**
+
+ - Add support for the *replicatefrom* and *clonefrom* tags for the patroni node (Oleksii Kliukin).
+
+ The tag *replicatefrom*  allows a replica to use an arbitrary node a source, not necessary the master. The *clonefrom* does the same for the initial backup. Together, they enable Patroni to fully support cascading replication.
+
+- Add support for running replication methods to initialize the replica even without a running replication connection (Oleksii).
+
+ This is useful in order to create replicas from the snapshots stored on S3 or FTP.  A replication method that does not require a running replication connection should supply *no_master: true* in the yaml configuration. Those scripts will still be called in order if the replication connection is present.
+
+**Patronictl, API and DCS improvements**
+
+- Implement scheduled failovers (Feike Steenbergen).
+
+  Failovers can be scheduled to happen at a certain time in the future, using either patronictl, or API calls.
+
+- Add support for *dbuser* and *password* parameters in patronictl (Feike).
+
+- Add PostgreSQL version to the health check output (Feike).
+
+- Improve Zookeeper support in patronictl (Oleksandr Shulgin)
+
+- Migrate to python-etcd 0.43 (Alexander Kukushkin)
+
+**Configuration**
+
+- Add a sample systems configuration script for Patroni (Jan Keirse).
+
+- Fix the problem of Patroni ignoring the superuser name specified in the configuration file for DB connections  (Alexander).
+
+- Fix the handling of CTRL-C by creating a separate session ID and process group for the postmaster launched by Patroni (Alexander).
+
+**Tests**
+
+- Add acceptance tests with *behave* in order to check real-world scenarios of running Patroni (Alexander, Oleksii).
+
+  The tests can be launched manually using the *behave* command. They are also launched automatically for pull requests and after commits.
+
+  Release notes for some older versions can be found on `project's github page <https://github.com/zalando/patroni/releases>`__.

docs/replica_bootstrap.rst

@@ -0,0 +1,98 @@
+Replica imaging and bootstrap
+=============================
+
+Patroni allows customizing creation of a new replica. It also supports defining what happens when the new empty cluster
+is being bootstrapped. The distinction between two is well defined: Patroni creates replicas only if the ``initialize``
+key is present in Etcd for the cluster. If there is no ``initialize`` key - Patroni calls bootstrap exclusively on the
+first node that takes the initialize key lock.
+
+.. _custom_bootstrap:
+
+Bootstrap
+---------
+
+PostgreSQL provides ``initdb`` command to initialize a new cluster and Patroni calls it by default. In certain cases,
+particularly when creating a new cluster as a copy of an existing one, it is necessary to replace a built-in method with
+custom actions. Patroni supports executing user-defined scripts to bootstrap new clusters, supplying some required
+arguments to them, i.e. the name of the cluster and the path to the data directory. This is configured in the
+``bootstrap`` section of the Patroni configuration. For example:
+
+.. code:: YAML
+
+    bootstrap:
+        method: <custom_bootstrap_method_name>
+        <custom_bootstrap_method_name>:
+            command: <path_to_custom_bootstrap_script> [param1 [, ...]]
+            recovery_conf:
+                recovery_target_action: promote
+                recovery_target_timeline: latest
+                restore_command: <method_specific_restore_command>
+
+
+Each bootstrap method must define at least a ``name`` and a ``command``. A special ``initdb`` method is available to trigger
+the default behavior, in which case ``method`` parameter can be omitted altogether. The ``command`` can be specified using either
+an absolute path, or the one relative to the ``patroni`` command location. In addition to the fixed parameters defined
+in the configuration files, Patroni supplies two cluster-specific ones:
+
+--scope
+    Name of the cluster to be bootstrapped
+--datadir
+    Path to the data directory of the cluster instance to be bootstrapped
+
+If the bootstrap script returns 0, Patroni tries to configure and start the PostgreSQL instance produced by it. If any
+of the intermediate steps fail, or the script returns a non-zero value, Patroni assumes that the bootstrap has failed,
+cleans up after itself and releases the initialize lock to give another node the opportunity to bootstrap.
+
+If a ``recovery_conf`` block is defined in the same section as the custom bootstrap method, Patroni will generate a
+``recovery.conf`` before starting the newly bootstrapped instance. Typically, such recovery.conf should contain at least
+one of the ``recovery_target_*`` parameters, together with the ``recovery_target_timeline`` set to ``promote``.
+
+ .. note:: Bootstrap methods are neither chained, nor fallen-back to the default one in case the primary one fails
+
+
+.. _custom_replica_creation:
+
+Building replicas
+-----------------
+
+Patroni uses tried and proven ``pg_basebackup`` in order to create new replicas. One downside of it is that it requires
+a running master node. Another one is the lack of 'on-the-fly' compression for the backup data and no built-in cleanup
+for outdated backup files. Some people prefer other backup solutions, such as ``WAL-E``, ``pgBackRest``, ``Barman`` and
+others, or simply roll their own scripts. In order to accommodate all those use-cases Patroni supports running custom
+scripts to clone a new replica. Those are configured in the ``postgresql`` configuration block:
+
+.. code:: YAML
+
+    postgresql:
+        create_replica_method:
+            - wal_e
+            - basebackup
+        wal_e:
+            command: patroni_wale_restore
+            no_master: 1
+            envdir: {{WALE_ENV_DIR}}
+            use_iam: 1
+
+
+The ``create_replica_method`` defines available replica creation methods and the order of executing them. Patroni will
+stop on the first one that returns 0. The basebackup is the built-in method and doesn't require any configuration. The
+rest of the methods should define a separate section in the configuration file, listing the command to execute and any
+custom parameters that should be passed to that command. All parameters will be passed in a ``--name=value`` format.
+Besides user-defined parameters, Patroni supplies a couple of cluster-specific ones:
+
+--scope
+    Which cluster this replica belongs to
+--datadir
+    Path to the data directory of the replica
+--role
+    Always 'replica'
+--connstring
+    Connection string to connect to the cluster member to clone from (master or other replica). The user in the
+    connection string can execute SQL and replication protocol commands.
+
+A special ``no_master`` parameter, if defined, allows Patroni to call the replica creation method even if there is no
+running master or replicas. In that case, an empty string will be passed in a connection string. This is useful for
+restoring the formerly running cluster from the binary backup.
+
+If all replica creation methods fail, Patroni will try again all methods in order during the next event loop cycle.
+

docs/replication_modes.rst

@@ -0,0 +1,68 @@
+.. _replication_modes:
+
+=================
+Replication modes
+=================
+
+Patroni uses PostgreSQL streaming replication. For more information about streaming replication, see the `Postgres documentation <http://www.postgresql.org/docs/current/static/warm-standby.html#STREAMING-REPLICATION>`__. By default Patroni configures PostgreSQL for asynchronous replication. Choosing your replication schema is dependent on your business considerations. Investigate both async and sync replication, as well as other HA solutions, to determine which solution is best for you.
+
+Asynchronous mode durability
+----------------------------
+
+In asynchronous mode the cluster is allowed to lose some committed transactions to ensure availability. When master server fails or becomes unavailable for any other reason Patroni will automatically promote a sufficiently healthy standby to master. Any transactions that have not been replicated to that standby remain in a "forked timeline" on the master, and are effectively unrecoverable [1]_.
+
+The amount of transactions that can be lost is controlled via ``maximum_lag_on_failover`` parameter. Because master transaction log position is not sampled in real time, in reality the amount of lost data on failover is worst case bounded by  ``maximum_lag_on_failover`` bytes of transaction log plus the amount that is written in the last ``ttl`` seconds (``loop_wait``/2 seconds in the average case). However typical steady state replication delay is well under a second.
+
+PostgreSQL synchronous replication
+----------------------------------
+
+You can use Postgres's `synchronous replication <http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION>`__ with Patroni. Synchronous replication ensures consistency across a cluster by confirming that writes are written to a secondary before returning to the connecting client with a success. The cost of synchronous replication: reduced throughput on writes. This throughput will be entirely based on network performance.
+
+In hosted datacenter environments (like AWS, Rackspace, or any network you do not control), synchronous replication significantly increases the variability of write performance. If followers become inaccessible from the leader, the leader effectively becomes read-only.
+
+To enable a simple synchronous replication test, add the follow lines to the ``parameters`` section of your YAML configuration files:
+
+.. code:: YAML
+
+        synchronous_commit: "on"
+        synchronous_standby_names: "*"
+
+When using PostgreSQL synchronous replication, use at least three Postgres data nodes to ensure write availability if one host fails.
+
+Using PostgreSQL synchronous replication does not guarantee zero lost transactions under all circumstances. When master and standby that is currently acting as synchronous fail simultaneously a third node that might not contain all transactions will be promoted.
+
+.. _synchronous_mode:
+
+Synchronous mode
+----------------
+
+For use cases where losing committed transactions is not permissible you can turn on Patronis ``synchronous_mode``. When ``synchronous_mode`` is turned on Patroni will not promote a standby unless it is certain that the standby contains all transactions that may have returned a successful commit status to client [2]_. This means that the system may be unavailable for writes even though some servers are available. System administrators can still use manual failover commmands to promote a standby even if it results in transaction loss.
+
+Turning on ``synchronous_mode`` does not guarantee multi node durability of commits under all circumstances. When no suitable standby is available, master server will still accept writes, but does not guarantee their replication. When the master fails in this mode no standby will be promote.  When the host that used to be master comes back it will get promoted automatically, unless system administrator performed a manual failover. This behavior makes synchronous mode usable with 2 node clusters.
+
+When ``synchronous_mode`` is on and a standby crashes, commits will block until next iteration of Patroni runs and switches master to standalone mode (worst case delay for writes ``ttl`` seconds, average case ``loop_wait``/2 seconds). Manually shutting down or restarting a standby will not cause a commit service interruption. Standby will signal the master to release itself from synchronous standby duties before PostgreSQL shutdown is initiated.
+
+You can ensure that a standby never becomes the synchronous standby by setting ``nosync`` tag to true. This is recommended to set for standbys that are behind slow network connections and would cause performance degradation when becoming a synchronous standby.
+
+Synchronous mode can be switched on and off via Patroni REST interface. See :ref:`dynamic configuration <dynamic_configuration>` for instructions.
+
+
+Synchronous mode implementation
+-------------------------------
+
+When in synchronous mode Patroni maintains synchronization state in the DCS, containing the latest master and current synchronous standby. This state is updated with strict ordering constraints to ensure the following invariants:
+
+- A node must be marked as the latest leader whenever it can accept write transactions. Patroni crashing or PostgreSQL not shutting down can cause violations of this invariant.
+
+- A node must be set as the synchronous standby in PostgreSQL as long as it is published as the synchronous standby.
+
+- A node that is not the leader or current synchronous standby is not allowed to promote itself automatically.
+
+Patroni will only ever assign one standby to ``synchronous_standby_names`` because with multiple candidates it is not possible to know which node was acting as synchronous during the failure.
+
+On each HA loop iteration Patroni re-evaluates synchronous standby choice. If the current synchronous standby is connected and has not requested its synchronous status to be removed it remains picked. Otherwise the cluster member avaiable for sync that is furthest ahead in replication is picked.
+
+
+.. [1] The data is still there, but recovering it requires a manual recovery effort by data recovery specialists. When Patroni is allowed to rewind with ``use_pg_rewind`` the forked timeline will be automatically erased to rejoin the failed master with the cluster.
+
+.. [2] Clients can change the behavior per transaction using PostgreSQL's ``synchronous_commit`` setting. Transactions with ``synchronous_commit`` values of ``off`` and ``local`` may be lost on fail over, but will not be blocked by replication delays.

docs/watchdog.rst

@@ -0,0 +1,40 @@
+.. _watchdog:
+
+================
+Watchdog support
+================
+
+Having multiple PostgreSQL servers running as master can result in transactions lost due to diverging timelines. This situation is also called a split-brain problem. To avoid split-brain Patroni needs to ensure PostgreSQL will not accept any transaction commits after leader key expires in the DCS. Under normal circumstances Patroni will try to achieve this by stopping PostgreSQL when leader lock update fails for any reason. However, this may fail to happen due to various reasons:
+
+- Patroni has crashed due to a bug, out-of-memory condition or by being accidentally killed by a system administrator.
+
+- Shutting down PostgreSQL is too slow.
+
+- Patroni does not get to run due to high load on the system, th VM being paused by the hypervisor, or other infrastructure issues.
+
+To guarantee correct behavior under these conditions Patroni supports watchdog devices. Watchdog devices are software or hardware mechanisms that will reset the whole system when they do not get a keepalive heartbeat within a specified timeframe. This adds an additional layer of fail safe in case usual Patroni split-brain protection mechanisms fail.
+
+Patroni will try to activate the watchdog before promoting PostgreSQL to master. If watchdog activation fails and watchdog mode is ``required`` then the node will refuse to become master. When deciding to participate in leader election Patroni will also check that watchdog configuration will allow it to become leader at all. After demoting PostgreSQL (for example due to a manual failover) Patroni will disable the watchdog again. Watchdog will also be disabled while Patroni is in paused state.
+
+By default Patroni will set up the watchdog to expire 5 seconds before TTL expires. With the default setup of ``loop_wait=10`` and ``ttl=30`` this gives HA loop at least 15 seconds (``ttl`` - ``safety_margin`` - ``loop_wait``) to complete before the system gets forcefully reset. By default accessing DCS is configured to time out after 10 seconds. This means that when DCS is unavailable, for example due to network issues, Patroni and PostgreSQL will have at least 5 seconds (``ttl`` - ``safety_margin`` - ``loop_wait`` - ``retry_timeout``) to come to a state where all client connections are terminated.
+
+Safety margin is the amount of time that Patroni reserves for time between leader key update and watchdog keepalive. Patroni will try to send a keepalive immediately after confirmation of leader key update. If Patroni process is suspended for extended amount of time at exactly the right moment the keepalive may be delayed for more than the safety margin without triggering the watchdog. This results in a window of time where watchdog will not trigger before leader key expiration, invalidating the guarantee. To be absolutely sure that watchdog will trigger under all circumstances set up the watchdog to expire after half of TTL by setting ``safety_margin`` to -1 to set watchdog timeout to ``ttl // 2``. If you need this guarantee you probably should increase ``ttl`` and/or reduce ``loop_wait`` and ``retry_timeout``.
+
+Currently watchdogs are only supported using Linux watchdog device interface.
+
+Setting up software watchdog on Linux
+-------------------------------------
+
+Default Patroni configuration will try to use ``/dev/watchdog`` on Linux if it is accessible to Patroni. For most use cases using software watchdog built into the Linux kernel is secure enough.
+
+To enable software watchdog issue the following commands as root before starting Patroni:
+
+.. code-block:: bash
+
+    modprobe softdog
+    # Replace postgres with the user you will be running patroni under
+    chown postgres /dev/watchdog
+
+For testing it may be helpful to disable rebooting by adding ``soft_noboot=1`` to the modprobe command line. In this case the watchdog will just log a line in kernel ring buffer, visible via `dmesg`.
+
+Patroni will log information about the watchdog when it is successfully enabled.

extras/startup-scripts/README.md

@@ -11,3 +11,12 @@ Upstart job for Ubuntu 12.04 or 14.04.  Requires Upstart > 1.4. Intended for sys
 
 ### patroni.service
 Systemd service file, to be copied to /etc/systemd/system/patroni.service, tested on Centos 7.1 with Patroni installed from pip. 
+
+### patroni
+Init.d service file for Debian-like distributions. Copy it to /etc/init.d/, make executable:
+```chmod 755 /etc/init.d/patroni``` and run with ```service patroni start```, or make it starting on boot with ```update-rc.d patroni defaults```. Also you might edit some configuration variables in it:
+PATRONI for patroni.py location
+CONF for configuration file
+LOGFILE for log (script creates it if does not exist)
+
+Note. If you have several versions of Postgres installed, please add to POSTGRES_VERSION the release number which you wish to run. Script uses this value to append PATH environment with correct path to Postgres bin.

extras/startup-scripts/patroni

@@ -0,0 +1,144 @@
+#!/bin/sh
+#
+### BEGIN INIT INFO
+# Provides:          patroni
+# Required-Start:    $remote_fs $syslog
+# Required-Stop:     $remote_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Patroni init script
+# Description:       Runners to orchestrate a high-availability PostgreSQL
+### END INIT INFO
+
+### BEGIN USER CONFIGURATION
+
+CONF="/etc/patroni/postgres.yml"
+LOGFILE="/var/log/patroni.log"
+USER="postgres"
+GROUP="postgres"
+
+NAME=patroni
+PATRONI="/opt/patroni/$NAME.py"
+PIDFILE="/var/run/$NAME.pid"
+
+# Set this parameter, if you have several Postgres versions installed
+# POSTGRES_VERSION="9.4"
+POSTGRES_VERSION=""
+
+### END USER CONFIGURATION
+
+. /lib/lsb/init-functions
+
+# Loading this library for get_versions() function
+if test ! -e /usr/share/postgresql-common/init.d-functions; then
+  log_failure_msg "Probably postgresql-common does not installed."
+  exit 1
+else
+  . /usr/share/postgresql-common/init.d-functions
+fi
+
+# Is there Patroni executable?
+if test ! -e $PATRONI; then
+  log_failure_msg "Patroni executable $PATRONI does not exist."
+  exit 1
+fi
+
+# Is there Patroni configuration file?
+if test ! -e $CONF; then
+  log_failure_msg "Patroni configuration file $CONF does not exist."
+  exit 1
+fi
+
+# Create logfile if doesn't exist
+if test ! -e $LOGFILE; then
+  log_action_msg "Creating logfile for Patroni..."
+  touch $LOGFILE
+  chown $USER:$GROUP $LOGFILE
+fi
+
+prepare_pgpath() {
+  if [ "$POSTGRES_VERSION" != "" ]; then
+    if [ -x /usr/lib/postgresql/$POSTGRES_VERSION/bin/pg_ctl ]; then
+      PGPATH="/usr/lib/postgresql/$POSTGRES_VERSION/bin"
+    else
+      log_failure_msg "Postgres version incorrect, check POSTGRES_VERSION variable."
+      exit 0
+    fi
+  else
+    get_versions
+    if echo $versions | grep -q -e "\s"; then
+      log_warning_msg "You have several Postgres versions installed. Please, use POSTGRES_VERSION to define correct environment."
+    else
+      versions=`echo $versions | sed -e 's/^[ \t]*//'`
+      PGPATH="/usr/lib/postgresql/$versions/bin"
+    fi
+  fi
+}
+
+get_pid() {
+  if test -e $PIDFILE; then
+    PID=`cat $PIDFILE`
+    CHILDPID=`ps --ppid $PID -o %p --no-headers`
+  else
+    log_failure_msg "Could not find PID file. Patroni probably down."
+    exit 1
+  fi
+}
+
+
+case "$1" in
+  start)
+    prepare_pgpath
+    PGPATH=$PATH:$PGPATH
+    log_success_msg "Starting Patroni\n"
+    exec start-stop-daemon --start --quiet \
+        --background \
+        --pidfile $PIDFILE --make-pidfile \
+        --chuid $USER:$GROUP \
+        --chdir `eval echo ~$USER` \
+        --exec $PATRONI \
+        --startas /bin/sh -- \
+        -c "/usr/bin/env PATH=$PGPATH /usr/bin/python $PATRONI $CONF >> $LOGFILE 2>&1"
+    ;;
+
+  stop)
+    log_success_msg "Stopping Patroni"
+    get_pid
+    start-stop-daemon --stop --pid $CHILDPID
+    start-stop-daemon --stop --pidfile $PIDFILE --remove-pidfile --quiet
+    ;;
+
+  reload)
+    log_success_msg "Reloading Patroni configuration"
+    get_pid
+    kill -HUP $CHILDPID
+    ;;
+
+  status)
+    get_pid
+    if start-stop-daemon -T --pid $CHILDPID; then
+      log_success_msg "Patroni is running\n"
+      exit 0
+    else
+      log_warning_msg "Patroni in not running\n"
+    fi
+    ;;
+
+  restart)
+    $0 stop
+    $0 start
+    ;;
+
+  *)
+    echo "Usage: /etc/init.d/$NAME {start|stop|restart|reload|status}"
+    exit 1
+    ;;
+esac
+
+if [ $? -eq 0 ]; then
+        echo .
+        exit 0
+else
+        echo " failed"
+        exit 1
+fi

extras/startup-scripts/patroni.service

@@ -21,7 +21,7 @@ ExecStart=/bin/patroni /etc/patroni.yml
 KillMode=process
 
 # Give a reasonable amount of time for the server to start up/shut down
-TimeoutSec=10
+TimeoutSec=30
 
 # Do not restart the service if it crashes, we want to manually inspect database on failure
 Restart=no

features/backup_create.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+while getopts ":-:" optchar; do
+    [[ "${optchar}" == "-" ]] || continue
+    case "${OPTARG}" in
+        datadir=* )
+            PGDATA=${OPTARG#*=}
+            ;;
+        dbname=* )
+            DBNAME=${OPTARG#*=}
+            ;;
+        walmethod=* )
+            WALMETHOD=${OPTARG#*=}
+            ;;
+    esac
+done
+
+[[ -z $PGDATA || -z $DBNAME || -z $WALMETHOD ]] && exit 1
+
+[[ $WALMETHOD != "none" ]] && WALMETHOD="-X $WALMETHOD" || WALMETHOD=""
+
+exec pg_basebackup -D $PGDATA $WALMETHOD -c fast -d $DBNAME

features/backup_restore.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -x
+
+while getopts ":-:" optchar; do
+    [[ "${optchar}" == "-" ]] || continue
+    case "${OPTARG}" in
+        datadir=* )
+            PGDATA=${OPTARG#*=}
+            ;;
+        sourcedir=* )
+            SOURCE=${OPTARG#*=}
+            ;;
+    esac
+done
+
+[[ -z $PGDATA || -z $SOURCE ]] && exit 1
+
+mkdir -p $(dirname $PGDATA)
+
+exec cp -af $SOURCE $PGDATA

features/basic_replication.feature

@@ -3,15 +3,39 @@ Feature: basic replication
 
   Scenario: check replication of a single table
     Given I start postgres0
-    And postgres0 is a leader after 10 seconds
-    And I start postgres1
-    When I add the table foo to postgres0
+    Then postgres0 is a leader after 10 seconds
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "loop_wait": 2, "synchronous_mode": true}
+    Then I receive a response code 200
+    When I start postgres1
+    And I configure and start postgres2 with a tag replicatefrom postgres0
+    And "sync" key in DCS has leader=postgres0 after 20 seconds
+    And I add the table foo to postgres0
     Then table foo is present on postgres1 after 20 seconds
+    Then table foo is present on postgres2 after 20 seconds
 
-  Scenario: check the basic failover
+  Scenario: check restart of sync replica
+    Given I run patronictl.py restart batman postgres2 --force
+    And "sync" key in DCS has sync_standby=postgres1 after 2 seconds
+    And I run patronictl.py restart batman postgres1 --force
+    Then I receive a response returncode 0
+    And "sync" key in DCS has sync_standby=postgres2 after 10 seconds
+
+  Scenario: check the basic failover in synchronous mode
     When I kill postgres0
-    Then postgres1 role is the primary after 32 seconds
-    When I start postgres0
+    Then postgres2 role is the primary after 22 seconds
+    When I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_mode": null, "master_start_timeout": 0}
+    Then I receive a response code 200
+    When I add the table bar to postgres2
+    Then table bar is present on postgres1 after 20 seconds
+
+  Scenario: check immediate failover when master_start_timeout=0
+    Given I kill postmaster on postgres2
+    Then postgres1 is a leader after 10 seconds
+    And postgres1 role is the primary after 10 seconds
+
+  Scenario: check rejoin of the former master with pg_rewind
+    Given I add the table splitbrain to postgres0
+    And I start postgres0
     Then postgres0 role is the secondary after 20 seconds
-    When I add the table bar to postgres1
-    Then table bar is present on postgres0 after 20 seconds
+    When I add the table buz to postgres1
+    Then table buz is present on postgres0 after 20 seconds

features/cascading_replication.feature

@@ -8,6 +8,7 @@ Scenario: check a base backup and streaming replication from a replica
 	And replication works from postgres0 to postgres1 after 20 seconds
 	And I create label with "postgres0" in postgres0 data directory
 	And I create label with "postgres1" in postgres1 data directory
+	And "members/postgres1" key in DCS has state=running after 12 seconds
 	And I configure and start postgres2 with a tag replicatefrom postgres1
 	Then replication works from postgres0 to postgres2 after 30 seconds
 	And there is a label with "postgres1" in postgres2 data directory

features/custom_bootstrap.feature

@@ -0,0 +1,17 @@
+Feature: custom bootstrap
+    We should check that patroni can bootstrap a new cluster from a backup
+
+Scenario: clone existing cluster using pg_basebackup
+    Given I start postgres0
+    Then postgres0 is a leader after 10 seconds
+    When I add the table foo to postgres0
+    And I start postgres1 in a cluster batman1 as a clone of postgres0
+    Then postgres1 is a leader of batman1 after 10 seconds
+    Then table foo is present on postgres1 after 10 seconds
+
+Scenario: make a backup and do a restore into a new cluster
+    Given I add the table bar to postgres1
+    And I do a backup of postgres1
+    When I start postgres2 in a cluster batman2 from backup
+    Then postgres2 is a leader of batman2 after 10 seconds
+    And table bar is present on postgres2 after 10 seconds

features/environment.py

@@ -1,14 +1,18 @@
 import abc
 import consul
+import datetime
 import etcd
 import kazoo.client
 import kazoo.exceptions
 import os
+import psutil
 import psycopg2
 import shutil
+import signal
 import six
 import subprocess
 import tempfile
+import threading
 import time
 import yaml
 
@@ -16,7 +20,8 @@ import yaml
 @six.add_metaclass(abc.ABCMeta)
 class AbstractController(object):
 
-    def __init__(self, name, work_directory, output_dir):
+    def __init__(self, context, name, work_directory, output_dir):
+        self._context = context
         self._name = name
         self._work_directory = work_directory
         self._output_dir = output_dir
@@ -46,6 +51,7 @@ class AbstractController(object):
 
         assert self._has_started(), "Process {0} is not running after being started".format(self._name)
 
+        max_wait_limit *= self._context.timeout_multiplier
         for _ in range(max_wait_limit):
             if self._is_accessible():
                 break
@@ -54,10 +60,11 @@ class AbstractController(object):
             assert False,\
                 "{0} instance is not available for queries after {1} seconds".format(self._name, max_wait_limit)
 
-    def stop(self, kill=False, timeout=15):
+    def stop(self, kill=False, timeout=15, _=False):
         term = False
         start_time = time.time()
 
+        timeout *= self._context.timeout_multiplier
         while self._handle and self._is_running():
             if kill:
                 self._handle.kill()
@@ -71,18 +78,28 @@ class AbstractController(object):
         if self._log:
             self._log.close()
 
+    def cancel_background(self):
+        pass
+
 
 class PatroniController(AbstractController):
     __PORT = 5440
     PATRONI_CONFIG = '{}.yml'
     """ starts and stops individual patronis"""
 
-    def __init__(self, dcs, name, work_directory, output_dir, tags=None):
-        super(PatroniController, self).__init__('patroni_' + name, work_directory, output_dir)
+    def __init__(self, context, name, work_directory, output_dir, custom_config=None):
+        super(PatroniController, self).__init__(context, 'patroni_' + name, work_directory, output_dir)
         PatroniController.__PORT += 1
         self._data_dir = os.path.join(work_directory, 'data', name)
         self._connstring = None
-        self._config = self._make_patroni_test_config(name, dcs, tags)
+        if custom_config and 'watchdog' in custom_config:
+            self.watchdog = WatchdogMonitor(name, work_directory, output_dir)
+            custom_config['watchdog'] = {'driver': 'testing', 'device': self.watchdog.fifo_path, 'mode': 'required'}
+        else:
+            self.watchdog = None
+
+        self._config = self._make_patroni_test_config(name, custom_config)
+        self._closables = []
 
         self._conn = None
         self._curs = None
@@ -106,13 +123,25 @@ class PatroniController(AbstractController):
                 yaml.safe_dump(config, w, default_flow_style=False)
 
     def _start(self):
+        if self.watchdog:
+            self.watchdog.start()
         return subprocess.Popen(['coverage', 'run', '--source=patroni', '-p', 'patroni.py', self._config],
                                 stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory)
 
-    def _is_accessible(self):
-        return self.query("SELECT 1", fail_ok=True) is not None
+    def stop(self, kill=False, timeout=15, postgres=False):
+        if postgres:
+            return subprocess.call(['pg_ctl', '-D', self._data_dir, 'stop', '-mi', '-w'])
+        super(PatroniController, self).stop(kill, timeout)
+        if self.watchdog:
+            self.watchdog.stop()
 
-    def _make_patroni_test_config(self, name, dcs, tags):
+    def _is_accessible(self):
+        cursor = self.query("SELECT 1", fail_ok=True)
+        if cursor is not None:
+            cursor.execute("SET synchronous_commit TO 'local'")
+            return True
+
+    def _make_patroni_test_config(self, name, custom_config):
         patroni_config_name = self.PATRONI_CONFIG.format(name)
         patroni_config_path = os.path.join(self._output_dir, patroni_config_name)
 
@@ -124,24 +153,37 @@ class PatroniController(AbstractController):
 
         config['postgresql']['listen'] = config['postgresql']['connect_address'] = '{0}:{1}'.format(host, self.__PORT)
 
+        config['name'] = name
+        config['postgresql']['data_dir'] = self._data_dir
+        config['postgresql']['use_unix_socket'] = True
+        config['postgresql']['parameters'].update({
+            'logging_collector': 'on', 'log_destination': 'csvlog', 'log_directory': self._output_dir,
+            'log_filename': name + '.log', 'log_statement': 'all', 'log_min_messages': 'debug1',
+            'unix_socket_directories': self._data_dir})
+
+        if 'bootstrap' in config:
+            config['bootstrap']['post_bootstrap'] = 'psql -w -c "SELECT 1"'
+            if 'initdb' in config['bootstrap']:
+                config['bootstrap']['initdb'].extend([{'auth': 'md5'}, {'auth-host': 'md5'}])
+
+        if custom_config is not None:
+            def recursive_update(dst, src):
+                for k, v in src.items():
+                    if k in dst and isinstance(dst[k], dict):
+                        recursive_update(dst[k], v)
+                    else:
+                        dst[k] = v
+            recursive_update(config, custom_config)
+
+        with open(patroni_config_path, 'w') as f:
+            yaml.safe_dump(config, f, default_flow_style=False)
+
         user = config['postgresql'].get('authentication', config['postgresql']).get('superuser', {})
         self._connkwargs = {k: user[n] for n, k in [('username', 'user'), ('password', 'password')] if n in user}
         self._connkwargs.update({'host': host, 'port': self.__PORT, 'database': 'postgres'})
 
-        config['name'] = name
-        config['postgresql']['data_dir'] = self._data_dir
-        config['postgresql']['parameters'].update({
-            'logging_collector': 'on', 'log_destination': 'csvlog', 'log_directory': self._output_dir,
-            'log_filename': name + '.log', 'log_statement': 'all', 'log_min_messages': 'debug1'})
-
-        if 'bootstrap' in config and 'initdb' in config['bootstrap']:
-            config['bootstrap']['initdb'].extend([{'auth': 'md5'}, {'auth-host': 'md5'}])
-
-        if tags:
-            config['tags'] = tags
-
-        with open(patroni_config_path, 'w') as f:
-            yaml.safe_dump(config, f, default_flow_style=False)
+        self._replication = config['postgresql'].get('authentication', config['postgresql']).get('replication', {})
+        self._replication.update({'host': host, 'port': self.__PORT, 'database': 'postgres'})
 
         return patroni_config_path
 
@@ -177,25 +219,120 @@ class PatroniController(AbstractController):
             time.sleep(1)
         return False
 
+    def get_watchdog(self):
+        return self.watchdog
+
+    def _get_pid(self):
+        try:
+            pidfile = os.path.join(self._data_dir, 'postmaster.pid')
+            if not os.path.exists(pidfile):
+                return None
+            return int(open(pidfile).readline().strip())
+        except:
+            return None
+
+    def database_is_running(self):
+        pid = self._get_pid()
+        if not pid:
+            return False
+        try:
+            os.kill(pid, 0)
+        except OSError:
+            return False
+        return True
+
+    def patroni_hang(self, timeout):
+        hang = ProcessHang(self._handle.pid, timeout)
+        self._closables.append(hang)
+        hang.start()
+
+    def checkpoint_hang(self, timeout):
+        pid = self._get_pid()
+        if not pid:
+            return False
+        proc = psutil.Process(pid)
+        for child in proc.children():
+            if 'checkpoint' in child.cmdline()[0]:
+                checkpointer = child
+                break
+        else:
+            return False
+        hang = ProcessHang(checkpointer.pid, timeout)
+        self._closables.append(hang)
+        hang.start()
+        return True
+
+    def cancel_background(self):
+        for obj in self._closables:
+            obj.close()
+        self._closables = []
+
+    def terminate_backends(self):
+        pid = self._get_pid()
+        if not pid:
+            return False
+        proc = psutil.Process(pid)
+        for p in proc.children():
+            if 'process' not in p.cmdline()[0]:
+                p.terminate()
+
+    @property
+    def backup_source(self):
+        return 'postgres://{username}:{password}@{host}:{port}/{database}'.format(**self._replication)
+
+    def backup(self, dest='basebackup'):
+        subprocess.call([PatroniPoolController.BACKUP_SCRIPT, '--walmethod=none',
+                         '--datadir=' + os.path.join(self._output_dir, dest),
+                         '--dbname=' + self.backup_source])
+
+
+class ProcessHang(object):
+
+    """A background thread implementing a cancelable process hang via SIGSTOP."""
+
+    def __init__(self, pid, timeout):
+        self._cancelled = threading.Event()
+        self._thread = threading.Thread(target=self.run)
+        self.pid = pid
+        self.timeout = timeout
+
+    def start(self):
+        self._thread.start()
+
+    def run(self):
+        os.kill(self.pid, signal.SIGSTOP)
+        try:
+            self._cancelled.wait(self.timeout)
+        finally:
+            os.kill(self.pid, signal.SIGCONT)
+
+    def close(self):
+        self._cancelled.set()
+        self._thread.join()
+
 
 class AbstractDcsController(AbstractController):
 
-    _CLUSTER_NODE = '/service/batman'
+    _CLUSTER_NODE = '/service/{0}'
+
+    def __init__(self, context, mktemp=True):
+        work_directory = mktemp and tempfile.mkdtemp() or None
+        super(AbstractDcsController, self).__init__(context, self.name(), work_directory, context.pctl.output_dir)
 
     def _is_accessible(self):
         return self._is_running()
 
-    def stop_and_remove_work_directory(self, timeout=15):
+    def stop(self, kill=False, timeout=15):
         """ terminate process and wipe out the temp work directory, but only if we actually started it"""
-        self.stop(timeout=timeout)
+        super(AbstractDcsController, self).stop(kill=kill, timeout=timeout)
         if self._work_directory:
             shutil.rmtree(self._work_directory)
 
-    def path(self, key=None):
-        return self._CLUSTER_NODE + (key and '/' + key or '')
+    def path(self, key=None, scope='batman'):
+        return self._CLUSTER_NODE.format(scope) + (key and '/' + key or '')
 
     @abc.abstractmethod
-    def query(self, key):
+    def query(self, key, scope='batman'):
         """ query for a value of a given key """
 
     @abc.abstractmethod
@@ -206,17 +343,37 @@ class AbstractDcsController(AbstractController):
     def cleanup_service_tree(self):
         """ clean all contents stored in the tree used for the tests """
 
+    @classmethod
+    def get_subclasses(cls):
+        for subclass in cls.__subclasses__():
+            for subsubclass in subclass.get_subclasses():
+                yield subsubclass
+            yield subclass
+
+    @classmethod
+    def name(cls):
+        return cls.__name__[:-10].lower()
+
 
 class ConsulController(AbstractDcsController):
 
-    def __init__(self, output_dir):
-        super(ConsulController, self).__init__('consul', tempfile.mkdtemp(), output_dir)
+    def __init__(self, context):
+        super(ConsulController, self).__init__(context)
         os.environ['PATRONI_CONSUL_HOST'] = 'localhost:8500'
         self._client = consul.Consul()
+        self._config_file = None
 
     def _start(self):
-        return subprocess.Popen(['consul', 'agent', '-server', '-bootstrap', '-advertise=127.0.0.1',
-                                 '-data-dir', self._work_directory], stdout=self._log, stderr=subprocess.STDOUT)
+        self._config_file = self._work_directory + '.json'
+        with open(self._config_file, 'wb') as f:
+            f.write(b'{"session_ttl_min":"5s","server":true,"bootstrap":true,"advertise_addr":"127.0.0.1"}')
+        return subprocess.Popen(['consul', 'agent', '-config-file', self._config_file, '-data-dir',
+                                 self._work_directory], stdout=self._log, stderr=subprocess.STDOUT)
+
+    def stop(self, kill=False, timeout=15):
+        super(ConsulController, self).stop(kill=kill, timeout=timeout)
+        if self._config_file:
+            os.unlink(self._config_file)
 
     def _is_running(self):
         try:
@@ -224,36 +381,39 @@ class ConsulController(AbstractDcsController):
         except Exception:
             return False
 
-    def path(self, key=None):
-        return super(ConsulController, self).path(key)[1:]
+    def path(self, key=None, scope='batman'):
+        return super(ConsulController, self).path(key, scope)[1:]
 
-    def query(self, key):
-        _, value = self._client.kv.get(self.path(key))
+    def query(self, key, scope='batman'):
+        _, value = self._client.kv.get(self.path(key, scope))
         return value and value['Value'].decode('utf-8')
 
     def set(self, key, value):
         self._client.kv.put(self.path(key), value)
 
     def cleanup_service_tree(self):
-        self._client.kv.delete(self.path(), recurse=True)
+        self._client.kv.delete(self.path(scope=''), recurse=True)
+
+    def start(self, max_wait_limit=15):
+        super(ConsulController, self).start(max_wait_limit)
 
 
 class EtcdController(AbstractDcsController):
 
     """ handles all etcd related tasks, used for the tests setup and cleanup """
 
-    def __init__(self, output_dir):
-        super(EtcdController, self).__init__('etcd', tempfile.mkdtemp(), output_dir)
-        os.environ['PATRONI_ETCD_HOST'] = 'localhost:4001'
-        self._client = etcd.Client()
+    def __init__(self, context):
+        super(EtcdController, self).__init__(context)
+        os.environ['PATRONI_ETCD_HOST'] = 'localhost:2379'
+        self._client = etcd.Client(port=2379)
 
     def _start(self):
         return subprocess.Popen(["etcd", "--debug", "--data-dir", self._work_directory],
                                 stdout=self._log, stderr=subprocess.STDOUT)
 
-    def query(self, key):
+    def query(self, key, scope='batman'):
         try:
-            return self._client.get(self.path(key)).value
+            return self._client.get(self.path(key, scope)).value
         except etcd.EtcdKeyNotFound:
             return None
 
@@ -262,7 +422,7 @@ class EtcdController(AbstractDcsController):
 
     def cleanup_service_tree(self):
         try:
-            self._client.delete(self.path(), recursive=True)
+            self._client.delete(self.path(scope=''), recursive=True)
         except (etcd.EtcdKeyNotFound, etcd.EtcdConnectionFailed):
             return
         except Exception as e:
@@ -280,8 +440,8 @@ class ZooKeeperController(AbstractDcsController):
 
     """ handles all zookeeper related tasks, used for the tests setup and cleanup """
 
-    def __init__(self, output_dir, export_env=True):
-        super(ZooKeeperController, self).__init__('zookeeper', None, output_dir)
+    def __init__(self, context, export_env=True):
+        super(ZooKeeperController, self).__init__(context, False)
         if export_env:
             os.environ['PATRONI_ZOOKEEPER_HOSTS'] = "'localhost:2181'"
         self._client = kazoo.client.KazooClient()
@@ -289,9 +449,9 @@ class ZooKeeperController(AbstractDcsController):
     def _start(self):
         pass  # TODO: implement later
 
-    def query(self, key):
+    def query(self, key, scope='batman'):
         try:
-            return self._client.get(self.path(key))[0].decode('utf-8')
+            return self._client.get(self.path(key, scope))[0].decode('utf-8')
         except kazoo.exceptions.NoNodeError:
             return None
 
@@ -300,7 +460,7 @@ class ZooKeeperController(AbstractDcsController):
 
     def cleanup_service_tree(self):
         try:
-            self._client.delete(self.path(), recursive=True)
+            self._client.delete(self.path(scope=''), recursive=True)
         except (kazoo.exceptions.NoNodeError):
             return
         except Exception as e:
@@ -318,22 +478,23 @@ class ZooKeeperController(AbstractDcsController):
 
 class ExhibitorController(ZooKeeperController):
 
-    def __init__(self, output_dir):
-        super(ExhibitorController, self).__init__(output_dir, False)
+    def __init__(self, context):
+        super(ExhibitorController, self).__init__(context, False)
         os.environ.update({'PATRONI_EXHIBITOR_HOSTS': 'localhost', 'PATRONI_EXHIBITOR_PORT': '8181'})
 
 
 class PatroniPoolController(object):
 
-    KNOWN_DCS = {'consul': ConsulController, 'etcd': EtcdController,
-                 'zookeeper': ZooKeeperController, 'exhibitor': ExhibitorController}
+    BACKUP_SCRIPT = 'features/backup_create.sh'
 
-    def __init__(self):
+    def __init__(self, context):
+        self._context = context
         self._dcs = None
         self._output_dir = None
         self._patroni_path = None
         self._processes = {}
         self.create_and_set_output_directory('')
+        self.known_dcs = {subclass.name(): subclass for subclass in AbstractDcsController.get_subclasses()}
 
     @property
     def patroni_path(self):
@@ -350,21 +511,25 @@ class PatroniPoolController(object):
     def output_dir(self):
         return self._output_dir
 
-    def start(self, pg_name, max_wait_limit=20, tags=None):
-        if pg_name not in self._processes:
-            self._processes[pg_name] = PatroniController(self.dcs, pg_name, self.patroni_path, self._output_dir, tags)
-        self._processes[pg_name].start(max_wait_limit)
+    def start(self, name, max_wait_limit=20, custom_config=None):
+        if name not in self._processes:
+            self._processes[name] = PatroniController(self._context, name, self.patroni_path,
+                                                      self._output_dir, custom_config)
+        self._processes[name].start(max_wait_limit)
 
     def __getattr__(self, func):
-        if func not in ['stop', 'query', 'write_label', 'read_label', 'check_role_has_changed_to', 'add_tag_to_config']:
+        if func not in ['stop', 'query', 'write_label', 'read_label', 'check_role_has_changed_to', 'add_tag_to_config',
+                        'get_watchdog', 'database_is_running', 'checkpoint_hang', 'patroni_hang',
+                        'terminate_backends', 'backup']:
             raise AttributeError("PatroniPoolController instance has no attribute '{0}'".format(func))
 
-        def wrapper(pg_name, *args, **kwargs):
-            return getattr(self._processes[pg_name], func)(*args, **kwargs)
+        def wrapper(name, *args, **kwargs):
+            return getattr(self._processes[name], func)(*args, **kwargs)
         return wrapper
 
     def stop_all(self):
         for ctl in self._processes.values():
+            ctl.cancel_background()
             ctl.stop()
         self._processes.clear()
 
@@ -375,28 +540,193 @@ class PatroniPoolController(object):
         os.makedirs(feature_dir)
         self._output_dir = feature_dir
 
+    def clone(self, from_name, cluster_name, to_name):
+        f = self._processes[from_name]
+        custom_config = {
+            'scope': cluster_name,
+            'bootstrap': {
+                'method': 'pg_basebackup',
+                'pg_basebackup': {
+                    'command': self.BACKUP_SCRIPT + ' --walmethod=stream --dbname=' + f.backup_source
+                }
+            },
+            'postgresql': {
+                'parameters': {
+                    'archive_mode': 'on',
+                    'archive_command': 'mkdir -p {0} && test ! -f {0}/%f && cp %p {0}/%f'.format(
+                            os.path.join(self._output_dir, 'wal_archive'))
+                },
+                'authentication': {
+                    'superuser': {'password': 'zalando1'},
+                    'replication': {'password': 'rep-pass1'}
+                }
+            }
+        }
+        self.start(to_name, custom_config=custom_config)
+
+    def bootstrap_from_backup(self, name, cluster_name):
+        custom_config = {
+            'scope': cluster_name,
+            'bootstrap': {
+                'method': 'backup_restore',
+                'backup_restore': {
+                    'command': 'features/backup_restore.sh --sourcedir=' + os.path.join(self._output_dir, 'basebackup'),
+                    'recovery_conf': {
+                        'recovery_target_action': 'promote',
+                        'recovery_target_timeline': 'latest',
+                        'restore_command': 'cp {0}/wal_archive/%f %p'.format(self._output_dir)
+                    }
+                }
+            },
+            'postgresql': {
+                'authentication': {
+                    'superuser': {'password': 'zalando2'},
+                    'replication': {'password': 'rep-pass2'}
+                }
+            }
+        }
+        self.start(name, custom_config=custom_config)
+
     @property
     def dcs(self):
         if self._dcs is None:
             self._dcs = os.environ.pop('DCS', 'etcd')
-            assert self._dcs in self.KNOWN_DCS, 'Unsupported dcs: ' + self._dcs
+            assert self._dcs in self.known_dcs, 'Unsupported dcs: ' + self._dcs
         return self._dcs
 
 
+class WatchdogMonitor(object):
+    """Testing harness for emulating a watchdog device as a named pipe. Because we can't easily emulate ioctl's we
+    require a custom driver on Patroni side. The device takes no action, only notes if it was pinged and/or triggered.
+    """
+    def __init__(self, name, work_directory, output_dir):
+        self.fifo_path = os.path.join(work_directory, 'data', 'watchdog.{0}.fifo'.format(name))
+        self.fifo_file = None
+        self._stop_requested = False  # Relying on bool setting being atomic
+        self._thread = None
+        self.last_ping = None
+        self.was_pinged = False
+        self.was_closed = False
+        self._was_triggered = False
+        self.timeout = 60
+        self._log_file = open(os.path.join(output_dir, 'watchdog.{0}.log'.format(name)), 'w')
+        self._log("watchdog {0} initialized".format(name))
+
+    def _log(self, msg):
+        tstamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")
+        self._log_file.write("{0}: {1}\n".format(tstamp, msg))
+
+    def start(self):
+        assert self._thread is None
+        self._stop_requested = False
+        self._log("starting fifo {0}".format(self.fifo_path))
+        fifo_dir = os.path.dirname(self.fifo_path)
+        if os.path.exists(self.fifo_path):
+            os.unlink(self.fifo_path)
+        elif not os.path.exists(fifo_dir):
+            os.mkdir(fifo_dir)
+        os.mkfifo(self.fifo_path)
+        self.last_ping = time.time()
+
+        self._thread = threading.Thread(target=self.run)
+        self._thread.start()
+
+    def run(self):
+        try:
+            while not self._stop_requested:
+                self._log("opening")
+                self.fifo_file = os.open(self.fifo_path, os.O_RDONLY)
+                try:
+                    self._log("Fifo {0} connected".format(self.fifo_path))
+                    self.was_closed = False
+                    while not self._stop_requested:
+                        c = os.read(self.fifo_file, 1)
+
+                        if c == b'X':
+                            self._log("Stop requested")
+                            return
+                        elif c == b'':
+                            self._log("Pipe closed")
+                            break
+                        elif c == b'C':
+                            command = b''
+                            c = os.read(self.fifo_file, 1)
+                            while c != b'\n' and c != b'':
+                                command += c
+                                c = os.read(self.fifo_file, 1)
+                            command = command.decode('utf8')
+
+                            if command.startswith('timeout='):
+                                self.timeout = int(command.split('=')[1])
+                                self._log("timeout={0}".format(self.timeout))
+                        elif c in [b'V', b'1']:
+                            cur_time = time.time()
+                            if cur_time - self.last_ping > self.timeout:
+                                self._log("Triggered")
+                                self._was_triggered = True
+                            if c == b'V':
+                                self._log("magic close")
+                                self.was_closed = True
+                            elif c == b'1':
+                                self.was_pinged = True
+                                self._log("ping after {0} seconds".format(cur_time - (self.last_ping or cur_time)))
+                                self.last_ping = cur_time
+                        else:
+                            self._log('Unknown command {0} received from fifo'.format(c))
+                finally:
+                    self.was_closed = True
+                    self._log("closing")
+                    os.close(self.fifo_file)
+        except Exception as e:
+            self._log("Error {0}".format(e))
+        finally:
+            self._log("stopping")
+            self._log_file.flush()
+            if os.path.exists(self.fifo_path):
+                os.unlink(self.fifo_path)
+
+    def stop(self):
+        self._log("Monitor stop")
+        self._stop_requested = True
+        try:
+            if os.path.exists(self.fifo_path):
+                fd = os.open(self.fifo_path, os.O_WRONLY)
+                os.write(fd, b'X')
+                os.close(fd)
+        except Exception as e:
+            self._log("err while closing: {0}".format(str(e)))
+        if self._thread:
+            self._thread.join()
+            self._thread = None
+
+    def reset(self):
+        self._log("reset")
+        self.was_pinged = self.was_closed = self._was_triggered = False
+
+    @property
+    def was_triggered(self):
+        delta = time.time() - self.last_ping
+        triggered = self._was_triggered or not self.was_closed and delta > self.timeout
+        self._log("triggered={0}, {1}s left".format(triggered, self.timeout - delta))
+        return triggered
+
+
 # actions to execute on start/stop of the tests and before running invidual features
 def before_all(context):
-    context.pctl = PatroniPoolController()
-    context.dcs_ctl = context.pctl.KNOWN_DCS[context.pctl.dcs](context.pctl.output_dir)
+    context.ci = 'TRAVIS_BUILD_NUMBER' in os.environ or 'BUILD_NUMBER' in os.environ
+    context.timeout_multiplier = 2 if context.ci else 1
+    context.pctl = PatroniPoolController(context)
+    context.dcs_ctl = context.pctl.known_dcs[context.pctl.dcs](context)
     context.dcs_ctl.start()
     try:
         context.dcs_ctl.cleanup_service_tree()
     except AssertionError:  # after_all handlers won't be executed in before_all
-        context.dcs_ctl.stop_and_remove_work_directory()
+        context.dcs_ctl.stop()
         raise
 
 
 def after_all(context):
-    context.dcs_ctl.stop_and_remove_work_directory()
+    context.dcs_ctl.stop()
     subprocess.call(['coverage', 'combine'])
     subprocess.call(['coverage', 'report'])
 

features/patroni_api.feature

@@ -34,7 +34,7 @@ Scenario: check local configuration reload
 	Then I receive a response code 202
 
 Scenario: check dynamic configuration change via DCS
-	Given I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "loop_wait": 2, "postgresql": {"parameters": {"max_connections": 101}}}
+	Given I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 10, "loop_wait": 2, "postgresql": {"parameters": {"max_connections": 101}}}
 	Then I receive a response code 200
 	And I receive a response loop_wait 2
 	And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 11 seconds
@@ -73,12 +73,12 @@ Scenario: check the failover via the API in the pause mode
 	And replication works from postgres1 to postgres0 after 20 seconds
 
 Scenario: check the scheduled failover
-	Given I issue a scheduled failover from postgres1 to postgres0 in 1 seconds
+	Given I issue a scheduled failover from postgres1 to postgres0 in 3 seconds
 	Then I receive a response returncode 1
 	And I receive a response output "Can't schedule failover in the paused state"
 	When I run patronictl.py resume batman
 	Then I receive a response returncode 0
-	Given I issue a scheduled failover from postgres1 to postgres0 in 1 seconds
+	Given I issue a scheduled failover from postgres1 to postgres0 in 3 seconds
 	Then I receive a response returncode 0
 	And postgres0 is a leader after 20 seconds
 	And postgres0 role is the primary after 10 seconds
@@ -89,11 +89,11 @@ Scenario: check the scheduled restart
 	Given I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"superuser_reserved_connections": "6"}}}
 	Then I receive a response code 200
 		And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 5 seconds
-	Given I issue a scheduled restart at http://127.0.0.1:8008 in 1 seconds with {"role": "replica"}
+	Given I issue a scheduled restart at http://127.0.0.1:8008 in 3 seconds with {"role": "replica"}
 	Then I receive a response code 202
-		And I sleep for 2 seconds
+		And I sleep for 4 seconds
 		And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 10 seconds
-	Given I issue a scheduled restart at http://127.0.0.1:8008 in 1 seconds with {"restart_pending": "True"}
+	Given I issue a scheduled restart at http://127.0.0.1:8008 in 3 seconds with {"restart_pending": "True"}
 	Then I receive a response code 202
 		And Response on GET http://127.0.0.1:8008/patroni does not contain pending_restart after 10 seconds
 

features/steps/basic_replication.py

@@ -11,7 +11,7 @@ def start_patroni(context, name):
 
 @step('I shut down {name:w}')
 def stop_patroni(context, name):
-    return context.pctl.stop(name)
+    return context.pctl.stop(name, timeout=60)
 
 
 @step('I kill {name:w}')
@@ -19,6 +19,11 @@ def kill_patroni(context, name):
     return context.pctl.stop(name, kill=True)
 
 
+@step('I kill postmaster on {name:w}')
+def stop_postgres(context, name):
+    return context.pctl.stop(name, postgres=True)
+
+
 @step('I add the table {table_name:w} to {pg_name:w}')
 def add_table(context, table_name, pg_name):
     # parse the configuration file and get the port
@@ -30,6 +35,7 @@ def add_table(context, table_name, pg_name):
 
 @then('Table {table_name:w} is present on {pg_name:w} after {max_replication_delay:d} seconds')
 def table_is_present_on(context, table_name, pg_name, max_replication_delay):
+    max_replication_delay *= context.timeout_multiplier
     for _ in range(int(max_replication_delay)):
         if context.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
             break
@@ -41,6 +47,7 @@ def table_is_present_on(context, table_name, pg_name, max_replication_delay):
 
 @then('{pg_name:w} role is the {pg_role:w} after {max_promotion_timeout:d} seconds')
 def check_role(context, pg_name, pg_role, max_promotion_timeout):
+    max_promotion_timeout *= context.timeout_multiplier
     assert context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)),\
         "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
 

features/steps/cascading_replication.py

@@ -1,9 +1,12 @@
+import json
+import time
+
 from behave import step, then
 
 
 @step('I configure and start {name:w} with a tag {tag_name:w} {tag_value:w}')
 def start_patroni_with_a_name_value_tag(context, name, tag_name, tag_value):
-    return context.pctl.start(name, tags={tag_name: tag_value})
+    return context.pctl.start(name, custom_config={'tags': {tag_name: tag_value}})
 
 
 @then('There is a label with "{content:w}" in {name:w} data directory')
@@ -15,3 +18,18 @@ def check_label(context, content, name):
 @step('I create label with "{content:w}" in {name:w} data directory')
 def write_label(context, content, name):
     context.pctl.write_label(name, content)
+
+
+@step('"{name}" key in DCS has {key:w}={value:w} after {time_limit:d} seconds')
+def check_member(context, name, key, value, time_limit):
+    time_limit *= context.timeout_multiplier
+    max_time = time.time() + int(time_limit)
+    while time.time() < max_time:
+        try:
+            response = json.loads(context.dcs_ctl.query(name))
+            if response.get(key) == value:
+                return
+        except Exception:
+            pass
+        time.sleep(1)
+    assert False, "{0} does not have {1}={2} in dcs after {3} seconds".format(name, key, value, time_limit)

features/steps/custom_bootstrap.py

@@ -0,0 +1,27 @@
+import time
+
+from behave import step, then
+
+
+@step('I start {name:w} in a cluster {cluster_name:w} as a clone of {name2:w}')
+def start_cluster_clone(context, name, cluster_name, name2):
+    context.pctl.clone(name2, cluster_name, name)
+
+
+@step('I start {name:w} in a cluster {cluster_name:w} from backup')
+def start_cluster_from_backup(context, name, cluster_name):
+    context.pctl.bootstrap_from_backup(name, cluster_name)
+
+
+@then('{name:w} is a leader of {cluster_name:w} after {time_limit:d} seconds')
+def is_a_leader(context, name, cluster_name, time_limit):
+    time_limit *= context.timeout_multiplier
+    max_time = time.time() + int(time_limit)
+    while (context.dcs_ctl.query("leader", scope=cluster_name) != name):
+        time.sleep(1)
+        assert time.time() < max_time, "{0} is not a leader in dcs after {1} seconds".format(name, time_limit)
+
+
+@step('I do a backup of {name:w}')
+def do_backup(context, name):
+    context.pctl.backup(name)

features/steps/patroni_api.py

@@ -1,6 +1,6 @@
 import json
+import os
 import parse
-import pytz
 import requests
 import shlex
 import subprocess
@@ -8,8 +8,11 @@ import time
 import yaml
 
 from behave import register_type, step, then
+from dateutil import tz
 from datetime import datetime, timedelta
 
+tzutc = tz.tzutc()
+
 
 @parse.with_pattern(r'https?://(?:\w|\.|:|/)+')
 def parse_url(text):
@@ -27,6 +30,7 @@ register_type(url=parse_url)
 @step('{name:w} is a leader after {time_limit:d} seconds')
 @then('{name:w} is a leader after {time_limit:d} seconds')
 def is_a_leader(context, name, time_limit):
+    time_limit *= context.timeout_multiplier
     max_time = time.time() + int(time_limit)
     while (context.dcs_ctl.query("leader") != name):
         time.sleep(1)
@@ -90,7 +94,10 @@ def do_request(context, request_method, url, data):
 def do_run(context, cmd):
     cmd = ['coverage', 'run', '--source=patroni', '-p'] + shlex.split(cmd)
     try:
-        response = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+        # XXX: Dirty hack! We need to take name/passwd from the config!
+        env = os.environ.copy()
+        env.update({'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password'})
+        response = subprocess.check_output(cmd, stderr=subprocess.STDOUT, env=env)
         context.status_code = 0
     except subprocess.CalledProcessError as e:
         response = e.output
@@ -104,7 +111,8 @@ def check_response(context, component, data):
         assert context.status_code == int(data),\
             "status code {0} != {1}, response: {2}".format(context.status_code, data, context.response)
     elif component == 'returncode':
-        assert context.status_code == int(data), "return code {0} != {1}".format(context.status_code, data)
+        assert context.status_code == int(data), "return code {0} != {1}, {2}".format(context.status_code,
+                                                                                      data, context.response)
     elif component == 'text':
         assert context.response == data.strip('"'), "response {0} does not contain {1}".format(context.response, data)
     elif component == 'output':
@@ -118,13 +126,13 @@ def check_response(context, component, data):
 def scheduled_failover(context, from_host, to_host, in_seconds):
     context.execute_steps(u"""
         Given I run patronictl.py failover batman --master {0} --candidate {1} --scheduled "{2}" --force
-    """.format(from_host, to_host, datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds))))
+    """.format(from_host, to_host, datetime.now(tzutc) + timedelta(seconds=int(in_seconds))))
 
 
 @step('I issue a scheduled restart at {url:url} in {in_seconds:d} seconds with {data}')
 def scheduled_restart(context, url, in_seconds, data):
     data = data and json.loads(data) or {}
-    data.update(schedule='{0}'.format((datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds))).isoformat()))
+    data.update(schedule='{0}'.format((datetime.now(tzutc) + timedelta(seconds=int(in_seconds))).isoformat()))
     context.execute_steps(u"""Given I issue a POST request to {0}/restart with {1}""".format(url, json.dumps(data)))
 
 
@@ -135,6 +143,7 @@ def add_tag_to_config(context, tag, value, pg_name):
 
 @then('Response on GET {url} contains {value} after {timeout:d} seconds')
 def check_http_response(context, url, value, timeout, negate=False):
+    timeout *= context.timeout_multiplier
     for _ in range(int(timeout)):
         r = requests.get(url)
         if (value in r.content.decode('utf-8')) != negate:

features/steps/watchdog.py

@@ -0,0 +1,74 @@
+from behave import step, then
+import time
+
+
+def polling_loop(timeout, interval=1):
+    """Returns an iterator that returns values until timeout has passed. Timeout is measured from start of iteration."""
+    start_time = time.time()
+    iteration = 0
+    end_time = start_time + timeout
+    while time.time() < end_time:
+        yield iteration
+        iteration += 1
+        time.sleep(interval)
+
+
+@step('I start {name:w} with watchdog')
+def start_patroni_with_watchdog(context, name):
+    return context.pctl.start(name, custom_config={'watchdog': True})
+
+
+@step('{name:w} watchdog has been pinged after {timeout:d} seconds')
+def watchdog_was_pinged(context, name, timeout):
+    for _ in polling_loop(timeout):
+        if context.pctl.get_watchdog(name).was_pinged:
+            return True
+    return False
+
+
+@then('{name:w} watchdog has been closed')
+def watchdog_was_closed(context, name):
+    assert context.pctl.get_watchdog(name).was_closed
+
+
+@step('I wait for next {name:w} watchdog ping')
+def watchdog_reset_pinged(context, name):
+    context.pctl.get_watchdog(name).reset()
+
+
+@then('{name:w} watchdog is triggered after {timeout:d} seconds')
+def watchdog_was_triggered(context, name, timeout):
+    for _ in polling_loop(timeout):
+        if context.pctl.get_watchdog(name).was_triggered:
+            return True
+    assert False
+
+
+@then('{name:w} watchdog was not triggered')
+def watchdog_was_not_triggered(context, name):
+    assert not context.pctl.get_watchdog(name).was_triggered
+
+
+@step('{name:w} checkpoint takes {timeout:d} seconds')
+def checkpoint_hang(context, name, timeout):
+    assert context.pctl.checkpoint_hang(name, timeout)
+
+
+@step('{name:w} hangs for {timeout:d} seconds')
+def patroni_hang(context, name, timeout):
+    return context.pctl.patroni_hang(name, timeout)
+
+
+@step('I terminate {name:w} user processes')
+def terminate_backends(context, name):
+    return context.pctl.terminate_backends(name)
+
+
+@step('Sleep for {timeout:d} seconds')
+def dcs_connection_lost(context, timeout):
+    time.sleep(timeout)
+
+
+@then('{name:w} database is running')
+def database_is_running(context, name):
+    assert context.pctl.database_is_running(name)

features/watchdog.feature

@@ -0,0 +1,19 @@
+Feature: watchdog
+  Verify that watchdog gets pinged and triggered under appropriate circumstances.
+
+  Scenario: watchdog is opened, pinged and closed
+    Given I start postgres0 with watchdog
+    Then postgres0 is a leader after 10 seconds
+    And postgres0 role is the primary after 10 seconds
+    And postgres0 watchdog has been pinged after 10 seconds
+    When I shut down postgres0
+    Then postgres0 watchdog has been closed
+
+  #TODO: test watchdog is disabled during pause
+  #TODO: test watchdog is disabled properly when shutting down
+
+  Scenario: watchdog is triggered if patroni stops responding
+    Given I start postgres0 with watchdog
+    Then postgres0 role is the primary after 10 seconds
+    When postgres0 hangs for 30 seconds
+    Then postgres0 watchdog is triggered after 30 seconds

haproxy.cfg

@@ -1,21 +1,25 @@
 global
-	maxconn 100
+    maxconn 100
 
 defaults
-	log	global
-	mode	tcp
-	retries 2
-	timeout client 30m
-	timeout connect 4s
-	timeout server 30m
-	timeout check 5s
+    log global
+    mode tcp
+    retries 2
+    timeout client 30m
+    timeout connect 4s
+    timeout server 30m
+    timeout check 5s
 
-frontend ft_postgresql
-	bind *:5000
-	default_backend bk_db
-
-backend bk_db
-	option httpchk
+listen stats
+    mode http
+    bind *:7000
+    stats enable
+    stats uri /
 
+listen batman
+    bind *:5000
+    option httpchk
+    http-check expect status 200
+    default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
     server postgresql_127.0.0.1_5432 127.0.0.1:5432 maxconn 100 check port 8008
     server postgresql_127.0.0.1_5433 127.0.0.1:5433 maxconn 100 check port 8009

patroni/__init__.py

@@ -1,28 +1,29 @@
 import logging
+import os
 import signal
 import sys
 import time
 
-from patroni.api import RestApiServer
-from patroni.config import Config
-from patroni.dcs import get_dcs
-from patroni.exceptions import DCSError
-from patroni.ha import Ha
-from patroni.postgresql import Postgresql
-from patroni.utils import reap_children, sigchld_handler
-from patroni.version import __version__
-
 logger = logging.getLogger(__name__)
 
 
 class Patroni(object):
 
     def __init__(self):
+        from patroni.api import RestApiServer
+        from patroni.config import Config
+        from patroni.dcs import get_dcs
+        from patroni.ha import Ha
+        from patroni.postgresql import Postgresql
+        from patroni.version import __version__
+        from patroni.watchdog import Watchdog
+
         self.setup_signal_handlers()
 
         self.version = __version__
         self.config = Config()
         self.dcs = get_dcs(self.config)
+        self.watchdog = Watchdog(self.config)
         self.load_dynamic_configuration()
 
         self.postgresql = Postgresql(self.config['postgresql'])
@@ -34,12 +35,14 @@ class Patroni(object):
         self.scheduled_restart = {}
 
     def load_dynamic_configuration(self):
+        from patroni.exceptions import DCSError
         while True:
             try:
                 cluster = self.dcs.get_cluster()
                 if cluster and cluster.config:
                     if self.config.set_dynamic_configuration(cluster.config):
                         self.dcs.reload_config(self.config)
+                        self.watchdog.reload_config(self.config)
                 elif not self.config.dynamic_configuration and 'bootstrap' in self.config:
                     if self.config.set_dynamic_configuration(self.config['bootstrap']['dcs']):
                         self.dcs.reload_config(self.config)
@@ -49,16 +52,21 @@ class Patroni(object):
 
     def get_tags(self):
         return {tag: value for tag, value in self.config.get('tags', {}).items()
-                if tag not in ('clonefrom', 'nofailover', 'noloadbalance') or value}
+                if tag not in ('clonefrom', 'nofailover', 'noloadbalance', 'nosync') or value}
 
     @property
     def nofailover(self):
         return bool(self.tags.get('nofailover', False))
 
+    @property
+    def nosync(self):
+        return bool(self.tags.get('nosync', False))
+
     def reload_config(self):
         try:
             self.tags = self.get_tags()
             self.dcs.reload_config(self.config)
+            self.watchdog.reload_config(self.config)
             self.api.reload_config(self.config['restapi'])
             self.postgresql.reload_config(self.config['postgresql'])
         except Exception:
@@ -90,7 +98,7 @@ class Patroni(object):
             time.sleep(0.001)
             # Warn user that Patroni is not keeping up
             logger.warning("Loop time exceeded, rescheduling immediately.")
-        elif self.dcs.watch(nap_time):
+        elif self.ha.watch(nap_time):
             self.next_run = time.time()
 
     def run(self):
@@ -109,10 +117,9 @@ class Patroni(object):
             if cluster and cluster.config and self.config.set_dynamic_configuration(cluster.config):
                 self.reload_config()
 
-            if not self.postgresql.data_directory_empty():
+            if self.postgresql.role != 'uninitialized':
                 self.config.save_cache()
 
-            reap_children()
             self.schedule_next_run()
 
     def setup_signal_handlers(self):
@@ -120,10 +127,13 @@ class Patroni(object):
         self._received_sigterm = False
         signal.signal(signal.SIGHUP, self.sighup_handler)
         signal.signal(signal.SIGTERM, self.sigterm_handler)
-        signal.signal(signal.SIGCHLD, sigchld_handler)
+
+    def shutdown(self):
+        self.api.shutdown()
+        self.ha.shutdown()
 
 
-def main():
+def patroni_main():
     logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
     logging.getLogger('requests').setLevel(logging.WARNING)
 
@@ -133,9 +143,61 @@ def main():
     except KeyboardInterrupt:
         pass
     finally:
-        patroni.api.shutdown()
-        if patroni.ha.is_paused():
-            logger.info('Leader key is not deleted and Postgresql is not stopped due paused state')
-        else:
-            patroni.postgresql.stop(checkpoint=False)
-            patroni.dcs.delete_leader()
+        patroni.shutdown()
+
+
+def pg_ctl_start(args):
+    import subprocess
+    postmaster = subprocess.Popen(args)
+    print(postmaster.pid)
+
+
+def call_self(args, **kwargs):
+    """This function executes Patroni once again with provided arguments.
+
+    :args: list of arguments to call Patroni with.
+    :returns: `Popen` object"""
+
+    exe = [sys.executable]
+    if not getattr(sys, 'frozen', False):  # Binary distribution?
+        exe.append(sys.argv[0])
+
+    import subprocess
+    return subprocess.Popen(exe + args, **kwargs)
+
+
+def main():
+    if os.getpid() != 1:
+        if len(sys.argv) > 5 and sys.argv[1] == 'pg_ctl_start':
+            return pg_ctl_start(sys.argv[2:])
+        return patroni_main()
+
+    pid = 0
+
+    # Looks like we are in a docker, so we will act like init
+    def sigchld_handler(signo, stack_frame):
+        try:
+            while True:
+                ret = os.waitpid(-1, os.WNOHANG)
+                if ret == (0, 0):
+                    break
+                elif ret[0] != pid:
+                    logging.info('Reaped pid=%s, exit status=%s', *ret)
+        except OSError:
+            pass
+
+    def passtochild(signo, stack_frame):
+        if pid:
+            os.kill(pid, signo)
+
+    signal.signal(signal.SIGCHLD, sigchld_handler)
+    signal.signal(signal.SIGHUP, passtochild)
+    signal.signal(signal.SIGINT, passtochild)
+    signal.signal(signal.SIGUSR1, passtochild)
+    signal.signal(signal.SIGUSR2, passtochild)
+    signal.signal(signal.SIGQUIT, passtochild)
+    signal.signal(signal.SIGTERM, passtochild)
+
+    patroni = call_self(sys.argv[1:])
+    pid = patroni.pid
+    patroni.wait()

patroni/api.py

@@ -6,10 +6,9 @@ import psycopg2
 import time
 import dateutil.parser
 import datetime
-import pytz
 
 from patroni.exceptions import PostgresConnectionException
-from patroni.utils import deep_compare, patch_config, Retry, RetryFailedError, is_valid_pg_version
+from patroni.utils import deep_compare, patch_config, Retry, RetryFailedError, is_valid_pg_version, parse_int, tzutc
 from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 from six.moves.socketserver import ThreadingMixIn
 from threading import Thread
@@ -57,7 +56,9 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
     def _write_status_response(self, status_code, response):
         patroni = self.server.patroni
-        response.update({'tags': patroni.tags} if patroni.tags else {})
+        tags = patroni.ha.get_effective_tags()
+        if tags:
+            response['tags'] = tags
         if patroni.postgresql.sysid:
             response['database_system_identifier'] = patroni.postgresql.sysid
         if patroni.postgresql.pending_restart:
@@ -67,6 +68,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
             response['scheduled_restart'] = patroni.scheduled_restart.copy()
             del response['scheduled_restart']['postmaster_start_time']
             response['scheduled_restart']['schedule'] = (response['scheduled_restart']['schedule']).isoformat()
+        if not patroni.ha.watchdog.is_healthy:
+            response['watchdog_failed'] = True
         self._write_json_response(status_code, response)
 
     def do_GET(self, write_status_code_only=False):
@@ -140,6 +143,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 value = json.dumps(data, separators=(',', ':'))
                 if not self.server.patroni.dcs.set_config_value(value, cluster.config.index):
                     return self.send_error(409)
+            self.server.patroni.ha.wakeup()
             self._write_json_response(200, data)
 
     @check_auth
@@ -178,7 +182,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
             if scheduled_at.tzinfo is None:
                 error = 'Timezone information is mandatory for the scheduled {0}'.format(action)
                 status_code = 400
-            elif scheduled_at < datetime.datetime.now(pytz.utc):
+            elif scheduled_at < datetime.datetime.now(tzutc):
                 error = 'Cannot schedule {0} in the past'.format(action)
                 status_code = 422
             else:
@@ -221,6 +225,12 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     status_code = 400
                     data = "PostgreSQL version should be in the first.major.minor format"
                     break
+            elif k == 'timeout':
+                request[k] = parse_int(request[k], 's')
+                if request[k] is None or request[k] <= 0:
+                    status_code = 400
+                    data = "Timeout should be a positive number of seconds"
+                    break
             elif k != 'restart_pending':
                 status_code = 400
                 data = "Unknown filter for the scheduled restart: {0}".format(k)
@@ -234,7 +244,6 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     logger.exception('Exception during restart')
                     status_code = 400
             else:
-                request['postmaster_start_time'] = self.server.patroni.ha.state_handler.postmaster_start_time()
                 if self.server.patroni.ha.schedule_future_restart(request):
                     data = "Restart scheduled"
                     status_code = 202
@@ -321,7 +330,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 if _:
                     status_code = _
                 elif self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
-                    self.server.patroni.dcs.event.set()
+                    self.server.patroni.ha.wakeup()
                     data = 'Failover scheduled'
                     status_code = 202
                 else:
@@ -331,7 +340,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 data = self.is_failover_possible(cluster, leader, candidate)
                 if not data:
                     if self.server.patroni.dcs.manual_failover(leader, candidate):
-                        self.server.patroni.dcs.event.set()
+                        self.server.patroni.ha.wakeup()
                         status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, candidate)
                     else:
                         data = 'failed to write failover key into DCS'
@@ -375,13 +384,17 @@ class RestApiHandler(BaseHTTPRequestHandler):
                                        pg_is_in_recovery(),
                                        CASE WHEN pg_is_in_recovery()
                                             THEN 0
-                                            ELSE pg_xlog_location_diff(pg_current_xlog_location(), '0/0')::bigint
+                                            ELSE pg_{0}_{1}_diff(pg_current_{0}_{1}(), '0/0')::bigint
                                        END,
-                                       pg_xlog_location_diff(pg_last_xlog_receive_location(), '0/0')::bigint,
-                                       pg_xlog_location_diff(pg_last_xlog_replay_location(), '0/0')::bigint,
+                                       pg_{0}_{1}_diff(COALESCE(pg_last_{0}_receive_{1}(),
+                                                                      pg_last_{0}_replay_{1}()), '0/0')::bigint,
+                                       pg_{0}_{1}_diff(pg_last_{0}_replay_{1}(), '0/0')::bigint,
                                        to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
-                                       pg_is_in_recovery() AND pg_is_xlog_replay_paused(),
-                                       (SELECT json_agg(row_to_json(ri)) FROM replication_info ri)""", retry=retry)[0]
+                                       pg_is_in_recovery() AND pg_is_{0}_replay_paused(),
+                                       (SELECT array_to_json(array_agg(row_to_json(ri)))
+                                          FROM replication_info ri)""".format(self.server.patroni.postgresql.wal_name,
+                                                                              self.server.patroni.postgresql.lsn_name),
+                             retry=retry)[0]
 
             result = {
                 'state': self.server.patroni.postgresql.state,

patroni/async_executor.py

@@ -1,15 +1,63 @@
 import logging
-from threading import RLock, Thread
+from threading import Lock, RLock, Thread
 
 logger = logging.getLogger(__name__)
 
 
+class CriticalTask(object):
+    """Represents a critical task in a background process that we either need to cancel or get the result of.
+
+    Fields of this object may be accessed only when holding a lock on it. To perform the critical task the background
+    thread must, while holding lock on this object, check `is_cancelled` flag, run the task and mark the task as
+    complete using `complete()`.
+
+    The main thread must hold async lock to prevent the task from completing, hold lock on critical task object,
+    call cancel. If the task has completed `cancel()` will return False and `result` field will contain the result of
+    the task. When cancel returns True it is guaranteed that the background task will notice the `is_cancelled` flag.
+    """
+    def __init__(self):
+        self._lock = Lock()
+        self.is_cancelled = False
+        self.result = None
+
+    def reset(self):
+        """Must be called every time the background task is finished.
+
+        Must be called from async thread. Caller must hold lock on async executor when calling."""
+        self.is_cancelled = False
+        self.result = None
+
+    def cancel(self):
+        """Tries to cancel the task, returns True if the task has already run.
+
+        Caller must hold lock on async executor and the task when calling."""
+        if self.result is not None:
+            return False
+        self.is_cancelled = True
+        return True
+
+    def complete(self, result):
+        """Mark task as completed along with a result.
+
+        Must be called from async thread. Caller must hold lock on task when calling."""
+        self.result = result
+
+    def __enter__(self):
+        self._lock.acquire()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._lock.release()
+
+
 class AsyncExecutor(object):
 
-    def __init__(self):
+    def __init__(self, ha_wakeup):
+        self._ha_wakeup = ha_wakeup
         self._thread_lock = RLock()
         self._scheduled_action = None
         self._scheduled_action_lock = RLock()
+        self.critical_task = CriticalTask()
 
     @property
     def busy(self):
@@ -32,13 +80,20 @@ class AsyncExecutor(object):
             self._scheduled_action = None
 
     def run(self, func, args=()):
+        wakeup = False
         try:
-            return func(*args) if args else func()
+            # if the func returned something (not None) - wake up main HA loop
+            wakeup = func(*args) if args else func()
+            return wakeup
         except:
             logger.exception('Exception during execution of long running task %s', self.scheduled_action)
         finally:
             with self:
                 self.reset_scheduled_action()
+                with self.critical_task:
+                    self.critical_task.reset()
+            if wakeup is not None:
+                self._ha_wakeup()
 
     def run_async(self, func, args=()):
         Thread(target=self.run, args=(func, args)).start()

patroni/callback_executor.py

@@ -0,0 +1,37 @@
+import logging
+import subprocess
+from threading import Event, Lock, Thread
+
+logger = logging.getLogger(__name__)
+
+
+class CallbackExecutor(Thread):
+
+    def __init__(self):
+        super(CallbackExecutor, self).__init__()
+        self.daemon = True
+        self._lock = Lock()
+        self._cmd = None
+        self._process = None
+        self._callback_event = Event()
+        self.start()
+
+    def call(self, cmd):
+        with self._lock:
+            if self._process and self._process.poll() is None:
+                self._process.kill()
+                logger.warning('Killed the old callback process because it was still running: %s', self._cmd)
+        self._cmd = cmd
+        self._callback_event.set()
+
+    def run(self):
+        while True:
+            self._callback_event.wait()
+            self._callback_event.clear()
+            with self._lock:
+                try:
+                    self._process = subprocess.Popen(self._cmd, close_fds=True)
+                except Exception:
+                    logger.exception('Failed to execute %s',  self._cmd)
+                    continue
+            self._process.wait()

patroni/config.py

@@ -41,10 +41,16 @@ class Config(object):
     __DEFAULT_CONFIG = {
         'ttl': 30, 'loop_wait': 10, 'retry_timeout': 10,
         'maximum_lag_on_failover': 1048576,
+        'master_start_timeout': 300,
+        'synchronous_mode': False,
+        'synchronous_mode_strict': False,
         'postgresql': {
             'bin_dir': '',
             'use_slots': True,
             'parameters': {p: v[0] for p, v in Postgresql.CMDLINE_OPTIONS.items()}
+        },
+        'watchdog': {
+            'mode': 'automatic',
         }
     }
 
@@ -172,7 +178,10 @@ class Config(object):
                     elif name not in ('connect_address', 'listen', 'data_dir', 'pgpass', 'authentication'):
                         config['postgresql'][name] = deepcopy(value)
             elif name in config:  # only variables present in __DEFAULT_CONFIG allowed to be overriden from DCS
-                config[name] = int(value)
+                if name in ('synchronous_mode', 'synchronous_mode_strict'):
+                    config[name] = value
+                else:
+                    config[name] = int(value)
         return config
 
     @staticmethod
@@ -232,8 +241,9 @@ class Config(object):
             if param.startswith(Config.PATRONI_ENV_PREFIX):
                 name, suffix = (param[8:].rsplit('_', 1) + [''])[:2]
                 if name and suffix:
-                    # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT)
-                    if suffix in ('HOST', 'HOSTS', 'PORT') and '_' not in name:
+                    # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..)
+                    if suffix in ('HOST', 'HOSTS', 'PORT', 'SRV', 'URL', 'PROXY', 'CACERT', 'CERT', 'KEY') \
+                            and '_' not in name:
                         value = os.environ.pop(param)
                         if suffix == 'PORT':
                             value = value and parse_int(value)
@@ -265,14 +275,12 @@ class Config(object):
                         config['postgresql'][name].update(self._process_postgresql_parameters(value, True))
                     elif name != 'use_slots':  # replication slots must be enabled/disabled globally
                         config['postgresql'][name] = deepcopy(value)
-            elif name not in config:
+            elif name not in config or name in ['watchdog']:
                 config[name] = deepcopy(value) if value else {}
 
         # restapi server expects to get restapi.auth = 'username:password'
         if 'authentication' in config['restapi']:
-            restapi = config['restapi']
-            auth = restapi['authentication']
-            restapi['auth'] = '{0}:{1}'.format(auth['username'], auth['password'])
+            config['restapi']['auth'] = '{username}:{password}'.format(**config['restapi']['authentication'])
 
         # special treatment for old config
 
@@ -295,7 +303,7 @@ class Config(object):
             config['name'] = pg_config['name']
 
         pg_config.update({p: config[p] for p in ('name', 'scope', 'retry_timeout',
-                          'maximum_lag_on_failover') if p in config})
+                          'synchronous_mode', 'maximum_lag_on_failover') if p in config})
 
         return config
 

patroni/ctl.py

@@ -4,35 +4,43 @@ Patroni Control
 
 import base64
 import click
+import codecs
 import datetime
 import dateutil.parser
+import cdiff
+import copy
+import difflib
+import io
 import json
 import logging
 import os
 import psycopg2
 import random
 import requests
+import subprocess
 import sys
+import tempfile
 import time
 import tzlocal
 import yaml
 
 from click import ClickException
+from contextlib import contextmanager
 from patroni.config import Config
 from patroni.dcs import get_dcs as _get_dcs
 from patroni.exceptions import PatroniException
 from patroni.postgresql import Postgresql
-from patroni.utils import is_valid_pg_version
+from patroni.utils import is_valid_pg_version, patch_config
 from prettytable import PrettyTable
 from six.moves.urllib_parse import urlparse
+from six import text_type
 
 CONFIG_DIR_PATH = click.get_app_dir('patroni')
 CONFIG_FILE_PATH = os.path.join(CONFIG_DIR_PATH, 'patronictl.yaml')
-LOGLEVEL = 'WARNING'
 DCS_DEFAULTS = {'zookeeper': {'port': 2181, 'template': "zookeeper:\n hosts: ['{host}:{port}']"},
                 'exhibitor': {'port': 8181, 'template': "exhibitor:\n hosts: [{host}]\n port: {port}"},
                 'consul': {'port': 8500, 'template': "consul:\n host: '{host}:{port}'"},
-                'etcd': {'port': 4001, 'template': "etcd:\n host: '{host}:{port}'"}}
+                'etcd': {'port': 2379, 'template': "etcd:\n host: '{host}:{port}'"}}
 
 
 class PatroniCtlException(ClickException):
@@ -88,25 +96,23 @@ def store_config(config, path):
         yaml.dump(config, fd)
 
 
-option_config_file = click.option('--config-file', '-c', help='Configuration file', default=CONFIG_FILE_PATH)
 option_format = click.option('--format', '-f', 'fmt', help='Output format (pretty, json)', default='pretty')
-option_dcs = click.option('--dcs', '-d', help='Use this DCS', envvar='DCS')
 option_watchrefresh = click.option('-w', '--watch', type=float, help='Auto update the screen every X seconds')
 option_watch = click.option('-W', is_flag=True, help='Auto update the screen every 2 seconds')
 option_force = click.option('--force', is_flag=True, help='Do not ask for confirmation at any point')
 
 
 @click.group()
+@click.option('--config-file', '-c', help='Configuration file', default=CONFIG_FILE_PATH)
+@click.option('--dcs', '-d', help='Use this DCS', envvar='DCS')
 @click.pass_context
-def ctl(ctx):
-    global LOGLEVEL
-    LOGLEVEL = os.environ.get('LOGLEVEL', LOGLEVEL)
-
-    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=LOGLEVEL)
+def ctl(ctx, config_file, dcs):
+    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=os.environ.get('LOGLEVEL', 'WARNING'))
+    ctx.obj = load_config(config_file, dcs)
 
 
 def get_dcs(config, scope):
-    config['scope'] = scope
+    config.update({'scope': scope, 'patronictl': True})
     config.setdefault('name', scope)
     try:
         return _get_dcs(config)
@@ -267,16 +273,15 @@ def get_members(cluster, cluster_name, member_names, role, force, action):
 @click.option('--role', '-r', help='Give a dsn of any member with this role', type=click.Choice(['master', 'replica',
               'any']), default=None)
 @click.option('--member', '-m', help='Generate a dsn for this member', type=str)
-@option_dcs
-@option_config_file
 @click.argument('cluster_name')
-def dsn(cluster_name, config_file, dcs, role, member):
+@click.pass_obj
+def dsn(obj, cluster_name, role, member):
     if role is not None and member is not None:
         raise PatroniCtlException('--role and --member are mutually exclusive options')
     if member is None and role is None:
         role = 'master'
 
-    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    cluster = get_dcs(obj, cluster_name).get_cluster()
     m = get_any_member(cluster, role=role, member=member)
     if m is None:
         raise PatroniCtlException('Can not find a suitable member')
@@ -287,13 +292,11 @@ def dsn(cluster_name, config_file, dcs, role, member):
 
 @ctl.command('query', help='Query a Patroni PostgreSQL member')
 @click.argument('cluster_name')
-@option_config_file
 @option_format
 @click.option('--format', 'fmt', help='Output format (pretty, json)', default='tsv')
 @click.option('--file', '-f', 'p_file', help='Execute the SQL commands from this file', type=click.File('rb'))
 @click.option('--password', help='force password prompt', is_flag=True)
 @click.option('-U', '--username', help='database user name', type=str)
-@option_dcs
 @option_watch
 @option_watchrefresh
 @click.option('--role', '-r', help='The role of the query', type=click.Choice(['master', 'replica', 'any']),
@@ -302,10 +305,10 @@ def dsn(cluster_name, config_file, dcs, role, member):
 @click.option('--delimiter', help='The column delimiter', default='\t')
 @click.option('--command', '-c', help='The SQL commands to execute')
 @click.option('-d', '--dbname', help='database name to connect to', type=str)
+@click.pass_obj
 def query(
+    obj,
     cluster_name,
-    config_file,
-    dcs,
     role,
     member,
     w,
@@ -329,7 +332,7 @@ def query(
     if p_file is None and command is None:
         raise PatroniCtlException('You need to specify either --command or --file')
 
-    connect_parameters = dict()
+    connect_parameters = {}
     if username:
         connect_parameters['username'] = username
     if password:
@@ -340,17 +343,16 @@ def query(
     if p_file is not None:
         command = p_file.read()
 
-    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    dcs = get_dcs(obj, cluster_name)
 
     cursor = None
     for _ in watching(w, watch, clear=False):
+        if cursor is None:
+            cluster = dcs.get_cluster()
 
         output, cursor = query_member(cluster, cursor, member, role, command, connect_parameters)
         print_output(None, output, fmt=fmt, delimiter=delimiter)
 
-        if cursor is None:
-            cluster = dcs.get_cluster()
-
 
 def query_member(cluster, cursor, member, role, command, connect_parameters):
     try:
@@ -385,11 +387,11 @@ def query_member(cluster, cursor, member, role, command, connect_parameters):
 
 @ctl.command('remove', help='Remove cluster from DCS')
 @click.argument('cluster_name')
-@option_config_file
 @option_format
-@option_dcs
-def remove(config_file, cluster_name, fmt, dcs):
-    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+@click.pass_obj
+def remove(obj, cluster_name, fmt):
+    dcs = get_dcs(obj, cluster_name)
+    cluster = dcs.get_cluster()
 
     output_members(cluster, cluster_name, fmt=fmt)
 
@@ -412,28 +414,6 @@ def remove(config_file, cluster_name, fmt, dcs):
     dcs.delete_cluster()
 
 
-def wait_for_leader(dcs, timeout=30):
-    t_stop = time.time() + timeout
-    timeout /= 2
-
-    while time.time() < t_stop:
-        dcs.watch(timeout)
-        cluster = dcs.get_cluster()
-
-        if cluster.leader:
-            return cluster
-
-    raise PatroniCtlException('Timeout occured')
-
-
-def ctl_load_config(cluster_name, config_file, dcs):
-    config = load_config(config_file, dcs)
-    dcs = get_dcs(config, cluster_name)
-    cluster = dcs.get_cluster()
-
-    return config, dcs, cluster
-
-
 def check_response(response, member_name, action_name, silent_success=False):
     if response.status_code >= 400:
         click.echo('Failed: {0} for member {1}, status code={2}, ({3})'.format(
@@ -468,11 +448,12 @@ def parse_scheduled(scheduled):
 @click.option('--pg-version', 'version', help='Restart if the PostgreSQL version is less than provided (e.g. 9.5.2)',
               default=None)
 @click.option('--pending', help='Restart if pending', is_flag=True)
-@option_config_file
+@click.option('--timeout',
+              help='Return error and fail over if necessary when restarting takes longer than this.')
 @option_force
-@option_dcs
-def restart(cluster_name, member_names, config_file, dcs, force, role, p_any, scheduled, version, pending):
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+@click.pass_obj
+def restart(obj, cluster_name, member_names, force, role, p_any, scheduled, version, pending, timeout):
+    cluster = get_dcs(obj, cluster_name).get_cluster()
 
     members = get_members(cluster, cluster_name, member_names, role, force, 'restart')
     if p_any:
@@ -503,13 +484,16 @@ def restart(cluster_name, member_names, config_file, dcs, force, role, p_any, sc
             raise PatroniCtlException("Can't schedule restart in the paused state")
         content['schedule'] = scheduled_at.isoformat()
 
+    if timeout is not None:
+        content['timeout'] = timeout
+
     for member in members:
         if 'schedule' in content:
             if force and member.data.get('scheduled_restart'):
-                r = request_patroni(member, 'delete', 'restart', headers=auth_header(config))
+                r = request_patroni(member, 'delete', 'restart', headers=auth_header(obj))
                 check_response(r, member.name, 'flush scheduled restart', True)
 
-        r = request_patroni(member, 'post', 'restart', content, auth_header(config))
+        r = request_patroni(member, 'post', 'restart', content, auth_header(obj))
         if r.status_code == 200:
             click.echo('Success: restart on member {0}'.format(member.name))
         elif r.status_code == 202:
@@ -525,15 +509,14 @@ def restart(cluster_name, member_names, config_file, dcs, force, role, p_any, sc
 @ctl.command('reinit', help='Reinitialize cluster member')
 @click.argument('cluster_name')
 @click.argument('member_names', nargs=-1)
-@option_config_file
 @option_force
-@option_dcs
-def reinit(cluster_name, member_names, config_file, dcs, force):
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+@click.pass_obj
+def reinit(obj, cluster_name, member_names, force):
+    cluster = get_dcs(obj, cluster_name).get_cluster()
     members = get_members(cluster, cluster_name, member_names, None, force, 'reinitialize')
 
     for member in members:
-        r = request_patroni(member, 'post', 'reinitialize', headers=auth_header(config))
+        r = request_patroni(member, 'post', 'reinitialize', headers=auth_header(obj))
         check_response(r, member.name, 'reinitialize')
 
 
@@ -544,9 +527,8 @@ def reinit(cluster_name, member_names, config_file, dcs, force):
 @click.option('--scheduled', help='Timestamp of a scheduled failover in unambiguous format (e.g. ISO 8601)',
               default=None)
 @option_force
-@option_config_file
-@option_dcs
-def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled):
+@click.pass_obj
+def failover(obj, cluster_name, master, candidate, force, scheduled):
     """
         We want to trigger a failover for the specified cluster name.
 
@@ -554,7 +536,8 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
         If so, we trigger a failover and keep the client up to date.
     """
 
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    dcs = get_dcs(obj, cluster_name)
+    cluster = dcs.get_cluster()
 
     if cluster.leader is None and not cluster.is_paused():
         raise PatroniCtlException('This cluster has no master')
@@ -590,12 +573,13 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
 
     scheduled_at = parse_scheduled(scheduled)
 
+    scheduled_at_str = None
     if scheduled_at:
         if cluster.is_paused():
             raise PatroniCtlException("Can't schedule failover in the paused state")
-        scheduled_at = scheduled_at.isoformat()
+        scheduled_at_str = scheduled_at.isoformat()
 
-    failover_value = {'leader': master, 'candidate': candidate, 'scheduled_at': scheduled_at}
+    failover_value = {'leader': master, 'candidate': candidate, 'scheduled_at': scheduled_at_str}
 
     logging.debug(failover_value)
 
@@ -614,7 +598,7 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
     try:
         member = cluster.leader.member if cluster.leader else [m for m in cluster.members if m.name == candidate][0]
 
-        r = request_patroni(member, 'post', 'failover', failover_value, auth_header(config))
+        r = request_patroni(member, 'post', 'failover', failover_value, auth_header(obj))
         if r.status_code in (200, 202):
             logging.debug(r)
             cluster = dcs.get_cluster()
@@ -628,7 +612,7 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
         logging.warning('Failing over to DCS')
         click.echo(timestamp() + ' Could not failover using Patroni api, falling back to DCS')
         click.echo(timestamp() + ' Initializing failover from master {0}'.format(master))
-        dcs.manual_failover(master, candidate, scheduled_at=failover_value)
+        dcs.manual_failover(master, candidate, scheduled_at=scheduled_at)
 
     output_members(cluster, cluster_name)
 
@@ -647,9 +631,11 @@ def output_members(cluster, name, extended=False, fmt='pretty'):
     for m in cluster.members:
         logging.debug(m)
 
-        leader = ''
+        role = ''
         if m.name == leader_name:
-            leader = '*'
+            role = 'Leader'
+        elif m.name == cluster.sync.sync_standby:
+            role = 'Sync standby'
 
         host = m.conn_kwargs()['host']
 
@@ -662,7 +648,7 @@ def output_members(cluster, name, extended=False, fmt='pretty'):
             name,
             m.name,
             host,
-            leader,
+            role,
             m.data.get('state', ''),
             lag,
         ]
@@ -682,7 +668,7 @@ def output_members(cluster, name, extended=False, fmt='pretty'):
         'Cluster',
         'Member',
         'Host',
-        'Leader',
+        'Role',
         'State',
         'Lag in MB',
     ]
@@ -698,19 +684,17 @@ def output_members(cluster, name, extended=False, fmt='pretty'):
 @ctl.command('list', help='List the Patroni members for a given Patroni')
 @click.argument('cluster_names', nargs=-1)
 @click.option('--extended', '-e', help='Show some extra information', is_flag=True)
-@option_config_file
 @option_format
 @option_watch
 @option_watchrefresh
-@option_dcs
-def members(config_file, cluster_names, fmt, watch, w, dcs, extended):
+@click.pass_obj
+def members(obj, cluster_names, fmt, watch, w, extended):
     if not cluster_names:
         logging.warning('Listing members: No cluster names were provided')
         return
 
-    config = load_config(config_file, dcs)
     for cluster_name in cluster_names:
-        dcs = get_dcs(config, cluster_name)
+        dcs = get_dcs(obj, cluster_name)
 
         for _ in watching(w, watch):
             cluster = dcs.get_cluster()
@@ -723,13 +707,10 @@ def timestamp(precision=6):
 
 @ctl.command('configure', help='Create configuration file')
 @click.option('--config-file', '-c', help='Configuration file', prompt='Configuration file', default=CONFIG_FILE_PATH)
-@click.option('--dcs', '-d', help='The DCS connect url', prompt='DCS connect url', default='etcd://localhost:4001')
+@click.option('--dcs', '-d', help='The DCS connect url', prompt='DCS connect url', default='etcd://localhost:2379')
 @click.option('--namespace', '-n', help='The namespace', prompt='Namespace', default='/service/')
 def configure(config_file, dcs, namespace):
-    config = dict()
-    config['dcs_api'] = str(dcs)
-    config['namespace'] = str(namespace)
-    store_config(config, config_file)
+    store_config({'dcs_api': str(dcs), 'namespace': str(namespace)}, config_file)
 
 
 def touch_member(config, dcs):
@@ -766,10 +747,10 @@ def set_defaults(config, cluster_name):
 @ctl.command('scaffold', help='Create a structure for the cluster in DCS')
 @click.argument('cluster_name')
 @click.option('--sysid', '-s', help='System ID of the cluster to put into the initialize key', default="")
-@option_config_file
-@option_dcs
-def scaffold(cluster_name, config_file, dcs, sysid):
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+@click.pass_obj
+def scaffold(obj, cluster_name, sysid):
+    dcs = get_dcs(obj, cluster_name)
+    cluster = dcs.get_cluster()
     if cluster and cluster.initialize is not None:
         raise PatroniCtlException("This cluster is already initialized")
 
@@ -777,10 +758,10 @@ def scaffold(cluster_name, config_file, dcs, sysid):
         # initialize key already exists, don't touch this cluster
         raise PatroniCtlException("Initialize key for cluster {0} already exists".format(cluster_name))
 
-    set_defaults(config, cluster_name)
+    set_defaults(obj, cluster_name)
 
     # make sure the leader keys will never expire
-    if not (touch_member(config, dcs) and dcs.attempt_to_acquire_leader(permanent=True)):
+    if not (touch_member(obj, dcs) and dcs.attempt_to_acquire_leader(permanent=True)):
         # we did initialize this cluster, but failed to write the leader or member keys, wipe it down completely.
         dcs.delete_cluster()
         raise PatroniCtlException("Unable to install permanent leader for cluster {0}".format(cluster_name))
@@ -793,47 +774,258 @@ def scaffold(cluster_name, config_file, dcs, sysid):
 @click.argument('target', type=click.Choice(['restart']))
 @click.option('--role', '-r', help='Flush only members with this role', default='any',
               type=click.Choice(['master', 'replica', 'any']))
-@option_config_file
 @option_force
-@option_dcs
-def flush(cluster_name, member_names, config_file, dcs, force, role, target):
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+@click.pass_obj
+def flush(obj, cluster_name, member_names, force, role, target):
+    cluster = get_dcs(obj, cluster_name).get_cluster()
 
     members = get_members(cluster, cluster_name, member_names, role, force, 'flush')
     for member in members:
         if target == 'restart':
             if member.data.get('scheduled_restart'):
-                r = request_patroni(member, 'delete', 'restart', None, auth_header(config))
+                r = request_patroni(member, 'delete', 'restart', None, auth_header(obj))
                 check_response(r, member.name, 'flush scheduled restart')
             else:
                 click.echo('No scheduled restart for member {0}'.format(member.name))
 
 
-def toggle_pause(config_file, cluster_name, dcs, paused):
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+def toggle_pause(config, cluster_name, paused):
+    cluster = get_dcs(config, cluster_name).get_cluster()
     if cluster.is_paused() == paused:
         raise PatroniCtlException('Cluster is {0} paused'.format(paused and 'already' or 'not'))
 
-    r = request_patroni(cluster.leader.member, 'patch', 'config', {'pause': paused or None}, auth_header(config))
+    members = []
+    if cluster.leader:
+        members.append(cluster.leader.member)
+    members.extend([m for m in cluster.members if m.api_url and (not members or members[0].name != m.name)])
 
-    if r.status_code == 200:
-        click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
+    for member in members:
+        try:
+            r = request_patroni(member, 'patch', 'config', {'pause': paused or None}, auth_header(config))
+        except Exception:
+            logging.warning('Member %s is not accessible', member.name)
+            continue
+
+        if r.status_code == 200:
+            click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
+        else:
+            click.echo('Failed: {0} cluster management status code={1}, ({2})'.format(
+                       paused and 'pause' or 'resume', r.status_code, r.text))
+        break
     else:
-        click.echo('Failed: {0} cluster management status code={1}, ({2})'.format(
-                   paused and 'pause' or 'resume', r.status_code, r.text))
+        raise PatroniCtlException('Can not find accessible cluster member')
 
 
 @ctl.command('pause', help='Disable auto failover')
 @click.argument('cluster_name')
-@option_config_file
-@option_dcs
-def pause(config_file, cluster_name, dcs):
-    return toggle_pause(config_file, cluster_name, dcs, True)
+@click.pass_obj
+def pause(obj, cluster_name):
+    return toggle_pause(obj, cluster_name, True)
 
 
 @ctl.command('resume', help='Resume auto failover')
 @click.argument('cluster_name')
-@option_config_file
-@option_dcs
-def resume(config_file, cluster_name, dcs):
-    return toggle_pause(config_file, cluster_name, dcs, False)
+@click.pass_obj
+def resume(obj, cluster_name):
+    return toggle_pause(obj, cluster_name, False)
+
+
+@contextmanager
+def temporary_file(contents, suffix='', prefix='tmp'):
+    """Creates a temporary file with specified contents that persists for the context.
+
+    :param contents: binary string that will be written to the file.
+    :param prefix: will be prefixed to the filename.
+    :param suffix: will be appended to the filename.
+    :returns path of the created file.
+    """
+    tmp = tempfile.NamedTemporaryFile(suffix=suffix, prefix=prefix, delete=False)
+    with tmp:
+        tmp.write(contents)
+
+    try:
+        yield tmp.name
+    finally:
+        os.unlink(tmp.name)
+
+
+def show_diff(before_editing, after_editing):
+    """Shows a diff between two strings.
+
+    If the output is to a tty the diff will be colored. Inputs are expected to be unicode strings.
+    """
+    def listify(string):
+        return [l+'\n' for l in string.rstrip('\n').split('\n')]
+
+    unified_diff = difflib.unified_diff(listify(before_editing), listify(after_editing))
+
+    if sys.stdout.isatty():
+        buf = io.StringIO()
+        for line in unified_diff:
+            # Force cast to unicode as difflib on Python 2.7 returns a mix of unicode and str.
+            buf.write(text_type(line))
+        buf.seek(0)
+
+        class opts:
+            side_by_side = False
+            width = 80
+            tab_width = 8
+        cdiff.markup_to_pager(cdiff.PatchStream(buf), opts)
+    else:
+        for line in unified_diff:
+            click.echo(line.rstrip('\n'))
+
+
+def format_config_for_editing(data):
+    """Formats configuration as YAML for human consumption.
+
+    :param data: configuration as nested dictionaries
+    :returns unicode YAML of the configuration"""
+    return yaml.safe_dump(data, default_flow_style=False, encoding=None, allow_unicode=True)
+
+
+def apply_config_changes(before_editing, data, kvpairs):
+    """Applies config changes specified as a list of key-value pairs.
+
+    Keys are interpreted as dotted paths into the configuration data structure. Except for paths beginning with
+    `postgresql.parameters` where rest of the path is used directly to allow for PostgreSQL GUCs containing dots.
+    Values are interpreted as YAML values.
+
+    :param before_editing: human representation before editing
+    :param data: configuration datastructure
+    :param kvpairs: list of strings containing key value pairs separated by =
+    :returns tuple of human readable and parsed datastructure after changes
+    """
+    changed_data = copy.deepcopy(data)
+
+    def set_path_value(config, path, value, prefix=()):
+        # Postgresql GUCs can't be nested, but can contain dots so we re-flatten the structure for this case
+        if prefix == ('postgresql', 'parameters'):
+            path = ['.'.join(path)]
+
+        if len(path) == 1:
+            if value is None:
+                config.pop(path[0], None)
+            else:
+                config[path[0]] = value
+        else:
+            key = path[0]
+            if key not in config:
+                config[key] = {}
+            set_path_value(config[key], path[1:], value, prefix + (key,))
+            if config[key] == {}:
+                del config[key]
+
+    for pair in kvpairs:
+        if not pair or "=" not in pair:
+            raise PatroniCtlException("Invalid parameter setting {0}".format(pair))
+        key_path, value = pair.split("=", 1)
+        set_path_value(changed_data, key_path.strip().split("."), yaml.safe_load(value))
+
+    return format_config_for_editing(changed_data), changed_data
+
+
+def apply_yaml_file(data, filename):
+    """Applies changes from a YAML file to configuration
+
+    :param data: configuration datastructure
+    :param filename: name of the YAML file, - is taken to mean standard input
+    :returns tuple of human readable and parsed datastructure after changes
+    """
+    changed_data = copy.deepcopy(data)
+
+    if filename == '-':
+        new_options = yaml.safe_load(sys.stdin)
+    else:
+        with open(filename) as fd:
+            new_options = yaml.safe_load(fd)
+
+    patch_config(changed_data, new_options)
+
+    return format_config_for_editing(changed_data), changed_data
+
+
+def invoke_editor(before_editing, cluster_name):
+    """Starts editor command to edit configuration in human readable format
+
+    :param before_editing: human representation before editing
+    :returns tuple of human readable and parsed datastructure after changes
+    """
+    editor_cmd = os.environ.get('EDITOR')
+    if not editor_cmd:
+        raise PatroniCtlException('EDITOR environment variable is not set')
+
+    with temporary_file(contents=before_editing.encode('utf-8'),
+                        suffix='.yaml',
+                        prefix='{0}-config-'.format(cluster_name)) as tmpfile:
+        ret = subprocess.call([editor_cmd, tmpfile])
+        if ret:
+            raise PatroniCtlException("Editor exited with return code {0}".format(ret))
+
+        with codecs.open(tmpfile, encoding='utf-8') as fd:
+            after_editing = fd.read()
+
+        return after_editing, yaml.safe_load(after_editing)
+
+
+@ctl.command('edit-config', help="Edit cluster configuration")
+@click.argument('cluster_name')
+@click.option('--quiet', '-q', is_flag=True, help='Do not show changes')
+@click.option('--set', '-s', 'kvpairs', multiple=True,
+              help='Set specific configuration value. Can be specified multiple times')
+@click.option('--pg', '-p', 'pgkvpairs', multiple=True,
+              help='Set specific PostgreSQL parameter value. Shorthand for -s postgresql.parameters. '
+                   'Can be specified multiple times')
+@click.option('--apply', 'apply_filename', help='Apply configuration from file. Use - for stdin.')
+@click.option('--replace', 'replace_filename', help='Apply configuration from file, replacing existing configuration.'
+              ' Use - for stdin.')
+@option_force
+@click.pass_obj
+def edit_config(obj, cluster_name, force, quiet, kvpairs, pgkvpairs, apply_filename, replace_filename):
+    dcs = get_dcs(obj, cluster_name)
+    cluster = dcs.get_cluster()
+
+    before_editing = format_config_for_editing(cluster.config.data)
+
+    after_editing = None  # Serves as a flag if any changes were requested
+    changed_data = cluster.config.data
+
+    if replace_filename:
+        after_editing, changed_data = apply_yaml_file({}, replace_filename)
+
+    if apply_filename:
+        after_editing, changed_data = apply_yaml_file(changed_data, apply_filename)
+
+    if kvpairs or pgkvpairs:
+        all_pairs = list(kvpairs) + ['postgresql.parameters.'+v.lstrip() for v in pgkvpairs]
+        after_editing, changed_data = apply_config_changes(before_editing, changed_data, all_pairs)
+
+    # If no changes were specified on the command line invoke editor
+    if after_editing is None:
+        after_editing, changed_data = invoke_editor(before_editing, cluster_name)
+
+    if cluster.config.data == changed_data:
+        if not quiet:
+            click.echo("Not changed")
+        return
+
+    if not quiet:
+        show_diff(before_editing, after_editing)
+
+    if (apply_filename == '-' or replace_filename == '-') and not force:
+        click.echo("Use --force option to apply changes")
+        return
+
+    if force or click.confirm('Apply these changes?'):
+        if not dcs.set_config_value(json.dumps(changed_data), cluster.config.modify_index):
+            raise PatroniCtlException("Config modification aborted due to concurrent changes")
+        click.echo("Configuration changed")
+
+
+@ctl.command('show-config', help="Show cluster configuration")
+@click.argument('cluster_name')
+@click.pass_obj
+def show_config(obj, cluster_name):
+    cluster = get_dcs(obj, cluster_name).get_cluster()
+
+    click.echo(format_config_for_editing(cluster.config.data))

patroni/dcs/__init__.py

@@ -3,6 +3,7 @@ import dateutil
 import importlib
 import inspect
 import json
+import logging
 import os
 import pkgutil
 import six
@@ -14,6 +15,8 @@ from random import randint
 from six.moves.urllib_parse import urlparse, urlunparse, parse_qsl
 from threading import Event, Lock
 
+logger = logging.getLogger(__name__)
+
 
 def parse_connection_string(value):
     """Original Governor stores connection strings for each cluster members if a following format:
@@ -51,18 +54,22 @@ def dcs_modules():
 def get_dcs(config):
     available_implementations = set()
     for module_name in dcs_modules():
-        module = importlib.import_module(module_name)
-        for name in filter(lambda name: not name.startswith('__'), dir(module)):  # iterate through module content
-            value = getattr(module, name)
-            name = name.lower()
-            # try to find implementation of AbstractDCS interface, class name must match with module_name
-            if inspect.isclass(value) and issubclass(value, AbstractDCS) and __package__ + '.' + name == module_name:
-                available_implementations.add(name)
-                if name in config:  # which has configuration section in the config file
-                    # propagate some parameters
-                    config[name].update({p: config[p] for p in ('namespace', 'name', 'scope',
-                                         'loop_wait', 'ttl', 'retry_timeout') if p in config})
-                    return value(config[name])
+        try:
+            module = importlib.import_module(module_name)
+            for name in filter(lambda name: not name.startswith('__'), dir(module)):  # iterate through module content
+                item = getattr(module, name)
+                name = name.lower()
+                # try to find implementation of AbstractDCS interface, class name must match with module_name
+                if inspect.isclass(item) and issubclass(item, AbstractDCS) and __package__ + '.' + name == module_name:
+                    available_implementations.add(name)
+                    if name in config:  # which has configuration section in the config file
+                        # propagate some parameters
+                        config[name].update({p: config[p] for p in ('namespace', 'name', 'scope', 'loop_wait',
+                                             'patronictl', 'ttl', 'retry_timeout') if p in config})
+                        return item(config[name])
+        except ImportError:
+            if not config.get('patronictl'):
+                logger.info('Failed to import %s', module_name)
     raise PatroniException("""Can not find suitable configuration of distributed configuration store
 Available implementations: """ + ', '.join(available_implementations))
 
@@ -142,6 +149,14 @@ class Member(namedtuple('Member', 'index,name,session,data')):
     def clonefrom(self):
         return self.tags.get('clonefrom', False) and bool(self.conn_url)
 
+    @property
+    def state(self):
+        return self.data.get('state', 'unknown')
+
+    @property
+    def is_running(self):
+        return self.state == 'running'
+
 
 class Leader(namedtuple('Leader', 'index,session,member')):
 
@@ -222,7 +237,59 @@ class ClusterConfig(namedtuple('ClusterConfig', 'index,data,modify_index')):
         return ClusterConfig(index, data, modify_index or index)
 
 
-class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operation,members,failover')):
+class SyncState(namedtuple('SyncState', 'index,leader,sync_standby')):
+    """Immutable object (namedtuple) which represents last observed synhcronous replication state
+
+    :param index: modification index of a synchronization key in a Configuration Store
+    :param leader: reference to member that was leader
+    :param sync_standby: standby that was last synchronized to leader
+    """
+
+    @staticmethod
+    def from_node(index, value):
+        """
+        >>> SyncState.from_node(1, None).leader is None
+        True
+        >>> SyncState.from_node(1, '{}').leader is None
+        True
+        >>> SyncState.from_node(1, '{').leader is None
+        True
+        >>> SyncState.from_node(1, '[]').leader is None
+        True
+        >>> SyncState.from_node(1, '{"leader": "leader"}').leader == "leader"
+        True
+        """
+        if value:
+            try:
+                data = json.loads(value)
+                if not isinstance(data, dict):
+                    data = {}
+            except (TypeError, ValueError):
+                data = {}
+        else:
+            data = {}
+        return SyncState(index, data.get('leader'), data.get('sync_standby'))
+
+    def matches(self, name):
+        """
+        Returns if a node name matches one of the nodes in the sync state
+
+        >>> s = SyncState(1, 'foo', 'bar')
+        >>> s.matches('foo')
+        True
+        >>> s.matches('bar')
+        True
+        >>> s.matches('baz')
+        False
+        >>> s.matches(None)
+        False
+        >>> SyncState(1, None, None).matches('foo')
+        False
+        """
+        return name is not None and name in (self.leader, self.sync_standby)
+
+
+class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operation,members,failover,sync')):
 
     """Immutable object (namedtuple) which represents PostgreSQL cluster.
     Consists of the following fields:
@@ -232,7 +299,9 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operat
     :param last_leader_operation: int or long object containing position of last known leader operation.
         This value is stored in `/optime/leader` key
     :param members: list of Member object, all PostgreSQL cluster members including leader
-    :param failover: reference to `Failover` object"""
+    :param failover: reference to `Failover` object
+    :param sync: reference to `SyncState` object, last observed synchronous replication state.
+    """
 
     def is_unlocked(self):
         return not (self.leader and self.leader.name)
@@ -243,8 +312,9 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operat
     def get_member(self, member_name, fallback_to_leader=True):
         return ([m for m in self.members if m.name == member_name] or [self.leader if fallback_to_leader else None])[0]
 
-    def get_clone_member(self):
-        candidates = [m for m in self.members if m.clonefrom and (not self.leader or m.name != self.leader.name)]
+    def get_clone_member(self, exclude):
+        exclude = [exclude] + [self.leader.name] if self.leader else []
+        candidates = [m for m in self.members if m.clonefrom and m.is_running and m.name not in exclude]
         return candidates[randint(0, len(candidates) - 1)] if candidates else self.leader
 
     def is_paused(self):
@@ -261,6 +331,7 @@ class AbstractDCS(object):
     _MEMBERS = 'members/'
     _OPTIME = 'optime'
     _LEADER_OPTIME = _OPTIME + '/' + _LEADER
+    _SYNC = 'sync'
 
     def __init__(self, config):
         """
@@ -272,8 +343,10 @@ class AbstractDCS(object):
         self._base_path = '/'.join([self._namespace, config['scope']])
         self._set_loop_wait(config.get('loop_wait', 10))
 
+        self._ctl = bool(config.get('patronictl', False))
         self._cluster = None
         self._cluster_thread_lock = Lock()
+        self._last_leader_operation = ''
         self.event = Event()
 
     def client_path(self, path):
@@ -307,6 +380,10 @@ class AbstractDCS(object):
     def leader_optime_path(self):
         return self.client_path(self._LEADER_OPTIME)
 
+    @property
+    def sync_path(self):
+        return self.client_path(self._SYNC)
+
     @abc.abstractmethod
     def set_ttl(self, ttl):
         """Set the new ttl value for leader key"""
@@ -356,9 +433,14 @@ class AbstractDCS(object):
             self._cluster = None
 
     @abc.abstractmethod
-    def write_leader_optime(self, last_operation):
+    def _write_leader_optime(self, last_operation):
         """write current xlog location into `/optime/leader` key in DCS
-        :param last_operation: absolute xlog location in bytes"""
+        :param last_operation: absolute xlog location in bytes
+        :returns: `!True` on success."""
+
+    def write_leader_optime(self, last_operation):
+        if self._last_leader_operation != last_operation and self._write_leader_optime(last_operation):
+            self._last_leader_operation = last_operation
 
     @abc.abstractmethod
     def update_leader(self):
@@ -445,10 +527,22 @@ class AbstractDCS(object):
     def delete_cluster(self):
         """Delete cluster from DCS"""
 
-    def watch(self, timeout):
+    def write_sync_state(self, leader, sync_standby, index=None):
+        return self.set_sync_state_value(json.dumps({'leader': leader, 'sync_standby': sync_standby}), index=index)
+
+    @abc.abstractmethod
+    def set_sync_state_value(self, value, index=None):
+        """"""
+
+    @abc.abstractmethod
+    def delete_sync_state(self, index=None):
+        """"""
+
+    def watch(self, leader_index, timeout):
         """If the current node is a master it should just sleep.
         Any other node should watch for changes of leader key with a given timeout
 
+        :param leader_index: index of a leader key
         :param timeout: timeout in seconds
         :returns: `!True` if you would like to reschedule the next run of ha cycle"""
 

patroni/dcs/consul.py

@@ -1,14 +1,17 @@
 from __future__ import absolute_import
 import logging
 import os
+import socket
 import time
-import six
+import urllib3
 
-from consul import ConsulException, NotFound, base, std
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member
+from consul import ConsulException, NotFound, base
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
 from patroni.exceptions import DCSError
-from patroni.utils import sleep
-from requests.exceptions import RequestException
+from patroni.utils import Retry, RetryFailedError
+from urllib3.exceptions import HTTPError
+from six.moves.urllib.parse import urlencode
+from six.moves.http_client import HTTPException
 
 logger = logging.getLogger(__name__)
 
@@ -17,38 +20,60 @@ class ConsulError(DCSError):
     pass
 
 
-class HTTPClient(std.HTTPClient):
+class ConsulInternalError(ConsulException):
+    """An internal Consul server error occurred"""
 
-    def __init__(self, *args, **kwargs):
-        super(HTTPClient, self).__init__(*args, **kwargs)
 
-    def patch_default_timeout(self, timeout):
-        # Set a default timeout for the `request.session.request` method, that is used
-        # internally  by the methods request.session.get, request.session.post and
-        # others. We monkey-patch here to avoid reimplementing each individual method from
-        # `std.HTTPClient`. By default, the timeout is not set. It means that a new
-        # session may hang almost indefinitely waiting for the server to respond,
-        # which is not what we want in Patroni.
+class HTTPClient(object):
 
-        request_func = getattr(self.session.request, '__func__' if six.PY3 else 'im_func')
-        defaults_attr_name = '__defaults__' if six.PY3 else 'func_defaults'
-        defaults = list(getattr(request_func, defaults_attr_name))
-        code = request_func.__code__ if six.PY3 else request_func.func_code
-        defaults[code.co_varnames[code.co_argcount - len(defaults):code.co_argcount].index('timeout')] = timeout
-        setattr(request_func, defaults_attr_name, tuple(defaults))  # monkeypatching
+    def __init__(self, host='127.0.0.1', port=8500, scheme='http', verify=True, timeout=10):
+        self.host = host
+        self.port = port
+        self.scheme = scheme
+        self.verify = verify
+        self.set_read_timeout(timeout)
+        self.base_uri = '{0}://{1}:{2}'.format(self.scheme, self.host, self.port)
+        self.http = urllib3.PoolManager(num_pools=10)
+        self._ttl = None
 
-    def get(self, callback, path, params=None):
-        # The get function is overridden to handle a special case of it being called
-        # with an index and wait parameters. That form indicates that a user needs to
-        # wait for the given key to change its value, with a wait timeout supplied. We
-        # don't want our monkey-patched timeout to be less than the value of the wait
-        # parameter, therefore, we set it to either the value of wait or a default of 5 minutes.
+    def set_read_timeout(self, timeout):
+        self._read_timeout = timeout/3.0
 
-        if isinstance(params, dict) and 'index' in params:
-            timeout = (float(params['wait'][:-1]) if 'wait' in params else 300) + 1
-        else:
-            timeout = None
-        return callback(self.response(self.session.get(self.uri(path, params), verify=self.verify, timeout=timeout)))
+    def set_ttl(self, ttl):
+        ret = self._ttl != ttl
+        self._ttl = ttl
+        return ret
+
+    @staticmethod
+    def response(response):
+        data = response.data.decode('utf-8')
+        if response.status == 500:
+            raise ConsulInternalError('{0} {1}'.format(response.status, data))
+        return base.Response(response.status, response.headers, data)
+
+    def uri(self, path, params=None):
+        return '{0}{1}{2}'.format(self.base_uri, path, params and '?' + urlencode(params) or '')
+
+    def __getattr__(self, method):
+        if method not in ('get', 'post', 'put', 'delete'):
+            raise AttributeError("HTTPClient instance has no attribute '{0}'".format(method))
+
+        def wrapper(callback, path, params=None, data=''):
+            # python-consul doesn't allow to specify ttl smaller then 10 seconds
+            # because session_ttl_min defaults to 10s, so we have to do this ugly dirty hack...
+            if method == 'put' and path == '/v1/session/create':
+                ttl = '"ttl": "{0}s"'.format(self._ttl)
+                if not data or data == '{}':
+                    data = '{' + ttl + '}'
+                else:
+                    data = data[:-1] + ', ' + ttl + '}'
+            kwargs = {'retries': 0, 'preload_content': False, 'body': data}
+            if method == 'get' and isinstance(params, dict) and 'index' in params:
+                kwargs['timeout'] = (float(params['wait'][:-1]) if 'wait' in params else 300) + 1
+            else:
+                kwargs['timeout'] = self._read_timeout
+            return callback(self.response(self.http.request(method.upper(), self.uri(path, params), **kwargs)))
+        return wrapper
 
 
 class ConsulClient(base.Consul):
@@ -62,7 +87,7 @@ def catch_consul_errors(func):
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
-        except (ConsulException, RequestException):
+        except (RetryFailedError, ConsulException, HTTPException, HTTPError, socket.error, socket.timeout):
             return False
     return wrapper
 
@@ -71,16 +96,24 @@ class Consul(AbstractDCS):
 
     def __init__(self, config):
         super(Consul, self).__init__(config)
-        self._ttl = None
+        self._scope = config['scope']
         self._session = None
+        self.__do_not_watch = False
+        self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
+                            retry_exceptions=(ConsulInternalError, HTTPException,
+                                              HTTPError, socket.error, socket.timeout))
+
         self._my_member_data = None
-        self.set_ttl(config.get('ttl') or 30)
         host, port = config.get('host', '127.0.0.1:8500').split(':')
         self._client = ConsulClient(host=host, port=port)
-        self._client.http.patch_default_timeout(config['retry_timeout']/2.0)
-        self._scope = config['scope']
-        self.create_session()
-        self.__do_not_watch = False
+        self.set_retry_timeout(config['retry_timeout'])
+        self.set_ttl(config.get('ttl') or 30)
+        self._last_session_refresh = 0
+        if not self._ctl:
+            self.create_session()
+
+    def retry(self, *args, **kwargs):
+        return self._retry.copy()(*args, **kwargs)
 
     def create_session(self):
         while not self._session:
@@ -88,34 +121,40 @@ class Consul(AbstractDCS):
                 self.refresh_session()
             except ConsulError:
                 logger.info('waiting on consul')
-                sleep(5)
+                time.sleep(5)
 
     def set_ttl(self, ttl):
-        ttl = ttl/2.0  # My experiments have shown that session expires after 2*ttl time
-        if self._ttl != ttl:
+        if self._client.http.set_ttl(ttl/2.0):  # Consul multiplies the TTL by 2x
             self._session = None
             self.__do_not_watch = True
-        self._ttl = ttl
 
     def set_retry_timeout(self, retry_timeout):
-        self._client.http.patch_default_timeout(retry_timeout/2.0)
+        self._retry.deadline = retry_timeout
+        self._client.http.set_read_timeout(retry_timeout)
 
-    def refresh_session(self):
+    def _do_refresh_session(self):
         """:returns: `!True` if it had to create new session"""
+        if self._session and self._last_session_refresh + self._loop_wait > time.time():
+            return False
+
         if self._session:
             try:
-                return self._client.session.renew(self._session) is None
+                self._client.session.renew(self._session)
             except NotFound:
                 self._session = None
-        if not self._session:
-            name = self._scope + '-' + self._name
-            try:
-                self._session = self._client.session.create(name=name, lock_delay=0, behavior='delete', ttl=self._ttl)
-            except (ConsulException, RequestException):
-                logger.exception('session.create')
-        if not self._session:
-            raise ConsulError('Failed to renew/create session')
-        return True
+        ret = not self._session
+        if ret:
+            self._session = self._client.session.create(name=self._scope + '-' + self._name,
+                                                        lock_delay=0.001, behavior='delete')
+        self._last_session_refresh = time.time()
+        return ret
+
+    def refresh_session(self):
+        try:
+            return self.retry(self._do_refresh_session)
+        except (ConsulException, RetryFailedError):
+            logger.exception('refresh_session')
+        raise ConsulError('Failed to renew/create session')
 
     def client_path(self, path):
         return super(Consul, self).client_path(path)[1:]
@@ -127,7 +166,7 @@ class Consul(AbstractDCS):
     def _load_cluster(self):
         try:
             path = self.client_path('/')
-            _, results = self._client.kv.get(path, recurse=True)
+            _, results = self.retry(self._client.kv.get, path, recurse=True)
 
             if results is None:
                 raise NotFound
@@ -154,7 +193,8 @@ class Consul(AbstractDCS):
 
             # get leader
             leader = nodes.get(self._LEADER)
-            if leader and leader['Value'] == self._name and self._session != leader.get('Session', 'x'):
+            if not self._ctl and leader and leader['Value'] == self._name \
+                    and self._session != leader.get('Session', 'x'):
                 logger.info('I am leader but not owner of the session. Removing leader node')
                 self._client.kv.delete(self.leader_path, cas=leader['ModifyIndex'])
                 leader = None
@@ -169,17 +209,22 @@ class Consul(AbstractDCS):
             if failover:
                 failover = Failover.from_node(failover['ModifyIndex'], failover['Value'])
 
-            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover)
+            # get synchronization state
+            sync = nodes.get(self._SYNC)
+            sync = SyncState.from_node(sync and sync['ModifyIndex'], sync and sync['Value'])
+
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
         except NotFound:
-            self._cluster = Cluster(None, None, None, None, [], None)
+            self._cluster = Cluster(None, None, None, None, [], None, None)
         except:
             logger.exception('get_cluster')
             raise ConsulError('Consul is not responding properly')
 
     def touch_member(self, data, **kwargs):
         cluster = self.cluster
-        member = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
+        member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
         create_member = self.refresh_session()
+
         if member and (create_member or member.session != self._session):
             try:
                 self._client.kv.delete(self.member_path)
@@ -201,8 +246,11 @@ class Consul(AbstractDCS):
 
     @catch_consul_errors
     def attempt_to_acquire_leader(self, permanent=False):
+        if not self._session and not permanent:
+            self.refresh_session()
+
         args = {} if permanent else {'acquire': self._session}
-        ret = self._client.kv.put(self.leader_path, self._name, **args)
+        ret = self.retry(self._client.kv.put, self.leader_path, self._name, **args)
         if not ret:
             logger.info('Could not take out TTL lock')
         return ret
@@ -219,25 +267,28 @@ class Consul(AbstractDCS):
         return self._client.kv.put(self.config_path, value, cas=index)
 
     @catch_consul_errors
-    def write_leader_optime(self, last_operation):
+    def _write_leader_optime(self, last_operation):
         return self._client.kv.put(self.leader_optime_path, last_operation)
 
-    @staticmethod
-    def update_leader():
-        return True
+    @catch_consul_errors
+    def update_leader(self):
+        if self._session:
+            self.retry(self._client.session.renew, self._session)
+            self._last_session_refresh = time.time()
+        return bool(self._session)
 
     @catch_consul_errors
     def initialize(self, create_new=True, sysid=''):
         kwargs = {'cas': 0} if create_new else {}
-        return self._client.kv.put(self.initialize_path, sysid, **kwargs)
+        return self.retry(self._client.kv.put, self.initialize_path, sysid, **kwargs)
 
     @catch_consul_errors
     def cancel_initialization(self):
-        return self._client.kv.delete(self.initialize_path)
+        return self.retry(self._client.kv.delete, self.initialize_path)
 
     @catch_consul_errors
     def delete_cluster(self):
-        return self._client.kv.delete(self.client_path(''), recurse=True)
+        return self.retry(self._client.kv.delete, self.client_path(''), recurse=True)
 
     @catch_consul_errors
     def delete_leader(self):
@@ -245,24 +296,31 @@ class Consul(AbstractDCS):
         if cluster and isinstance(cluster.leader, Leader) and cluster.leader.name == self._name:
             return self._client.kv.delete(self.leader_path, cas=cluster.leader.index)
 
-    def watch(self, timeout):
+    @catch_consul_errors
+    def set_sync_state_value(self, value, index=None):
+        return self._client.kv.put(self.sync_path, value, cas=index)
+
+    @catch_consul_errors
+    def delete_sync_state(self, index=None):
+        return self._client.kv.delete(self.sync_path, cas=index)
+
+    def watch(self, leader_index, timeout):
         if self.__do_not_watch:
             self.__do_not_watch = False
             return True
 
-        cluster = self.cluster
-        if cluster and cluster.leader and cluster.leader.name != self._name and cluster.leader.index:
+        if leader_index:
             end_time = time.time() + timeout
             while timeout >= 1:
                 try:
-                    idx, _ = self._client.kv.get(self.leader_path, index=cluster.leader.index, wait=str(timeout) + 's')
-                    return str(idx) != str(cluster.leader.index)
-                except (ConsulException, RequestException):
+                    idx, _ = self._client.kv.get(self.leader_path, index=leader_index, wait=str(timeout) + 's')
+                    return str(idx) != str(leader_index)
+                except (ConsulException, HTTPException, HTTPError, socket.error, socket.timeout):
                     logging.exception('watch')
 
                 timeout = end_time - time.time()
 
         try:
-            return super(Consul, self).watch(timeout)
+            return super(Consul, self).watch(None, timeout)
         finally:
             self.event.clear()

patroni/dcs/etcd.py

@@ -2,6 +2,7 @@ from __future__ import absolute_import
 import etcd
 import logging
 import os
+import urllib3.util.connection
 import random
 import requests
 import socket
@@ -9,12 +10,15 @@ import time
 
 from dns.exception import DNSException
 from dns import resolver
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
 from patroni.exceptions import DCSError
-from patroni.utils import Retry, RetryFailedError, sleep
+from patroni.utils import Retry, RetryFailedError
 from urllib3.exceptions import HTTPError, ReadTimeoutError
 from requests.exceptions import RequestException
+from six.moves.queue import Queue
 from six.moves.http_client import HTTPException
+from six.moves.urllib_parse import urlparse
+from threading import Thread
 
 logger = logging.getLogger(__name__)
 
@@ -23,13 +27,63 @@ class EtcdError(DCSError):
     pass
 
 
+class DnsCachingResolver(Thread):
+
+    def __init__(self, cache_time=600.0, cache_fail_time=30.0):
+        super(DnsCachingResolver, self).__init__()
+        self._cache = {}
+        self._cache_time = cache_time
+        self._cache_fail_time = cache_fail_time
+        self._resolve_queue = Queue()
+        self.daemon = True
+        self.start()
+
+    def run(self):
+        while True:
+            (host, port), attempt = self._resolve_queue.get()
+            response = self._do_resolve(host, port)
+            if response:
+                self._cache[(host, port)] = (time.time(), response)
+            else:
+                if attempt < 10:
+                    self.resolve_async(host, port, attempt + 1)
+                    time.sleep(1)
+
+    def resolve(self, host, port):
+        current_time = time.time()
+        cached_time, response = self._cache.get((host, port), (0, []))
+        time_passed = current_time - cached_time
+        if time_passed > self._cache_time or (not response and time_passed > self._cache_fail_time):
+            new_response = self._do_resolve(host, port)
+            if new_response:
+                self._cache[(host, port)] = (current_time, new_response)
+                response = new_response
+        return response
+
+    def resolve_async(self, host, port, attempt=0):
+        self._resolve_queue.put(((host, port), attempt))
+
+    @staticmethod
+    def _do_resolve(host, port):
+        try:
+            return socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, socket.IPPROTO_TCP)
+        except socket.gaierror:
+            logger.warning('failed to resolve host %s', host)
+            return []
+
+
 class Client(etcd.Client):
 
-    def __init__(self, config):
-        super(Client, self).__init__(read_timeout=config['retry_timeout'])
+    def __init__(self, config, dns_resolver, cache_ttl=300):
+        self._dns_resolver = dns_resolver
+        self.set_machines_cache_ttl(cache_ttl)
+        self._machines_cache_updated = 0
+        args = {p: config.get(p) for p in ('host', 'port', 'protocol', 'use_proxies', 'username', 'password',
+                                           'cert', 'ca_cert') if config.get(p)}
+        super(Client, self).__init__(read_timeout=config['retry_timeout'], **args)
         self._config = config
         self._load_machines_cache()
-        self._allow_reconnect = True
+        self._allow_reconnect = not self._use_proxies
 
     def _build_request_parameters(self):
         kwargs = {'headers': self._get_headers(), 'redirect': self.allow_redirect}
@@ -48,6 +102,9 @@ class Client(etcd.Client):
         kwargs['timeout'] = self.read_timeout/float(kwargs['retries'] + 1)/etcd_nodes
         return kwargs
 
+    def set_machines_cache_ttl(self, cache_ttl):
+        self._machines_cache_ttl = cache_ttl
+
     @property
     def machines(self):
         """Original `machines` method(property) of `etcd.Client` class raise exception
@@ -68,6 +125,10 @@ class Client(etcd.Client):
                 machines = [n.strip() for n in self._handle_server_response(response).data.decode('utf-8').split(',')]
                 logger.debug("Retrieved list of machines: %s", machines)
                 random.shuffle(machines)
+                for url in machines:
+                    r = urlparse(url)
+                    port = r.port or (443 if r.scheme == 'https' else 80)
+                    self._dns_resolver.resolve_async(r.hostname, port)
                 return machines
             except Exception as e:
                 # We can't get the list of machines, if one server is in the
@@ -119,6 +180,11 @@ class Client(etcd.Client):
         # Update machines_cache if previous attempt of update has failed
         if self._update_machines_cache:
             self._load_machines_cache()
+        elif time.time() - self._machines_cache_updated > self._machines_cache_ttl:
+            self._machines_cache = self.machines
+            if self._base_uri in self._machines_cache:
+                self._machines_cache.remove(self._base_uri)
+            self._machines_cache_updated = time.time()
 
         kwargs.update(self._build_request_parameters())
 
@@ -147,40 +213,51 @@ class Client(etcd.Client):
     @staticmethod
     def get_srv_record(host):
         try:
-            return [(str(r.target).rstrip('.'), r.port) for r in resolver.query('_etcd-server._tcp.' + host, 'SRV')]
+            return [(r.target.to_text(True), r.port) for r in resolver.query(host, 'SRV')]
         except DNSException:
-            logger.exception('Can not resolve SRV for %s', host)
-        return []
+            return []
 
-    def _get_machines_cache_from_srv(self, discovery_srv):
+    def _get_machines_cache_from_srv(self, srv):
         """Fetch list of etcd-cluster member by resolving _etcd-server._tcp. SRV record.
         This record should contain list of host and peer ports which could be used to run
         'GET http://{host}:{port}/members' request (peer protocol)"""
 
         ret = []
-        for host, port in self.get_srv_record(discovery_srv):
-            url = '{0}://{1}:{2}/members'.format(self._protocol, host, port)
-            try:
-                response = requests.get(url, timeout=self.read_timeout)
-                if response.ok:
-                    for member in response.json():
-                        ret.extend(member['clientURLs'])
-                    break
-            except RequestException:
-                logger.exception('GET %s', url)
+        for r in ['-client-ssl', '-client', '-ssl', '', '-server-ssl', '-server']:
+            protocol = 'https' if '-ssl' in r else 'http'
+            endpoint = '/members' if '-server' in r else ''
+            for host, port in self.get_srv_record('_etcd{0}._tcp.{1}'.format(r, srv)):
+                url = '{0}://{1}:{2}{3}'.format(protocol, host, port, endpoint)
+                if endpoint:
+                    try:
+                        response = requests.get(url, timeout=self.read_timeout, verify=False)
+                        if response.ok:
+                            for member in response.json():
+                                ret.extend(member['clientURLs'])
+                            break
+                    except RequestException:
+                        logger.exception('GET %s', url)
+                else:
+                    ret.append(url)
+            if ret:
+                self._protocol = protocol
+                break
+        else:
+            logger.warning('Can not resolve SRV for %s', srv)
         return list(set(ret))
 
-    def _get_machines_cache_from_dns(self, addr):
+    def _get_machines_cache_from_dns(self, host, port):
         """One host might be resolved into multiple ip addresses. We will make list out of it"""
-
-        ret = []
-        host, port = addr.split(':')
-        try:
-            for r in set(socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)):
-                ret.append('{0}://{1}:{2}'.format(self._protocol, r[4][0], r[4][1]))
-        except socket.error:
-            logger.exception('Can not resolve %s', host)
-        return list(set(ret)) if ret else ['{0}://{1}:{2}'.format(self._protocol, host, port)]
+        if self.protocol == 'http':
+            ret = []
+            for af, _, _, _, sa in self._dns_resolver.resolve(host, port):
+                host, port = sa[:2]
+                if af == socket.AF_INET6:
+                    host = '[{0}]'.format(host)
+                ret.append('{0}://{1}:{2}'.format(self.protocol, host, port))
+            if ret:
+                return list(set(ret))
+        return ['{0}://{1}:{2}'.format(self.protocol, host, port)]
 
     def _load_machines_cache(self):
         """This method should fill up `_machines_cache` from scratch.
@@ -190,42 +267,33 @@ class Client(etcd.Client):
 
         self._update_machines_cache = True
 
-        if 'discovery_srv' not in self._config and 'host' not in self._config:
-            raise Exception('Neither discovery_srv nor host are defined in etcd section of config')
+        if 'srv' not in self._config and 'host' not in self._config:
+            raise Exception('Neither srv nor host url are defined in etcd section of config')
 
-        self._machines_cache = []
+        if self._use_proxies:
+            self._machines_cache = ['{0}://{1}:{2}'.format(self.protocol, self._config['host'], self._config['port'])]
+        else:
+            self._machines_cache = []
 
-        if 'discovery_srv' in self._config:
-            self._machines_cache = self._get_machines_cache_from_srv(self._config['discovery_srv'])
+            if 'srv' in self._config:
+                self._machines_cache = self._get_machines_cache_from_srv(self._config['srv'])
 
-        if not self._machines_cache and 'host' in self._config:
-            self._machines_cache = self._get_machines_cache_from_dns(self._config['host'])
+            if not self._machines_cache and 'host' in self._config:
+                self._machines_cache = self._get_machines_cache_from_dns(self._config['host'], self._config['port'])
 
         # Can not bootstrap list of etcd-cluster members, giving up
         if not self._machines_cache:
             raise etcd.EtcdException
 
         # After filling up initial list of machines_cache we should ask etcd-cluster about actual list
-        self._base_uri = self._machines_cache.pop(0)
+        self._base_uri = self._next_server()
         self._machines_cache = self.machines
 
         if self._base_uri in self._machines_cache:
             self._machines_cache.remove(self._base_uri)
 
         self._update_machines_cache = False
-
-
-def catch_etcd_errors(func):
-    def wrapper(*args, **kwargs):
-        try:
-            return func(*args, **kwargs) is not None
-        except (RetryFailedError, etcd.EtcdException):
-            return False
-        except:
-            logger.exception("")
-            raise EtcdError("unexpected error")
-
-    return wrapper
+        self._machines_cache_updated = time.time()
 
 
 class Etcd(AbstractDCS):
@@ -239,25 +307,111 @@ class Etcd(AbstractDCS):
                                               etcd.EtcdEventIndexCleared))
         self._client = self.get_etcd_client(config)
         self.__do_not_watch = False
+        self._has_failed = False
 
     def retry(self, *args, **kwargs):
         return self._retry.copy()(*args, **kwargs)
 
+    def _handle_exception(self, e, name='', do_sleep=False, raise_ex=None):
+        if not self._has_failed:
+            logger.exception(name)
+        else:
+            logger.error(e)
+            if do_sleep:
+                time.sleep(1)
+        self._has_failed = True
+        if isinstance(raise_ex, Exception):
+            raise raise_ex
+
+    def catch_etcd_errors(func):
+        def wrapper(self, *args, **kwargs):
+            try:
+                retval = func(self, *args, **kwargs) is not None
+                self._has_failed = False
+                return retval
+            except (RetryFailedError, etcd.EtcdException) as e:
+                self._handle_exception(e)
+                return False
+            except Exception as e:
+                self._handle_exception(e, raise_ex=EtcdError('unexpected error'))
+
+        return wrapper
+
     @staticmethod
     def get_etcd_client(config):
+        if 'proxy' in config:
+            config['use_proxies'] = True
+            config['url'] = config['proxy']
+
+        if 'url' in config:
+            r = urlparse(config['url'])
+            config.update({'protocol': r.scheme, 'host': r.hostname, 'port': r.port or 2379,
+                           'username': r.username, 'password': r.password})
+        elif 'host' in config:
+            host, port = (config['host'] + ':2379').split(':')[:2]
+            config['host'] = host
+            if 'port' not in config:
+                config['port'] = int(port)
+
+        if config.get('cacert'):
+            config['ca_cert'] = config.pop('cacert')
+
+        if config.get('key') and config.get('cert'):
+            config['cert'] = (config['cert'], config['key'])
+
+        for p in ('discovery_srv', 'srv_domain'):
+            if p in config:
+                config['srv'] = config.pop(p)
+
+        dns_resolver = DnsCachingResolver()
+
+        def create_connection_patched(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                                      source_address=None, socket_options=None):
+            host, port = address
+            if host.startswith('['):
+                host = host.strip('[]')
+            err = None
+            for af, socktype, proto, _, sa in dns_resolver.resolve(host, port):
+                sock = None
+                try:
+                    sock = socket.socket(af, socktype, proto)
+                    if socket_options:
+                        for opt in socket_options:
+                            sock.setsockopt(*opt)
+                    if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
+                        sock.settimeout(timeout)
+                    if source_address:
+                        sock.bind(source_address)
+                    sock.connect(sa)
+                    return sock
+
+                except socket.error as e:
+                    err = e
+                    if sock is not None:
+                        sock.close()
+                        sock = None
+
+            if err is not None:
+                raise err
+
+            raise socket.error("getaddrinfo returns an empty list")
+
+        urllib3.util.connection.create_connection = create_connection_patched
+
         client = None
         while not client:
             try:
-                client = Client(config)
+                client = Client(config, dns_resolver)
             except etcd.EtcdException:
                 logger.info('waiting on etcd')
-                sleep(5)
+                time.sleep(5)
         return client
 
     def set_ttl(self, ttl):
         ttl = int(ttl)
         self.__do_not_watch = self._ttl != ttl
         self._ttl = ttl
+        self._client.set_machines_cache_ttl(ttl*10)
 
     def set_retry_timeout(self, retry_timeout):
         self._retry.deadline = retry_timeout
@@ -300,12 +454,16 @@ class Etcd(AbstractDCS):
             if failover:
                 failover = Failover.from_node(failover.modifiedIndex, failover.value)
 
-            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover)
+            # get synchronization state
+            sync = nodes.get(self._SYNC)
+            sync = SyncState.from_node(sync and sync.modifiedIndex, sync and sync.value)
+
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
         except etcd.EtcdKeyNotFound:
-            self._cluster = Cluster(None, None, None, None, [], None)
-        except:
-            logger.exception('get_cluster')
-            raise EtcdError('Etcd is not responding properly')
+            self._cluster = Cluster(None, None, None, None, [], None, None)
+        except Exception as e:
+            self._handle_exception(e, 'get_cluster', raise_ex=EtcdError('Etcd is not responding properly'))
+        self._has_failed = False
 
     @catch_etcd_errors
     def touch_member(self, data, ttl=None, permanent=False):
@@ -337,7 +495,7 @@ class Etcd(AbstractDCS):
         return self._client.write(self.config_path, value, prevIndex=index or 0)
 
     @catch_etcd_errors
-    def write_leader_optime(self, last_operation):
+    def _write_leader_optime(self, last_operation):
         return self._client.set(self.leader_optime_path, last_operation)
 
     @catch_etcd_errors
@@ -360,31 +518,42 @@ class Etcd(AbstractDCS):
     def delete_cluster(self):
         return self.retry(self._client.delete, self.client_path(''), recursive=True)
 
-    def watch(self, timeout):
+    @catch_etcd_errors
+    def set_sync_state_value(self, value, index=None):
+        return self._client.write(self.sync_path, value, prevIndex=index or 0)
+
+    @catch_etcd_errors
+    def delete_sync_state(self, index=None):
+        return self.retry(self._client.delete, self.sync_path, prevIndex=index or 0)
+
+    def watch(self, leader_index, timeout):
         if self.__do_not_watch:
             self.__do_not_watch = False
             return True
 
-        cluster = self.cluster
-        # watch on leader key changes if it is defined and current node is not lock owner
-        if cluster and cluster.leader and cluster.leader.name != self._name and cluster.leader.index:
+        if leader_index:
             end_time = time.time() + timeout
 
             while timeout >= 1:  # when timeout is too small urllib3 doesn't have enough time to connect
                 try:
-                    self._client.watch(self.leader_path, index=cluster.leader.index, timeout=timeout + 0.5)
+                    self._client.watch(self.leader_path, index=leader_index, timeout=timeout + 0.5)
+                    self._has_failed = False
                     # Synchronous work of all cluster members with etcd is less expensive
                     # than reestablishing http connection every time from every replica.
                     return True
                 except etcd.EtcdWatchTimedOut:
                     self._client.http.clear()
+                    self._has_failed = False
                     return False
-                except etcd.EtcdException:
-                    logging.exception('watch')
+                except (etcd.EtcdEventIndexCleared, etcd.EtcdWatcherCleared):  # Watch failed
+                    self._has_failed = False
+                    return True  # leave the loop, because watch with the same parameters will fail anyway
+                except etcd.EtcdException as e:
+                    self._handle_exception(e, 'watch', True)
 
                 timeout = end_time - time.time()
 
         try:
-            return super(Etcd, self).watch(timeout)
+            return super(Etcd, self).watch(None, timeout)
         finally:
             self.event.clear()

patroni/dcs/exhibitor.py

@@ -4,7 +4,6 @@ import requests
 import time
 
 from patroni.dcs.zookeeper import ZooKeeper
-from patroni.utils import sleep
 from requests.exceptions import RequestException
 
 logger = logging.getLogger(__name__)
@@ -24,7 +23,7 @@ class ExhibitorEnsembleProvider(object):
         self._next_poll = None
         while not self.poll():
             logger.info('waiting on exhibitor')
-            sleep(5)
+            time.sleep(5)
 
     def poll(self):
         if self._next_poll and self._next_poll > time.time():

patroni/dcs/zookeeper.py

@@ -1,9 +1,10 @@
 import logging
+import time
 
-from kazoo.client import KazooClient, KazooState
+from kazoo.client import KazooClient, KazooState, KazooRetry
 from kazoo.exceptions import NoNodeError, NodeExistsError
 from kazoo.handlers.threading import SequentialThreadingHandler
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
 from patroni.exceptions import DCSError
 
 logger = logging.getLogger(__name__)
@@ -51,13 +52,13 @@ class ZooKeeper(AbstractDCS):
             hosts = ','.join(hosts)
 
         self._client = KazooClient(hosts, handler=PatroniSequentialThreadingHandler(config['retry_timeout']),
-                                   timeout=config['ttl'], connection_retry={'max_delay': 1, 'max_tries': -1},
-                                   command_retry={'deadline': config['retry_timeout'], 'max_delay': 1, 'max_tries': -1})
+                                   timeout=config['ttl'], connection_retry=KazooRetry(max_delay=1, max_tries=-1,
+                                   sleep_func=time.sleep), command_retry=KazooRetry(deadline=config['retry_timeout'],
+                                   max_delay=1, max_tries=-1, sleep_func=time.sleep))
         self._client.add_listener(self.session_listener)
 
         self._my_member_data = None
         self._fetch_cluster = True
-        self._last_leader_operation = 0
 
         self._orig_kazoo_connect = self._client._connection._connect
         self._client._connection._connect = self._kazoo_connect
@@ -65,7 +66,6 @@ class ZooKeeper(AbstractDCS):
         self._client.start()
 
     def _kazoo_connect(self, host, port):
-
         """Kazoo is using Ping's to determine health of connection to zookeeper. If there is no
         response on Ping after Ping interval (1/2 from read_timeout) it will consider current
         connection dead and try to connect to another node. Without this "magic" it was taking
@@ -116,7 +116,8 @@ class ZooKeeper(AbstractDCS):
             return True
 
     def set_retry_timeout(self, retry_timeout):
-        self._client._retry.deadline = retry_timeout
+        retry = self._client.retry if isinstance(self._client.retry, KazooRetry) else self._client._retry
+        retry.deadline = retry_timeout
 
     def get_node(self, key, watch=None):
         try:
@@ -135,10 +136,11 @@ class ZooKeeper(AbstractDCS):
         except NoNodeError:
             return []
 
-    def load_members(self):
+    def load_members(self, sync_standby):
         members = []
         for member in self.get_children(self.members_path, self.cluster_watcher):
-            data = self.get_node(self.members_path + member)
+            watch = member == sync_standby and self.cluster_watcher or None
+            data = self.get_node(self.members_path + member, watch)
             if data is not None:
                 members.append(self.member(member, *data))
         return members
@@ -157,14 +159,24 @@ class ZooKeeper(AbstractDCS):
         config = self.get_node(self.config_path, watch=self.cluster_watcher) if self._CONFIG in nodes else None
         config = config and ClusterConfig.from_node(config[1].version, config[0], config[1].mzxid)
 
+        # get last leader operation
+        last_leader_operation = self._OPTIME in nodes and self._fetch_cluster and self.get_node(self.leader_optime_path)
+        last_leader_operation = last_leader_operation and int(last_leader_operation[0]) or 0
+
+        # get synchronization state
+        sync = self.get_node(self.sync_path, watch=self.cluster_watcher) if self._SYNC in nodes else None
+        sync = SyncState.from_node(sync and sync[1].version, sync and sync[0])
+
         # get list of members
-        members = self.load_members() if self._MEMBERS[:-1] in nodes else []
+        sync_standby = sync.leader == self._name and sync.sync_standby or None
+        members = self.load_members(sync_standby) if self._MEMBERS[:-1] in nodes else []
 
         # get leader
         leader = self.get_node(self.leader_path) if self._LEADER in nodes else None
         if leader:
             client_id = self._client.client_id
-            if leader[0] == self._name and client_id is not None and client_id[0] != leader[1].ephemeralOwner:
+            if not self._ctl and leader[0] == self._name and client_id is not None \
+                    and client_id[0] != leader[1].ephemeralOwner:
                 logger.info('I am leader but not owner of the session. Removing leader node')
                 self._client.delete(self.leader_path)
                 leader = None
@@ -179,10 +191,7 @@ class ZooKeeper(AbstractDCS):
         failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
         failover = failover and Failover.from_node(failover[1].version, failover[0])
 
-        # get last leader operation
-        optime = self.get_node(self.leader_optime_path) if self._OPTIME in nodes and self._fetch_cluster else None
-        self._last_leader_operation = 0 if optime is None else int(optime[0])
-        self._cluster = Cluster(initialize, config, leader, self._last_leader_operation, members, failover)
+        self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
 
     def _load_cluster(self):
         if self._fetch_cluster or self._cluster is None:
@@ -228,11 +237,11 @@ class ZooKeeper(AbstractDCS):
 
     def initialize(self, create_new=True, sysid=""):
         return self._create(self.initialize_path, sysid, makepath=True) if create_new \
-            else self._client.retry(self._client.set, self.initialize_path,  sysid.encode("utf-8"))
+            else self._client.retry(self._client.set, self.initialize_path, sysid.encode("utf-8"))
 
     def touch_member(self, data, ttl=None, permanent=False):
         cluster = self.cluster
-        member = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
+        member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
         data = data.encode('utf-8')
         if member and self._client.client_id is not None and member.session != self._client.client_id[0]:
             try:
@@ -267,20 +276,20 @@ class ZooKeeper(AbstractDCS):
     def take_leader(self):
         return self.attempt_to_acquire_leader()
 
-    def write_leader_optime(self, last_operation):
+    def _write_leader_optime(self, last_operation):
         last_operation = last_operation.encode('utf-8')
-        if last_operation != self._last_leader_operation:
+        try:
+            self._client.set_async(self.leader_optime_path, last_operation).get(timeout=1)
+            return True
+        except NoNodeError:
             try:
-                self._client.set_async(self.leader_optime_path, last_operation).get(timeout=1)
-                self._last_leader_operation = last_operation
-            except NoNodeError:
-                try:
-                    self._client.create_async(self.leader_optime_path, last_operation, makepath=True).get(timeout=1)
-                    self._last_leader_operation = last_operation
-                except:
-                    logger.exception('Failed to create %s', self.leader_optime_path)
+                self._client.create_async(self.leader_optime_path, last_operation, makepath=True).get(timeout=1)
+                return True
             except:
-                logger.exception('Failed to update %s', self.leader_optime_path)
+                logger.exception('Failed to create %s', self.leader_optime_path)
+        except:
+            logger.exception('Failed to update %s', self.leader_optime_path)
+        return False
 
     def update_leader(self):
         return True
@@ -307,7 +316,20 @@ class ZooKeeper(AbstractDCS):
         except NoNodeError:
             return True
 
-    def watch(self, timeout):
-        if super(ZooKeeper, self).watch(timeout):
+    def set_sync_state_value(self, value, index=None):
+        try:
+            self._client.retry(self._client.set, self.sync_path, value.encode('utf-8'), version=index or -1)
+            return True
+        except NoNodeError:
+            return value == '' or (index is None and self._create(self.sync_path, value))
+        except:
+            logging.exception('set_sync_state_value')
+            return False
+
+    def delete_sync_state(self, index=None):
+        return self.set_sync_state_value("{}", index)
+
+    def watch(self, leader_index, timeout):
+        if super(ZooKeeper, self).watch(leader_index, timeout):
             self._fetch_cluster = True
         return self._fetch_cluster

patroni/exceptions.py

@@ -23,3 +23,7 @@ class DCSError(PatroniException):
 
 class PostgresConnectionException(PostgresException):
     pass
+
+
+class WatchdogError(PatroniException):
+    pass

patroni/scripts/aws.py

@@ -6,63 +6,64 @@ from requests.exceptions import RequestException
 import sys
 import boto.ec2
 
+from patroni.utils import Retry, RetryFailedError
+
 logger = logging.getLogger(__name__)
 
 
 class AWSConnection(object):
+
     def __init__(self, cluster_name):
         self.available = False
         self.cluster_name = cluster_name if cluster_name is not None else 'unknown'
+        self._retry = Retry(deadline=300, max_delay=30, max_tries=-1, retry_exceptions=(boto.exception.StandardError,))
         try:
             # get the instance id
-            r = requests.get('http://169.254.169.254/latest/dynamic/instance-identity/document', timeout=0.1)
+            r = requests.get('http://169.254.169.254/latest/dynamic/instance-identity/document', timeout=2.1)
         except RequestException:
-            logger.info("cannot query AWS meta-data")
+            logger.error('cannot query AWS meta-data')
             return
+
         if r.ok:
             try:
                 content = r.json()
                 self.instance_id = content['instanceId']
                 self.region = content['region']
-            except Exception as e:
-                logger.info('unable to fetch instance id and region from AWS meta-data: {}'.format(e))
+            except Exception:
+                logger.exception('unable to fetch instance id and region from AWS meta-data')
                 return
             self.available = True
 
+    def retry(self, *args, **kwargs):
+        return self._retry.copy()(*args, **kwargs)
+
     def aws_available(self):
         return self.available
 
-    def _tag_ebs(self, role):
+    def _tag_ebs(self, conn, role):
         """ set tags, carrying the cluster name, instance role and instance id for the EBS storage """
-        if not self.available:
-            return False
-
         tags = {'Name': 'spilo_' + self.cluster_name, 'Role': role, 'Instance': self.instance_id}
-        try:
-            conn = boto.ec2.connect_to_region(self.region)
-            volumes = conn.get_all_volumes(filters={'attachment.instance-id': self.instance_id})
-            conn.create_tags([v.id for v in volumes], tags)
-        except Exception as e:
-            logger.info('could not set tags for EBS storage devices attached: {}'.format(e))
-            return False
-        return True
+        volumes = conn.get_all_volumes(filters={'attachment.instance-id': self.instance_id})
+        conn.create_tags([v.id for v in volumes], tags)
 
-    def _tag_ec2(self, role):
+    def _tag_ec2(self, conn, role):
         """ tag the current EC2 instance with a cluster role """
-        if not self.available:
-            return False
         tags = {'Role': role}
-        try:
-            conn = boto.ec2.connect_to_region(self.region)
-            conn.create_tags([self.instance_id], tags)
-        except Exception as e:
-            logger.info("could not set tags for EC2 instance %s: %s", self.instance_id, e)
-            return False
-        return True
+        conn.create_tags([self.instance_id], tags)
 
     def on_role_change(self, new_role):
-        ret = self._tag_ec2(new_role)
-        return self._tag_ebs(new_role) and ret
+        if not self.available:
+            return False
+        try:
+            conn = self.retry(boto.ec2.connect_to_region, self.region)
+            self.retry(self._tag_ec2, conn, new_role)
+            self.retry(self._tag_ebs, conn, new_role)
+        except RetryFailedError:
+            logger.warning("Unable to communicate to AWS "
+                           "when setting tags for the EC2 instance {0} "
+                           "and attached EBS volumes".format(self.instance_id))
+            return False
+        return True
 
 
 def main():
@@ -72,5 +73,6 @@ def main():
     else:
         sys.exit("Usage: {0} action role name".format(sys.argv[0]))
 
+
 if __name__ == '__main__':
     main()

patroni/scripts/wale_restore.py

@@ -23,77 +23,180 @@
 # currently also requires that you configure the restore_command to use wal_e, example:
 #       recovery_conf:
 #               restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1
-
-from collections import namedtuple
+import argparse
+import csv
 import logging
 import os
 import psycopg2
 import subprocess
 import sys
-import argparse
+import time
 
-
-if sys.hexversion >= 0x0300000:
-    long = int
+from collections import namedtuple
 
 logger = logging.getLogger(__name__)
 
+RETRY_SLEEP_INTERVAL = 1
+si_prefixes = ['K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']
+
+
+# Meaningful names to the exit codes used by WALERestore
+ExitCode = type('Enum', (), {
+    'SUCCESS': 0,  #: Succeeded
+    'RETRY_LATER': 1,  #: External issue, retry later
+    'FAIL': 2  #: Don't try again unless configuration changes
+})
+
+
+# We need to know the current PG version in order to figure out the correct WAL directory name
+def get_major_version(data_dir):
+    version_file = os.path.join(data_dir, 'PG_VERSION')
+    if os.path.isfile(version_file):  # version file exists
+        try:
+            with open(version_file) as f:
+                return float(f.read())
+        except Exception:
+            logger.exception('Failed to read PG_VERSION from %s', data_dir)
+    return 0.0
+
+
+def repr_size(n_bytes):
+    """
+    >>> repr_size(1000)
+    '1000 Bytes'
+    >>> repr_size(8257332324597)
+    '7.5 TiB'
+    """
+    if n_bytes < 1024:
+        return '{0} Bytes'.format(n_bytes)
+    i = -1
+    while n_bytes > 1023:
+        n_bytes /= 1024.0
+        i += 1
+    return '{0} {1}iB'.format(round(n_bytes, 1), si_prefixes[i])
+
+
+def size_as_bytes(size_, prefix):
+    """
+    >>> size_as_bytes(7.5, 'T')
+    8246337208320
+    """
+    prefix = prefix.upper()
+
+    assert prefix in si_prefixes
+
+    exponent = si_prefixes.index(prefix) + 1
+
+    return int(size_ * (1024.0 ** exponent))
+
+
+WALEConfig = namedtuple(
+    'WALEConfig',
+    [
+        'env_dir',
+        'threshold_mb',
+        'threshold_pct',
+        'cmd',
+    ]
+)
+
 
 class WALERestore(object):
-
-    def __init__(self, scope, datadir, connstring, env_dir, threshold_mb, threshold_pct, use_iam, no_master):
+    def __init__(self, scope, datadir, connstring, env_dir, threshold_mb,
+                 threshold_pct, use_iam, no_master, retries):
         self.scope = scope
         self.master_connection = connstring
         self.data_dir = datadir
-        self.wal_e = namedtuple('wale', 'dir,threshold_mb,threshold_pct,iam_string,cmd')
-        self.wal_e.dir = env_dir
-        self.wal_e.threshold_mb = threshold_mb
-        self.wal_e.threshold_pct = threshold_pct
-        self.wal_e.iam_string = ' --aws-instance-profile ' if use_iam == 1 else ''
         self.no_master = no_master
-        self.wal_e.cmd = 'envdir {0} wal-e {1} '.format(self.wal_e.dir, self.wal_e.iam_string)
-        self.init_error = (not os.path.exists(self.wal_e.dir))
+
+        wale_cmd = [
+            'envdir',
+            env_dir,
+            'wal-e',
+        ]
+
+        if use_iam == 1:
+            wale_cmd += ['--aws-instance-profile']
+
+        self.wal_e = WALEConfig(
+            env_dir=env_dir,
+            threshold_mb=threshold_mb,
+            threshold_pct=threshold_pct,
+            cmd=wale_cmd,
+        )
+
+        self.init_error = (not os.path.exists(self.wal_e.env_dir))
+        self.retries = retries
 
     def run(self):
-        """ creates a new replica using WAL-E """
-        if not self.init_error and self.should_use_s3_to_create_replica():
-            return self.create_replica_with_s3()
-        return 2
+        """
+        Creates a new replica using WAL-E
+
+        Returns
+        -------
+        ExitCode
+            0 = Success
+            1 = Error, try again
+            2 = Error, don't try again
+
+        """
+        if self.init_error:
+            logger.error('init error: %r did not exist at initialization time',
+                         self.wal_e.env_dir)
+            return ExitCode.FAIL
+
+        try:
+            should_use_s3 = self.should_use_s3_to_create_replica()
+            if should_use_s3 is None:  # Need to retry
+                return ExitCode.RETRY_LATER
+            elif should_use_s3:
+                return self.create_replica_with_s3()
+            elif not should_use_s3:
+                return ExitCode.FAIL
+        except Exception:
+            logger.exception("Unhandled exception when running WAL-E restore")
+        return ExitCode.FAIL
 
     def should_use_s3_to_create_replica(self):
         """ determine whether it makes sense to use S3 and not pg_basebackup """
 
         threshold_megabytes = self.wal_e.threshold_mb
-        threshold_backup_size_percentage = self.wal_e.threshold_pct
+        threshold_percent = self.wal_e.threshold_pct
 
         try:
-            latest_backup = subprocess.check_output(self.wal_e.cmd.split() + ['backup-list', '--detail', 'LATEST'])
-            # name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start
-            #                                                   wal_segment_backup_stop wal_segment_offset_backup_stop
-            # base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z
-            # 20310671    00000001000000000000007F    00000040
-            # 00000001000000000000007F    00000240
-            backup_strings = latest_backup.decode('utf-8').splitlines() if latest_backup else ()
-            if len(backup_strings) != 2:
+            cmd = self.wal_e.cmd + ['backup-list', '--detail', 'LATEST']
+
+            logger.debug('calling %r', cmd)
+            wale_output = subprocess.check_output(cmd)
+
+            reader = csv.DictReader(wale_output.decode('utf-8').splitlines(),
+                                    dialect='excel-tab')
+            rows = list(reader)
+            if not len(rows):
+                logger.warning('wal-e did not find any backups')
                 return False
 
-            names = backup_strings[0].split()
-            vals = backup_strings[1].split()
-            if (len(names) != len(vals)) or (len(names) != 7):
+            # This check might not add much, it was performed in the previous
+            # version of this code. since the old version rolled CSV parsing the
+            # check may have been part of the CSV parsing.
+            if len(rows) > 1:
+                logger.warning(
+                    'wal-e returned more than one row of backups: %r',
+                    rows)
                 return False
 
-            backup_info = dict(zip(names, vals))
-        except subprocess.CalledProcessError as e:
-            logger.error("could not query wal-e latest backup: {}".format(e))
-            return False
+            backup_info = rows[0]
+        except subprocess.CalledProcessError:
+            logger.exception("could not query wal-e latest backup")
+            return None
 
         try:
-            backup_size = backup_info['expanded_size_bytes']
+            backup_size = int(backup_info['expanded_size_bytes'])
             backup_start_segment = backup_info['wal_segment_backup_start']
             backup_start_offset = backup_info['wal_segment_offset_backup_start']
-        except Exception as e:
-            logger.error("unable to get some of WALE backup parameters: {}".format(e))
-            return False
+        except KeyError:
+            logger.exception("unable to get some of WALE backup parameters")
+            return None
 
         # WAL filename is XXXXXXXXYYYYYYYY000000ZZ, where X - timeline, Y - LSN logical log file,
         # ZZ - 2 high digits of LSN offset. The rest of the offset is the provided decimal offset,
@@ -101,41 +204,131 @@ class WALERestore(object):
 
         lsn_segment = backup_start_segment[8:16]
         # first 2 characters of the result are 0x and the last one is L
-        lsn_offset = hex((long(backup_start_segment[16:32], 16) << 24) + long(backup_start_offset))[2:-1]
+        lsn_offset = hex((int(backup_start_segment[16:32], 16) << 24) + int(backup_start_offset))[2:-1]
 
         # construct the LSN from the segment and offset
         backup_start_lsn = '{0}/{1}'.format(lsn_segment, lsn_offset)
 
-        diff_in_bytes = long(backup_size)
-        if not self.no_master:
-            try:
-                # get the difference in bytes between the current WAL location and the backup start offset
-                with psycopg2.connect(self.master_connection) as con:
-                    con.autocommit = True
-                    with con.cursor() as cur:
-                        cur.execute("SELECT pg_xlog_location_diff(pg_current_xlog_location(), %s)", (backup_start_lsn,))
-                        diff_in_bytes = long(cur.fetchone()[0])
-            except psycopg2.Error as e:
-                logger.error('could not determine difference with the master location: %s', e)
-                return False
-        else:
-            # always try to use WAL-E if base backup is available
-            diff_in_bytes = 0
+        diff_in_bytes = backup_size
+        attempts_no = 0
+        while True:
+            if self.master_connection:
+                try:
+                    # get the difference in bytes between the current WAL location and the backup start offset
+                    with psycopg2.connect(self.master_connection) as con:
+                        if con.server_version >= 100000:
+                            wal_name = 'wal'
+                            lsn_name = 'lsn'
+                        else:
+                            wal_name = 'xlog'
+                            lsn_name = 'location'
+                        con.autocommit = True
+                        with con.cursor() as cur:
+                            cur.execute("""SELECT CASE WHEN pg_is_in_recovery()
+                                                       THEN GREATEST(
+                                                                pg_{0}_{1}_diff(COALESCE(
+                                                                  pg_last_{0}_receive_{1}(), '0/0'), %s)::bigint,
+                                                                pg_{0}_{1}_diff(pg_last_{0}_replay_{1}(), %s)::bigint)
+                                                       ELSE pg_{0}_{1}_diff(pg_current_{0}_{1}(), %s)::bigint
+                                                   END""".format(wal_name, lsn_name),
+                                        (backup_start_lsn, backup_start_lsn, backup_start_lsn))
+
+                            diff_in_bytes = int(cur.fetchone()[0])
+                except psycopg2.Error:
+                    logger.exception('could not determine difference with the master location')
+                    if attempts_no < self.retries:  # retry in case of a temporarily connection issue
+                        attempts_no = attempts_no + 1
+                        time.sleep(RETRY_SLEEP_INTERVAL)
+                        continue
+                    else:
+                        if not self.no_master:
+                            return False  # do no more retries on the outer level
+                        logger.info("continue with base backup from S3 since master is not available")
+                        diff_in_bytes = 0
+                        break
+            else:
+                # always try to use WAL-E if master connection string is not available
+                diff_in_bytes = 0
+            break
 
         # if the size of the accumulated WAL segments is more than a certan percentage of the backup size
         # or exceeds the pre-determined size - pg_basebackup is chosen instead.
-        return (diff_in_bytes < long(threshold_megabytes) * 1048576) and\
-            (diff_in_bytes < long(backup_size) * float(threshold_backup_size_percentage) / 100)
+        is_size_thresh_ok = diff_in_bytes < int(threshold_megabytes) * 1048576
+        threshold_pct_bytes = backup_size * threshold_percent / 100.0
+        is_percentage_thresh_ok = float(diff_in_bytes) < int(threshold_pct_bytes)
+        are_thresholds_ok = is_size_thresh_ok and is_percentage_thresh_ok
+
+        class Size(object):
+            def __init__(self, n_bytes, prefix=None):
+                self.n_bytes = n_bytes
+                self.prefix = prefix
+
+            def __repr__(self):
+                if self.prefix is not None:
+                    n_bytes = size_as_bytes(self.n_bytes, self.prefix)
+                else:
+                    n_bytes = self.n_bytes
+                return repr_size(n_bytes)
+
+        class HumanContext(object):
+            def __init__(self, items):
+                self.items = items
+
+            def __repr__(self):
+                return ', '.join('{}={!r}'.format(key, value)
+                                 for key, value in self.items)
+
+        human_context = repr(HumanContext([
+            ('threshold_size', Size(threshold_megabytes, 'M')),
+            ('threshold_percent', threshold_percent),
+            ('threshold_percent_size', Size(threshold_pct_bytes)),
+            ('backup_size', Size(backup_size)),
+            ('backup_diff', Size(diff_in_bytes)),
+            ('is_size_thresh_ok', is_size_thresh_ok),
+            ('is_percentage_thresh_ok', is_percentage_thresh_ok),
+        ]))
+
+        if not are_thresholds_ok:
+            logger.info('wal-e backup size diff is over threshold, falling back '
+                        'to other means of restore: %s', human_context)
+        else:
+            logger.info('Thresholds are OK, using wal-e basebackup: %s', human_context)
+        return are_thresholds_ok
+
+    def fix_subdirectory_path_if_broken(self, dirname):
+        # in case it is a symlink pointing to a non-existing location, remove it and create the actual directory
+        path = os.path.join(self.data_dir, dirname)
+        if not os.path.exists(path):
+            if os.path.islink(path):  # broken xlog symlink, to remove
+                try:
+                    os.remove(path)
+                except OSError:
+                    logger.exception("could not remove broken %s symlink pointing to %s",
+                                     dirname, os.readlink(path))
+                    return False
+            try:
+                os.mkdir(path)
+            except OSError:
+                logger.exception("coud not create missing %s directory path", dirname)
+                return False
+        return True
 
     def create_replica_with_s3(self):
         # if we're set up, restore the replica using fetch latest
         try:
-            ret = subprocess.call(self.wal_e.cmd.split() + ['backup-fetch', '{}'.format(self.data_dir), 'LATEST'])
+            cmd = self.wal_e.cmd + ['backup-fetch',
+                                    '{}'.format(self.data_dir),
+                                    'LATEST']
+            logger.debug('calling: %r', cmd)
+            exit_code = subprocess.call(cmd)
         except Exception as e:
             logger.error('Error when fetching backup with WAL-E: {0}'.format(e))
-            return 1
+            return ExitCode.RETRY_LATER
 
-        return ret
+        if (exit_code == 0 and not
+           self.fix_subdirectory_path_if_broken('pg_xlog' if get_major_version(self.data_dir) < 10 else 'pg_wal')):
+            return ExitCode.FAIL
+        return exit_code
 
 
 def main():
@@ -153,17 +346,27 @@ def main():
     parser.add_argument('--no_master', type=int, default=0)
     args = parser.parse_args()
 
-    # retry cloning in a loop
+    exit_code = None
+    assert args.retries >= 0
+
+    # Retry cloning in a loop. We do separate retries for the master
+    # connection attempt inside should_use_s3_to_create_replica,
+    # because we need to differentiate between the last attempt and
+    # the rest and make a decision when the last attempt fails on
+    # whether to use WAL-E or not depending on the no_master flag.
     for _ in range(0, args.retries + 1):
         restore = WALERestore(scope=args.scope, datadir=args.datadir, connstring=args.connstring,
                               env_dir=args.envdir, threshold_mb=args.threshold_megabytes,
                               threshold_pct=args.threshold_backup_size_percentage, use_iam=args.use_iam,
-                              no_master=args.no_master)
-        ret = restore.run()
-        if ret == 0:
+                              no_master=args.no_master, retries=args.retries)
+        exit_code = restore.run()
+        if not exit_code == ExitCode.RETRY_LATER:  # only WAL-E failures lead to the retry
+            logger.debug('exit_code is %r, not retrying', exit_code)
             break
+        time.sleep(RETRY_SLEEP_INTERVAL)
+
+    return exit_code
 
-    sys.exit(ret)
 
 if __name__ == '__main__':
-    main()
+    sys.exit(main())

patroni/utils.py

@@ -1,16 +1,12 @@
-import os
+import contextlib
 import random
-import sys
 import time
 import re
 
+from dateutil import tz
 from patroni.exceptions import PatroniException
 
-if sys.hexversion >= 0x0300000:
-    long = int
-
-__interrupted_sleep = False
-__reap_children = False
+tzutc = tz.tzutc()
 
 
 def deep_compare(obj1, obj2):
@@ -122,7 +118,7 @@ def strtol(value, strict=True):
         while i <= l:
             try:  # try to find maximally long number
                 i += 1  # by giving to `int` longer and longer strings
-                ret = long(value[:i], base)
+                ret = int(value[:i], base)
             except ValueError:  # until we will not get an exception or end of the string
                 i -= 1
                 break
@@ -195,36 +191,8 @@ def compare_values(vartype, unit, old_value, new_value):
     return old_value is not None and new_value is not None and old_value == new_value
 
 
-def sigchld_handler(signo, stack_frame):
-    global __interrupted_sleep, __reap_children
-    __reap_children = __interrupted_sleep = True
-
-
-def sleep(interval):
-    global __interrupted_sleep
-    current_time = time.time()
-    end_time = current_time + interval
-    while current_time < end_time:
-        __interrupted_sleep = False
-        time.sleep(end_time - current_time)
-        if not __interrupted_sleep:  # we will ignore only sigchld
-            break
-        current_time = time.time()
-    __interrupted_sleep = False
-
-
-def reap_children():
-    global __reap_children
-    if __reap_children:
-        try:
-            while True:
-                ret = os.waitpid(-1, os.WNOHANG)
-                if ret == (0, 0):
-                    break
-        except OSError:
-            pass
-        finally:
-            __reap_children = False
+def _sleep(interval):
+    time.sleep(interval)
 
 
 def is_valid_pg_version(version):
@@ -241,7 +209,7 @@ class Retry(object):
     """Helper for retrying a method in the face of retry-able exceptions"""
 
     def __init__(self, max_tries=1, delay=0.1, backoff=2, max_jitter=0.8, max_delay=3600,
-                 sleep_func=sleep, deadline=None, retry_exceptions=PatroniException):
+                 sleep_func=_sleep, deadline=None, retry_exceptions=PatroniException):
         """Create a :class:`Retry` instance for retrying function calls
 
         :param max_tries: How many times to retry the command. -1 means infinite tries.
@@ -302,3 +270,19 @@ class Retry(object):
                 else:
                     self.sleep_func(sleeptime)
                 self._cur_delay = min(self._cur_delay * self.backoff, self.max_delay)
+
+
+def polling_loop(timeout, interval=1):
+    """Returns an iterator that returns values until timeout has passed. Timeout is measured from start of iteration."""
+    start_time = time.time()
+    iteration = 0
+    end_time = start_time + timeout
+    while time.time() < end_time:
+        yield iteration
+        iteration += 1
+        time.sleep(interval)
+
+
+@contextlib.contextmanager
+def null_context():
+    yield

patroni/version.py

@@ -1 +1 @@
-__version__ = '1.1'
+__version__ = '1.3'

patroni/watchdog/__init__.py

@@ -0,0 +1,2 @@
+from patroni.watchdog.base import WatchdogError, Watchdog
+__all__ = ['WatchdogError', 'Watchdog']

patroni/watchdog/base.py

@@ -0,0 +1,313 @@
+import abc
+import logging
+import platform
+import six
+import sys
+from threading import RLock
+
+from patroni.exceptions import WatchdogError
+
+__all__ = ['WatchdogError', 'Watchdog']
+
+logger = logging.getLogger(__name__)
+
+MODE_REQUIRED = 'required'    # Will not run if a watchdog is not available
+MODE_AUTOMATIC = 'automatic'  # Will use a watchdog if one is available
+MODE_OFF = 'off'              # Will not try to use a watchdog
+
+
+def parse_mode(mode):
+    if mode is False:
+        return MODE_OFF
+    mode = mode.lower()
+    if mode in ['require', 'required']:
+        return MODE_REQUIRED
+    elif mode in ['auto', 'automatic']:
+        return MODE_AUTOMATIC
+    else:
+        if mode not in ['off', 'disable', 'disabled']:
+            logger.warning("Watchdog mode {0} not recognized, disabling watchdog".format(mode))
+        return MODE_OFF
+
+
+def synchronized(func):
+    def wrapped(self, *args, **kwargs):
+        with self._lock:
+            return func(self, *args, **kwargs)
+    return wrapped
+
+
+class WatchdogConfig(object):
+    """Helper to contain a snapshot of configuration"""
+    def __init__(self, config):
+        self.mode = parse_mode(config['watchdog'].get('mode', 'automatic'))
+        self.ttl = config['ttl']
+        self.loop_wait = config['loop_wait']
+        self.safety_margin = config['watchdog'].get('safety_margin', 5)
+        self.driver = config['watchdog'].get('driver', 'default')
+        self.driver_config = dict((k, v) for k, v in config['watchdog'].items()
+                                  if k not in ['mode', 'safety_margin', 'driver'])
+
+    def __eq__(self, other):
+        return isinstance(other, WatchdogConfig) and \
+            all(getattr(self, attr) == getattr(other, attr) for attr in
+                ['mode', 'ttl', 'loop_wait', 'safety_margin', 'driver', 'driver_config'])
+
+    def __ne__(self, other):
+        return not self == other
+
+    def get_impl(self):
+        if self.driver == 'testing':
+            from patroni.watchdog.linux import TestingWatchdogDevice
+            return TestingWatchdogDevice.from_config(self.driver_config)
+        elif platform.system() == 'Linux' and self.driver == 'default':
+            from patroni.watchdog.linux import LinuxWatchdogDevice
+            return LinuxWatchdogDevice.from_config(self.driver_config)
+        else:
+            return NullWatchdog()
+
+    @property
+    def timeout(self):
+        if self.safety_margin == -1:
+            return int(self.ttl // 2)
+        else:
+            return self.ttl - self.safety_margin
+
+    @property
+    def timing_slack(self):
+        return self.timeout - self.loop_wait
+
+
+class Watchdog(object):
+    """Facade to dynamically manage watchdog implementations and handle config changes.
+
+    When activation fails underlying implementation will be switched to a Null implementation. To avoid log spam
+    activation will only be retried when watchdog configuration is changed."""
+    def __init__(self, config):
+        self.active_config = self.config = WatchdogConfig(config)
+        self._lock = RLock()
+        self.active = False
+
+        if self.config.mode == MODE_OFF:
+            self.impl = NullWatchdog()
+        else:
+            self.impl = self.config.get_impl()
+            if self.config.mode == MODE_REQUIRED and self.impl.is_null:
+                logger.error("Configuration requires a watchdog, but watchdog is not supported on this platform.")
+                sys.exit(1)
+
+    @synchronized
+    def reload_config(self, config):
+        self.config = WatchdogConfig(config)
+        # Turning a watchdog off can always be done immediately
+        if self.config.mode == MODE_OFF:
+            if self.active:
+                self._disable()
+            self.active_config = self.config
+            self.impl = NullWatchdog()
+        # If watchdog is not active we can apply config immediately to show any warnings early. Otherwise we need to
+        # delay until next time a keepalive is sent so timeout matches up with leader key update.
+        if not self.active:
+            if self.config.driver != self.active_config.driver or \
+               self.config.driver_config != self.active_config.driver_config:
+                self.impl = self.config.get_impl()
+            self.active_config = self.config
+
+    @synchronized
+    def activate(self):
+        """Activates the watchdog device with suitable timeouts. While watchdog is active keepalive needs
+        to be called every time loop_wait expires.
+
+        :returns False if a safe watchdog could not be configured, but is required.
+        """
+        self.active = True
+        return self._activate()
+
+    def _activate(self):
+        self.active_config = self.config
+
+        if self.config.timing_slack < 0:
+            logger.warning('Watchdog not supported because leader TTL {0} is less than 2x loop_wait {1}'
+                           .format(self.config.ttl, self.config.loop_wait))
+            self.impl = NullWatchdog()
+
+        try:
+            self.impl.open()
+        except WatchdogError as e:
+            logger.warning("Could not activate %s: %s", self.impl.describe(), e)
+            self.impl = NullWatchdog()
+
+        if self.impl.is_running and not self.impl.can_be_disabled:
+            logger.warning("Watchdog implementation can't be disabled."
+                           " Watchdog will trigger after Patroni loses leader key.")
+
+        actual_timeout = self._set_timeout()
+
+        if not self.impl.is_running or actual_timeout > self.config.timeout:
+            if self.config.mode == MODE_REQUIRED:
+                if self.impl.is_null:
+                    logger.error("Configuration requires watchdog, but watchdog could not be configured.")
+                else:
+                    logger.error("Configuration requires watchdog, but a safe watchdog timeout {0} could"
+                                 " not be configured. Watchdog timeout is {1}.".format(
+                                     self.config.timeout, actual_timeout))
+                return False
+            else:
+                if not self.impl.is_null:
+                    logger.warning("Watchdog timeout {0} seconds does not ensure safe termination within {1} seconds"
+                                   .format(actual_timeout, self.config.timeout))
+
+        if self.is_running:
+            logger.info("{0} activated with {1} second timeout, timing slack {2} seconds"
+                        .format(self.impl.describe(), actual_timeout, self.config.timing_slack))
+        else:
+            if self.config.mode == MODE_REQUIRED:
+                logger.error("Configuration requires watchdog, but watchdog could not be activated")
+                return False
+
+        return True
+
+    def _set_timeout(self):
+        if self.impl.has_set_timeout():
+            self.impl.set_timeout(self.config.timeout)
+
+        # Safety checks for watchdog implementations that don't support configurable timeouts
+        actual_timeout = self.impl.get_timeout()
+        if self.impl.is_running and actual_timeout < self.config.loop_wait:
+            logger.error('loop_wait of {0} seconds is too long for watchdog {1} second timeout'
+                         .format(self.config.loop_wait, actual_timeout))
+            if self.impl.can_be_disabled:
+                logger.info('Disabling watchdog due to unsafe timeout.')
+                self.impl.close()
+                self.impl = NullWatchdog()
+                return None
+        return actual_timeout
+
+    @synchronized
+    def disable(self):
+        self._disable()
+        self.active = False
+
+    def _disable(self):
+        try:
+            if self.impl.is_running and not self.impl.can_be_disabled:
+                # Give sysadmin some extra time to clean stuff up.
+                self.impl.keepalive()
+                logger.warning("Watchdog implementation can't be disabled. System will reboot after "
+                               "{0} seconds when watchdog times out.".format(self.impl.get_timeout()))
+            self.impl.close()
+        except WatchdogError as e:
+            logger.error("Error while disabling watchdog: %s", e)
+
+    @synchronized
+    def keepalive(self):
+        try:
+            self.impl.keepalive()
+            # In case there are any pending configuration changes apply them now.
+            if self.active and self.config != self.active_config:
+                if self.config.mode != MODE_OFF and self.active_config.mode == MODE_OFF:
+                    self.impl = self.config.get_impl()
+                    self._activate()
+                if self.config.driver != self.active_config.driver \
+                   or self.config.driver_config != self.active_config.driver_config:
+                    self._disable()
+                    self.impl = self.config.get_impl()
+                    self._activate()
+                if self.config.timeout != self.active_config.timeout:
+                    self.impl.set_timeout(self.config.timeout)
+        except WatchdogError as e:
+            logger.error("Error while sending keepalive: %s", e)
+
+    @property
+    @synchronized
+    def is_running(self):
+        return self.impl.is_running
+
+    @property
+    @synchronized
+    def is_healthy(self):
+        if self.config.mode != MODE_REQUIRED:
+            return True
+        return self.config.timing_slack >= 0 and self.impl.is_healthy
+
+
+@six.add_metaclass(abc.ABCMeta)
+class WatchdogBase(object):
+    """A watchdog object when opened requires periodic calls to keepalive.
+    When keepalive is not called within a timeout the system will be terminated."""
+    is_null = False
+
+    @property
+    def is_running(self):
+        """Returns True when watchdog is activated and capable of performing it's task."""
+        return False
+
+    @property
+    def is_healthy(self):
+        """Returns False when calling open() is known to fail."""
+        return False
+
+    @property
+    def can_be_disabled(self):
+        """Returns True when watchdog will be disabled by calling close(). Some watchdog devices
+        will keep running no matter what once activated. May raise WatchdogError if called without
+        calling open() first."""
+        return True
+
+    @abc.abstractmethod
+    def open(self):
+        """Open watchdog device.
+
+        When watchdog is opened keepalive must be called. Returns nothing on success
+        or raises WatchdogError if the device could not be opened."""
+
+    @abc.abstractmethod
+    def close(self):
+        """Gracefully close watchdog device."""
+
+    @abc.abstractmethod
+    def keepalive(self):
+        """Resets the watchdog timer.
+
+        Watchdog must be open when keepalive is called."""
+
+    @abc.abstractmethod
+    def get_timeout(self):
+        """Returns the current keepalive timeout in effect."""
+
+    @staticmethod
+    def has_set_timeout():
+        """Returns True if setting a timeout is supported."""
+        return False
+
+    def set_timeout(self, timeout):
+        """Set the watchdog timer timeout.
+
+        :param timeout: watchdog timeout in seconds"""
+        raise WatchdogError("Setting timeout is not supported on {0}".format(self.describe()))
+
+    def describe(self):
+        """Human readable name for this device"""
+        return self.__class__.__name__
+
+    @classmethod
+    def from_config(cls, config):
+        return cls()
+
+
+class NullWatchdog(WatchdogBase):
+    """Null implementation when watchdog is not supported."""
+    is_null = True
+
+    def open(self):
+        return
+
+    def close(self):
+        return
+
+    def keepalive(self):
+        return
+
+    def get_timeout(self):
+        # A big enough number to not matter
+        return 1000000000

patroni/watchdog/linux.py

@@ -0,0 +1,224 @@
+import collections
+import ctypes
+import fcntl
+import os
+import platform
+from patroni.watchdog.base import WatchdogBase, WatchdogError
+
+# Pythonification of linux/ioctl.h
+IOC_NONE = 0
+IOC_WRITE = 1
+IOC_READ = 2
+
+IOC_NRBITS = 8
+IOC_TYPEBITS = 8
+IOC_SIZEBITS = 14
+IOC_DIRBITS = 2
+
+# Non-generic platform special cases
+machine = platform.machine()
+if machine in ['mips', 'sparc', 'powerpc', 'ppc64']:
+    IOC_SIZEBITS = 13
+    IOC_DIRBITS = 3
+    IOC_NONE, IOC_WRITE, IOC_READ = 1, 2, 4
+elif machine == 'parisc':
+    IOC_WRITE, IOC_READ = 2, 1
+
+IOC_NRSHIFT = 0
+IOC_TYPESHIFT = IOC_NRSHIFT + IOC_NRBITS
+IOC_SIZESHIFT = IOC_TYPESHIFT + IOC_TYPEBITS
+IOC_DIRSHIFT = IOC_SIZESHIFT + IOC_SIZEBITS
+
+
+def IOW(type_, nr, size):
+    return IOC(IOC_WRITE, type_, nr, size)
+
+
+def IOR(type_, nr, size):
+    return IOC(IOC_READ, type_, nr, size)
+
+
+def IOWR(type_, nr, size):
+    return IOC(IOC_READ | IOC_WRITE, type_, nr, size)
+
+
+def IOC(dir_, type_, nr, size):
+    return (dir_ << IOC_DIRSHIFT) \
+        | (ord(type_) << IOC_TYPESHIFT) \
+        | (nr << IOC_NRSHIFT) \
+        | (size << IOC_SIZESHIFT)
+
+
+# Pythonification of linux/watchdog.h
+
+WATCHDOG_IOCTL_BASE = 'W'
+
+
+class watchdog_info(ctypes.Structure):
+    _fields_ = [
+        ('options', ctypes.c_uint32),           # Options the card/driver supports
+        ('firmware_version', ctypes.c_uint32),  # Firmware version of the card
+        ('identity', ctypes.c_uint8 * 32),      # Identity of the board
+    ]
+
+
+struct_watchdog_info_size = ctypes.sizeof(watchdog_info)
+int_size = ctypes.sizeof(ctypes.c_int)
+
+WDIOC_GETSUPPORT = IOR(WATCHDOG_IOCTL_BASE, 0, struct_watchdog_info_size)
+WDIOC_GETSTATUS = IOR(WATCHDOG_IOCTL_BASE, 1, int_size)
+WDIOC_GETBOOTSTATUS = IOR(WATCHDOG_IOCTL_BASE, 2, int_size)
+WDIOC_GETTEMP = IOR(WATCHDOG_IOCTL_BASE, 3, int_size)
+WDIOC_SETOPTIONS = IOR(WATCHDOG_IOCTL_BASE, 4, int_size)
+WDIOC_KEEPALIVE = IOR(WATCHDOG_IOCTL_BASE, 5, int_size)
+WDIOC_SETTIMEOUT = IOWR(WATCHDOG_IOCTL_BASE, 6, int_size)
+WDIOC_GETTIMEOUT = IOR(WATCHDOG_IOCTL_BASE, 7, int_size)
+WDIOC_SETPRETIMEOUT = IOWR(WATCHDOG_IOCTL_BASE, 8, int_size)
+WDIOC_GETPRETIMEOUT = IOR(WATCHDOG_IOCTL_BASE, 9, int_size)
+WDIOC_GETTIMELEFT = IOR(WATCHDOG_IOCTL_BASE, 10, int_size)
+
+
+WDIOF_UNKNOWN = -1  # Unknown flag error
+WDIOS_UNKNOWN = -1  # Unknown status error
+
+WDIOF = {
+    "OVERHEAT": 0x0001,       # Reset due to CPU overheat
+    "FANFAULT": 0x0002,       # Fan failed
+    "EXTERN1": 0x0004,        # External relay 1
+    "EXTERN2": 0x0008,        # External relay 2
+    "POWERUNDER": 0x0010,     # Power bad/power fault
+    "CARDRESET": 0x0020,      # Card previously reset the CPU
+    "POWEROVER": 0x0040,      # Power over voltage
+    "SETTIMEOUT": 0x0080,     # Set timeout (in seconds)
+    "MAGICCLOSE": 0x0100,     # Supports magic close char
+    "PRETIMEOUT": 0x0200,     # Pretimeout (in seconds), get/set
+    "ALARMONLY": 0x0400,      # Watchdog triggers a management or other external alarm not a reboot
+    "KEEPALIVEPING": 0x8000,  # Keep alive ping reply
+}
+
+WDIOS = {
+    "DISABLECARD": 0x0001,    # Turn off the watchdog timer
+    "ENABLECARD": 0x0002,     # Turn on the watchdog timer
+    "TEMPPANIC": 0x0004,      # Kernel panic on temperature trip
+}
+
+# Implementation
+
+
+class WatchdogInfo(collections.namedtuple('WatchdogInfo', 'options,version,identity')):
+    """Watchdog descriptor from the kernel"""
+    def __getattr__(self, name):
+        """Convenience has_XYZ attributes for checking WDIOF bits in options"""
+        if name.startswith('has_') and name[4:] in WDIOF:
+            return bool(self.options & WDIOF[name[4:]])
+
+        raise AttributeError("WatchdogInfo instance has no attribute '{0}'".format(name))
+
+
+class LinuxWatchdogDevice(WatchdogBase):
+    DEFAULT_DEVICE = '/dev/watchdog'
+
+    def __init__(self, device):
+        self.device = device
+        self._support_cache = None
+        self._fd = None
+
+    @classmethod
+    def from_config(cls, config):
+        device = config.get('device', cls.DEFAULT_DEVICE)
+        return cls(device)
+
+    @property
+    def is_running(self):
+        return self._fd is not None
+
+    @property
+    def is_healthy(self):
+        return os.path.exists(self.device) and os.access(self.device, os.W_OK)
+
+    def open(self):
+        try:
+            self._fd = os.open(self.device, os.O_WRONLY)
+        except OSError as e:
+            raise WatchdogError("Can't open watchdog device: {0}".format(e))
+
+    def close(self):
+        if self.is_running:
+            try:
+                os.write(self._fd, b'V')
+                os.close(self._fd)
+                self._fd = None
+            except OSError as e:
+                raise WatchdogError("Error while closing {0}: {1}".format(self.describe(), e))
+
+    @property
+    def can_be_disabled(self):
+        return self.get_support().has_MAGICCLOSE
+
+    def _ioctl(self, func, arg, mutate_arg=False):
+        if self._fd is None:
+            raise WatchdogError("Watchdog device is closed")
+
+        result = fcntl.ioctl(self._fd, func, arg, mutate_arg)
+        if result < 0:
+            raise IOError(result)
+
+    def get_support(self):
+        if self._support_cache is None:
+            info = watchdog_info()
+            self._ioctl(WDIOC_GETSUPPORT, info, True)
+            self._support_cache = WatchdogInfo(info.options,
+                                               info.firmware_version,
+                                               str(bytearray(info.identity)).rstrip('\x00'))
+        return self._support_cache
+
+    def describe(self):
+        dev_str = " at {0}".format(self.device) if self.device != self.DEFAULT_DEVICE else ""
+        ver_str = ""
+        identity = "Linux watchdog device"
+        if self._fd:
+            try:
+                _, version, identity = self.get_support()
+                ver_str = " (firmware {0})".format(version) if version else ""
+            except WatchdogError:  # XXX: Can it really be raise when self._fd is not None?
+                pass
+
+        return identity + ver_str + dev_str
+
+    def keepalive(self):
+        try:
+            os.write(self._fd, b'1')
+        except OSError as e:
+            raise WatchdogError("Could not send watchdog keepalive: {0}".format(e))
+
+    def has_set_timeout(self):
+        """Returns True if setting a timeout is supported."""
+        return self.get_support().has_SETTIMEOUT
+
+    def set_timeout(self, timeout):
+        timeout = int(timeout)
+        if not 0 < timeout < 0xFFFF:
+            raise WatchdogError("Invalid timeout {0}. Supported values are between 1 and 65535".format(timeout))
+        self._ioctl(WDIOC_SETTIMEOUT, ctypes.c_int(timeout))
+
+    def get_timeout(self):
+        timeout = ctypes.c_int()
+        self._ioctl(WDIOC_GETTIMEOUT, timeout, True)
+        return timeout.value
+
+
+class TestingWatchdogDevice(LinuxWatchdogDevice):
+    """Converts timeout ioctls to regular writes that can be intercepted from a named pipe."""
+    timeout = 60
+
+    def get_support(self):
+        return WatchdogInfo(WDIOF['MAGICCLOSE'] | WDIOF['SETTIMEOUT'], 0, "Watchdog test harness")
+
+    def set_timeout(self, timeout):
+        buf = "Ctimeout={0}\n".format(timeout).encode('utf8')
+        while len(buf):
+            buf = buf[os.write(self._fd, buf):]
+        self.timeout = timeout
+
+    def get_timeout(self):
+        return self.timeout

postgres0.yml

@@ -12,7 +12,7 @@ restapi:
 #    password: password
 
 etcd:
-  host: 127.0.0.1:4001
+  host: 127.0.0.1:2379
 
 bootstrap:
   # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
@@ -22,6 +22,8 @@ bootstrap:
     loop_wait: 10
     retry_timeout: 10
     maximum_lag_on_failover: 1048576
+#    master_start_timeout: 300
+#    synchronous_mode: false
     postgresql:
       use_pg_rewind: true
 #      use_slots: true
@@ -48,6 +50,9 @@ bootstrap:
   - host all all 0.0.0.0/0 md5
 #  - hostssl all all 0.0.0.0/0 md5
 
+  # Additional script to be launched after initial cluster creation (will be passed the connection URL as parameter)
+# post_init: /usr/local/bin/setup_cluster.sh
+
   # Some additional users users which needs to be created after initializing new cluster
   users:
     admin:
@@ -61,6 +66,7 @@ postgresql:
   connect_address: 127.0.0.1:5432
   data_dir: data/postgresql0
 #  bin_dir:
+#  config_dir:
   pgpass: /tmp/pgpass0
   authentication:
     replication:
@@ -71,7 +77,14 @@ postgresql:
       password: zalando
   parameters:
     unix_socket_directories: '.'
+
+#watchdog:
+#  mode: automatic # Allowed values: off, automatic, required
+#  device: /dev/watchdog
+#  safety_margin: 5
+
 tags:
     nofailover: false
     noloadbalance: false
     clonefrom: false
+    nosync: false

postgres1.yml

@@ -12,7 +12,7 @@ restapi:
 #    password: password
 
 etcd:
-  host: 127.0.0.1:4001
+  host: 127.0.0.1:2379
 
 bootstrap:
   # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
@@ -48,6 +48,9 @@ bootstrap:
   - host all all 0.0.0.0/0 md5
 #  - hostssl all all 0.0.0.0/0 md5
 
+  # Additional script to be launched after initial cluster creation (will be passed the connection URL as parameter)
+# post_init: /usr/local/bin/setup_cluster.sh
+
   # Some additional users users which needs to be created after initializing new cluster
   users:
     admin:
@@ -61,6 +64,7 @@ postgresql:
   connect_address: 127.0.0.1:5433
   data_dir: data/postgresql1
 #  bin_dir:
+#  config_dir:
   pgpass: /tmp/pgpass1
   authentication:
     replication:

postgres2.yml

@@ -12,7 +12,7 @@ restapi:
     password: password
 
 etcd:
-  host: 127.0.0.1:4001
+  host: 127.0.0.1:2379
 
 bootstrap:
   # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
@@ -61,6 +61,7 @@ postgresql:
   connect_address: 127.0.0.1:5434
   data_dir: data/postgresql2
 #  bin_dir:
+#  config_dir:
   pgpass: /tmp/pgpass2
   authentication:
     replication:

requirements.txt

@@ -1,12 +1,15 @@
+urllib3>=1.9
 boto
 psycopg2>=2.6.1
 PyYAML
 requests
 six >= 1.7
 kazoo==2.2.1
-python-etcd==0.4.3
-python-consul==0.6.0
+python-etcd>=0.4.3,<0.5
+python-consul==0.7.0
 click>=4.1
 prettytable>=0.7
 tzlocal
-python-dateutil
+python-dateutil
+psutil
+cdiff

setup.py

@@ -24,6 +24,7 @@ def read_version(package):
         exec(fd.read(), data)
     return data['__version__']
 
+
 NAME = 'patroni'
 MAIN_PACKAGE = NAME
 SCRIPTS = 'scripts'

tests/test_api.py

@@ -1,19 +1,19 @@
 import datetime
 import json
 import psycopg2
-import pytz
 import unittest
 
 from mock import Mock, patch
 from patroni.api import RestApiHandler, RestApiServer
 from patroni.dcs import ClusterConfig, Member
+from patroni.utils import tzutc
 from six import BytesIO as IO
 from six.moves import BaseHTTPServer
 from test_postgresql import psycopg2_connect, MockCursor
 
 
-future_restart_time = datetime.datetime.now(pytz.utc) + datetime.timedelta(days=5)
-postmaster_start_time = datetime.datetime.now(pytz.utc)
+future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5)
+postmaster_start_time = datetime.datetime.now(tzutc)
 
 
 class MockPostgresql(object):
@@ -25,6 +25,8 @@ class MockPostgresql(object):
     sysid = 'dummysysid'
     scope = 'dummy'
     pending_restart = True
+    wal_name = 'wal'
+    lsn_name = 'lsn'
 
     @staticmethod
     def connection():
@@ -35,9 +37,14 @@ class MockPostgresql(object):
         return str(postmaster_start_time)
 
 
+class MockWatchdog(object):
+    is_healthy = True
+
+
 class MockHa(object):
 
     state_handler = MockPostgresql()
+    watchdog = MockWatchdog()
 
     @staticmethod
     def reinitialize():
@@ -63,6 +70,18 @@ class MockHa(object):
     def schedule_future_restart(data):
         return True
 
+    @staticmethod
+    def is_lagging(wal):
+        return False
+
+    @staticmethod
+    def get_effective_tags():
+        return {'nosync': True}
+
+    @staticmethod
+    def wakeup():
+        pass
+
 
 class MockPatroni(object):
 
@@ -89,6 +108,9 @@ class MockRequest(object):
     def makefile(self, *args, **kwargs):
         return IO(self.request)
 
+    def sendall(self, *args, **kwargs):
+        pass
+
 
 class MockRestApiServer(RestApiServer):
 
@@ -225,6 +247,10 @@ class TestRestApiHandler(unittest.TestCase):
 
         mock_dcs.get_cluster.return_value.is_paused.return_value = True
         MockRestApiServer(RestApiHandler, make_request(schedule='2016-08-42 12:45TZ+1', role='master'))
+        # Valid timeout
+        MockRestApiServer(RestApiHandler, make_request(timeout='60s'))
+        # Invalid timeout
+        MockRestApiServer(RestApiHandler, make_request(timeout='42towels'))
 
     def test_do_DELETE_restart(self):
         for retval in (True, False):

tests/test_async_executor.py

@@ -1,14 +1,14 @@
 import unittest
 
 from mock import Mock, patch
-from patroni.async_executor import AsyncExecutor
+from patroni.async_executor import AsyncExecutor, CriticalTask
 from threading import Thread
 
 
 class TestAsyncExecutor(unittest.TestCase):
 
     def setUp(self):
-        self.a = AsyncExecutor()
+        self.a = AsyncExecutor(Mock())
 
     @patch.object(Thread, 'start', Mock())
     def test_run_async(self):
@@ -16,3 +16,11 @@ class TestAsyncExecutor(unittest.TestCase):
 
     def test_run(self):
         self.a.run(Mock(side_effect=Exception()))
+
+
+class TestCriticalTask(unittest.TestCase):
+
+    def test_completed_task(self):
+        ct = CriticalTask()
+        ct.complete(1)
+        self.assertFalse(ct.cancel())

tests/test_aws.py

@@ -1,5 +1,4 @@
 import boto.ec2
-import requests
 import sys
 import unittest
 
@@ -11,82 +10,60 @@ from requests.exceptions import RequestException
 
 class MockEc2Connection(object):
 
-    def __init__(self, error=False):
-        self.error = error
-
-    def get_all_volumes(self, filters):
-        if self.error:
-            raise Exception("get_all_volumes")
+    @staticmethod
+    def get_all_volumes(*args, **kwargs):
         oid = namedtuple('Volume', 'id')
         return [oid(id='a'), oid(id='b')]
 
-    def create_tags(self, objects, tags):
-        if self.error or len(objects) == 0:
-            raise Exception("create_tags")
+    @staticmethod
+    def create_tags(objects, *args, **kwargs):
+        if len(objects) == 0:
+            raise boto.exception.BotoServerError(503, 'Service Unavailable', 'Request limit exceeded')
         return True
 
 
 class MockResponse(object):
+    ok = True
 
     def __init__(self, content):
         self.content = content
-        self.ok = True
 
     def json(self):
         return self.content
 
 
+def requests_get(url, **kwargs):
+    if url.split('/')[-1] == 'document':
+        result = {"instanceId": "012345", "region": "eu-west-1"}
+    else:
+        result = 'foo'
+    return MockResponse(result)
+
+
+@patch('boto.ec2.connect_to_region', Mock(return_value=MockEc2Connection()))
 class TestAWSConnection(unittest.TestCase):
 
-    def boto_ec2_connect_to_region(self, region):
-        return MockEc2Connection(self.error)
-
-    def requests_get(self, url, **kwargs):
-        if self.error:
-            raise RequestException("foo")
-        result = namedtuple('Request', 'ok content')
-        result.ok = True
-        if url.split('/')[-1] == 'document' and not self.json_error:
-            result = {"instanceId": "012345", "region": "eu-west-1"}
-        else:
-            result = 'foo'
-        return MockResponse(result)
-
+    @patch('requests.get', requests_get)
     def setUp(self):
-        self.error = False
-        self.json_error = False
-        requests.get = self.requests_get
-        boto.ec2.connect_to_region = self.boto_ec2_connect_to_region
         self.conn = AWSConnection('test')
 
-    def test_aws_available(self):
-        self.assertTrue(self.conn.aws_available())
-
     def test_on_role_change(self):
-        self.assertTrue(self.conn._tag_ebs('master'))
-        self.assertTrue(self.conn._tag_ec2('master'))
         self.assertTrue(self.conn.on_role_change('master'))
+        with patch.object(MockEc2Connection, 'get_all_volumes', Mock(return_value=[])):
+            self.conn._retry.max_tries = 1
+            self.assertFalse(self.conn.on_role_change('master'))
 
+    @patch('requests.get', Mock(side_effect=RequestException('foo')))
     def test_non_aws(self):
-        self.error = True
         conn = AWSConnection('test')
-        self.assertFalse(conn.aws_available())
-        self.assertFalse(conn._tag_ebs('master'))
-        self.assertFalse(conn._tag_ec2('master'))
+        self.assertFalse(conn.on_role_change("master"))
 
+    @patch('requests.get', Mock(return_value=MockResponse('foo')))
     def test_aws_bizare_response(self):
-        self.json_error = True
         conn = AWSConnection('test')
         self.assertFalse(conn.aws_available())
 
-    def test_aws_tag_ebs_error(self):
-        self.error = True
-        self.assertFalse(self.conn._tag_ebs("master"))
-
-    def test_aws_tag_ec2_error(self):
-        self.error = True
-        self.assertFalse(self.conn._tag_ec2("master"))
-
+    @patch('requests.get', requests_get)
     @patch('sys.exit', Mock())
     def test_main(self):
         self.assertIsNone(_main())

tests/test_callback_executor.py

@@ -0,0 +1,24 @@
+import unittest
+
+from mock import Mock, patch
+from patroni.callback_executor import CallbackExecutor
+
+
+class TestCallbackExecutor(unittest.TestCase):
+
+    @patch('subprocess.Popen')
+    def test_callback_executor(self, mock_popen):
+        mock_popen.return_value.wait.side_effect = Exception
+        mock_popen.return_value.poll.return_value = None
+
+        ce = CallbackExecutor()
+        self.assertIsNone(ce.call([]))
+        ce.join()
+
+        self.assertIsNone(ce.call([]))
+
+        mock_popen.side_effect = Exception
+        ce = CallbackExecutor()
+        ce._callback_event.wait = Mock(side_effect=[None, Exception])
+        self.assertIsNone(ce.call([]))
+        ce.join()

tests/test_config.py

@@ -20,9 +20,10 @@ class TestConfig(unittest.TestCase):
     def test_no_config(self):
         self.assertRaises(SystemExit, Config)
 
-    @patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception))
     def test_set_dynamic_configuration(self):
-        self.assertIsNone(self.config.set_dynamic_configuration({'foo': 'bar'}))
+        with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
+            self.assertIsNone(self.config.set_dynamic_configuration({'foo': 'bar'}))
+        self.assertTrue(self.config.set_dynamic_configuration({'synchronous_mode': True}))
 
     def test_reload_local_configuration(self):
         os.environ.update({
@@ -38,8 +39,15 @@ class TestConfig(unittest.TestCase):
             'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432',
             'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432',
             'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0',
+            'PATRONI_POSTGRESQL_CONFIG_DIR': 'data/postgres0',
             'PATRONI_POSTGRESQL_PGPASS': '/tmp/pgpass0',
             'PATRONI_ETCD_HOST': '127.0.0.1:2379',
+            'PATRONI_ETCD_URL': 'https://127.0.0.1:2379',
+            'PATRONI_ETCD_PROXY': 'http://127.0.0.1:2379',
+            'PATRONI_ETCD_SRV': 'test',
+            'PATRONI_ETCD_CACERT': '/cacert',
+            'PATRONI_ETCD_CERT': '/cert',
+            'PATRONI_ETCD_KEY': '/key',
             'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
             'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
             'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',

tests/test_consul.py

@@ -1,8 +1,9 @@
 import consul
 import unittest
 
+from consul import ConsulException, NotFound
 from mock import Mock, patch
-from patroni.dcs.consul import AbstractDCS, Cluster, Consul, ConsulError, ConsulException, HTTPClient, NotFound
+from patroni.dcs.consul import AbstractDCS, Cluster, Consul, ConsulInternalError, ConsulError, HTTPClient
 from test_etcd import SleepException
 
 
@@ -30,17 +31,34 @@ def kv_get(self, key, **kwargs):
                   'Value': ('postgres://replicator:[email protected]:5433/postgres' +
                             '?application_name=http://127.0.0.1:8009/patroni').encode('utf-8')},
                  {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'optime/leader', 'LockIndex': 0,
-                  'ModifyIndex': 6429, 'Value': b'4496294792'}])
+                  'ModifyIndex': 6429, 'Value': b'4496294792'},
+                 {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'sync', 'LockIndex': 0,
+                  'ModifyIndex': 6429, 'Value': b'{"leader": "leader", "sync_standby": null}'}])
     raise ConsulException
 
 
 class TestHTTPClient(unittest.TestCase):
 
-    def test_get(self):
+    def setUp(self):
         self.client = HTTPClient('127.0.0.1', '8500', 'http', False)
-        self.client.session.get = Mock()
+        self.client.http.request = Mock()
+
+    def test_get(self):
         self.client.get(Mock(), '')
         self.client.get(Mock(), '', {'wait': '1s', 'index': 1})
+        self.client.http.request.return_value.status = 500
+        self.assertRaises(ConsulInternalError, self.client.get, Mock(), '')
+
+    def test_unknown_method(self):
+        try:
+            self.client.bla(Mock(), '')
+            self.assertFail()
+        except Exception as e:
+            self.assertTrue(isinstance(e, AttributeError))
+
+    def test_put(self):
+        self.client.put(Mock(), '/v1/session/create')
+        self.client.put(Mock(), '/v1/session/create', data='{"foo": "bar"}')
 
 
 @patch.object(consul.Consul.KV, 'get', kv_get)
@@ -65,7 +83,8 @@ class TestConsul(unittest.TestCase):
     @patch.object(consul.Consul.Session, 'create', Mock(side_effect=ConsulException))
     def test_referesh_session(self):
         self.c._session = '1'
-        self.c._name = ''
+        self.assertFalse(self.c.refresh_session())
+        self.c._last_session_refresh = 0
         self.assertRaises(ConsulError, self.c.refresh_session)
 
     @patch.object(consul.Consul.KV, 'delete', Mock())
@@ -91,6 +110,8 @@ class TestConsul(unittest.TestCase):
 
     @patch.object(consul.Consul.KV, 'put', Mock(return_value=False))
     def test_take_leader(self):
+        self.c.set_ttl(20)
+        self.c.refresh_session = Mock()
         self.c.take_leader()
 
     @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
@@ -103,8 +124,9 @@ class TestConsul(unittest.TestCase):
 
     @patch.object(consul.Consul.KV, 'put', Mock(side_effect=ConsulException))
     def test_write_leader_optime(self):
-        self.c.write_leader_optime('')
+        self.c.write_leader_optime('1')
 
+    @patch.object(consul.Consul.Session, 'renew', Mock())
     def test_update_leader(self):
         self.c.update_leader()
 
@@ -126,15 +148,17 @@ class TestConsul(unittest.TestCase):
 
     @patch.object(AbstractDCS, 'watch', Mock())
     def test_watch(self):
+        self.c.watch(None, 1)
         self.c._name = ''
-        self.c.watch(1)
+        self.c.watch(6429, 1)
         with patch.object(consul.Consul.KV, 'get', Mock(side_effect=ConsulException)):
-            self.c.watch(1)
-
-    @patch.object(consul.Consul.Session, 'destroy', Mock(side_effect=ConsulException))
-    def test_set_ttl(self):
-        self.c.set_ttl(20)
-        self.assertTrue(self.c.watch(1))
+            self.c.watch(6429, 1)
 
     def test_set_retry_timeout(self):
         self.c.set_retry_timeout(10)
+
+    @patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
+    @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
+    def test_sync_state(self):
+        self.assertTrue(self.c.set_sync_state_value('{}'))
+        self.assertTrue(self.c.delete_sync_state())

tests/test_ctl.py

@@ -7,7 +7,8 @@ import unittest
 from click.testing import CliRunner
 from mock import patch, Mock
 from patroni.ctl import ctl, members, store_config, load_config, output_members, request_patroni, get_dcs, parse_dcs, \
-    wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException
+    get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException, apply_config_changes, \
+    format_config_for_editing, show_diff, invoke_editor
 from patroni.dcs.etcd import Client
 from psycopg2 import OperationalError
 from test_etcd import etcd_read, requests_get, socket_getaddrinfo, MockResponse
@@ -31,7 +32,7 @@ def test_rw_config():
 
 @patch('patroni.ctl.load_config',
        Mock(return_value={'postgresql': {'data_dir': '.', 'parameters': {}, 'retry_timeout': 5},
-                          'restapi': {'auth': 'u:p', 'listen': ''}, 'etcd': {'host': 'localhost:4001'}}))
+                          'restapi': {'auth': 'u:p', 'listen': ''}, 'etcd': {'host': 'localhost:2379'}}))
 class TestCtl(unittest.TestCase):
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
@@ -54,8 +55,8 @@ class TestCtl(unittest.TestCase):
 
     def test_parse_dcs(self):
         assert parse_dcs(None) is None
-        assert parse_dcs('localhost') == {'etcd': {'host': 'localhost:4001'}}
-        assert parse_dcs('') == {'etcd': {'host': 'localhost:4001'}}
+        assert parse_dcs('localhost') == {'etcd': {'host': 'localhost:2379'}}
+        assert parse_dcs('') == {'etcd': {'host': 'localhost:2379'}}
         assert parse_dcs('localhost:8500') == {'consul': {'host': 'localhost:8500'}}
         assert parse_dcs('zookeeper://localhost') == {'zookeeper': {'hosts': ['localhost:2181']}}
         assert parse_dcs('exhibitor://dummy') == {'exhibitor': {'hosts': ['dummy'], 'port': 8181}}
@@ -73,18 +74,15 @@ class TestCtl(unittest.TestCase):
     def test_failover(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        mock_get_dcs.return_value.set_failover_value = Mock()
         result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
         assert 'leader' in result.output
 
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2100-01-01T12:23:00\ny')
-        assert result.exit_code == 0
-
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2030-01-01T12:23:00\ny')
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
         assert result.exit_code == 0
 
         with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
-            result = self.runner.invoke(ctl,
-                                        ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
+            result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00'])
             assert result.exit_code == 1
 
         # Aborting failover,as we anser NO to the confirmation
@@ -119,7 +117,7 @@ class TestCtl(unittest.TestCase):
 
         with patch('patroni.ctl.request_patroni', Mock(side_effect=Exception)):
             # Non-responding patroni
-            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
+            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
             assert 'falling back to DCS' in result.output
 
         with patch('patroni.ctl.request_patroni') as mocked:
@@ -240,6 +238,9 @@ class TestCtl(unittest.TestCase):
         assert 'Error: PostgreSQL version' in result.output
         assert result.exit_code == 1
 
+        result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force', '--timeout', '10min'])
+        assert result.exit_code == 0
+
         with patch('requests.delete', Mock(return_value=MockResponse(500))):
             # normal restart, the schedule is actually parsed, but not validated in patronictl
             result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force',
@@ -308,14 +309,6 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['remove', 'alpha'], input='alpha\nYes I am aware\nleader')
         assert result.exit_code == 0
 
-    @patch('patroni.dcs.AbstractDCS.watch', Mock(return_value=None))
-    @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
-    def test_wait_for_leader(self):
-        self.assertRaises(PatroniCtlException, wait_for_leader, self.e, 0)
-
-        cluster = wait_for_leader(self.e, timeout=2)
-        assert cluster.leader.member.name == 'leader'
-
     @patch('requests.post', Mock(side_effect=requests.exceptions.ConnectionError('foo')))
     def test_request_patroni(self):
         member = get_cluster_initialized_with_leader().leader.member
@@ -364,6 +357,7 @@ class TestCtl(unittest.TestCase):
         mock_get_dcs.return_value.initialize = Mock(return_value=True)
         mock_get_dcs.return_value.touch_member = Mock(return_value=True)
         mock_get_dcs.return_value.attempt_to_acquire_leader = Mock(return_value=True)
+        mock_get_dcs.return_value.delete_cluster = Mock()
 
         with patch.object(self.e, 'initialize', return_value=False):
             result = self.runner.invoke(ctl, ['scaffold', 'alpha'])
@@ -383,7 +377,8 @@ class TestCtl(unittest.TestCase):
     @patch('patroni.ctl.get_dcs')
     def test_list_extended(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
+        mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster)
 
         result = self.runner.invoke(ctl, ['list', 'dummy', '--extended'])
         assert '2100' in result.output
@@ -440,3 +435,74 @@ class TestCtl(unittest.TestCase):
                     patch('patroni.dcs.Cluster.is_paused', Mock(return_value=False)):
                 result = self.runner.invoke(ctl, ['resume', 'dummy'])
                 assert 'Cluster is not paused' in result.output
+
+            with patch('requests.patch', Mock(side_effect=Exception)):
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Can not find accessible cluster member' in result.output
+
+    def test_apply_config_changes(self):
+        config = {"postgresql": {"parameters": {"work_mem": "4MB"}, "use_pg_rewind": True}, "ttl": 30}
+
+        before_editing = format_config_for_editing(config)
+
+        # Spaces are allowed and stripped, numbers and booleans are interpreted
+        after_editing, changed_config = apply_config_changes(before_editing, config,
+                                                             ["postgresql.parameters.work_mem = 5MB",
+                                                              "ttl=15", "postgresql.use_pg_rewind=off", 'a.b=c'])
+        self.assertEquals(changed_config, {"a": {"b": "c"}, "postgresql": {"parameters": {"work_mem": "5MB"},
+                                                                           "use_pg_rewind": False}, "ttl": 15})
+
+        # postgresql.parameters namespace is flattened
+        after_editing, changed_config = apply_config_changes(before_editing, config,
+                                                             ["postgresql.parameters.work_mem.sub = x"])
+        self.assertEquals(changed_config, {"postgresql": {"parameters": {"work_mem": "4MB", "work_mem.sub": "x"},
+                                                          "use_pg_rewind": True}, "ttl": 30})
+
+        # Setting to null deletes
+        after_editing, changed_config = apply_config_changes(before_editing, config,
+                                                             ["postgresql.parameters.work_mem=null"])
+        self.assertEquals(changed_config, {"postgresql": {"use_pg_rewind": True}, "ttl": 30})
+        after_editing, changed_config = apply_config_changes(before_editing, config,
+                                                             ["postgresql.use_pg_rewind=null",
+                                                              "postgresql.parameters.work_mem=null"])
+        self.assertEquals(changed_config, {"ttl": 30})
+
+        self.assertRaises(PatroniCtlException, apply_config_changes, before_editing, config, ['a'])
+
+    @patch('sys.stdout.isatty', return_value=False)
+    @patch('cdiff.markup_to_pager')
+    def test_show_diff(self, mock_markup_to_pager, mock_isatty):
+        show_diff("foo:\n  bar: 1\n", "foo:\n  bar: 2\n")
+        mock_markup_to_pager.assert_not_called()
+
+        mock_isatty.return_value = True
+        show_diff("foo:\n  bar: 1\n", "foo:\n  bar: 2\n")
+        mock_markup_to_pager.assert_called_once()
+
+        # Test that unicode handling doesn't fail with an exception
+        show_diff(b"foo:\n  bar: \xc3\xb6\xc3\xb6\n".decode('utf-8'),
+                  b"foo:\n  bar: \xc3\xbc\xc3\xbc\n".decode('utf-8'))
+
+    def test_invoke_editor(self):
+        for e in ('', 'false'):
+            os.environ['EDITOR'] = e
+            self.assertRaises(PatroniCtlException, invoke_editor, 'foo: bar\n', 'test')
+
+    @patch('patroni.ctl.get_dcs')
+    def test_show_config(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        self.runner.invoke(ctl, ['show-config', 'dummy'])
+
+    @patch('patroni.ctl.get_dcs')
+    def test_edit_config(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        os.environ['EDITOR'] = 'true'
+        self.runner.invoke(ctl, ['edit-config', 'dummy'])
+        self.runner.invoke(ctl, ['edit-config', 'dummy', '-s', 'foo=bar'])
+        self.runner.invoke(ctl, ['edit-config', 'dummy', '--replace', 'postgres0.yml'])
+        self.runner.invoke(ctl, ['edit-config', 'dummy', '--apply', '-'], input='foo: bar')
+        self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar')
+        mock_get_dcs.return_value.set_config_value = Mock(return_value=True)
+        self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar')

tests/test_etcd.py

@@ -1,12 +1,13 @@
 import etcd
 import json
+import urllib3.util.connection
 import requests
 import socket
 import unittest
 
 from dns.exception import DNSException
 from mock import Mock, patch
-from patroni.dcs.etcd import AbstractDCS, Client, Cluster, Etcd, EtcdError
+from patroni.dcs.etcd import AbstractDCS, Client, Cluster, Etcd, EtcdError, DnsCachingResolver
 from patroni.exceptions import DCSError
 from urllib3.exceptions import ReadTimeoutError
 
@@ -42,7 +43,7 @@ def requests_get(url, **kwargs):
     if url.startswith('http://local'):
         raise requests.exceptions.RequestException()
     elif ':8011/patroni' in url:
-        response.content = '{"role": "replica", "xlog": {"replayed_location": 0}, "tags": {}}'
+        response.content = '{"role": "replica", "xlog": {"received_location": 0}, "tags": {}}'
     elif url.endswith('/members'):
         response.content = '[{}]' if url.startswith('http://error') else members
     elif url.startswith('http://exhibitor'):
@@ -58,8 +59,10 @@ def etcd_watch(self, key, index=None, timeout=None, recursive=None):
         raise etcd.EtcdWatchTimedOut
     elif timeout == 5.0:
         return etcd.EtcdResult('delete', {})
-    elif timeout == 10.0:
+    elif 5 < timeout <= 10.0:
         raise etcd.EtcdException
+    elif timeout == 20.0:
+        raise etcd.EtcdEventIndexCleared
 
 
 def etcd_write(self, key, value, **kwargs):
@@ -91,6 +94,8 @@ def etcd_read(self, key, **kwargs):
                     {"key": "/service/batman5/optime/leader", "value": "2164261704",
                      "modifiedIndex": 20729, "createdIndex": 20729}],
                  "modifiedIndex": 20437, "createdIndex": 20437},
+                {"key": "/service/batman5/sync", "value": '{"leader": "leader"}',
+                 "modifiedIndex": 1582, "createdIndex": 1582},
                 {"key": "/service/batman5/members", "dir": True, "nodes": [
                     {"key": "/service/batman5/members/postgresql1",
                      "value": "postgres://replicator:[email protected]:5434/postgres" +
@@ -112,23 +117,23 @@ class SleepException(Exception):
     pass
 
 
-class MockSRV(object):
-    port = 2380
-    target = '127.0.0.1'
-
-
 def dns_query(name, _):
+    if '-server' not in name or '-ssl' in name:
+        return []
     if name == '_etcd-server._tcp.blabla':
         return []
     elif name == '_etcd-server._tcp.exception':
         raise DNSException()
-    return [MockSRV()]
+    srv = Mock()
+    srv.port = 2380
+    srv.target.to_text.return_value = 'localhost' if name == '_etcd-server._tcp.foobar' else '127.0.0.1'
+    return [srv]
 
 
 def socket_getaddrinfo(*args):
-    if args[0] == 'ok':
-        return [(2, 1, 6, '', ('127.0.0.1', 2379)), (2, 1, 6, '', ('127.0.0.1', 2379))]
-    raise socket.error
+    if args[0] in ('ok', 'localhost', '127.0.0.1'):
+        return [(socket.AF_INET, 1, 6, '', ('127.0.0.1', 0)), (socket.AF_INET6, 1, 6, '', ('::1', 0))]
+    raise socket.gaierror
 
 
 def http_request(method, url, **kwargs):
@@ -143,17 +148,28 @@ def http_request(method, url, **kwargs):
     raise socket.error
 
 
+class TestDnsCachingResolver(unittest.TestCase):
+
+    @patch('time.sleep', Mock(side_effect=SleepException))
+    @patch('socket.getaddrinfo', Mock(side_effect=socket.gaierror))
+    def test_run(self):
+        r = DnsCachingResolver()
+        self.assertIsNone(r.resolve_async('', 0))
+        r.join()
+
+
 @patch('dns.resolver.query', dns_query)
 @patch('socket.getaddrinfo', socket_getaddrinfo)
 @patch('requests.get', requests_get)
 class TestClient(unittest.TestCase):
 
     @patch('dns.resolver.query', dns_query)
+    @patch('socket.getaddrinfo', socket_getaddrinfo)
     @patch('requests.get', requests_get)
     def setUp(self):
         with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
-            self.client = Client({'discovery_srv': 'test', 'retry_timeout': 3})
+            self.client = Client({'srv': 'test', 'retry_timeout': 3}, DnsCachingResolver())
             self.client.http.request = http_request
             self.client.http.request_encode_body = http_request
 
@@ -179,6 +195,9 @@ class TestClient(unittest.TestCase):
         self.client._base_uri = 'http://localhost:4001'
         self.client._machines_cache = ['http://localhost:2379']
         self.client.api_execute('/', 'POST', timeout=0)
+        mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
+        self.client._machines_cache_updated = 0
+        self.client.api_execute('/', 'POST', timeout=0)
         self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
         self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '')
         self.client._update_machines_cache = True
@@ -186,31 +205,43 @@ class TestClient(unittest.TestCase):
             self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'GET')
 
     def test_get_srv_record(self):
-        self.assertEquals(self.client.get_srv_record('blabla'), [])
-        self.assertEquals(self.client.get_srv_record('exception'), [])
+        self.assertEquals(self.client.get_srv_record('_etcd-server._tcp.blabla'), [])
+        self.assertEquals(self.client.get_srv_record('_etcd-server._tcp.exception'), [])
 
     def test__get_machines_cache_from_srv(self):
+        self.client._get_machines_cache_from_srv('foobar')
         self.client.get_srv_record = Mock(return_value=[('localhost', 2380)])
         self.client._get_machines_cache_from_srv('blabla')
 
     def test__get_machines_cache_from_dns(self):
-        self.client._get_machines_cache_from_dns('error:2379')
+        self.client._get_machines_cache_from_dns('error', 2379)
 
     @patch.object(Client, 'machines')
     def test__load_machines_cache(self, mock_machines):
         mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
         self.client._config = {}
         self.assertRaises(Exception, self.client._load_machines_cache)
-        self.client._config = {'discovery_srv': 'blabla'}
+        self.client._config = {'srv': 'blabla'}
         self.assertRaises(etcd.EtcdException, self.client._load_machines_cache)
 
+    @patch.object(socket.socket, 'connect')
+    def test_create_connection_patched(self, mock_connect):
+        self.assertRaises(socket.error, urllib3.util.connection.create_connection, ('fail', 2379))
+        urllib3.util.connection.create_connection(('[localhost]', 2379))
+        mock_connect.side_effect = socket.error
+        self.assertRaises(socket.error, urllib3.util.connection.create_connection, ('[localhost]', 2379),
+                          timeout=1, source_address=('localhost', 53333),
+                          socket_options=[(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)])
+
 
 @patch('requests.get', requests_get)
+@patch('socket.getaddrinfo', socket_getaddrinfo)
 @patch.object(etcd.Client, 'write', etcd_write)
 @patch.object(etcd.Client, 'read', etcd_read)
 @patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
 class TestEtcd(unittest.TestCase):
 
+    @patch('socket.getaddrinfo', socket_getaddrinfo)
     def setUp(self):
         with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
@@ -226,7 +257,11 @@ class TestEtcd(unittest.TestCase):
             mock_machines.__get__ = Mock(side_effect=etcd.EtcdException)
             with patch('time.sleep', Mock(side_effect=SleepException)):
                 self.assertRaises(SleepException, self.etcd.get_etcd_client,
-                                  {'discovery_srv': 'test', 'retry_timeout': 10})
+                                  {'discovery_srv': 'test', 'retry_timeout': 10, 'cacert': '1', 'key': '1', 'cert': 1})
+                self.assertRaises(SleepException, self.etcd.get_etcd_client,
+                                  {'url': 'https://test:2379', 'retry_timeout': 10})
+                self.assertRaises(SleepException, self.etcd.get_etcd_client,
+                                  {'proxy': 'https://user:password@test:2379', 'retry_timeout': 10})
 
     def test_get_cluster(self):
         self.assertIsInstance(self.etcd.get_cluster(), Cluster)
@@ -267,14 +302,16 @@ class TestEtcd(unittest.TestCase):
     def test_delete_cluster(self):
         self.assertFalse(self.etcd.delete_cluster())
 
+    @patch('time.sleep', Mock(side_effect=SleepException))
     @patch.object(etcd.Client, 'watch', etcd_watch)
     def test_watch(self):
-        self.etcd.watch(0)
+        self.etcd.watch(None, 0)
         self.etcd.get_cluster()
-        self.etcd.watch(1.5)
-        self.etcd.watch(4.5)
+        self.etcd.watch(20729, 1.5)
+        self.etcd.watch(20729, 4.5)
         with patch.object(AbstractDCS, 'watch', Mock()):
-            self.etcd.watch(9.5)
+            self.assertTrue(self.etcd.watch(20729, 19.5))
+            self.assertRaises(SleepException, self.etcd.watch, 20729, 9.5)
 
     def test_other_exceptions(self):
         self.etcd.retry = Mock(side_effect=AttributeError('foo'))
@@ -282,4 +319,8 @@ class TestEtcd(unittest.TestCase):
 
     def test_set_ttl(self):
         self.etcd.set_ttl(20)
-        self.assertTrue(self.etcd.watch(1))
+        self.assertTrue(self.etcd.watch(None, 1))
+
+    def test_sync_state(self):
+        self.assertFalse(self.etcd.write_sync_state('leader', None))
+        self.assertFalse(self.etcd.delete_sync_state())

tests/test_ha.py

@@ -1,17 +1,19 @@
 import datetime
 import etcd
 import os
-import pytz
 import unittest
 
 from mock import Mock, MagicMock, PropertyMock, patch
 from patroni.config import Config
-from patroni.dcs import Cluster, Failover, Leader, Member, get_dcs
+from patroni.dcs import Cluster, ClusterConfig, Failover, Leader, Member, get_dcs, SyncState
 from patroni.dcs.etcd import Client
-from patroni.exceptions import DCSError, PostgresException
-from patroni.ha import Ha
+from patroni.exceptions import DCSError, PostgresConnectionException, PatroniException
+from patroni.ha import Ha, _MemberStatus
 from patroni.postgresql import Postgresql
+from patroni.watchdog import Watchdog
+from patroni.utils import tzutc
 from test_etcd import socket_getaddrinfo, etcd_read, etcd_write, requests_get
+from test_postgresql import psycopg2_connect
 
 
 def true(*args, **kwargs):
@@ -22,36 +24,47 @@ def false(*args, **kwargs):
     return False
 
 
-def get_cluster(initialize, leader, members, failover):
-    return Cluster(initialize, None, leader, 10, members, failover)
+def get_cluster(initialize, leader, members, failover, sync):
+    return Cluster(initialize, ClusterConfig(1, {1: 2}, 1), leader, 10, members, failover, sync)
 
 
 def get_cluster_not_initialized_without_leader():
-    return get_cluster(None, None, [], None)
+    return get_cluster(None, None, [], None, SyncState(None, None, None))
 
 
-def get_cluster_initialized_without_leader(leader=False, failover=None):
+def get_cluster_initialized_without_leader(leader=False, failover=None, sync=None):
     m1 = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres',
                                   'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location': 4})
     l = Leader(0, 0, m1) if leader else None
     m2 = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
                                  'api_url': 'http://127.0.0.1:8011/patroni',
+                                 'state': 'running',
                                  'tags': {'clonefrom': True},
                                  'scheduled_restart': {'schedule': "2100-01-01 10:53:07.560445+00:00",
                                                        'postgres_version': '99.0.0'}})
-    return get_cluster(True, l, [m1, m2], failover)
+    syncstate = SyncState(0 if sync else None, sync and sync[0], sync and sync[1])
+    return get_cluster(True, l, [m1, m2], failover, syncstate)
 
 
-def get_cluster_initialized_with_leader(failover=None):
-    return get_cluster_initialized_without_leader(leader=True, failover=failover)
+def get_cluster_initialized_with_leader(failover=None, sync=None):
+    return get_cluster_initialized_without_leader(leader=True, failover=failover, sync=sync)
 
 
 def get_cluster_initialized_with_only_leader(failover=None):
     l = get_cluster_initialized_without_leader(leader=True, failover=failover).leader
-    return get_cluster(True, l, [l], failover)
+    return get_cluster(True, l, [l], failover, None)
 
-future_restart_time = datetime.datetime.now(pytz.utc) + datetime.timedelta(days=5)
-postmaster_start_time = datetime.datetime.now(pytz.utc)
+
+def get_node_status(reachable=True, in_recovery=True, wal_position=10, nofailover=False, watchdog_failed=False):
+    def fetch_node_status(e):
+        tags = {}
+        if nofailover:
+            tags['nofailover'] = True
+        return _MemberStatus(e, reachable, in_recovery, wal_position, tags, watchdog_failed)
+    return fetch_node_status
+
+future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5)
+postmaster_start_time = datetime.datetime.now(tzutc)
 
 
 class MockPatroni(object):
@@ -72,6 +85,8 @@ postgresql:
   pg_rewind:
     username: postgres
     password: postgres
+watchdog:
+  mode: off
 zookeeper:
   exhibitor:
     hosts: [localhost]
@@ -86,8 +101,10 @@ zookeeper:
         self.replicatefrom = None
         self.api.connection_string = 'http://127.0.0.1:8008'
         self.clonefrom = None
+        self.nosync = False
         self.scheduled_restart = {'schedule': future_restart_time,
                                   'postmaster_start_time': str(postmaster_start_time)}
+        self.watchdog = Watchdog(self.config)
 
 
 def run_async(self, func, args=()):
@@ -96,31 +113,36 @@ def run_async(self, func, args=()):
 
 @patch.object(Postgresql, 'is_running', Mock(return_value=True))
 @patch.object(Postgresql, 'is_leader', Mock(return_value=True))
-@patch.object(Postgresql, 'xlog_position', Mock(return_value=0))
+@patch.object(Postgresql, 'wal_position', Mock(return_value=10))
 @patch.object(Postgresql, 'call_nowait', Mock(return_value=True))
 @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
 @patch.object(Postgresql, 'controldata', Mock(return_value={'Database system identifier': '1234567890'}))
 @patch.object(Postgresql, 'sync_replication_slots', Mock())
 @patch.object(Postgresql, 'write_pg_hba', Mock())
-@patch.object(Postgresql, 'write_pgpass', Mock())
+@patch.object(Postgresql, 'write_pgpass', Mock(return_value={}))
 @patch.object(Postgresql, 'write_recovery_conf', Mock())
 @patch.object(Postgresql, 'query', Mock())
 @patch.object(Postgresql, 'checkpoint', Mock())
+@patch.object(Postgresql, 'call_nowait', Mock())
 @patch.object(etcd.Client, 'write', etcd_write)
 @patch.object(etcd.Client, 'read', etcd_read)
 @patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
 @patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=False))
 @patch('patroni.async_executor.AsyncExecutor.run_async', run_async)
 @patch('subprocess.call', Mock(return_value=0))
+@patch('time.sleep', Mock())
 class TestHa(unittest.TestCase):
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
+    @patch('psycopg2.connect', psycopg2_connect)
+    @patch('patroni.dcs.dcs_modules', Mock(return_value=['foo', 'patroni.dcs.etcd']))
     @patch.object(etcd.Client, 'read', etcd_read)
     def setUp(self):
         with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
             self.p = Postgresql({'name': 'postgresql0', 'scope': 'dummy', 'listen': '127.0.0.1:5432',
                                  'data_dir': 'data/postgresql0', 'retry_timeout': 10,
+                                 'maximum_lag_on_failover': 5,
                                  'authentication': {'superuser': {'username': 'foo', 'password': 'bar'},
                                                     'replication': {'username': '', 'password': ''}},
                                  'parameters': {'wal_level': 'hot_standby', 'max_replication_slots': 5, 'foo': 'bar',
@@ -128,7 +150,6 @@ class TestHa(unittest.TestCase):
             self.p.set_state('running')
             self.p.set_role('replica')
             self.p.postmaster_start_time = MagicMock(return_value=str(postmaster_start_time))
-            self.p.check_replication_lag = true
             self.p.can_create_replica_without_replication_connection = MagicMock(return_value=False)
             self.e = get_dcs({'etcd': {'ttl': 30, 'host': 'ok:2379', 'scope': 'test',
                                        'name': 'foo', 'retry_timeout': 10}})
@@ -136,13 +157,14 @@ class TestHa(unittest.TestCase):
             self.ha.old_cluster = self.e.get_cluster()
             self.ha.cluster = get_cluster_not_initialized_without_leader()
             self.ha.load_cluster_from_dcs = Mock()
+            self.ha.is_synchronous_mode = false
 
     def test_update_lock(self):
-        self.p.last_operation = Mock(side_effect=PostgresException(''))
-        self.assertTrue(self.ha.update_lock())
+        self.p.last_operation = Mock(side_effect=PostgresConnectionException(''))
+        self.assertTrue(self.ha.update_lock(True))
 
     def test_touch_member(self):
-        self.p.xlog_position = Mock(side_effect=Exception)
+        self.p.wal_position = Mock(side_effect=Exception)
         self.ha.touch_member()
 
     def test_start_as_replica(self):
@@ -151,7 +173,6 @@ class TestHa(unittest.TestCase):
 
     def test_recover_replica_failed(self):
         self.p.controldata = lambda: {'Database cluster state': 'in production'}
-        self.p.is_healthy = false
         self.p.is_running = false
         self.p.follow = false
         self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
@@ -159,7 +180,6 @@ class TestHa(unittest.TestCase):
 
     def test_recover_master_failed(self):
         self.p.follow = false
-        self.p.is_healthy = false
         self.p.is_running = false
         self.p.name = 'leader'
         self.p.set_role('master')
@@ -167,8 +187,11 @@ class TestHa(unittest.TestCase):
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.assertEquals(self.ha.run_cycle(), 'starting as readonly because i had the session lock')
 
-    def test_do_not_recover_in_pause(self):
-        pass
+    @patch.object(Postgresql, 'rewind_needed_and_possible', Mock(return_value=True))
+    def test_recover_with_rewind(self):
+        self.p.is_running = false
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.assertEquals(self.ha.run_cycle(), 'running pg_rewind from leader')
 
     @patch('sys.exit', return_value=1)
     @patch('patroni.ha.Ha.sysid_valid', MagicMock(return_value=True))
@@ -216,6 +239,15 @@ class TestHa(unittest.TestCase):
         self.p.is_leader = false
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader because i had the session lock')
 
+    def test_promote_without_watchdog(self):
+        self.ha.cluster.is_unlocked = false
+        self.ha.has_lock = true
+        self.p.is_leader = true
+        with patch.object(Watchdog, 'activate', Mock(return_value=False)):
+            self.assertEquals(self.ha.run_cycle(), 'Demoting self because watchdog could not be activated')
+            self.p.is_leader = false
+            self.assertEquals(self.ha.run_cycle(), 'Not promoting self because watchdog could not be actived')
+
     def test_leader_with_lock(self):
         self.ha.cluster.is_unlocked = false
         self.ha.has_lock = true
@@ -223,13 +255,16 @@ class TestHa(unittest.TestCase):
 
     def test_demote_because_not_having_lock(self):
         self.ha.cluster.is_unlocked = false
-        self.assertEquals(self.ha.run_cycle(), 'demoting self because i do not have the lock and i was a leader')
+        with patch.object(Watchdog, 'is_running', PropertyMock(return_value=True)):
+            self.assertEquals(self.ha.run_cycle(), 'demoting self because i do not have the lock and i was a leader')
 
     def test_demote_because_update_lock_failed(self):
         self.ha.cluster.is_unlocked = false
         self.ha.has_lock = true
         self.ha.update_lock = false
-        self.assertEquals(self.ha.run_cycle(), 'demoting self because i do not have the lock and i was a leader')
+        self.assertEquals(self.ha.run_cycle(), 'demoted self because failed to update leader lock in DCS')
+        self.p.is_leader = false
+        self.assertEquals(self.ha.run_cycle(), 'not promoting because failed to update leader lock in DCS')
 
     def test_follow(self):
         self.ha.cluster.is_unlocked = false
@@ -245,10 +280,18 @@ class TestHa(unittest.TestCase):
         self.p.is_leader = false
         self.assertEquals(self.ha.run_cycle(), 'PAUSE: no action')
 
+    @patch.object(Postgresql, 'rewind_needed_and_possible', Mock(return_value=True))
+    def test_follow_triggers_rewind(self):
+        self.p.is_leader = false
+        self.p.trigger_check_diverged_lsn()
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.assertEquals(self.ha.run_cycle(), 'running pg_rewind from leader')
+
     def test_no_etcd_connection_master_demote(self):
         self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
         self.assertEquals(self.ha.run_cycle(), 'demoted self because DCS is not accessible and i was a leader')
 
+    @patch('time.sleep', Mock())
     def test_bootstrap_from_another_member(self):
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap from replica \'other\'')
@@ -269,14 +312,31 @@ class TestHa(unittest.TestCase):
     def test_bootstrap_initialized_new_cluster(self):
         self.ha.cluster = get_cluster_not_initialized_without_leader()
         self.e.initialize = true
-        self.assertEquals(self.ha.bootstrap(), 'initialized a new cluster')
+        self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap a new cluster')
+        self.p.is_leader = false
+        self.assertEquals(self.ha.run_cycle(), 'waiting for end of recovery after bootstrap')
+        self.p.is_leader = true
+        self.assertEquals(self.ha.run_cycle(), 'running post_bootstrap')
+        self.assertEquals(self.ha.run_cycle(), 'initialized a new cluster')
 
     def test_bootstrap_release_initialize_key_on_failure(self):
         self.ha.cluster = get_cluster_not_initialized_without_leader()
         self.e.initialize = true
-        self.p.bootstrap = Mock(side_effect=PostgresException("Could not bootstrap master PostgreSQL"))
-        self.assertRaises(PostgresException, self.ha.bootstrap)
+        self.ha.bootstrap()
+        self.p.is_running = false
+        self.assertRaises(PatroniException, self.ha.post_bootstrap)
 
+    def test_bootstrap_release_initialize_key_on_watchdog_failure(self):
+        self.ha.cluster = get_cluster_not_initialized_without_leader()
+        self.e.initialize = true
+        self.ha.bootstrap()
+        self.p.is_running = true
+        self.p.is_leader = true
+        with patch.object(Watchdog, 'activate', Mock(return_value=False)):
+            self.assertEquals(self.ha.post_bootstrap(), 'running post_bootstrap')
+            self.assertRaises(PatroniException, self.ha.post_bootstrap)
+
+    @patch('psycopg2.connect', psycopg2_connect)
     def test_reinitialize(self):
         self.assertIsNotNone(self.ha.reinitialize())
 
@@ -288,21 +348,25 @@ class TestHa(unittest.TestCase):
         self.ha.state_handler.name = self.ha.cluster.leader.name
         self.assertIsNotNone(self.ha.reinitialize())
 
+    @patch('time.sleep', Mock())
     def test_restart(self):
-        self.assertEquals(self.ha.restart(), (True, 'restarted successfully'))
+        self.assertEquals(self.ha.restart({}), (True, 'restarted successfully'))
+        self.p.restart = Mock(return_value=None)
+        self.assertEquals(self.ha.restart({}), (False, 'postgres is still starting'))
         self.p.restart = false
-        self.assertEquals(self.ha.restart(), (False, 'restart failed'))
+        self.assertEquals(self.ha.restart({}), (False, 'restart failed'))
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.ha.reinitialize()
-        self.assertEquals(self.ha.restart(), (False, 'reinitialize already in progress'))
+        self.assertEquals(self.ha.restart({}), (False, 'reinitialize already in progress'))
         with patch.object(self.ha, "restart_matches", return_value=False):
             self.assertEquals(self.ha.restart({'foo': 'bar'}), (False, "restart conditions are not satisfied"))
 
+    @patch('os.kill', Mock())
     def test_restart_in_progress(self):
         with patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=True)):
-            self.ha.restart(run_async=True)
+            self.ha.restart({}, run_async=True)
             self.assertTrue(self.ha.restart_scheduled())
-            self.assertEquals(self.ha.run_cycle(), 'not healthy enough for leader race')
+            self.assertEquals(self.ha.run_cycle(), 'restart in progress')
 
             self.ha.cluster = get_cluster_initialized_with_leader()
             self.assertEquals(self.ha.run_cycle(), 'restart in progress')
@@ -311,11 +375,15 @@ class TestHa(unittest.TestCase):
             self.assertEquals(self.ha.run_cycle(), 'updated leader lock during restart')
 
             self.ha.update_lock = false
-            self.assertEquals(self.ha.run_cycle(), 'failed to update leader lock during restart')
+            self.p.set_role('master')
+            with patch('patroni.async_executor.CriticalTask.cancel', Mock(return_value=False)):
+                with patch('patroni.postgresql.Postgresql.stop') as stop_mock:
+                    self.assertEquals(self.ha.run_cycle(), 'lost leader lock during restart')
+                    stop_mock.assert_called()
 
     @patch('requests.get', requests_get)
-    @patch('time.sleep', Mock())
     def test_manual_failover_from_leader(self):
+        self.ha.fetch_node_status = get_node_status()
         self.ha.has_lock = true
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None))
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
@@ -326,7 +394,13 @@ class TestHa(unittest.TestCase):
         f = Failover(0, self.p.name, '', None)
         self.ha.cluster = get_cluster_initialized_with_leader(f)
         self.assertEquals(self.ha.run_cycle(), 'manual failover: demoting myself')
-        self.ha.fetch_node_status = lambda e: (e, True, True, 0, {'nofailover': 'True'})
+        self.p.rewind_needed_and_possible = true
+        self.assertEquals(self.ha.run_cycle(), 'manual failover: demoting myself')
+        self.ha.fetch_node_status = get_node_status(nofailover=True)
+        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
+        self.ha.fetch_node_status = get_node_status(watchdog_failed=True)
+        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
+        self.ha.fetch_node_status = get_node_status(wal_position=1)
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
         # manual failover from the previous leader to us won't happen if we hold the nofailover flag
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, None))
@@ -337,7 +411,7 @@ class TestHa(unittest.TestCase):
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
         self.ha.run_cycle()
 
-        scheduled = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
+        scheduled = datetime.datetime.utcnow().replace(tzinfo=tzutc)
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
         self.assertEquals('no action.  i am the leader with the lock', self.ha.run_cycle())
 
@@ -364,7 +438,6 @@ class TestHa(unittest.TestCase):
         self.assertEquals('PAUSE: no action.  i am the leader with the lock', self.ha.run_cycle())
 
     @patch('requests.get', requests_get)
-    @patch('time.sleep', Mock())
     def test_manual_failover_process_no_leader(self):
         self.p.is_leader = false
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', self.p.name, None))
@@ -372,24 +445,23 @@ class TestHa(unittest.TestCase):
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader', None))
         self.p.set_role('replica')
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
-        self.ha.fetch_node_status = lambda e: (e, True, True, 0, {})  # accessible, in_recovery
+        self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
         self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, self.p.name, '', None))
         self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
-        self.ha.fetch_node_status = lambda e: (e, False, True, 0, {})  # inaccessible, in_recovery
+        self.ha.fetch_node_status = get_node_status(reachable=False)  # inaccessible, in_recovery
         self.p.set_role('replica')
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
         # set failover flag to True for all members of the cluster
         # this should elect the current member, as we are not going to call the API for it.
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
-        self.ha.fetch_node_status = lambda e: (e, True, True, 0, {'nofailover': 'True'})  # accessible, in_recovery
+        self.ha.fetch_node_status = get_node_status(nofailover=True)  # accessible, in_recovery
         self.p.set_role('replica')
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
         # same as previous, but set the current member to nofailover. In no case it should be elected as a leader
         self.ha.patroni.nofailover = True
         self.assertEquals(self.ha.run_cycle(), 'following a different leader because I am not allowed to promote')
 
-    @patch('time.sleep', Mock())
     def test_manual_failover_process_no_leader_in_pause(self):
         self.ha.is_paused = true
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
@@ -406,22 +478,26 @@ class TestHa(unittest.TestCase):
     def test_is_healthiest_node(self):
         self.ha.state_handler.is_leader = false
         self.ha.patroni.nofailover = False
-        self.ha.fetch_node_status = lambda e: (e, True, True, 0, {})
+        self.ha.fetch_node_status = get_node_status()
         self.assertTrue(self.ha.is_healthiest_node())
+        with patch.object(Watchdog, 'is_healthy', PropertyMock(return_value=False)):
+            self.assertFalse(self.ha.is_healthiest_node())
+        with patch('patroni.postgresql.Postgresql.is_starting', return_value=True):
+            self.assertFalse(self.ha.is_healthiest_node())
         self.ha.is_paused = true
         self.assertFalse(self.ha.is_healthiest_node())
 
     def test__is_healthiest_node(self):
         self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.p.is_leader = false
-        self.ha.fetch_node_status = lambda e: (e, True, True, 0, {})  # accessible, in_recovery
+        self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
         self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
-        self.ha.fetch_node_status = lambda e: (e, True, False, 0, {})  # accessible, not in_recovery
+        self.ha.fetch_node_status = get_node_status(in_recovery=False)  # accessible, not in_recovery
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
-        self.ha.fetch_node_status = lambda e: (e, True, True, 1, {})  # accessible, in_recovery, xlog location ahead
-        self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
-        self.p.check_replication_lag = false
+        self.ha.fetch_node_status = get_node_status(wal_position=11)  # accessible, in_recovery, wal position ahead
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
+        with patch('patroni.postgresql.Postgresql.wal_position', return_value=1):
+            self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.patroni.nofailover = True
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.patroni.nofailover = False
@@ -453,6 +529,9 @@ class TestHa(unittest.TestCase):
 
     def test_evaluate_scheduled_restart(self):
         self.p.postmaster_start_time = Mock(return_value=str(postmaster_start_time))
+        # restart already in progres
+        with patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=True)):
+            self.assertIsNone(self.ha.evaluate_scheduled_restart())
         # restart while the postmaster has been already restarted, fails
         with patch.object(self.ha,
                           'future_restart_scheduled',
@@ -507,3 +586,261 @@ class TestHa(unittest.TestCase):
         self.ha.is_paused = true
         self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
         self.assertEquals(self.ha.run_cycle(), 'PAUSE: DCS is not accessible')
+
+    @patch('patroni.ha.Ha.update_lock', return_value=True)
+    @patch('patroni.ha.Ha.demote')
+    def test_starting_timeout(self, demote, update_lock):
+        def check_calls(seq):
+            for mock, called in seq:
+                if called:
+                    mock.assert_called_once()
+                else:
+                    mock.assert_not_called()
+                mock.reset_mock()
+        self.ha.has_lock = true
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.p.check_for_startup = true
+        self.p.time_in_state = lambda: 30
+        self.assertEquals(self.ha.run_cycle(), 'PostgreSQL is still starting up, 270 seconds until timeout')
+        check_calls([(update_lock, True), (demote, False)])
+
+        self.p.time_in_state = lambda: 350
+        self.ha.fetch_node_status = get_node_status(reachable=False)  # inaccessible, in_recovery
+        self.assertEquals(self.ha.run_cycle(),
+                          'master start has timed out, but continuing to wait because failover is not possible')
+        check_calls([(update_lock, True), (demote, False)])
+
+        self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
+        self.assertEquals(self.ha.run_cycle(), 'stopped PostgreSQL because of startup timeout')
+        check_calls([(update_lock, True), (demote, True)])
+
+        update_lock.return_value = False
+        self.assertEquals(self.ha.run_cycle(), 'stopped PostgreSQL while starting up because leader key was lost')
+        check_calls([(update_lock, True), (demote, True)])
+
+        self.ha.has_lock = false
+        self.p.is_leader = false
+        self.assertEquals(self.ha.run_cycle(), 'no action.  i am a secondary and i am following a leader')
+        check_calls([(update_lock, False), (demote, False)])
+
+    def test_manual_failover_while_starting(self):
+        self.ha.has_lock = true
+        self.p.check_for_startup = true
+        f = Failover(0, self.p.name, '', None)
+        self.ha.cluster = get_cluster_initialized_with_leader(f)
+        self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
+        self.assertEquals(self.ha.run_cycle(), 'manual failover: demoting myself')
+
+    @patch('patroni.ha.Ha.demote')
+    def test_failover_immediately_on_zero_master_start_timeout(self, demote):
+        self.p.is_running = false
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.ha.patroni.config.set_dynamic_configuration({'master_start_timeout': 0})
+        self.ha.has_lock = true
+        self.ha.update_lock = true
+        self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
+        self.assertEquals(self.ha.run_cycle(), 'stopped PostgreSQL to fail over after a crash')
+        demote.assert_called_once()
+
+    @patch('patroni.postgresql.Postgresql.follow')
+    def test_demote_immediate(self, follow):
+        self.ha.has_lock = true
+        self.e.get_cluster = Mock(return_value=get_cluster_initialized_without_leader())
+        self.ha.demote('immediate')
+        follow.assert_called_once_with(None)
+
+    def test_process_sync_replication(self):
+        self.ha.has_lock = true
+        mock_set_sync = self.p.set_synchronous_standby = Mock()
+        self.p.name = 'leader'
+
+        # Test sync key removed when sync mode disabled
+        self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
+        with patch.object(self.ha.dcs, 'delete_sync_state') as mock_delete_sync:
+            self.ha.run_cycle()
+            mock_delete_sync.assert_called_once()
+            mock_set_sync.assert_called_once_with(None)
+
+        mock_set_sync.reset_mock()
+        # Test sync key not touched when not there
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        with patch.object(self.ha.dcs, 'delete_sync_state') as mock_delete_sync:
+            self.ha.run_cycle()
+            mock_delete_sync.assert_not_called()
+            mock_set_sync.assert_called_once_with(None)
+
+        mock_set_sync.reset_mock()
+
+        self.ha.is_synchronous_mode = true
+
+        # Test sync standby not touched when picking the same node
+        self.p.pick_synchronous_standby = Mock(return_value=('other', True))
+        self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
+        self.ha.run_cycle()
+        mock_set_sync.assert_not_called()
+
+        mock_set_sync.reset_mock()
+
+        # Test sync standby is replaced when switching standbys
+        self.p.pick_synchronous_standby = Mock(return_value=('other2', False))
+        self.ha.dcs.write_sync_state = Mock(return_value=True)
+        self.ha.run_cycle()
+        mock_set_sync.assert_called_once_with('other2')
+
+        mock_set_sync.reset_mock()
+        # Test sync standby is not disabled when updating dcs fails
+        self.ha.dcs.write_sync_state = Mock(return_value=False)
+        self.ha.run_cycle()
+        mock_set_sync.assert_not_called()
+
+        mock_set_sync.reset_mock()
+        # Test changing sync standby
+        self.ha.dcs.write_sync_state = Mock(return_value=True)
+        self.ha.dcs.get_cluster = Mock(return_value=get_cluster_initialized_with_leader(sync=('leader', 'other')))
+        # self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
+        self.p.pick_synchronous_standby = Mock(return_value=('other2', True))
+        self.ha.run_cycle()
+        self.ha.dcs.get_cluster.assert_called_once()
+        self.assertEquals(self.ha.dcs.write_sync_state.call_count, 2)
+
+        # Test updating sync standby key failed due to race
+        self.ha.dcs.write_sync_state = Mock(side_effect=[True, False])
+        self.ha.run_cycle()
+        self.assertEquals(self.ha.dcs.write_sync_state.call_count, 2)
+
+        # Test changing sync standby failed due to race
+        self.ha.dcs.write_sync_state = Mock(return_value=True)
+        self.ha.dcs.get_cluster = Mock(return_value=get_cluster_initialized_with_leader(sync=('somebodyelse', None)))
+        self.ha.run_cycle()
+        self.assertEquals(self.ha.dcs.write_sync_state.call_count, 1)
+
+        # Test sync set to '*' when synchronous_mode_strict is enabled
+        mock_set_sync.reset_mock()
+        self.ha.is_synchronous_mode_strict = true
+        self.p.pick_synchronous_standby = Mock(return_value=(None, False))
+        self.ha.run_cycle()
+        mock_set_sync.assert_called_once_with('*')
+
+    def test_sync_replication_become_master(self):
+        self.ha.is_synchronous_mode = true
+
+        mock_set_sync = self.p.set_synchronous_standby = Mock()
+        self.p.is_leader = false
+        self.p.set_role('replica')
+        self.ha.has_lock = true
+        mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=True)
+        self.p.name = 'leader'
+        self.ha.cluster = get_cluster_initialized_with_leader(sync=('other', None))
+
+        # When we just became master nobody is sync
+        self.assertEquals(self.ha.enforce_master_role('msg', 'promote msg'), 'promote msg')
+        mock_set_sync.assert_called_once_with(None)
+        mock_write_sync.assert_called_once_with('leader', None, index=0)
+
+        mock_set_sync.reset_mock()
+
+        # When we just became master nobody is sync
+        self.p.set_role('replica')
+        mock_write_sync.return_value = False
+        self.assertTrue(self.ha.enforce_master_role('msg', 'promote msg') != 'promote msg')
+        mock_set_sync.assert_not_called()
+
+    def test_unhealthy_sync_mode(self):
+        self.ha.is_synchronous_mode = true
+
+        self.p.is_leader = false
+        self.p.set_role('replica')
+        self.p.name = 'other'
+        self.ha.cluster = get_cluster_initialized_without_leader(sync=('leader', 'other2'))
+        mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=True)
+        mock_acquire = self.ha.acquire_lock = Mock(return_value=True)
+        mock_follow = self.p.follow = Mock()
+        mock_promote = self.p.promote = Mock()
+
+        # If we don't match the sync replica we are not allowed to acquire lock
+        self.ha.run_cycle()
+        mock_acquire.assert_not_called()
+        mock_follow.assert_called_once()
+        self.assertEquals(mock_follow.call_args[0][0], None)
+        mock_write_sync.assert_not_called()
+
+        mock_follow.reset_mock()
+        # If we do match we will try to promote
+        self.ha._is_healthiest_node = true
+
+        self.ha.cluster = get_cluster_initialized_without_leader(sync=('leader', 'other'))
+        self.ha.run_cycle()
+        mock_acquire.assert_called_once()
+        mock_follow.assert_not_called()
+        mock_promote.assert_called_once()
+        mock_write_sync.assert_called_once_with('other', None, index=0)
+
+    def test_disable_sync_when_restarting(self):
+        self.ha.is_synchronous_mode = true
+
+        self.p.name = 'other'
+        self.p.is_leader = false
+        self.p.set_role('replica')
+        mock_restart = self.p.restart = Mock(return_value=True)
+        self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
+        self.ha.touch_member = Mock(return_value=True)
+        self.ha.dcs.get_cluster = Mock(side_effect=[
+            get_cluster_initialized_with_leader(sync=('leader', syncstandby))
+            for syncstandby in ['other', None]])
+
+        with patch('time.sleep') as mock_sleep:
+            self.ha.restart({})
+            mock_restart.assert_called_once()
+            mock_sleep.assert_called()
+
+        # Restart is still called when DCS connection fails
+        mock_restart.reset_mock()
+        self.ha.dcs.get_cluster = Mock(side_effect=DCSError("foo"))
+        self.ha.restart({})
+
+        mock_restart.assert_called_once()
+
+        # We don't try to fetch the cluster state when touch_member fails
+        mock_restart.reset_mock()
+        self.ha.dcs.get_cluster.reset_mock()
+        self.ha.touch_member = Mock(return_value=False)
+
+        self.ha.restart({})
+
+        mock_restart.assert_called_once()
+        self.ha.dcs.get_cluster.assert_not_called()
+
+    def test_effective_tags(self):
+        self.ha._disable_sync = True
+        self.assertEquals(self.ha.get_effective_tags(), {'foo': 'bar', 'nosync': True})
+        self.ha._disable_sync = False
+        self.assertEquals(self.ha.get_effective_tags(), {'foo': 'bar'})
+
+    def test_restore_cluster_config(self):
+        self.ha.cluster.config.data.clear()
+        self.ha.has_lock = true
+        self.ha.cluster.is_unlocked = false
+        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
+
+    def test_watch(self):
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.ha.watch(0)
+
+    def test_wakup(self):
+        self.ha.wakeup()
+
+    def test_shutdown(self):
+        self.p.is_running = false
+        self.ha.shutdown()
+
+    @patch('time.sleep', Mock())
+    def test_leader_with_empty_directory(self):
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.ha.has_lock = true
+        self.p.data_directory_empty = true
+        self.assertEquals(self.ha.run_cycle(), 'released leader key voluntarily as data dir empty and currently leader')
+
+        # as has_lock is mocked out, we need to fake the leader key release
+        self.ha.has_lock = false
+        # will not say bootstrap from leader as replica can't self elect
+        self.assertEquals(self.ha.run_cycle(), "trying to bootstrap from replica 'other'")

tests/test_patroni.py

@@ -1,14 +1,15 @@
 import etcd
+import signal
 import sys
 import time
 import unittest
 
-from mock import Mock, patch
+from mock import Mock, PropertyMock, patch
 from patroni.api import RestApiServer
 from patroni.async_executor import AsyncExecutor
 from patroni.dcs.etcd import Client
 from patroni.exceptions import DCSError
-from patroni import Patroni, main as _main
+from patroni import Patroni, main as _main, patroni_main
 from six.moves import BaseHTTPServer
 from test_etcd import SleepException, etcd_read, etcd_write
 from test_postgresql import Postgresql, psycopg2_connect
@@ -26,6 +27,7 @@ class MockFrozenImporter(object):
 @patch.object(Postgresql, '_write_postgresql_conf', Mock())
 @patch.object(Postgresql, 'write_recovery_conf', Mock())
 @patch.object(Postgresql, 'is_running', Mock(return_value=True))
+@patch.object(Postgresql, 'call_nowait', Mock())
 @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
 @patch.object(AsyncExecutor, 'run', Mock())
 @patch.object(etcd.Client, 'write', etcd_write)
@@ -53,20 +55,57 @@ class TestPatroni(unittest.TestCase):
     @patch('time.sleep', Mock(side_effect=SleepException))
     @patch.object(etcd.Client, 'delete', Mock())
     @patch.object(Client, 'machines')
-    def test_patroni_main(self, mock_machines):
+    def test_patroni_patroni_main(self, mock_machines):
         with patch('subprocess.call', Mock(return_value=1)):
             sys.argv = ['patroni.py', 'postgres0.yml']
 
             mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
             with patch.object(Patroni, 'run', Mock(side_effect=SleepException)):
-                self.assertRaises(SleepException, _main)
+                self.assertRaises(SleepException, patroni_main)
             with patch.object(Patroni, 'run', Mock(side_effect=KeyboardInterrupt())):
                 with patch('patroni.ha.Ha.is_paused', Mock(return_value=True)):
-                    _main()
+                    patroni_main()
+
+    @patch('os.getpid')
+    @patch('subprocess.Popen', )
+    @patch('patroni.patroni_main', Mock())
+    def test_patroni_main(self, mock_popen, mock_getpid):
+        mock_getpid.return_value = 2
+        _main()
+
+        with patch('sys.frozen', Mock(return_value=True), create=True):
+            sys.argv = ['/patroni', 'pg_ctl_start', 'postgres', '-D', '/data', '--max_connections=100']
+            _main()
+
+        mock_getpid.return_value = 1
+
+        def mock_signal(signo, handler):
+            handler(signo, None)
+
+        with patch('signal.signal', mock_signal):
+            with patch('os.waitpid', Mock(side_effect=[(1, 0), (0, 0)])):
+                _main()
+            with patch('os.waitpid', Mock(side_effect=OSError)):
+                _main()
+
+        ref = {'passtochild': lambda signo, stack_frame: 0}
+
+        def mock_sighup(signo, handler):
+            if signo == signal.SIGHUP:
+                ref['passtochild'] = handler
+
+        def mock_wait():
+            ref['passtochild'](0, None)
+
+        mock_popen.return_value.wait = mock_wait
+        with patch('signal.signal', mock_sighup), patch('os.kill', Mock()):
+            self.assertIsNone(_main())
 
     @patch('patroni.config.Config.save_cache', Mock())
     @patch('patroni.config.Config.reload_local_configuration', Mock(return_value=True))
+    @patch.object(Postgresql, 'state', PropertyMock(return_value='running'))
     def test_run(self):
+        self.p.postgresql.set_role('replica')
         self.p.sighup_handler()
         self.p.ha.dcs.watch = Mock(side_effect=SleepException)
         self.p.api.start = Mock()
@@ -105,3 +144,9 @@ class TestPatroni(unittest.TestCase):
         self.p.reload_config()
         self.p.get_tags = Mock(side_effect=Exception)
         self.p.reload_config()
+
+    def test_nosync(self):
+        self.p.tags['nosync'] = True
+        self.assertTrue(self.p.nosync)
+        self.p.tags['nosync'] = None
+        self.assertFalse(self.p.nosync)

tests/test_postgresql.py

@@ -1,3 +1,4 @@
+import errno
 import mock  # for the mock.call method, importing it without a namespace breaks python3
 import os
 import psycopg2
@@ -6,12 +7,13 @@ import subprocess
 import unittest
 
 from mock import Mock, MagicMock, PropertyMock, patch, mock_open
-from patroni.dcs import Cluster, Leader, Member
-from patroni.exceptions import PostgresException, PostgresConnectionException
-from patroni.postgresql import Postgresql
+from patroni.async_executor import CriticalTask
+from patroni.dcs import Cluster, Leader, Member, SyncState
+from patroni.exceptions import PostgresConnectionException
+from patroni.postgresql import Postgresql, STATE_REJECT, STATE_NO_RESPONSE
 from patroni.utils import RetryFailedError
 from six.moves import builtins
-from test_ha import false
+from threading import Thread
 
 
 class MockCursor(object):
@@ -19,17 +21,20 @@ class MockCursor(object):
     def __init__(self, connection):
         self.connection = connection
         self.closed = False
+        self.rowcount = 0
         self.results = []
 
     def execute(self, sql, *params):
-        if sql.startswith('blabla') or sql == 'CHECKPOINT':
+        if sql.startswith('blabla'):
+            raise psycopg2.ProgrammingError()
+        elif sql == 'CHECKPOINT':
             raise psycopg2.OperationalError()
         elif sql.startswith('RetryFailedError'):
             raise RetryFailedError('retry')
         elif sql.startswith('SELECT slot_name'):
             self.results = [('blabla',), ('foobar',)]
-        elif sql.startswith('SELECT pg_xlog_location_diff'):
-            self.results = [(0,)]
+        elif sql.startswith('SELECT CASE WHEN pg_is_in_recovery()'):
+            self.results = [(2,)]
         elif sql == 'SELECT pg_is_in_recovery()':
             self.results = [(False, )]
         elif sql.startswith('WITH replication_info AS ('):
@@ -41,7 +46,15 @@ class MockCursor(object):
                             ('search_path', 'public', None, 'string', 'user'),
                             ('port', '5433', None, 'integer', 'postmaster'),
                             ('listen_addresses', '*', None, 'string', 'postmaster'),
-                            ('autovacuum', 'on', None, 'bool', 'sighup')]
+                            ('autovacuum', 'on', None, 'bool', 'sighup'),
+                            ('unix_socket_directories', '/tmp', None, 'string', 'postmaster')]
+        elif sql.startswith('IDENTIFY_SYSTEM'):
+            self.results = [('1', 2, '0/402EEC0', '')]
+        elif sql.startswith('TIMELINE_HISTORY '):
+            self.results = [('', b'x\t0/40159C0\tno recovery target specified\n\n' +
+                                 b'1\t0/40159C0\tno recovery target specified\n\n' +
+                                 b'2\t0/402DD98\tno recovery target specified\n\n' +
+                                 b'3\t0/403DD98\tno recovery target specified\n')]
         else:
             self.results = [(None, None, None, None, None, None, None, None, None, None)]
 
@@ -64,7 +77,7 @@ class MockCursor(object):
 
 class MockConnect(object):
 
-    server_version = '99999'
+    server_version = 99999
     autocommit = False
     closed = 0
 
@@ -137,51 +150,42 @@ Data page checksum version:           0
 """
 
 
-def postmaster_opts_string(*args, **kwargs):
-    return '/usr/local/pgsql/bin/postgres "-D" "data/postgresql0" "--listen_addresses=127.0.0.1" \
-"--port=5432" "--hot_standby=on" "--wal_keep_segments=8" "--wal_level=hot_standby" \
-"--archive_command=mkdir -p ../wal_archive && cp %p ../wal_archive/%f" "--wal_log_hints=on" \
-"--max_wal_senders=5" "--archive_timeout=1800s" "--archive_mode=on" "--max_replication_slots=5"\n'
-
-
 def psycopg2_connect(*args, **kwargs):
     return MockConnect()
 
 
-def fake_listdir(path):
-    return ["a", "b", "c"] if path.endswith('pg_xlog/archive_status') else []
-
-
 @patch('subprocess.call', Mock(return_value=0))
 @patch('psycopg2.connect', psycopg2_connect)
 class TestPostgresql(unittest.TestCase):
     _PARAMETERS = {'wal_level': 'hot_standby', 'max_replication_slots': 5, 'f.oo': 'bar',
                    'search_path': 'public', 'hot_standby': 'on', 'max_wal_senders': 5,
                    'wal_keep_segments': 8, 'wal_log_hints': 'on', 'max_locks_per_transaction': 64,
-                   'max_worker_processes': 8, 'max_connections': 100, 'max_prepared_transactions': 0}
+                   'max_worker_processes': 8, 'max_connections': 100, 'max_prepared_transactions': 0,
+                   'track_commit_timestamp': 'off', 'unix_socket_directories': '/tmp'}
 
     @patch('subprocess.call', Mock(return_value=0))
     @patch('psycopg2.connect', psycopg2_connect)
     @patch('os.rename', Mock())
-    @patch.object(Postgresql, 'get_major_version', Mock(return_value=9.4))
+    @patch.object(Postgresql, 'get_major_version', Mock(return_value=90600))
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def setUp(self):
         self.data_dir = 'data/test0'
+        self.config_dir = self.data_dir
         if not os.path.exists(self.data_dir):
             os.makedirs(self.data_dir)
-        self.p = Postgresql({'name': 'test0', 'scope': 'batman', 'data_dir': self.data_dir, 'retry_timeout': 10,
-                             'listen': '127.0.0.1, *:5432', 'connect_address': '127.0.0.2:5432',
+        self.p = Postgresql({'name': 'test0', 'scope': 'batman', 'data_dir': self.data_dir,
+                             'config_dir': self.config_dir, 'retry_timeout': 10,
+                             'listen': '127.0.0.2, 127.0.0.3:5432', 'connect_address': '127.0.0.2:5432',
                              'authentication': {'superuser': {'username': 'test', 'password': 'test'},
                                                 'replication': {'username': 'replicator', 'password': 'rep-pass'}},
                              'remove_data_directory_on_rewind_failure': True,
                              'use_pg_rewind': True, 'pg_ctl_timeout': 'bla',
                              'parameters': self._PARAMETERS,
                              'recovery_conf': {'foo': 'bar'},
-                             'callbacks': {'on_start': 'true', 'on_stop': 'true',
-                                           'on_restart': 'true', 'on_role_change': 'true',
-                                           'on_reload': 'true'
-                                           },
-                             'restore': 'true'})
+                             'pg_hba': ['host all all 0.0.0.0/0 md5'],
+                             'callbacks': {'on_start': 'true', 'on_stop': 'true', 'on_reload': 'true',
+                                           'on_restart': 'true', 'on_role_change': 'true'}})
+        self.p._callback_executor = Mock()
         self.leadermem = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres'})
         self.leader = Leader(-1, 28, self.leadermem)
         self.other = Member(0, 'test-1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres',
@@ -203,34 +207,99 @@ class TestPostgresql(unittest.TestCase):
     def test_delete_trigger_file(self):
         self.p.delete_trigger_file()
 
+    @patch('subprocess.Popen')
+    @patch.object(Postgresql, 'wait_for_startup')
+    @patch.object(Postgresql, 'wait_for_port_open')
     @patch.object(Postgresql, 'is_running')
-    def test_start(self, mock_is_running):
+    def test_start(self, mock_is_running, mock_wait_for_port_open, mock_wait_for_startup, mock_popen):
         mock_is_running.return_value = True
+        mock_wait_for_port_open.return_value = True
+        mock_wait_for_startup.return_value = False
+        mock_popen.return_value.stdout.readline.return_value = '123'
         self.assertTrue(self.p.start())
         mock_is_running.return_value = False
         open(os.path.join(self.data_dir, 'postmaster.pid'), 'w').close()
         pg_conf = os.path.join(self.data_dir, 'postgresql.conf')
         open(pg_conf, 'w').close()
-        self.assertTrue(self.p.start())
+        self.assertFalse(self.p.start(task=CriticalTask()))
         with open(pg_conf) as f:
             lines = f.readlines()
             self.assertTrue("f.oo = 'bar'\n" in lines)
 
+        mock_wait_for_startup.return_value = None
+        self.assertFalse(self.p.start(10))
+        self.assertIsNone(self.p.start())
+
+        mock_wait_for_port_open.return_value = False
+        self.assertFalse(self.p.start())
+        task = CriticalTask()
+        task.cancel()
+        self.assertFalse(self.p.start(task=task))
+
+    @patch.object(Postgresql, 'pg_isready')
+    @patch.object(Postgresql, 'read_pid_file')
+    @patch.object(Postgresql, 'is_pid_running')
+    @patch('patroni.postgresql.polling_loop', Mock(return_value=range(1)))
+    def test_wait_for_port_open(self, mock_is_pid_running, mock_read_pid_file, mock_pg_isready):
+        mock_is_pid_running.return_value = False
+        mock_pg_isready.return_value = STATE_NO_RESPONSE
+
+        # No pid file and postmaster death
+        mock_read_pid_file.return_value = {}
+        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
+
+        mock_is_pid_running.return_value = True
+
+        # timeout
+        mock_read_pid_file.return_value = {'pid', 1}
+        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
+
+        # Garbage pid
+        mock_read_pid_file.return_value = {'pid': 'garbage', 'start_time': '101', 'data_dir': '',
+                                           'socket_dir': '', 'port': '', 'listen_addr': ''}
+        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
+
+        # Not ready
+        mock_read_pid_file.return_value = {'pid': '42', 'start_time': '101', 'data_dir': '',
+                                           'socket_dir': '', 'port': '', 'listen_addr': ''}
+        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
+
+        # pg_isready failure
+        mock_pg_isready.return_value = 'garbage'
+        self.assertTrue(self.p.wait_for_port_open(42, 100., 1))
+
+    @patch('time.sleep', Mock())
     @patch.object(Postgresql, 'is_running')
-    def test_stop(self, mock_is_running):
+    @patch.object(Postgresql, 'get_pid')
+    def test_stop(self, mock_get_pid, mock_is_running):
+        mock_callback = Mock()
+        mock_is_running.return_value = False
+        self.assertTrue(self.p.stop(on_safepoint=mock_callback))
+        mock_callback.assert_called()
         mock_is_running.return_value = True
-        self.assertTrue(self.p.stop())
-        with patch('subprocess.call', Mock(return_value=1)):
-            mock_is_running.return_value = False
+        mock_get_pid.return_value = 0
+        mock_callback.reset_mock()
+        self.assertTrue(self.p.stop(on_safepoint=mock_callback))
+        mock_callback.assert_called()
+        mock_get_pid.return_value = -1
+        self.assertFalse(self.p.stop())
+        mock_get_pid.return_value = 123
+        with patch('os.kill', Mock(side_effect=[OSError(errno.ESRCH, ''), OSError, None])):
             self.assertTrue(self.p.stop())
+            self.assertFalse(self.p.stop())
+            self.assertTrue(self.p.stop())
+        with patch.object(Postgresql, '_signal_postmaster_stop', Mock(return_value=(123, None))):
+            with patch.object(Postgresql, 'is_pid_running', Mock(side_effect=[True, False, False])):
+                self.assertTrue(self.p.stop())
 
     def test_restart(self):
-        self.p.start = false
+        self.p.start = Mock(return_value=False)
         self.assertFalse(self.p.restart())
         self.assertEquals(self.p.state, 'restart failed (restarting)')
 
     @patch.object(builtins, 'open', MagicMock())
     def test_write_pgpass(self):
+        self.p.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo'})
         self.p.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo', 'password': 'bar'})
 
     def test_checkpoint(self):
@@ -244,39 +313,84 @@ class TestPostgresql(unittest.TestCase):
     @patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
     def test_pg_rewind(self, mock_call):
         r = {'user': '', 'host': '', 'port': '', 'database': '', 'password': ''}
-        self.assertTrue(self.p.rewind(r))
+        self.assertTrue(self.p.pg_rewind(r))
         subprocess.call = mock_call
-        self.assertFalse(self.p.rewind(r))
+        self.assertFalse(self.p.pg_rewind(r))
 
-    @patch('os.unlink', Mock(return_value=True))
-    @patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string))
-    @patch.object(Postgresql, 'remove_data_directory', Mock(return_value=True))
-    @patch.object(Postgresql, 'single_user_mode', Mock(return_value=1))
-    @patch.object(Postgresql, 'write_pgpass', Mock(return_value={}))
-    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
+    def test_check_recovery_conf(self):
+        self.p.write_recovery_conf({'primary_conninfo': 'foo'})
+        self.assertFalse(self.p.check_recovery_conf(None))
+        self.p.write_recovery_conf({})
+        self.assertTrue(self.p.check_recovery_conf(None))
+
+    @patch.object(Postgresql, 'start', Mock())
     @patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
-    @patch.object(Postgresql, 'rewind', return_value=False)
-    def test_follow(self, mock_pg_rewind):
-        with patch.object(Postgresql, 'check_recovery_conf', Mock(return_value=True)):
-            self.assertTrue(self.p.follow(None, None))  # nothing to do, recovery.conf has good primary_conninfo
+    def test__get_local_timeline_lsn(self):
+        self.p.trigger_check_diverged_lsn()
+        with patch.object(Postgresql, 'controldata', Mock(return_value={'Database cluster state': 'shut down'})):
+            self.p.rewind_needed_and_possible(self.leader)
+        with patch.object(Postgresql, 'controldata',
+                          Mock(return_value={'Database cluster state': 'shut down in recovery'})):
+            self.p.rewind_needed_and_possible(self.leader)
+        with patch.object(Postgresql, 'is_running', Mock(return_value=True)):
+            with patch.object(MockCursor, 'fetchone', Mock(side_effect=[(False, ), Exception])):
+                self.p.rewind_needed_and_possible(self.leader)
 
-        self.p.follow(self.me, self.me)  # follow is called when the node is holding leader lock
+    @patch.object(Postgresql, 'start', Mock())
+    @patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
+    @patch.object(Postgresql, '_get_local_timeline_lsn', Mock(return_value=(2, '0/40159C1')))
+    @patch.object(Postgresql, 'check_leader_is_not_in_recovery')
+    def test__check_timeline_and_lsn(self, mock_check_leader_is_not_in_recovery):
+        mock_check_leader_is_not_in_recovery.return_value = False
+        self.p.trigger_check_diverged_lsn()
+        self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
+        mock_check_leader_is_not_in_recovery.return_value = True
+        self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
+        self.p.trigger_check_diverged_lsn()
+        with patch('psycopg2.connect', Mock(side_effect=Exception)):
+            self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
+        with patch.object(MockCursor, 'fetchone',
+                          Mock(side_effect=[('', 2, '0/0'), ('', b'2\tG/40159C0\tno recovery target specified\n\n')])):
+            self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
+        self.p.trigger_check_diverged_lsn()
+        with patch.object(MockCursor, 'fetchone',
+                          Mock(side_effect=[('', 2, '0/0'), ('', b'3\t040159C0\tno recovery target specified\n')])):
+            self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
+        self.p.trigger_check_diverged_lsn()
+        with patch.object(MockCursor, 'fetchone', Mock(return_value=('', 1, '0/0'))):
+            with patch.object(Postgresql, '_get_local_timeline_lsn', Mock(return_value=(1, '0/0'))):
+                self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
+            self.p.trigger_check_diverged_lsn()
+            self.assertTrue(self.p.rewind_needed_and_possible(self.leader))
 
-        with patch.object(Postgresql, 'restart', Mock(return_value=False)):
-            self.p.set_role('replica')
-            self.p.follow(None, None)  # restart without rewind
+    @patch.object(MockCursor, 'fetchone', Mock(side_effect=[(True,), Exception]))
+    def test_check_leader_is_not_in_recovery(self):
+        self.p.check_leader_is_not_in_recovery()
+        self.p.check_leader_is_not_in_recovery()
 
-        with patch.object(Postgresql, 'stop', Mock(return_value=False)):
-            self.p.follow(self.leader, self.leader, need_rewind=True)  # failed to stop postgres
+    @patch.object(Postgresql, 'checkpoint', side_effect=['', '1'])
+    @patch.object(Postgresql, 'stop', Mock(return_value=False))
+    @patch.object(Postgresql, 'start', Mock())
+    def test_rewind(self, mock_checkpoint):
+        self.p.rewind(self.leader)
+        with patch.object(Postgresql, 'pg_rewind', Mock(return_value=False)):
+            mock_checkpoint.side_effect = ['1', '', '', '']
+            self.p.rewind(self.leader)
+            self.p.rewind(self.leader)
+            with patch.object(Postgresql, 'check_leader_is_not_in_recovery', Mock(return_value=False)):
+                self.p.rewind(self.leader)
+            self.p.config['remove_data_directory_on_rewind_failure'] = False
+            self.p.trigger_check_diverged_lsn()
+            self.p.rewind(self.leader)
+        with patch.object(Postgresql, 'is_running', Mock(return_value=True)):
+            self.p.rewind(self.leader)
+            self.p.is_leader = Mock(return_value=False)
+            self.p.rewind(self.leader)
 
-        self.p.follow(self.leader, self.leader)  # "leader" is not accessible or is_in_recovery
-
-        with patch.object(Postgresql, 'checkpoint', Mock(return_value=None)):
-            self.p.follow(self.leader, self.leader)
-            mock_pg_rewind.return_value = True
-            self.p.follow(self.leader, self.leader, need_rewind=True)
-
-        self.p.follow(None, None)  # check_recovery_conf...
+    @patch.object(Postgresql, 'is_running', Mock(return_value=False))
+    @patch.object(Postgresql, 'start', Mock())
+    def test_follow(self):
+        self.p.follow(None)
 
     @patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string))
     def test_can_rewind(self):
@@ -290,6 +404,7 @@ class TestPostgresql(unittest.TestCase):
         self.assertFalse(self.p.can_rewind)
 
     @patch('time.sleep', Mock())
+    @patch.object(Postgresql, 'remove_data_directory', Mock(return_value=True))
     def test_create_replica(self):
         self.p.delete_trigger_file = Mock(side_effect=OSError)
         with patch('subprocess.call', Mock(side_effect=[1, 0])):
@@ -307,10 +422,13 @@ class TestPostgresql(unittest.TestCase):
         with patch('subprocess.call', Mock(side_effect=Exception("foo"))):
             self.assertEquals(self.p.create_replica(self.leader), 1)
 
+        with patch('subprocess.call', Mock(return_value=1)):
+            self.assertEquals(self.p.create_replica(self.leader), 1)
+
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def test_sync_replication_slots(self):
         self.p.start()
-        cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None)
+        cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None, None)
         with mock.patch('patroni.postgresql.Postgresql._query', Mock(side_effect=psycopg2.OperationalError)):
             self.p.sync_replication_slots(cluster)
         self.p.sync_replication_slots(cluster)
@@ -326,7 +444,7 @@ class TestPostgresql(unittest.TestCase):
             assert "test-3" in errorlog_mock.call_args[0][1]
             assert "test.3" in errorlog_mock.call_args[0][1]
 
-    @patch.object(MockConnect, 'closed', 2)
+    @patch.object(MockCursor, 'execute', Mock(side_effect=psycopg2.OperationalError))
     def test__query(self):
         self.assertRaises(PostgresConnectionException, self.p._query, 'blabla')
         self.p._state = 'restarting'
@@ -335,10 +453,13 @@ class TestPostgresql(unittest.TestCase):
     def test_query(self):
         self.p.query('select 1')
         self.assertRaises(PostgresConnectionException, self.p.query, 'RetryFailedError')
-        self.assertRaises(psycopg2.OperationalError, self.p.query, 'blabla')
+        self.assertRaises(psycopg2.ProgrammingError, self.p.query, 'blabla')
 
+    @patch.object(Postgresql, 'pg_isready', Mock(return_value=STATE_REJECT))
     def test_is_leader(self):
         self.assertTrue(self.p.is_leader())
+        with patch.object(Postgresql, '_query', Mock(side_effect=RetryFailedError(''))):
+            self.assertRaises(PostgresConnectionException, self.p.is_leader)
 
     def test_reload(self):
         self.assertTrue(self.p.reload())
@@ -351,12 +472,13 @@ class TestPostgresql(unittest.TestCase):
         self.assertFalse(self.p.is_healthy())
 
     def test_promote(self):
-        self.p._role = 'replica'
+        self.p.set_role('replica')
         self.assertTrue(self.p.promote())
         self.assertTrue(self.p.promote())
 
     def test_last_operation(self):
-        self.assertEquals(self.p.last_operation(), '0')
+        self.assertEquals(self.p.last_operation(), '2')
+        Thread(target=self.p.last_operation).start()
 
     @patch('os.path.isfile', Mock(return_value=True))
     @patch('os.kill', Mock(side_effect=Exception))
@@ -367,9 +489,12 @@ class TestPostgresql(unittest.TestCase):
     def test_is_running(self):
         self.assertFalse(self.p.is_running())
 
-    @patch('subprocess.Popen', Mock(side_effect=OSError))
+    @patch('shlex.split', Mock(side_effect=OSError))
     def test_call_nowait(self):
-        self.assertFalse(self.p.call_nowait('on_start'))
+        self.p.set_role('replica')
+        self.assertIsNone(self.p.call_nowait('on_start'))
+        self.p.bootstrapping = True
+        self.assertIsNone(self.p.call_nowait('on_start'))
 
     def test_non_existing_callback(self):
         self.assertFalse(self.p.call_nowait('foobar'))
@@ -380,9 +505,6 @@ class TestPostgresql(unittest.TestCase):
         self.p.query = Mock(side_effect=psycopg2.OperationalError("not supported"))
         self.assertTrue(self.p.stop())
 
-    def test_check_replication_lag(self):
-        self.assertTrue(self.p.check_replication_lag(0))
-
     @patch('os.rename', Mock())
     @patch('os.path.isdir', Mock(return_value=True))
     def test_move_data_directory(self):
@@ -393,16 +515,95 @@ class TestPostgresql(unittest.TestCase):
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def test_bootstrap(self):
         with patch('subprocess.call', Mock(return_value=1)):
-            self.assertRaises(PostgresException, self.p.bootstrap, {})
+            self.assertFalse(self.p.bootstrap({}))
 
-        self.p.bootstrap({'users': {'replicator': {'password': 'rep-pass', 'options': ['replication']}},
-                          'pg_hba': ['host replication replicator 127.0.0.1/32 md5',
-                                     'hostssl all all 0.0.0.0/0 md5',
-                                     'host all all 0.0.0.0/0 md5']})
+        config = {'users': {'replicator': {'password': 'rep-pass', 'options': ['replication']}}}
+
+        self.p.bootstrap(config)
+        with open(os.path.join(self.config_dir, 'pg_hba.conf')) as f:
+            lines = f.readlines()
+            self.assertTrue('host all all 0.0.0.0/0 md5\n' in lines)
+
+        self.p.config.pop('pg_hba')
+        config.update({'post_init': '/bin/false',
+                       'pg_hba': ['host replication replicator 127.0.0.1/32 md5',
+                                  'hostssl all all 0.0.0.0/0 md5',
+                                  'host all all 0.0.0.0/0 md5']})
+        self.p.bootstrap(config)
         with open(os.path.join(self.data_dir, 'pg_hba.conf')) as f:
             lines = f.readlines()
-            assert 'host replication replicator 127.0.0.1/32 md5\n' in lines
-            assert 'host all all 0.0.0.0/0 md5\n' in lines
+            self.assertTrue('host replication replicator 127.0.0.1/32 md5\n' in lines)
+
+    def test_custom_bootstrap(self):
+        config = {'method': 'foo', 'foo': {'command': 'bar'}}
+        with patch('subprocess.call', Mock(return_value=1)):
+            self.assertFalse(self.p.bootstrap(config))
+        with patch('subprocess.call', Mock(side_effect=Exception)):
+            self.assertFalse(self.p.bootstrap(config))
+        with patch('subprocess.call', Mock(return_value=0)),\
+                patch('subprocess.Popen', Mock(side_effect=Exception("42"))),\
+                patch('os.path.isfile', Mock(return_value=True)),\
+                patch('os.unlink', Mock()),\
+                patch.object(Postgresql, 'save_configuration_files', Mock()),\
+                patch.object(Postgresql, 'restore_configuration_files', Mock()),\
+                patch.object(Postgresql, 'write_recovery_conf', Mock()):
+            with self.assertRaises(Exception) as e:
+                self.p.bootstrap(config)
+            self.assertEqual(str(e.exception), '42')
+
+            config['foo']['recovery_conf'] = {'foo': 'bar'}
+
+            with self.assertRaises(Exception) as e:
+                self.p.bootstrap(config)
+            self.assertEqual(str(e.exception), '42')
+
+    @patch('time.sleep', Mock())
+    @patch('os.unlink', Mock())
+    @patch.object(Postgresql, 'run_bootstrap_post_init', Mock(return_value=True))
+    @patch.object(Postgresql, '_custom_bootstrap', Mock(return_value=True))
+    @patch.object(Postgresql, 'start', Mock(return_value=True))
+    def test_post_bootstrap(self):
+        config = {'method': 'foo', 'foo': {'command': 'bar'}}
+        self.p.bootstrap(config)
+
+        task = CriticalTask()
+        with patch.object(Postgresql, 'create_or_update_role', Mock(side_effect=Exception)):
+            self.p.post_bootstrap({}, task)
+            self.assertFalse(task.result)
+
+        self.p.config.pop('pg_hba')
+        self.p.post_bootstrap({}, task)
+        self.assertTrue(task.result)
+
+        self.p.bootstrap(config)
+        self.p.set_state('stopped')
+        self.p.reload_config({'authentication': {'superuser': {'username': 'p', 'password': 'p'},
+                                                 'replication': {'username': 'r', 'password': 'r'}},
+                              'listen': '*', 'retry_timeout': 10, 'parameters': {'hba_file': 'foo'}})
+        with patch.object(Postgresql, 'restart', Mock()) as mock_restart:
+            self.p.post_bootstrap({}, task)
+            mock_restart.assert_called_once()
+
+    def test_run_bootstrap_post_init(self):
+        with patch('subprocess.call', Mock(return_value=1)):
+            self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+
+        with patch('subprocess.call', Mock(side_effect=OSError)):
+            self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+
+        with patch('subprocess.call', Mock(return_value=0)) as mock_method:
+            self.p._superuser.pop('username')
+            self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+            mock_method.assert_called()
+            args, kwargs = mock_method.call_args
+            self.assertTrue('PGPASSFILE' in kwargs['env'])
+            self.assertEquals(args[0], ['/bin/false', 'postgres://127.0.0.2:5432/postgres'])
+
+            mock_method.reset_mock()
+            self.p._local_address.pop('host')
+            self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+            mock_method.assert_called()
+            self.assertEquals(mock_method.call_args[0][0], ['/bin/false', 'postgres://:5432/postgres'])
 
     @patch('patroni.postgresql.Postgresql.create_replica', Mock(return_value=0))
     def test_clone(self):
@@ -434,65 +635,6 @@ class TestPostgresql(unittest.TestCase):
         with patch('subprocess.check_output', Mock(side_effect=subprocess.CalledProcessError(1, ''))):
             self.assertEquals(self.p.controldata(), {})
 
-    def test_read_postmaster_opts(self):
-        m = mock_open(read_data=postmaster_opts_string())
-        with patch.object(builtins, 'open', m):
-            data = self.p.read_postmaster_opts()
-            self.assertEquals(data['wal_level'], 'hot_standby')
-            self.assertEquals(int(data['max_replication_slots']), 5)
-            self.assertEqual(data.get('D'), None)
-
-            m.side_effect = IOError
-            data = self.p.read_postmaster_opts()
-            self.assertEqual(data, dict())
-
-    @patch('subprocess.Popen')
-    @patch.object(builtins, 'open', MagicMock(return_value=42))
-    def test_single_user_mode(self, subprocess_popen_mock):
-        subprocess_popen_mock.return_value.wait.return_value = 0
-        self.assertEquals(self.p.single_user_mode(options=dict(archive_mode='on', archive_command='false')), 0)
-        subprocess_popen_mock.assert_called_once_with(['postgres', '--single', '-D', self.data_dir,
-                                                      '-c', 'archive_command=false', '-c', 'archive_mode=on',
-                                                       'postgres'], stdin=subprocess.PIPE,
-                                                      stdout=42,
-                                                      stderr=subprocess.STDOUT)
-        subprocess_popen_mock.reset_mock()
-        self.assertEquals(self.p.single_user_mode(command="CHECKPOINT"), 0)
-        subprocess_popen_mock.assert_called_once_with(['postgres', '--single', '-D', self.data_dir,
-                                                      'postgres'], stdin=subprocess.PIPE,
-                                                      stdout=42,
-                                                      stderr=subprocess.STDOUT)
-        subprocess_popen_mock.return_value = None
-        self.assertEquals(self.p.single_user_mode(), 1)
-
-    @patch('os.listdir', MagicMock(side_effect=fake_listdir))
-    @patch('os.unlink', return_value=True)
-    @patch('os.remove', return_value=True)
-    @patch('os.path.islink', return_value=False)
-    @patch('os.path.isfile', return_value=True)
-    def test_cleanup_archive_status(self, mock_file, mock_link, mock_remove, mock_unlink):
-        ap = os.path.join(self.data_dir, 'pg_xlog', 'archive_status/')
-        self.p.cleanup_archive_status()
-        mock_remove.assert_has_calls([mock.call(ap + 'a'), mock.call(ap + 'b'), mock.call(ap + 'c')])
-        mock_unlink.assert_not_called()
-
-        mock_remove.reset_mock()
-
-        mock_file.return_value = False
-        mock_link.return_value = True
-        self.p.cleanup_archive_status()
-        mock_unlink.assert_has_calls([mock.call(ap + 'a'), mock.call(ap + 'b'), mock.call(ap + 'c')])
-        mock_remove.assert_not_called()
-
-        mock_unlink.reset_mock()
-        mock_remove.reset_mock()
-
-        mock_file.side_effect = OSError
-        mock_link.side_effect = OSError
-        self.p.cleanup_archive_status()
-        mock_unlink.assert_not_called()
-        mock_remove.assert_not_called()
-
     @patch('patroni.postgresql.Postgresql._version_file_exists', Mock(return_value=True))
     @patch('subprocess.check_output', MagicMock(return_value=0, side_effect=pg_controldata_string))
     def test_sysid(self):
@@ -527,24 +669,217 @@ class TestPostgresql(unittest.TestCase):
     def test_reload_config(self):
         parameters = self._PARAMETERS.copy()
         parameters.pop('f.oo')
-        self.p.reload_config({'retry_timeout': 10, 'listen': '*', 'parameters': parameters})
+        config = {'pg_hba': [''], 'use_unix_socket': True, 'authentication': {},
+                  'retry_timeout': 10, 'listen': '*', 'parameters': parameters}
+        self.p.reload_config(config)
         parameters['b.ar'] = 'bar'
-        self.p.reload_config({'retry_timeout': 10, 'listen': '*', 'parameters': parameters})
+        self.p.reload_config(config)
         parameters['autovacuum'] = 'on'
-        self.p.reload_config({'retry_timeout': 10, 'listen': '*', 'parameters': parameters})
+        self.p.reload_config(config)
         parameters['autovacuum'] = 'off'
         parameters.pop('search_path')
-        self.p.reload_config({'retry_timeout': 10, 'listen': '*:5433', 'parameters': parameters})
+        config['listen'] = '*:5433'
+        self.p.reload_config(config)
+        parameters['unix_socket_directories'] = '.'
+        self.p.reload_config(config)
+        self.p.resolve_connection_addresses()
 
     @patch.object(Postgresql, '_version_file_exists', Mock(return_value=True))
     def test_get_major_version(self):
         with patch.object(builtins, 'open', mock_open(read_data='9.4')):
-            self.assertEquals(self.p.get_major_version(), 9.4)
+            self.assertEquals(self.p.get_major_version(), 90400)
         with patch.object(builtins, 'open', Mock(side_effect=Exception)):
-            self.assertEquals(self.p.get_major_version(), 0.0)
+            self.assertEquals(self.p.get_major_version(), 0)
 
     def test_postmaster_start_time(self):
         with patch.object(MockCursor, "fetchone", Mock(return_value=('foo', True, '', '', '', '', False))):
             self.assertEqual(self.p.postmaster_start_time(), 'foo')
         with patch.object(MockCursor, "execute", side_effect=psycopg2.Error):
             self.assertIsNone(self.p.postmaster_start_time())
+
+    def test_check_for_startup(self):
+        with patch('subprocess.call', return_value=0):
+            self.p._state = 'starting'
+            self.assertFalse(self.p.check_for_startup())
+            self.assertEquals(self.p.state, 'running')
+
+        with patch('subprocess.call', return_value=1):
+            self.p._state = 'starting'
+            self.assertTrue(self.p.check_for_startup())
+            self.assertEquals(self.p.state, 'starting')
+
+        with patch('subprocess.call', return_value=2):
+            self.p._state = 'starting'
+            self.assertFalse(self.p.check_for_startup())
+            self.assertEquals(self.p.state, 'start failed')
+
+        with patch('subprocess.call', return_value=0):
+            self.p._state = 'running'
+            self.assertFalse(self.p.check_for_startup())
+            self.assertEquals(self.p.state, 'running')
+
+        with patch('subprocess.call', return_value=127):
+            self.p._state = 'running'
+            self.assertFalse(self.p.check_for_startup())
+            self.assertEquals(self.p.state, 'running')
+
+            self.p._state = 'starting'
+            self.assertFalse(self.p.check_for_startup())
+            self.assertEquals(self.p.state, 'running')
+
+    def test_wait_for_startup(self):
+        state = {'sleeps': 0, 'num_rejects': 0, 'final_return': 0}
+
+        def increment_sleeps(*args):
+            print("Sleep")
+            state['sleeps'] += 1
+
+        def isready_return(*args):
+            ret = 1 if state['sleeps'] < state['num_rejects'] else state['final_return']
+            print("Isready {0} {1}".format(ret, state))
+            return ret
+
+        def time_in_state(*args):
+            return state['sleeps']
+
+        with patch('subprocess.call', side_effect=isready_return):
+            with patch('time.sleep', side_effect=increment_sleeps):
+                self.p.time_in_state = Mock(side_effect=time_in_state)
+
+                self.p._state = 'stopped'
+                self.assertTrue(self.p.wait_for_startup())
+                self.assertEquals(state['sleeps'], 0)
+
+                self.p._state = 'starting'
+                state['num_rejects'] = 5
+                self.assertTrue(self.p.wait_for_startup())
+                self.assertEquals(state['sleeps'], 5)
+
+                self.p._state = 'starting'
+                state['sleeps'] = 0
+                state['final_return'] = 2
+                self.assertFalse(self.p.wait_for_startup())
+
+                self.p._state = 'starting'
+                state['sleeps'] = 0
+                state['final_return'] = 0
+                self.assertFalse(self.p.wait_for_startup(timeout=2))
+                self.assertEquals(state['sleeps'], 3)
+
+    def test_read_pid_file(self):
+        pidfile = os.path.join(self.data_dir, 'postmaster.pid')
+        if os.path.exists(pidfile):
+            os.remove(pidfile)
+        self.assertEquals(self.p.read_pid_file(), {})
+
+    @patch('os.kill')
+    def test_is_pid_running(self, mock_kill):
+        mock_kill.return_value = True
+        self.assertTrue(self.p.is_pid_running(-100))
+        self.assertFalse(self.p.is_pid_running(0))
+        self.assertFalse(self.p.is_pid_running(None))
+
+    def test_pick_sync_standby(self):
+        cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None,
+                          SyncState(0, self.me.name, self.leadermem.name))
+
+        with patch.object(Postgresql, "query", return_value=[
+                    (self.leadermem.name, 'streaming', 'sync'),
+                    (self.me.name, 'streaming', 'async'),
+                    (self.other.name, 'streaming', 'async'),
+                ]):
+            self.assertEquals(self.p.pick_synchronous_standby(cluster), (self.leadermem.name, True))
+
+        with patch.object(Postgresql, "query", return_value=[
+                    (self.me.name, 'streaming', 'async'),
+                    (self.leadermem.name, 'streaming', 'potential'),
+                    (self.other.name, 'streaming', 'async'),
+                ]):
+            self.assertEquals(self.p.pick_synchronous_standby(cluster), (self.leadermem.name, False))
+
+        with patch.object(Postgresql, "query", return_value=[
+                    (self.me.name, 'streaming', 'async'),
+                    (self.other.name, 'streaming', 'async'),
+                ]):
+            self.assertEquals(self.p.pick_synchronous_standby(cluster), (self.me.name, False))
+
+        with patch.object(Postgresql, "query", return_value=[
+                    ('missing', 'streaming', 'sync'),
+                    (self.me.name, 'streaming', 'async'),
+                    (self.other.name, 'streaming', 'async'),
+                ]):
+            self.assertEquals(self.p.pick_synchronous_standby(cluster), (self.me.name, False))
+
+        with patch.object(Postgresql, "query", return_value=[]):
+            self.assertEquals(self.p.pick_synchronous_standby(cluster), (None, False))
+
+    def test_set_sync_standby(self):
+        def value_in_conf():
+            with open(os.path.join(self.data_dir, 'postgresql.conf')) as f:
+                for line in f:
+                    if line.startswith('synchronous_standby_names'):
+                        return line.strip()
+
+        mock_reload = self.p.reload = Mock()
+        self.p.set_synchronous_standby('n1')
+        self.assertEquals(value_in_conf(), "synchronous_standby_names = 'n1'")
+        mock_reload.assert_called()
+
+        mock_reload.reset_mock()
+        self.p.set_synchronous_standby('n1')
+        mock_reload.assert_not_called()
+        self.assertEquals(value_in_conf(), "synchronous_standby_names = 'n1'")
+
+        self.p.set_synchronous_standby('n2')
+        mock_reload.assert_called()
+        self.assertEquals(value_in_conf(), "synchronous_standby_names = 'n2'")
+
+        mock_reload.reset_mock()
+        self.p.set_synchronous_standby(None)
+        mock_reload.assert_called()
+        self.assertEquals(value_in_conf(), None)
+
+    def test_get_server_parameters(self):
+        config = {'synchronous_mode': True, 'parameters': {'wal_level': 'hot_standby'}, 'listen': '0'}
+        self.p.get_server_parameters(config)
+        config['synchronous_mode_strict'] = True
+        self.p.get_server_parameters(config)
+        self.p.set_synchronous_standby('foo')
+        self.p.get_server_parameters(config)
+
+    @patch.object(Postgresql, 'read_pid_file', Mock(return_value={'pid': 'z'}))
+    def test_get_pid(self):
+        self.p.get_pid()
+
+    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
+    @patch.object(Postgresql, '_signal_postmaster_stop', Mock(return_value=(123, None)))
+    @patch.object(Postgresql, 'get_pid', Mock(return_value=123))
+    @patch('time.sleep', Mock())
+    @patch.object(Postgresql, 'is_pid_running')
+    def test__wait_for_connection_close(self, mock_is_pid_running):
+        mock_is_pid_running.side_effect = [True, False, False]
+        mock_callback = Mock()
+        self.p.stop(on_safepoint=mock_callback)
+
+        mock_is_pid_running.side_effect = [True, False, False]
+        with patch.object(MockCursor, "execute", Mock(side_effect=psycopg2.Error)):
+            self.p.stop(on_safepoint=mock_callback)
+
+    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
+    @patch.object(Postgresql, '_signal_postmaster_stop', Mock(return_value=(123, None)))
+    @patch.object(Postgresql, 'get_pid', Mock(return_value=123))
+    @patch.object(Postgresql, 'is_pid_running', Mock(return_value=False))
+    @patch('psutil.Process')
+    def test__wait_for_user_backends_to_close(self, mock_psutil):
+        child = Mock()
+        child.cmdline.return_value = ['foo']
+        mock_psutil.return_value.children.return_value = [child]
+        mock_callback = Mock()
+        self.p.stop(on_safepoint=mock_callback)
+
+    @patch('os.kill', Mock(side_effect=[OSError(errno.ESRCH, ''), OSError]))
+    @patch('time.sleep', Mock())
+    @patch.object(Postgresql, 'is_pid_running', Mock(side_effect=[True, False]))
+    def test_terminate_starting_postmaster(self):
+        self.p.terminate_starting_postmaster(123)
+        self.p.terminate_starting_postmaster(123)

tests/test_utils.py

@@ -2,25 +2,13 @@ import unittest
 
 from mock import Mock, patch
 from patroni.exceptions import PatroniException
-from patroni.utils import reap_children, Retry, RetryFailedError, sigchld_handler, sleep
-
-
-def time_sleep(_):
-    sigchld_handler(None, None)
+from patroni.utils import Retry, RetryFailedError, polling_loop
 
 
 class TestUtils(unittest.TestCase):
 
-    @patch('time.sleep', Mock())
-    def test_reap_children(self):
-        self.assertIsNone(reap_children())
-        with patch('os.waitpid', Mock(return_value=(0, 0))):
-            sigchld_handler(None, None)
-            self.assertIsNone(reap_children())
-
-    @patch('time.sleep', time_sleep)
-    def test_sleep(self):
-        self.assertIsNone(sleep(0.01))
+    def test_polling_loop(self):
+        self.assertEquals(list(polling_loop(0.001, interval=0.001)), [0])
 
 
 @patch('time.sleep', Mock())

tests/test_wale_restore.py

@@ -2,61 +2,130 @@ import psycopg2
 import subprocess
 import unittest
 
-from mock import MagicMock, patch, PropertyMock
-from patroni.scripts.wale_restore import WALERestore, main as _main
+from mock import Mock, PropertyMock, patch, mock_open
+from patroni.scripts import wale_restore
+from patroni.scripts.wale_restore import WALERestore, main as _main, get_major_version
+from six.moves import builtins
+from test_postgresql import MockConnect, psycopg2_connect
 
 
-wale_output = b'name last_modified expanded_size_bytes wal_segment_backup_start ' +\
-              b'wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop\n' +\
-              b'base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 ' +\
-              b'00000001000000000000007F 00000040 00000001000000000000007F 00000240\n'
+wale_output_header = (
+    b'name\tlast_modified\t'
+    b'expanded_size_bytes\t'
+    b'wal_segment_backup_start\twal_segment_offset_backup_start\t'
+    b'wal_segment_backup_stop\twal_segment_offset_backup_stop\n'
+)
+
+wale_output_values = (
+    b'base_00000001000000000000007F_00000040\t2015-05-18T10:13:25.000Z\t'
+    b'167772160\t'
+    b'00000001000000000000007F\t00000040\t'
+    b'00000001000000000000007F\t00000240\n'
+)
+
+wale_output = wale_output_header + wale_output_values
+
+wale_restore.RETRY_SLEEP_INTERVAL = 0.001  # Speed up retries
+WALE_TEST_RETRIES = 2
 
 
-@patch('os.access', MagicMock(return_value=True))
-@patch('os.makedirs', MagicMock(return_value=True))
-@patch('os.path.exists', MagicMock(return_value=True))
-@patch('os.path.isdir', MagicMock(return_value=True))
-@patch('psycopg2.extensions.cursor', MagicMock(autospec=True))
-@patch('psycopg2.extensions.connection', MagicMock(autospec=True))
-@patch('psycopg2.connect', MagicMock(autospec=True))
-@patch('subprocess.check_output', MagicMock(return_value=wale_output))
+@patch('os.access', Mock(return_value=True))
+@patch('os.makedirs', Mock(return_value=True))
+@patch('os.path.exists', Mock(return_value=True))
+@patch('os.path.isdir', Mock(return_value=True))
+@patch('psycopg2.connect', psycopg2_connect)
+@patch('subprocess.check_output', Mock(return_value=wale_output))
 class TestWALERestore(unittest.TestCase):
 
     def setUp(self):
-        self.wale_restore = WALERestore("batman", "/data", "host=batman port=5432 user=batman", "/etc", 100, 100, 1, 0)
+        self.wale_restore = WALERestore('batman', '/data', 'host=batman port=5432 user=batman',
+                                        '/etc', 100, 100, 1, 0, WALE_TEST_RETRIES)
 
     def test_should_use_s3_to_create_replica(self):
-        with patch('psycopg2.connect', MagicMock(side_effect=psycopg2.Error("foo"))):
-            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output', MagicMock(side_effect=subprocess.CalledProcessError(1, "cmd", "foo"))):
-            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output', MagicMock(return_value=wale_output.split(b'\n')[0])):
-            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output',
-                   MagicMock(return_value=wale_output.replace(b' wal_segment_offset_backup_stop', b''))):
-            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output',
-                   MagicMock(return_value=wale_output.replace(b'expanded_size_bytes', b'expanded_size_foo'))):
+        self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
+        with patch.object(MockConnect, 'server_version', PropertyMock(return_value=100000)):
+            self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
+
+        with patch('subprocess.check_output', Mock(return_value=wale_output.replace(b'167772160', b'1'))):
             self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
 
-        self.wale_restore.should_use_s3_to_create_replica()
-        self.wale_restore.no_master = 1
-        self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
+        with patch('psycopg2.connect', Mock(side_effect=psycopg2.Error("foo"))):
+            save_no_master = self.wale_restore.no_master
+            save_master_connection = self.wale_restore.master_connection
+
+            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
+
+            with patch('time.sleep', Mock(return_value=None)) as mock_sleep:
+                self.wale_restore.no_master = 1
+                assert self.wale_restore.should_use_s3_to_create_replica()
+                # verify retries
+                mock_sleep.assert_has_calls(
+                    [((wale_restore.RETRY_SLEEP_INTERVAL,),)] * WALE_TEST_RETRIES
+                )
+
+            self.wale_restore.master_connection = ''
+            self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
+
+            self.wale_restore.no_master = save_no_master
+            self.wale_restore.master_connection = save_master_connection
+
+        with patch('subprocess.check_output', Mock(side_effect=subprocess.CalledProcessError(1, "cmd", "foo"))):
+            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
+        with patch('subprocess.check_output', Mock(return_value=wale_output_header)):
+            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
+        with patch('subprocess.check_output', Mock(return_value=wale_output + wale_output_values)):
+            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
+        with patch('subprocess.check_output',
+                   Mock(return_value=wale_output.replace(b'expanded_size_bytes', b'expanded_size_foo'))):
+            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
 
     def test_create_replica_with_s3(self):
-        with patch('subprocess.call', MagicMock(return_value=0)):
+        with patch('subprocess.call', Mock(return_value=0)):
             self.assertEqual(self.wale_restore.create_replica_with_s3(), 0)
-        with patch('subprocess.call', MagicMock(side_effect=Exception("foo"))):
+            with patch.object(self.wale_restore, 'fix_subdirectory_path_if_broken', Mock(return_value=False)):
+                self.assertEqual(self.wale_restore.create_replica_with_s3(), 2)
+
+        with patch('subprocess.call', Mock(side_effect=Exception("foo"))):
             self.assertEqual(self.wale_restore.create_replica_with_s3(), 1)
 
     def test_run(self):
-        with patch.object(self.wale_restore, 'init_error', PropertyMock(return_value=True)):
-            self.assertEqual(self.wale_restore.run(), 2)
-        with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', MagicMock(return_value=True)):
-            with patch.object(self.wale_restore, 'create_replica_with_s3', MagicMock(return_value=0)):
+        self.wale_restore.init_error = True
+        self.assertEqual(self.wale_restore.run(), 2)  # this would do 2 retries 1 sec each
+        self.wale_restore.init_error = False
+        with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(return_value=True)):
+            with patch.object(self.wale_restore, 'create_replica_with_s3', Mock(return_value=0)):
                 self.assertEqual(self.wale_restore.run(), 0)
+        with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(return_value=False)):
+            self.assertEqual(self.wale_restore.run(), 2)
+        with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(return_value=None)):
+            self.assertEqual(self.wale_restore.run(), 1)
+        with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(side_effect=Exception)):
+            self.assertEqual(self.wale_restore.run(), 2)
 
-    @patch('sys.exit', MagicMock())
-    @patch.object(WALERestore, 'run', MagicMock(return_value=0))
+    @patch('sys.exit', Mock())
     def test_main(self):
-        self.assertEqual(_main(), None)
+        with patch.object(WALERestore, 'run', Mock(return_value=0)):
+            self.assertEqual(_main(), 0)
+
+        with patch.object(WALERestore, 'run', Mock(return_value=1)), \
+                patch('time.sleep', Mock(return_value=None)) as mock_sleep:
+            self.assertEqual(_main(), 1)
+            assert mock_sleep.call_count == WALE_TEST_RETRIES
+
+    @patch('os.path.isfile', Mock(return_value=True))
+    def test_get_major_version(self):
+        with patch.object(builtins, 'open', mock_open(read_data='9.4')):
+            self.assertEqual(get_major_version("data"), 9.4)
+        with patch.object(builtins, 'open', side_effect=OSError):
+            self.assertEqual(get_major_version("data"), 0.0)
+
+    @patch('os.path.islink', Mock(return_value=True))
+    @patch('os.readlink', Mock(return_value="foo"))
+    @patch('os.remove', Mock())
+    @patch('os.mkdir', Mock())
+    def test_fix_subdirectory_path_if_broken(self):
+        with patch('os.path.exists', Mock(return_value=False)):  # overriding the class-wide mock
+            self.assertTrue(self.wale_restore.fix_subdirectory_path_if_broken("data1"))
+            for fn in ('os.remove', 'os.mkdir'):
+                with patch(fn, side_effect=OSError):
+                    self.assertFalse(self.wale_restore.fix_subdirectory_path_if_broken("data3"))

tests/test_watchdog.py

@@ -0,0 +1,206 @@
+import ctypes
+import patroni.watchdog.linux as linuxwd
+import sys
+import unittest
+
+from mock import patch, Mock, PropertyMock
+from patroni.watchdog import Watchdog, WatchdogError
+from patroni.watchdog.base import NullWatchdog
+from patroni.watchdog.linux import LinuxWatchdogDevice
+
+
+class MockDevice(object):
+    def __init__(self, fd, filename, flag):
+        self.fd = fd
+        self.filename = filename
+        self.flag = flag
+        self.timeout = 60
+        self.open = True
+        self.writes = []
+
+
+mock_devices = [None]
+
+
+def mock_open(filename, flag):
+    fd = len(mock_devices)
+    mock_devices.append(MockDevice(fd, filename, flag))
+    return fd
+
+
+def mock_ioctl(fd, op, arg=None, mutate_flag=False):
+    assert 0 < fd < len(mock_devices)
+    dev = mock_devices[fd]
+    sys.stderr.write("Ioctl %d %d %r\n" % (fd, op, arg))
+    if op == linuxwd.WDIOC_GETSUPPORT:
+        sys.stderr.write("Get support\n")
+        assert(mutate_flag is True)
+        arg.options = sum(map(linuxwd.WDIOF.get, ['SETTIMEOUT', 'KEEPALIVEPING']))
+        arg.identity = (ctypes.c_ubyte*32)(*map(ord, 'Mock Watchdog'))
+    elif op == linuxwd.WDIOC_GETTIMEOUT:
+        arg.value = dev.timeout
+    elif op == linuxwd.WDIOC_SETTIMEOUT:
+        sys.stderr.write("Set timeout called with %s\n" % arg.value)
+        assert 0 < arg.value < 65535
+        dev.timeout = arg.value - 1
+    else:
+        raise Exception("Unknown op %d", op)
+    return 0
+
+
+def mock_write(fd, string):
+    assert 0 < fd < len(mock_devices)
+    assert len(string) == 1
+    assert mock_devices[fd].open
+    mock_devices[fd].writes.append(string)
+
+
+def mock_close(fd):
+    assert 0 < fd < len(mock_devices)
+    assert mock_devices[fd].open
+    mock_devices[fd].open = False
+
+
+@patch('os.open', mock_open)
+@patch('os.write', mock_write)
+@patch('os.close', mock_close)
+@patch('fcntl.ioctl', mock_ioctl)
+class TestWatchdog(unittest.TestCase):
+    def setUp(self):
+        mock_devices[:] = [None]
+
+    @patch('platform.system', Mock(return_value='Linux'))
+    @patch.object(LinuxWatchdogDevice, 'can_be_disabled', PropertyMock(return_value=True))
+    def test_unsafe_timeout_disable_watchdog_and_exit(self):
+        watchdog = Watchdog({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'required', 'safety_margin': -1}})
+        self.assertEquals(watchdog.activate(), False)
+        self.assertEquals(watchdog.is_running, False)
+
+    @patch('platform.system', Mock(return_value='Linux'))
+    @patch.object(LinuxWatchdogDevice, 'get_timeout', Mock(return_value=16))
+    def test_timeout_does_not_ensure_safe_termination(self):
+        Watchdog({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'auto', 'safety_margin': -1}}).activate()
+        self.assertEquals(len(mock_devices), 2)
+
+    @patch('platform.system', Mock(return_value='Linux'))
+    @patch.object(Watchdog, 'is_running', PropertyMock(return_value=False))
+    def test_watchdog_not_activated(self):
+        self.assertFalse(Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'required'}}).activate())
+
+    @patch('platform.system', Mock(return_value='Linux'))
+    @patch.object(LinuxWatchdogDevice, 'is_running', PropertyMock(return_value=False))
+    def test_watchdog_activate(self):
+        with patch.object(LinuxWatchdogDevice, 'open', Mock(side_effect=WatchdogError(''))):
+            self.assertTrue(Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'auto'}}).activate())
+        self.assertFalse(Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'required'}}).activate())
+
+    @patch('platform.system', Mock(return_value='Linux'))
+    def test_basic_operation(self):
+        watchdog = Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'required'}})
+        watchdog.activate()
+
+        self.assertEquals(len(mock_devices), 2)
+        device = mock_devices[-1]
+        self.assertTrue(device.open)
+
+        self.assertEquals(device.timeout, 24)
+
+        watchdog.keepalive()
+        self.assertEquals(len(device.writes), 1)
+
+        watchdog.disable()
+        self.assertFalse(device.open)
+        self.assertEquals(device.writes[-1], b'V')
+
+    def test_invalid_timings(self):
+        watchdog = Watchdog({'ttl': 30, 'loop_wait': 20, 'watchdog': {'mode': 'automatic', 'safety_margin': -1}})
+        watchdog.activate()
+        self.assertEquals(len(mock_devices), 1)
+        self.assertFalse(watchdog.is_running)
+
+    def test_parse_mode(self):
+        with patch('patroni.watchdog.base.logger.warning', new_callable=Mock()) as warning_mock:
+            watchdog = Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'bad'}})
+            self.assertEquals(watchdog.config.mode, 'off')
+            warning_mock.assert_called_once()
+
+    @patch('platform.system', Mock(return_value='Unknown'))
+    def test_unsupported_platform(self):
+        self.assertRaises(SystemExit, Watchdog, {'ttl': 30, 'loop_wait': 10,
+                                                 'watchdog': {'mode': 'required', 'driver': 'bad'}})
+
+    def test_exceptions(self):
+        wd = Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'bad'}})
+        wd.impl.close = wd.impl.keepalive = Mock(side_effect=WatchdogError(''))
+        self.assertIsNone(wd.disable())
+        self.assertIsNone(wd.keepalive())
+
+    def test_config_reload(self):
+        watchdog = Watchdog({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'required'}})
+        self.assertTrue(watchdog.activate())
+        self.assertTrue(watchdog.is_running)
+
+        watchdog.reload_config({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'off'}})
+        self.assertFalse(watchdog.is_running)
+
+        watchdog.reload_config({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'required'}})
+        self.assertFalse(watchdog.is_running)
+        watchdog.keepalive()
+        self.assertTrue(watchdog.is_running)
+
+        watchdog.disable()
+        watchdog.reload_config({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'required', 'driver': 'unknown'}})
+        self.assertFalse(watchdog.is_healthy)
+
+        self.assertFalse(watchdog.activate())
+        watchdog.reload_config({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'required'}})
+        self.assertFalse(watchdog.is_running)
+        watchdog.keepalive()
+        self.assertTrue(watchdog.is_running)
+
+        watchdog.reload_config({'ttl': 60, 'loop_wait': 15, 'watchdog': {'mode': 'required'}})
+        watchdog.keepalive()
+
+
+class TestNullWatchdog(unittest.TestCase):
+
+    def test_basics(self):
+        watchdog = NullWatchdog()
+        self.assertTrue(watchdog.can_be_disabled)
+        self.assertRaises(WatchdogError, watchdog.set_timeout, 1)
+        self.assertEquals(watchdog.describe(), 'NullWatchdog')
+        self.assertIsInstance(NullWatchdog.from_config({}), NullWatchdog)
+
+
+class TestLinuxWatchdogDevice(unittest.TestCase):
+
+    def setUp(self):
+        self.impl = LinuxWatchdogDevice.from_config({})
+
+    @patch('os.open', Mock(return_value=3))
+    @patch('os.write', Mock(side_effect=OSError))
+    @patch('fcntl.ioctl', Mock(return_value=0))
+    def test_basics(self):
+        self.impl.open()
+        try:
+            if self.impl.get_support().has_foo:
+                self.assertFail()
+        except Exception as e:
+            self.assertTrue(isinstance(e, AttributeError))
+        self.assertRaises(WatchdogError, self.impl.close)
+        self.assertRaises(WatchdogError, self.impl.keepalive)
+        self.assertRaises(WatchdogError, self.impl.set_timeout, -1)
+
+    @patch('os.open', Mock(return_value=3))
+    @patch('fcntl.ioctl', Mock(return_value=-1))
+    def test__ioctl(self):
+        self.assertRaises(WatchdogError, self.impl.get_support)
+        self.impl.open()
+        self.assertRaises(IOError, self.impl.get_support)
+
+    def test_is_healthy(self):
+        self.assertFalse(self.impl.is_healthy)
+
+    @patch('os.open', Mock(side_effect=OSError))
+    def test_open(self):
+        self.assertRaises(WatchdogError, self.impl.open)

tests/test_zookeeper.py

@@ -50,7 +50,7 @@ class MockKazooClient(Mock):
         if path.startswith('/no_node'):
             raise NoNodeError
         elif path in ['/service/bla/', '/service/test/']:
-            return ['initialize', 'leader', 'members', 'optime', 'failover']
+            return ['initialize', 'leader', 'members', 'optime', 'failover', 'sync']
         return ['foo', 'bar', 'buzz']
 
     def create(self, path, value=b"", acl=None, ephemeral=False, sequence=False, makepath=False):
@@ -78,7 +78,7 @@ class MockKazooClient(Mock):
             raise Exception
         if path == '/service/test/members/bar' and value == b'retry':
             return
-        if path in ('/service/test/failover', '/service/test/config'):
+        if path in ('/service/test/failover', '/service/test/config', '/service/test/sync'):
             if value == b'Exception':
                 raise Exception
             elif value == b'ok':
@@ -203,11 +203,17 @@ class TestZooKeeper(unittest.TestCase):
         self.assertTrue(self.zk.delete_cluster())
 
     def test_watch(self):
-        self.zk.watch(0)
-        self.zk.event.isSet = lambda: True
-        self.zk.watch(0)
+        self.zk.watch(None, 0)
+        self.zk.event.isSet = Mock(return_value=True)
+        self.zk.watch(None, 0)
 
     def test__kazoo_connect(self):
         self.zk._client._retry.deadline = 1
         self.zk._orig_kazoo_connect = Mock(return_value=(0, 0))
         self.zk._kazoo_connect(None, None)
+
+    def test_sync_state(self):
+        self.zk.set_sync_state_value('')
+        self.zk.set_sync_state_value('ok')
+        self.zk.set_sync_state_value('Exception')
+        self.zk.delete_sync_state()

tox.ini

@@ -1,8 +1,2 @@
 [flake8]
 max-line-length=120
-
-[testenv:py27]
-deps = -rrequirements-py2.txt
-
-[testenv:py33]
-deps = -rrequirements-py3.txt