Bumped version to 1.2.5

When all etcd servers refuse connections during watch the call will fail with an exception and will be immediately retried. This creates a huge amount of log spam potentially creating additional issues on top of losing the DCS. This patch takes note if etcd failures are repeating and starting from the second failure will sleep for a second before retrying. It additionally omits the stack trace after the first failure in a streak of failures.

.gitignore

@@ -46,3 +46,8 @@ dummy
 
 pgpass
 scm-source.json
+
+# Sphinx-generated documentation
+docs/build/
+docs/source/_static/
+docs/source/_templates/

.travis.yml

@@ -1,15 +1,12 @@
 sudo: false
+dist: trusty
 language: python
 python:
-- "3.5"
-addons:
-  apt:
-    packages:
-    - postgresql-contrib-9.5
-  postgresql: "9.5"
+- "3.4"  # 2.7 and 3.5 are preinstalled by default
 env:
   global:
-  - ETCDVERSION=2.3.2 ZKVERSION=3.4.6 CONSULVERSION=0.6.4
+  - ETCDVERSION=3.0.17 ZKVERSION=3.4.9 CONSULVERSION=0.7.4
+  - PYVERSIONS="2.7 3.4 3.5"
   matrix:
   - TEST_SUITE="python setup.py"
   - DCS="etcd" TEST_SUITE="behave"
@@ -17,62 +14,97 @@ env:
   - DCS="consul" TEST_SUITE="behave"
 cache:
   directories:
-  - $HOME/virtualenv/python2.7.9
-  - $HOME/virtualenv/python3.4.2
-  - $HOME/virtualenv/python3.5.2
+  - $HOME/mycache
+before_cache:
+- |
+  rm -fr $HOME/mycache/python*
+  for pv in $PYVERSIONS; do
+      if [[ $TEST_SUITE != "behave" || $pv != "3.4" ]]; then
+          fpv=$(basename $(readlink $HOME/virtualenv/python${pv}))
+          mv $HOME/virtualenv/${fpv} $HOME/mycache/${fpv}
+      fi
+  done
 install:
   - |
     set -e
 
     if [[ $TEST_SUITE == "behave" ]]; then
-        if [[ $DCS == "consul" ]]; then
-            curl -L https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip \
-                | gunzip > consul
-            chmod +x consul
-        fi
+        function get_consul() {
+            CC=~/mycache/consul_${CONSULVERSION}
+            if [[ ! -x $CC ]]; then
+                curl -L https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip \
+                    | gunzip > $CC
+                [[ ${PIPESTATUS[0]} == 0 ]] || return 1
+                chmod +x $CC
+            fi
+            ln -s $CC consul
+        }
 
-        if [[ $DCS == "etcd" ]]; then
-            curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
-                | tar xz -C . --strip=1 --wildcards --no-anchored etcd
-        fi
+        function get_etcd() {
+            EC=~/mycache/etcd_${ETCDVERSION}
+            if [[ ! -x $EC ]]; then
+                curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
+                    | tar xz -C . --strip=1 --wildcards --no-anchored etcd
+                [[ ${PIPESTATUS[0]} == 0 ]] || return 1
+                mv etcd $EC
+            fi
+            ln -s $EC etcd
+        }
 
-        if [[ $DCS == "exhibitor" ]]; then
-            curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz
-            mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg
-            zookeeper-${ZKVERSION}/bin/zkServer.sh start
+        function get_exhibitor() {
+            ZC=~/mycache/zookeeper-${ZKVERSION}
+            if [[ ! -d $ZC ]]; then
+                curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz
+                [[ ${PIPESTATUS[0]} == 0 ]] || return 1
+                mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg
+                mv zookeeper-${ZKVERSION} $ZC
+            fi
+            $ZC/bin/zkServer.sh start
             # following lines are 'emulating' exhibitor REST API
             while true; do
                 echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \
                     | nc -l 8181 &> /dev/null
             done&
-        fi
+        }
+
+        attempt_num=1
+        until get_${DCS}; do
+            [[ $attempt_num -ge 3 ]] && exit 1
+            echo "Attempt $attempt_num failed! Trying again in $attempt_num seconds..."
+            sleep $(( attempt_num++ ))
+        done
     fi
 
-    for pv in "2.7" "3.4" "3.5"; do
-        source ~/virtualenv/python${pv}/bin/activate
-        # explicitly install all needed python modules to cache them
-        for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8==2.6.0 mock>=2.0.0 pytest-cov pytest'; do
-            pip install $p
-        done
+    for pv in $PYVERSIONS; do
+        if [[ $TEST_SUITE != "behave" || $pv != "3.4" ]]; then
+            fpv=$(basename $(readlink $HOME/virtualenv/python$pv))
+            if [[ -d ~/mycache/${fpv} ]]; then
+                mv ~/virtualenv/${fpv} ~/virtualenv/${fpv}.bckp
+                mv ~/mycache/${fpv} ~/virtualenv/${fpv}
+            fi
+            source ~/virtualenv/python${pv}/bin/activate
+            # explicitly install all needed python modules to cache them
+            for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8 mock pytest-cov pytest setuptools'; do
+                pip install $p --upgrade
+            done
+        fi
     done
 script:
   - |
-    for pv in "2.7" "3.4" "3.5"; do
+    for pv in $PYVERSIONS; do
         source ~/virtualenv/python${pv}/bin/activate
 
-        if [[ $TEST_SUITE == "behave" ]]; then
-            if [[ $pv != "3.4" ]]; then
-                echo Running acceptance tests using python${pv}
-                if ! PATH=.:$PATH $TEST_SUITE; then
-                    # output all log files when tests are failing
-                    grep . features/output/*/*postgres?.*
-                    exit 1
-                fi
-            fi
-        else
+        if [[ $TEST_SUITE != "behave" ]]; then
             echo Running unit tests using python${pv}
             $TEST_SUITE test
             $TEST_SUITE flake8
+        elif [[ $pv != "3.4" ]]; then
+            echo Running acceptance tests using python${pv}
+            if ! PATH=.:/usr/lib/postgresql/9.6/bin:$PATH $TEST_SUITE; then
+                # output all log files when tests are failing
+                grep . features/output/*/*postgres?.*
+                exit 1
+            fi
         fi
     done
 

Dockerfile

@@ -1,26 +1,27 @@
 ## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine
 ## It has all the necessary components to play/debug with a single node appliance, running etcd
-FROM ubuntu:16.04
-MAINTAINER Feike Steenbergen <feike.steenberge[email protected]>
+FROM postgres:9.6
+MAINTAINER Alexander Kukushkin <alexander.kukushki[email protected]>
 
-RUN echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/01norecommend \
-    && echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/01norecommend
-
-ENV PGVERSION 9.5
-ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:$PATH
-RUN apt-get update -y \
+RUN echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
+    && apt-get update -y \
     && apt-get upgrade -y \
-    && apt-get install -y curl jq haproxy zookeeper postgresql-${PGVERSION} python-psycopg2 python-yaml \
-        python-requests python-six python-click python-dateutil python-tzlocal python-urllib3 \
-        python-dnspython python-pip python-setuptools python-kazoo python-prettytable python \
-    && pip install python-etcd==0.4.3 python-consul==0.6.0 --upgrade \
+    && apt-get install -y curl jq haproxy python-psycopg2 python-yaml python-requests \
+        python-six python-dateutil python-urllib3 python-dnspython \
+        python-pip python-setuptools python-kazoo python-prettytable python-wheel python \
+
+    && pip install python-etcd==0.4.3 python-consul==0.7.0 click tzlocal --upgrade \
+
+    && mkdir -p /home/postgres \
+    && chown postgres:postgres /home/postgres \
+
+    # Clean up
     && apt-get remove -y python-pip python-setuptools \
     && apt-get autoremove -y \
-        # Clean up
     && apt-get clean -y \
     && rm -rf /var/lib/apt/lists/* /root/.cache
 
-ENV ETCDVERSION 2.3.6
+ENV ETCDVERSION 3.1.2
 RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
     | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl
 
@@ -35,8 +36,7 @@ RUN ln -s /patronictl.py /usr/local/bin/patronictl
 
 ### Setting up a simple script that will serve as an entrypoint
 RUN mkdir /data/ && touch /pgpass /patroni.yml \
-    && chown postgres:postgres -R /patroni/ /data/ /pgpass /patroni.yml /etc/haproxy /var/run/ /var/lib/ /var/log/ \
-    && echo 1 > /etc/zookeeper/conf/myid
+    && chown postgres:postgres -R /patroni/ /data/ /pgpass /patroni.yml /etc/haproxy /var/run/ /var/lib/ /var/log/
 
 EXPOSE 2379 5432 8008
 

README.rst

@@ -2,13 +2,17 @@
 
 Patroni: A Template for PostgreSQL HA with ZooKeeper, etcd or Consul
 ------------------------------------------------------------
+
+You can find a version of this documentation that is searchable and also easier to navigate at `patroni.readthedocs.io <https://patroni.readthedocs.io>`__.
+
+
 There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
 
 Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__ or `Consul <https://github.com/hashicorp/consul>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
 
 We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely.
 
-**Note to Kubernetes users**: We're currently developing Patroni to be as useful as possible for teams running Kubernetes on top of Google Compute Engine; Patroni can be the HA solution for Postgres in such an environment. Please contact us via our Issues Tracker if this describes your team's current setup, and we'll follow up.
+**Note to Kubernetes users**: We're currently developing Patroni to be as useful as possible for teams running Kubernetes on top of Google Compute Engine; Patroni can be the HA solution for Postgres in such an environment. To this end, there is a `Helm chart <https://github.com/kubernetes/charts/tree/master/incubator/patroni>`__ that uses Patroni and  `Spilo  <https://github.com/zalando/spilo/>`__ to provision a five-node PostgreSQL HA cluster in a Kubernetes+GCE environment. (The Helm chart deploys Spilo Docker images, not just "bare" Patroni.)
 
 .. contents::
     :local:
@@ -19,12 +23,13 @@ We call Patroni a "template" because it is far from being a one-size-fits-all or
 How Patroni Works
 ==============
 
-Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features. 
+Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features.
 
 For an example of a Docker-based deployment with Patroni, see `Spilo <https://github.com/zalando/spilo>`__, currently in use at Zalando.
 
 For additional background info, see:
 
+* `Elephants on Automatic: HA Clustered PostgreSQL with Helm <https://www.youtube.com/watch?v=CftcVhFMGSY>`_, talk by Josh Berkus and Oleksii Kliukin at KubeCon Berlin 2017 
 * `PostgreSQL HA with Kubernetes and Patroni <https://www.youtube.com/watch?v=iruaCgeG7qs>`__, talk by Josh Berkus at KubeCon 2016 (video)
 * `Feb. 2016 Zalando Tech blog post <https://tech.zalando.de/blog/zalandos-patroni-a-template-for-high-availability-postgresql/>`__
 
@@ -32,7 +37,9 @@ For additional background info, see:
 Development Status
 ================
 
-Patroni is in active development and accepts contributions. See our `Contributing <https://github.com/zalando/patroni/blob/master/README.rst#contributing>`__ section below for more details. 
+Patroni is in active development and accepts contributions. See our `Contributing <https://github.com/zalando/patroni/blob/master/docs/CONTRIBUTING.rst>`__ section below for more details.
+
+We report new releases information `here <https://github.com/zalando/patroni/releases>`__.
 
 ===========================
 Technical Requirements/Installation
@@ -77,7 +84,7 @@ run:
 YAML Configuration
 ===============
 
-Go `here <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__. 
+Go `here <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
 
 =========================
 Environment Configuration
@@ -89,24 +96,7 @@ Go `here <https://github.com/zalando/patroni/blob/master/docs/ENVIRONMENT.rst>`_
 Replication Choices
 ===============
 
-Patroni uses Postgres' streaming replication, which is asynchronous by default. For more information, see the `Postgres documentation on streaming replication <http://www.postgresql.org/docs/current/static/warm-standby.html#STREAMING-REPLICATION>`__.
-
-Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the follower. This setting should be increased or decreased based on business requirements.
-
-When asynchronous replication is not optimal for your use case, investigate Postgres's `synchronous replication <http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION>`__. Synchronous replication ensures consistency across a cluster by confirming that writes are written to a secondary before returning to the connecting client with a success. The cost of synchronous replication: reduced throughput on writes. This throughput will be entirely based on network performance. 
-
-In hosted datacenter environments (like AWS, Rackspace, or any network you do not control), synchronous replication significantly increases the variability of write performance. If followers become inaccessible from the leader, the leader effectively becomes read-only.
-
-To enable a simple synchronous replication test, add the follow lines to the ``parameters`` section of your YAML configuration files:
-
-.. code:: YAML
-
-        synchronous_commit: "on"
-        synchronous_standby_names: "*"
-
-When using synchronous replication, use at least three Postgres data nodes to ensure write availability if one host fails.
-
-Choosing your replication schema is dependent on your business considerations. Investigate both async and sync replication, as well as other HA solutions, to determine which solution is best for you.
+Patroni uses Postgres' streaming replication, which is asynchronous by default. Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the leader. This setting should be increased or decreased based on business requirements. It's also possible to use synchronous replication for better durability guarantees. See `replication modes documentation <https://github.com/zalando/patroni/blob/master/docs/replication_modes.rst>`__ for details.
 
 ===============================
 Applications Should Not Use Superusers
@@ -114,14 +104,6 @@ Applications Should Not Use Superusers
 
 When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
 
-================
-Contributing
-================
-Patroni accepts contributions from the open-source community; see the `Issues Tracker <https://github.com/zalando/patroni/issues>`__ for current needs. 
-
-Before making a contribution, please let us know by posting a comment to the relevant issue. 
-If you would like to propose a new feature, please first file a new issue explaining the feature you'd like to create.
-
 .. |Build Status| image:: https://travis-ci.org/zalando/patroni.svg?branch=master
    :target: https://travis-ci.org/zalando/patroni
 .. |Coverage Status| image:: https://coveralls.io/repos/zalando/patroni/badge.svg?branch=master

docker-compose.yml

@@ -50,8 +50,8 @@ haproxy:
     links:
         - patroni_etcd:patroni_etcd
     ports:
-        - "5000"
-        - "5001"
+        - "5000:5000"
+        - "5001:5001"
     environment:
         PATRONI_ETCD_HOST: patroni_etcd:2379
         PATRONI_SCOPE: testcluster

docs/CONTRIBUTING.rst

@@ -0,0 +1,24 @@
+.. _contributing:
+
+Contributing guidelines
+=======================
+
+Wanna contribute to Patroni? Yay - here is how!
+
+Reporting issues
+----------------
+
+If you have a question about patroni or have a problem using it, please read the :ref:`README <readme>` before filing an issue.
+Also double check with the current issues on our `Issues Tracker <https://github.com/zalando/patroni/issues>`__.
+
+Contributing a pull request
+---------------------------
+
+1) Submit a comment to the relevant issue or create a new issue describing your proposed change.
+2) Do a fork, develop and test your code changes.
+3) Include documentation
+4) Submit a pull request.
+
+You'll get feedback about your pull request as soon as possible.
+
+Happy Patroni hacking ;-)

docs/ENVIRONMENT.rst

@@ -1,3 +1,5 @@
+.. _environment:
+
 ==================================
 Environment Configuration Settings
 ==================================
@@ -27,6 +29,12 @@ Consul
 Etcd
 ----
 -  **PATRONI\_ETCD\_HOST**: the host:port for the etcd endpoint.
+-  **PATRONI\_ETCD\_URL**: url for the etcd, in format: http(s)://(username:password@)host:port
+-  **PATRONI\_ETCD\_PROXY**: proxy url for the etcd. If you are connecting to the etcd using proxy, use this parameter instead of **PATRONI\_ETCD\_URL**
+-  **PATRONI\_ETCD\_SRV**: Domain to search the SRV record(s) for cluster autodiscovery.
+-  **PATRONI\_ETCD\_CACERT**: The ca certificate. If pressent it will enable validation.
+-  **PATRONI\_ETCD\_CERT**: File with the client certificate
+-  **PATRONI\_ETCD\_KEY**: File with the client key. Can be empty if the key is part of certificate.
 
 Exhibitor
 ---------

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = Patroni
+SOURCEDIR     = .
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/README.rst

@@ -0,0 +1,90 @@
+.. _readme:
+
+=================
+How Patroni Works
+=================
+
+Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features.
+
+For an example of a Docker-based deployment with Patroni, see `Spilo <https://github.com/zalando/spilo>`__, currently in use at Zalando.
+
+For additional background info, see:
+
+* `PostgreSQL HA with Kubernetes and Patroni <https://www.youtube.com/watch?v=iruaCgeG7qs>`__, talk by Josh Berkus at KubeCon 2016 (video)
+* `Feb. 2016 Zalando Tech blog post <https://tech.zalando.de/blog/zalandos-patroni-a-template-for-high-availability-postgresql/>`__
+
+==================
+Development Status
+==================
+
+Patroni is in active development and accepts contributions. See our :ref:`Contributing <contributing>` section below for more details.
+
+We report new releases information :ref:`here <releases>`.
+
+===================================
+Technical Requirements/Installation
+===================================
+
+**For Mac**
+
+To install requirements on a Mac, run the following:
+
+::
+
+    brew install postgresql etcd haproxy libyaml python
+    pip install psycopg2 pyyaml
+
+=======================
+Running and Configuring
+=======================
+
+To get started, do the following from different terminals:
+::
+
+    > etcd --data-dir=data/etcd
+    > ./patroni.py postgres0.yml
+    > ./patroni.py postgres1.yml
+
+You will then see a high-availability cluster start up. Test different settings in the YAML files to see how the cluster's behavior changes. Kill some of the components to see how the system behaves.
+
+Add more ``postgres*.yml`` files to create an even larger cluster.
+
+Patroni provides an `HAProxy <http://www.haproxy.org/>`__ configuration, which will give your application a single endpoint for connecting to the cluster's leader. To configure,
+run:
+
+::
+
+    > haproxy -f haproxy.cfg
+
+::
+
+    > psql --host 127.0.0.1 --port 5000 postgres
+
+==================
+YAML Configuration
+==================
+
+Go :ref:`here <settings>` for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
+
+=========================
+Environment Configuration
+=========================
+
+Go :ref:`here <environment>` for comprehensive information about configuring(overriding) settings via environment variables.
+
+===================
+Replication Choices
+===================
+
+Patroni uses Postgres' streaming replication, which is asynchronous by default. Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the leader. This setting should be increased or decreased based on business requirements. It's also possible to use synchronous replication for better durability guarantees. See :ref:`replication modes documentation <replication_modes>` for details.
+
+======================================
+Applications Should Not Use Superusers
+======================================
+
+When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
+
+.. |Build Status| image:: https://travis-ci.org/zalando/patroni.svg?branch=master
+   :target: https://travis-ci.org/zalando/patroni
+.. |Coverage Status| image:: https://coveralls.io/repos/zalando/patroni/badge.svg?branch=master
+   :target: https://coveralls.io/r/zalando/patroni?branch=master

docs/SETTINGS.rst

@@ -1,3 +1,5 @@
+.. _settings:
+
 ===========================
 YAML Configuration Settings
 ===========================
@@ -14,6 +16,8 @@ Bootstrap configuration
     -  **loop\_wait**: the number of seconds the loop will sleep. Default value: 10
     -  **ttl**: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process. Default value: 30
     -  **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election.
+    -  **master\_start\_timeout**: the amount of time a master is allowed to recover from failures before failover is triggered. Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Best worst case failover time for master failure is: loop\_wait + master\_start\_timeout + loop\_wait, unless master\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff.
+    -  **synchronous\_mode**: turns on synchronous replication mode. In this mode a replica will be chosen as synchronous and only the latest leader and synchronous replica are able to participate in leader election. Synchronous mode makes sure that succesfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See `replication modes documentation <https://github.com/zalando/patroni/blob/master/docs/replication_modes.rst>`__ for details.
     -  **postgresql**:
         -  **use\_pg\_rewind**:whether or not to use pg_rewind
         -  **use\_slots**: whether or not to use replication_slots. Must be False for PostgreSQL 9.3. You should comment out max_replication_slots before it becomes ineligible for leader status.
@@ -32,6 +36,7 @@ Bootstrap configuration
         -  **options**: list of options for CREATE USER statement
             -  **- createrole**
             -  **- createdb**
+-  **post_init**: An additional script that will be executed after initializing the cluster. The script receives a connection string URL (with the cluster superuser as a user name). The PGPASSFILE variable is set to the location of pgpass file.
 
 Consul
 ------
@@ -39,7 +44,17 @@ Consul
 
 Etcd
 ----
+Most of the parameters are optional, but you have to specify one of the **host**, **url**, **proxy** or **srv**
 -  **host**: the host:port for the etcd endpoint.
+-  **url**: url for the etcd
+-  **proxy**: proxy url for the etcd. If you are connecting to the etcd using proxy, use this parameter instead of **url**
+-  **srv**: Domain to search the SRV record(s) for cluster autodiscovery.
+-  **protocol**: (optional) http or https, if not specified http is used. If the **url** or **proxy** is specified - will take protocol from them.
+-  **username**: (optional) username for etcd authentication
+-  **password**: (optional) password for etcd authentication.
+-  **cacert**: (optional) The ca certificate. If pressent it will enable validation.
+-  **cert**: (optional) file with the client certificate
+-  **key**: (optional) file with the client key. Can be empty if the key is part of **cert**.
 
 Exhibitor
 ---------
@@ -47,6 +62,8 @@ Exhibitor
 -  **poll\_interval**: how often the list of ZooKeeper and Exhibitor nodes should be updated from Exhibitor
 -  **port**: Exhibitor port.
 
+.. _postgresql_settings:
+
 PostgreSQL
 ----------
 -  **authentication**:
@@ -67,7 +84,7 @@ PostgreSQL
 -  **data\_dir**: The location of the Postgres data directory, either existing or to be initialized by Patroni.
 -  **bin\_dir**: Path to PostgreSQL binaries. (pg_ctl, pg_rewind, pg_basebackup, postgres) The  default value is an empty string meaning that PATH environment variable will be used to find the executables.
 -  **listen**: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
--  **pgpass**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni.
+-  **pgpass**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup, the post_init script and under some other circumstances. The location must be writable by Patroni.
 -  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower.
 -  **custom_conf** : path to an optional custom ``postgresql.conf`` file, that will be used in place of ``postgresql.base.conf``. The file must exist on all cluster nodes, be readable by PostgreSQL and will be included from its location on the real ``postgresql.conf``. Note that Patroni will not monitor this file for changes, nor backup it. However, its settings can still be overriden by Patroni's own configuration facilities - see `dynamic configuration <https://github.com/zalando/patroni/blob/master/docs/dynamic_configuration.rst>`__ for details.
 -  **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work.

docs/_static/custom.css

@@ -0,0 +1,3 @@
+li {
+   margin-bottom: 0.5em
+}

docs/conf.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Patroni documentation build configuration file, created by
+# sphinx-quickstart on Mon Dec 19 16:54:09 2016.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
+    'sphinx.ext.mathjax',
+    'sphinx.ext.ifconfig',
+    'sphinx.ext.viewcode']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'Patroni'
+copyright = '2016, Zalando SE'
+author = 'Zalando SE'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '1.2'
+# The full version, including alpha/beta/rc tags.
+release = '1.2.2'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+if not on_rtd:  # only import and set the theme if we're building docs locally
+    import sphinx_rtd_theme
+    html_theme = 'sphinx_rtd_theme'
+    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Patronidoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'Patroni.tex', 'Patroni Documentation',
+     'Zalando SE', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'patroni', 'Patroni Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'Patroni', 'Patroni Documentation',
+     author, 'Patroni', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+
+# -- Options for Epub output ----------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+epub_author = author
+epub_publisher = author
+epub_copyright = copyright
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {'https://docs.python.org/': None}
+
+# A possibility to have an own stylesheet, to add new rules or override existing ones
+# For the latter case, the CSS specificity of the rules should be higher than the default ones
+def setup(app):
+   app.add_stylesheet("custom.css")

docs/dynamic_configuration.rst

@@ -1,3 +1,5 @@
+.. _dynamic_configuration:
+
 Patroni configuration
 =====================
 
@@ -10,11 +12,11 @@ Patroni configuration is stored in the DCS (Distributed Configuration Store). Th
 	have changed), a special flag, ``pending_restart`` indicating this, is set in the members.data JSON.
 	Additionally, the node status also indicates this, by showing ``"restart_pending": true``.
 
-- Local `configuration <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ (patroni.yml).
+- Local :ref:`configuration <settings>` (patroni.yml).
 	These options are defined in the configuration file and take precedence over dynamic configuration.
 	patroni.yml could be changed and reload in runtime (without restart of Patroni) by sending SIGHUP to the Patroni process or by performing ``POST /reload`` REST-API request.
 
-- Environment `configuration <https://github.com/zalando/patroni/blob/master/docs/ENVIRONMENT.rst>`__ .
+- Environment :ref:`configuration <environment>` .
 	It is possible to set/override some of the "Local" configuration parameters with environment variables.
 	Environment configuration is very useful when you are running in a dynamic environment and you don't know some of the parameters in advance (for example it's not possible to know you external IP address when you are running inside ``docker``).
 

docs/ha_loop_diagram.dot

@@ -0,0 +1,148 @@
+// Graphviz source for ha_loop_diagram.png
+// recompile with:
+// dot -Tpng ha_loop_diagram.dot -o ha_loop_diagram.png
+
+digraph G {
+  rankdir=TB;
+  fontname="sans-serif";
+  penwidth="0.3";
+  layout="dot";
+  newrank=true;
+  edge [fontname="sans-serif", 
+        fontsize=12, 
+        color=black, 
+        fontcolor=black];
+  node [fontname=serif, 
+        fontsize=12, 
+        fillcolor=white, 
+        color=black, 
+        fontcolor=black,
+        style=filled];
+  "start" [label=Start, shape="rectangle", fillcolor="green"]
+  "start" -> "load_cluster_from_dcs";
+  "update_member" [label="Persist node state in DCS"]
+  "update_member" -> "start"
+  
+  subgraph cluster_run_cycle {
+    label="run_cycle"
+    "load_cluster_from_dcs" [label="Load cluster from DCS"];
+    "touch_member" [label="Persist node in DCS"];
+    "cluster.has_member" [shape="diamond", label="Is node registered on DCS?"]
+    "cluster.has_member" -> "touch_member"  [label="no" color="red"]
+    "long_action_in_progress?" [shape="diamond" label="Is the PostgreSQL currently being\nstopping/starting/restarting/reinitializing?"]
+    "load_cluster_from_dcs" -> "cluster.has_member";
+    "touch_member" -> "long_action_in_progress?";
+    "cluster.has_member" -> "long_action_in_progress?"  [label="yes" color="green"];
+    "long_action_in_progress?" -> "recovering?" [label="no" color="red"]
+    "recovering?" [label="Was cluster recovering and failed?", shape="diamond"];
+    "recovering?" -> "post_recover" [label="yes" color="green"];
+    "recovering?" -> "data_directory_empty" [label="no" color="red"];
+    "post_recover" [label="Remove leader key (if I was the leader)"];
+    "data_directory_empty" [label="Is data folder empty?", shape="diamond"];
+    "data_directory_empty" -> "cluster_initialize" [label="no" color="red"];
+    "data_belongs_to_cluster" [label="Does data dir belong to cluster?", shape="diamond"];
+    "data_belongs_to_cluster" -> "exit" [label="no" color="red"];
+    "data_belongs_to_cluster" -> "is_healthy" [label="yes" color="green"]
+    "exit" [label="Fail and exit", fillcolor=red];
+    "cluster_initialize" [label="Is cluster initialized on DCS?" shape="diamond"]
+    "cluster_initialize" -> "cluster.has_leader" [label="no" color="red"]
+    "cluster.has_leader" [label="Does the cluster has leader?", shape="diamond"]
+    "cluster.has_leader" -> "dcs.initialize" [label="no", color="red"]
+    "cluster.has_leader" -> "is_healthy" [label="yes", color="green"]
+    "cluster_initialize" -> "data_belongs_to_cluster" [label="yes" color="green"]
+    "dcs.initialize" [label="Initialize new cluster"];
+    "dcs.initialize" -> "is_healthy"
+    "is_healthy" [label="Is node healthy?\n(running Postgres)", shape="diamond"];
+    "recover" [label="Start as read-only\nand set Recover flag"]
+    "is_healthy" -> "recover" [label="no" color="red"]; 
+    "is_healthy" -> "cluster.is_unlocked" [label="yes" color="green"]; 
+    "cluster.is_unlocked" [label="Does the cluster has a leader?", shape="diamond"]
+  }
+
+  "post_recover" -> "update_member"
+  "recover" -> "update_member"
+  "long_action_in_progress?" -> "async_has_lock?" [label="yes" color="green"];
+  "cluster.is_unlocked" -> "unhealthy_is_healthiest" [label="no" color="red"]
+  "cluster.is_unlocked" -> "healthy_has_lock" [label="yes" color="green"]
+  "data_directory_empty" -> "bootstrap.is_unlocked" [label="yes" color="green"]
+
+  subgraph cluster_async {
+    label = "Long action in progress\n(Start/Stop/Restart/Reinitialize)"
+    "async_has_lock?" [label="Do I have the leader lock?", shape="diamond"]
+    "async_update_lock" [label="Renew leader lock"]
+    "async_has_lock?" -> "async_update_lock" [label="yes" color="green"]
+  }
+  "async_update_lock" -> "update_member"
+  "async_has_lock?" -> "update_member" [label="no" color="red"]
+
+  subgraph cluster_bootstrap {
+    label = "Node bootstrap";
+    "bootstrap.is_unlocked" [label="Does the cluster has a leader?", shape="diamond"]
+    "bootstrap.is_initialized" [label="Does the cluster has an initialize key?", shape="diamond"]
+    "bootstrap.is_unlocked" -> "bootstrap.is_initialized" [label="no" color="red"]
+    "bootstrap.is_unlocked" -> "bootstrap.select_node" [label="yes" color="green"]
+    "bootstrap.select_node" [label="Select a node to take a backup from"]
+    "bootstrap.do_bootstrap" [label="Run pg_basebackup\n(async)"]
+    "bootstrap.select_node" -> "bootstrap.do_bootstrap"
+    "bootstrap.is_initialized" -> "bootstrap.initialization_race" [label="no" color="red"]
+    "bootstrap.is_initialized" -> "bootstrap.wait_for_leader" [label="yes" color="green"]
+    "bootstrap.initialization_race" [label="Race for initialize key"]
+    "bootstrap.initialization_race" -> "bootstrap.won_initialize_race?"
+    "bootstrap.won_initialize_race?" [label="Do I won initialize race?", shape="diamond"]
+    "bootstrap.won_initialize_race?" -> "bootstrap.initdb_and_start" [label="yes" color="green"]
+    "bootstrap.won_initialize_race?" -> "bootstrap.wait_for_leader" [label="no" color="red"]
+    "bootstrap.wait_for_leader" [label="Need to wait for leader key"]
+    "bootstrap.initdb_and_start" [label="Run initdb, start postgres and create roles"]
+    "bootstrap.initdb_and_start" -> "bootstrap.success?"
+    "bootstrap.success?" [label="Success", shape="diamond"]
+    "bootstrap.success?" -> "bootstrap.take_leader_key" [label="yes" color="green"]
+    "bootstrap.success?" -> "bootstrap.clean" [label="no" color="red"]
+    "bootstrap.clean" [label="Remove initialize key from DCS\nand data directory from filesystem"]
+    "bootstrap.take_leader_key" [label="Take a leader key in DCS"]
+  }
+
+  "bootstrap.do_bootstrap" -> "update_member"
+  "bootstrap.wait_for_leader" -> "update_member"
+  "bootstrap.clean" -> "update_member"
+  "bootstrap.take_leader_key" -> "update_member"
+
+  subgraph cluster_process_healthy_cluster {
+    label = "process_healthy_cluster"
+    "healthy_has_lock" [label="Am I the owner of the leader lock?", shape=diamond]
+    "healthy_is_leader" [label="Is Postgres running as master?", shape=diamond]
+    "healthy_no_lock" [label="Follow the leader (async,\ncreate/update recovery.conf and restart if necessary)"]
+    "healthy_has_lock" -> "healthy_no_lock" [label="no" color="red"]
+    "healthy_has_lock" -> "healthy_update_leader_lock" [label="yes" color="green"]
+    "healthy_update_leader_lock" [label="Try to update leader lock"]
+    "healthy_update_leader_lock" -> "healthy_update_success"
+    "healthy_update_success" [label="Success?", shape=diamond]
+    "healthy_update_success" -> "healthy_is_leader" [label="yes" color="green"]
+    "healthy_update_success" -> "healthy_demote" [label="no" color="red"]
+    "healthy_demote" [label="Demote (async,\nrestart in read-only)"]
+    "healthy_failover" [label="Promote Postgres to master"]
+    "healthy_is_leader" -> "healthy_failover" [label="no" color="red"]
+  }
+  "healthy_demote" -> "update_member"
+  "healthy_is_leader" -> "update_member" [label="yes" color="green"]
+  "healthy_failover" -> "update_member"
+  "healthy_no_lock" -> "update_member"
+
+  subgraph cluster_process_unhealthy_cluster {
+    label = "process_unhealthy_cluster"
+    "unhealthy_is_healthiest" [label="Am I the healthiest node?", shape="diamond"]
+    "unhealthy_is_healthiest" -> "unhealthy_leader_race" [label="yes", color="green"]
+    "unhealthy_leader_race" [label="Try to create leader key"]
+    "unhealthy_leader_race" -> "unhealthy_acquire_lock"
+    "unhealthy_acquire_lock" [label="Was I able to get the lock?", shape="diamond"]
+    "unhealthy_is_leader" [label="Is Postgres running as master?", shape=diamond]
+    "unhealthy_acquire_lock" -> "unhealthy_is_leader" [label="yes" color="green"]
+    "unhealthy_is_leader" -> "unhealthy_promote" [label="no" color="red"]
+    "unhealthy_promote" [label="Promote to master"]
+    "unhealthy_is_healthiest" -> "unhealthy_follow" [label="no" color="red"]
+    "unhealthy_follow" [label="try to follow somebody else()"]
+    "unhealthy_acquire_lock" -> "unhealthy_follow" [label="no" color="red"]
+  }
+  "unhealthy_follow" -> "update_member"
+  "unhealthy_promote" -> "update_member"
+  "unhealthy_is_leader" -> "update_member" [label="yes" color="green"]
+}

docs/index.rst

@@ -0,0 +1,36 @@
+.. Patroni documentation master file, created by
+   sphinx-quickstart on Mon Dec 19 16:54:09 2016.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Introduction
+============
+
+Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__ or `Consul <https://github.com/hashicorp/consul>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
+
+We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely. There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
+
+**Note to Kubernetes users**: We're currently developing Patroni to be as useful as possible for teams running Kubernetes on top of Google Compute Engine; Patroni can be the HA solution for Postgres in such an environment. To this end, we've created a `Helm Chart <https://github.com/kubernetes/charts/tree/master/incubator/patroni>`__ that enables you to deploy a five-node Patroni cluster using a Kubernetes PetSet.
+
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   README
+   dynamic_configuration
+   ENVIRONMENT
+   SETTINGS
+   replication_modes
+   pause
+   releases
+   CONTRIBUTING
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+
+

docs/pause.rst

@@ -1,3 +1,5 @@
+.. _pause:
+
 Pause/Resume mode for the cluster
 =================================
 

docs/releases.rst

@@ -0,0 +1,394 @@
+.. _releases:
+
+Release notes
+=============
+
+Version 1.2
+-----------
+
+This version introduces significant improvements over the handling of synchronous replication, makes the startup process and failover more reliable, adds PostgreSQL 9.6 support and fixes plenty of bugs.
+In addition, the documentation, including these release notes, has been moved to https://patroni.readthedocs.io.
+
+**Synchronous replication**
+
+- Add synchronous replication support. (Ants Aasma)
+
+  Adds a new configuration variable ``synchronous_mode``. When enabled, Patroni will manage ``synchronous_standby_names`` to enable synchronous replication whenever there are healthy standbys available. When synchronous mode is enabled, Patroni will automatically fail over only to a standby that was synchronously replicating at the time of the master failure. This effectively means that no user visible transaction gets lost in such a case. See the
+  :ref:`feature documentation <synchronous_mode>` for the detailed description and implementation details.
+
+**Reliability improvements**
+
+- Do not try to update the leader position stored in the ``leader optime`` key when PostgreSQL is not 100% healthy. Demote immediately when the update of the leader key failed. (Alexander Kukushkin)
+
+- Exclude unhealthy nodes from the list of targets to clone the new replica from. (Alexander)
+
+- Implement retry and timeout strategy for Consul similar to how it is done for Etcd. (Alexander)
+
+- Make ``--dcs`` and ``--config-file`` apply to all options in ``patronictl``. (Alexander)
+
+- Write all postgres parameters into postgresql.conf. (Alexander)
+
+  It allows starting PostgreSQL configured by Patroni with just ``pg_ctl``.
+
+- Avoid exceptions when there are no users in the config. (Kirill Pushkin)
+
+- Allow pausing an unhealthy cluster. Before this fix, ``patronictl`` would bail out if the node it tries to execute pause on is unhealthy. (Alexander)
+
+- Improve the leader watch functionality. (Alexander)
+
+  Previously the replicas were always watching the leader key (sleeping until the timeout or the leader key changes). With this change, they only watch
+  when the replica's PostgreSQL is in the ``running`` state and not when it is stopped/starting or restarting PostgreSQL.
+
+- Avoid running into race conditions when handling SIGCHILD as a PID 1. (Alexander)
+
+  Previously a race condition could occur when running inside the Docker containers, since the same process inside Patroni both spawned new processes
+  and handled SIGCHILD from them. This change uses fork/execs for Patroni and leaves the original PID 1 process responsible for handling signals from children.
+
+- Fix WAL-E restore. (Oleksii Kliukin)
+
+  Previously WAL-E restore used the ``no_master`` flag to avoid consulting with the master altogether, making Patroni always choose restoring
+  from WAL over the ``pg_basebackup``. This change reverts it to the original meaning of ``no_master``, namely Patroni WAL-E restore may be selected as a replication method if the master is not running.
+  The latter is checked by examining the connection string passed to the method. In addition, it makes the retry mechanism more robust and handles other minutia.
+
+- Implement asynchronous DNS resolver cache. (Alexander)
+
+  Avoid failing when DNS is temporary unavailable (for instance, due to an excessive traffic received by the node).
+
+- Implement starting state and master start timeout. (Ants, Alexander)
+
+  Previously ``pg_ctl`` waited for a timeout and then happily trodded on considering PostgreSQL to be running. This caused PostgreSQL to show up in listings as running when it was actually not and caused a race condition that   resulted in either a failover, or a crash recovery, or a crash recovery interrupted by failover and a missed rewind.
+  This change adds a ``master_start_timeout`` parameter and introduces a new state for the main HA loop: ``starting``. When ``master_start_timeout`` is 0 we will failover immediately when the master crashes as soon as there is a failover candidate. Otherwise, Patroni will wait after attempting to start PostgreSQL on the master for the duration of the timeout; when it expires, it will failover if possible. Manual failover requests will be honored during the crash of the master even before the timeout expiration.
+
+  Introduce the ``timeout`` parameter to the ``restart`` API endpoint and ``patronictl``. When it is set and restart takes longer than the timeout, PostgreSQL is considered unhealthy and the other nodes becomes eligible to take the leader lock.
+
+- Fix ``pg_rewind`` behavior in a pause mode. (Ants)
+
+  Avoid unnecessary restart in a pause mode when Patroni thinks it needs to rewind but rewind is not possible (i.e. ``pg_rewind`` is not present). Fallback to default ``libpq`` values for the ``superuser`` (default OS user) if ``superuser`` authentication is missing from the ``pg_rewind`` related Patroni configuration section.
+
+- Serialize callback execution. Kill the previous callback of the same type when the new one is about to run. Fix the issue of spawning zombie processes when running callbacks. (Alexander)
+
+- Avoid promoting a former master when the leader key is set in DCS but update to this leader key fails. (Alexander)
+
+  This avoids the issue of a current master continuing to keep its role when it is partitioned together with the minority of nodes in Etcd and other DCSs that allow "inconsistent reads".
+
+**Miscellaneous**
+
+- Add ``post_init`` configuration option on bootstrap. (Alejandro Martínez)
+
+  Patroni will call the script argument of this option right after running ``initdb`` and starting up PostgreSQL for a new cluster. The script receives a connection URL with ``superuser``
+  and sets ``PGPASSFILE`` to point to the ``.pgpass`` file containing the password. If the script fails, Patroni initialization fails as well. It is useful for adding
+  new users or creating extensions in the new cluster.
+
+- Implement PostgreSQL 9.6 support. (Alexander)
+
+  Use ``wal_level = replica`` as a synonym for ``hot_standby``, avoiding pending_restart flag when it changes from one to another. (Alexander)
+
+**Documentation improvements**
+
+- Add a Patroni main `loop workflow diagram <https://raw.githubusercontent.com/zalando/patroni/master/docs/ha_loop_diagram.png>`__. (Alejandro, Alexander)
+
+- Improve README, adding the Helm chart and links to release notes. (Lauri Apple)
+
+- Move Patroni documentation to ``Read the Docs``. The up-to-date documentation is available at https://patroni.readthedocs.io. (Oleksii)
+
+  Makes the documentation easily viewable from different devices (including smartphones) and searchable.
+
+- Move the package to the semantic versioning. (Oleksii)
+
+  Patroni will follow the major.minor.patch version schema to avoid releasing the new minor version on small but critical bugfixes. We will only publish the release notes for the minor version, which will include all patches.
+
+
+Version 1.1
+-----------
+
+This release improves management of Patroni cluster by bring in pause mode, improves maintenance with scheduled and conditional restarts, makes Patroni interaction with Etcd or Zookeeper more resilient and greatly enhances patronictl.
+
+**Upgrade notice**
+
+When upgrading from releases below 1.0 read about changing of credentials and configuration format at 1.0 release notes.
+
+**Pause mode**
+
+- Introduce pause mode to temporary detach Patroni from managing PostgreSQL instance (Murat Kabilov, Alexander Kukushkin, Oleksii Kliukin).
+
+  Previously, one had to send SIGKILL signal to Patroni to stop it without terminating PostgreSQL. The new pause mode detaches Patroni from PostgreSQL cluster-wide without terminating Patroni. It is similar to the maintenance mode in Pacemaker. Patroni is still responsible for updating member and leader keys in DCS, but it will not start, stop or restart PostgreSQL server in the process. There are a few exceptions, for instance, manual failovers, reinitializes and restarts are still allowed. You can read :ref:`a detailed description of this feature <pause>`.
+
+In addition, patronictl supports new ``pause`` and ``resume`` commands to toggle the pause mode.
+
+**Scheduled and conditional restarts**
+
+- Add conditions to the restart API command (Oleksii)
+
+  This change enhances Patroni restarts by adding a couple of conditions that can be verified in order to do the restart. Among the conditions are restarting when PostgreSQL role is either a master or a replica, checking the PostgreSQL version number or restarting only when restart is necessary in order to apply configuration changes.
+
+- Add scheduled restarts (Oleksii)
+
+  It is now possible to schedule a restart in the future. Only one scheduled restart per node is supported. It is possible to clear the scheduled restart if it is not needed anymore. A combination of scheduled and conditional restarts is supported, making it possible, for instance, to scheduled minor PostgreSQL upgrades in the night, restarting only the instances that are running the outdated minor version without adding postgres-specific logic to administration scripts.
+
+- Add support for conditional and scheduled restarts to patronictl (Murat).
+
+  patronictl restart supports several new options. There is also patronictl flush command to clean the scheduled actions.
+
+**Robust DCS interaction**
+
+- Set Kazoo timeouts depending on the loop_wait (Alexander)
+
+  Originally, ping_timeout and connect_timeout values were calculated from the negotiated session timeout. Patroni loop_wait was not taken into account. As
+  a result, a single retry could take more time than the session timeout, forcing Patroni to release the lock and demote.
+
+  This change set ping and connect timeout to half of the value of loop_wait, speeding up detection of connection issues and  leaving enough time to retry the connection attempt before loosing the lock.
+
+- Update Etcd topology only after original request succeed (Alexander)
+
+  Postpone updating the Etcd topology known to the client until after the original request. When retrieving the cluster topology, implement the retry timeouts depending on the known number of nodes in the Etcd cluster. This makes our client prefer to get the results of the request to having the up-to-date list of nodes.
+
+  Both changes make Patroni connections to DCS more robust in the face of network issues.
+
+**Patronictl, monitoring and configuration**
+
+- Return information about streaming replicas via the API (Feike Steenbergen)
+
+Previously, there was no reliable way to query Patroni about PostgreSQL instances that fail to stream changes (for instance, due to connection issues). This change exposes the contents of pg_stat_replication via the /patroni endpoint.
+
+- Add patronictl scaffold command (Oleksii)
+
+  Add a command to create cluster structure in Etcd. The cluster is created with user-specified sysid and leader, and both leader and member keys are made persistent. This command is useful to create so-called master-less configurations, where Patroni cluster consisting of only replicas replicate  from the external master node that is unaware of Patroni. Subsequently, one
+  may remove the leader key, promoting one of the Patroni nodes and replacing
+  the original master with the Patroni-based HA cluster.
+
+- Add configuration option ``bin_dir`` to locate PostgreSQL binaries (Ants Aasma)
+
+  It is useful to be able to specify the location of PostgreSQL binaries explicitly when Linux distros that support installing multiple PostgreSQL versions at the same time.
+
+- Allow configuration file path to be overridden using ``custom_conf`` of (Alejandro Martínez)
+
+  Allows for custom configuration file paths, which will be unmanaged by Patroni, :ref:`details <postgresql_settings>`.
+
+**Bug fixes and code improvements**
+
+- Make Patroni compatible with new version schema in PostgreSQL 10 and above (Feike)
+
+  Make sure that Patroni understand 2-digits version numbers when doing conditional restarts based on the PostgreSQL version.
+
+- Use pkgutil to find DCS modules (Alexander)
+
+  Use the dedicated python module instead of traversing directories manually in order to find DCS modules.
+
+- Always call on_start callback when starting Patroni (Alexander)
+
+  Previously, Patroni did not call any callbacks when attaching to the already running node with the correct role. Since callbacks are often used to route
+  client connections that could result in the failure to register the running
+  node in the connection routing scheme. With this fix, Patroni calls on_start
+  callback even when attaching to the already running node.
+
+- Do not drop active replication slots (Murat, Oleksii)
+
+  Avoid dropping active physical replication slots on master. PostgreSQL cannot
+  drop such slots anyway. This change makes possible to run non-Patroni managed
+  replicas/consumers on the master.
+
+- Close Patroni connections during start of the PostgreSQL instance (Alexander)
+
+  Forces Patroni to close all former connections when PostgreSQL node is started. Avoids the trap of reusing former connections if postmaster was killed with SIGKILL.
+
+- Replace invalid characters when constructing slot names from member names (Ants)
+
+  Make sure that standby names that do not comply with the slot naming rules don't cause the slot creation and standby startup to fail. Replace the dashes in the slot names with underscores and all other characters not allowed in slot names with their unicode codepoints.
+
+Version 1.0
+-----------
+
+This release introduces the global dynamic configuration that allows dynamic changes of the PostgreSQL and Patroni configuration parameters for the entire HA cluster. It also delivers numerous bugfixes.
+
+**Upgrade notice**
+
+When upgrading from v0.90 or below, always upgrade all replicas before the master. Since we don't store replication credentials in DCS anymore, an old replica won't be able to connect to the new master.
+
+**Dynamic Configuration**
+
+- Implement the dynamic global configuration (Alexander Kukushkin)
+
+  Introduce new REST API endpoint /config to provide PostgreSQL and Patroni configuration parameters that should be set globally for the entire HA cluster (master and all the replicas). Those parameters are set in DCS and in many cases can be applied without disrupting PostgreSQL or Patroni. Patroni sets a special flag called "pending restart" visible via the API when some of the values require the PostgreSQL restart. In that case, restart should be issued manually via the API.
+
+  Patroni SIGHUP or POST to /reload will make it re-read the configuration file.
+
+  See the :ref:`dynamic configuration <dynamic_configuration>`  for the details on which parameters can be changed and the order of processing difference configuration sources.
+
+  The configuration file format *has changed* since the v0.90. Patroni is still compatible with the old configuration files, but in order to take advantage of the bootstrap parameters one needs to change it. Users are encourage to update them by referring to the :ref:`dynamic configuraton documentation page <dynamic_configuration>`.
+
+**More flexible configuration***
+
+- Make postgresql configuration and database name Patroni connects to configurable (Misja Hoebe)
+
+  Introduce `database` and `config_base_name` configuration parameters. Among others, it makes possible to run Patroni with PipelineDB and other PostgreSQL forks.
+
+- Implement possibility to configure some Patroni configuration parameters via environment (Alexander)
+
+  Those include the scope, the node name and the namespace, as well as the secrets and makes it easier to run Patroni in a dynamic environment, i.e. Kubernetes  Please, refer to the :ref:`supported environment variables <environment>` for further details.
+
+- Update the built-in Patroni docker container  to take advantage of environment-based configuration (Feike Steenbergen).
+
+- Add Zookeeper support to Patroni docker image (Alexander)
+
+- Split the Zookeeper and Exhibitor configuration options (Alexander)
+
+- Make patronictl reuse the code from Patroni to read configuration (Alexander)
+
+  This allows patronictl to take advantage of environment-based configuration.
+
+- Set application name to node name in primary_conninfo (Alexander)
+
+  This simplifies identification and configuration of synchronous replication for a given node.
+
+**Stability, security and usability improvements**
+
+- Reset sysid and do not call pg_controldata when restore of backup in progress (Alexander)
+
+  This change reduces the amount of noise generated by Patroni API health checks during the lengthy initialization of this node from the backup.
+
+- Fix a bunch of pg_rewind corner-cases (Alexander)
+
+  Avoid running pg_rewind if the source cluster is not the master.
+
+  In addition, avoid removing the data directory on an unsuccessful rewind, unless the new parameter *remove_data_directory_on_rewind_failure* is set to true. By default it is false.
+
+- Remove passwords from the replication connection string in DCS (Alexander)
+
+  Previously, Patroni always used the replication credentials from the Postgres URL in DCS. That is now changed to take the credentials from the patroni configuration. The secrets (replication username and password) and no longer exposed in DCS.
+
+- Fix the asynchronous machinery around the demote call (Alexander)
+
+  Demote now runs totally asynchronously without blocking the DCS interactions.
+
+- Make patronictl always send the authorization header if it is configured (Alexander)
+
+  This allows patronictl to issue "protected" requests, i.e. restart or reinitialize, when Patroni is configured to require authorization on those.
+
+- Handle the SystemExit exception correctly (Alexander)
+
+  Avoids the issues of Patroni not stopping properly when receiving the SIGTERM
+
+- Sample haproxy templates for confd (Alexander)
+
+  Generates and dynamically changes haproxy configuration from the patroni state in the DCS using confide
+
+- Improve and restructure the documentation to make it more friendly to the new users (Lauri Apple)
+
+- API must report role=master during pg_ctl stop (Alexander)
+
+  Makes the callback calls more reliable, particularly in the cluster stop case. In addition, introduce the `pg_ctl_timeout` option to set the timeout for the start, stop and restart calls via the `pg_ctl`.
+
+- Fix the retry logic in etcd (Alexander)
+
+  Make retries more predictable and robust.
+
+- Make Zookeeper code more resilient against short network hiccups (Alexander)
+
+  Reduce the connection timeouts to make Zookeeper connection attempts more frequent.
+
+Version 0.90
+------------
+
+This releases adds support for Consul, includes a new *noloadbalance* tag, changes the behavior of the *clonefrom* tag, improves *pg_rewind* handling and improves *patronictl* control program.
+
+**Consul support**
+
+- Implement Consul support (Alexander Kukushkin)
+
+  Patroni runs against Consul, in addition to Etcd and Zookeeper. the connection parameters can be configured in the YAML file.
+
+**New and improved tags**
+
+- Implement *noloadbalance* tag (Alexander)
+
+  This tag makes Patroni always return that the replica is not available to the load balancer.
+
+- Change the implementation of the *clonefrom* tag (Alexander)
+
+  Previously, a node name had to be supplied to the *clonefrom*, forcing a tagged replica to clone from the specific node. The new implementation makes *clonefrom* a boolean tag: if it is set to true, the replica becomes a candidate for other replicas to clone from it. When multiple candidates are present, the replicas picks one randomly.
+
+**Stability and security improvements**
+
+- Numerous reliability improvements (Alexander)
+
+  Removes some spurious error messages, improves the stability of the failover, addresses some corner cases with reading data from DCS, shutdown, demote and reattaching of the former leader.
+
+- Improve systems script to avoid killing Patroni children on stop (Jan Keirse, Alexander Kukushkin)
+
+  Previously, when stopping Patroni, *systemd* also sent a signal to PostgreSQL. Since Patroni also tried to stop PostgreSQL by itself, it resulted in sending to different shutdown requests (the smart shutdown, followed by the fast shutdown). That resulted in replicas disconnecting too early and a former master not being able to rejoin after demote. Fix by Jan with prior research by Alexander.
+
+- Eliminate some cases where the former master was unable to call pg_rewind before rejoining as a replica (Oleksii Kliukin)
+
+  Previously, we only called *pg_rewind* if the former master had crashed. Change this to always run pg_rewind for the former master as long as pg_rewind is present in the system. This fixes the case when the master is shut down before the replicas managed to get the latest changes (i.e. during the "smart" shutdown).
+
+- Numerous improvements to unit- and acceptance- tests, in particular, enable support for Zookeeper and Consul (Alexander).
+
+- Make Travis CI faster and implement support for running tests against Zookeeper (Exhibitor) and Consul (Alexander)
+
+  Both unit and acceptance tests run automatically against Etcd, Zookeeper and Consul on each commit or pull-request.
+
+- Clear environment variables before calling PostgreSQL commands from Patroni (Feike Steenbergen)
+
+  This prevents  a possibility of reading system environment variables by connecting to the PostgreSQL cluster managed by Patroni.
+
+**Configuration and control changes**
+
+- Unify patronictl and Patroni configuration (Feike)
+
+  patronictl can use the same configuration file as Patroni itself.
+
+- Enable Patroni to read the configuration from the environment variables (Oleksii)
+
+  This simplifies generating configuration for Patroni automatically, or merging a single configuration from different sources.
+
+- Include database system identifier in the information returned by the API (Feike)
+
+- Implement *delete_cluster* for all available DCSs (Alexander)
+
+  Enables support for DCSs other than Etcd in patronictl.
+
+
+Version 0.80
+------------
+
+This release adds support for *cascading replication* and simplifies Patroni management by providing *scheduled failovers*. One may use older versions of Patroni (in particular, 0.78) combined with this one in order to migrate to the new release. Note that the scheduled failover and cascading replication related features will only work with Patroni 0.80 and above.
+
+**Cascading replication**
+
+ - Add support for the *replicatefrom* and *clonefrom* tags for the patroni node (Oleksii Kliukin).
+
+ The tag *replicatefrom*  allows a replica to use an arbitrary node a source, not necessary the master. The *clonefrom* does the same for the initial backup. Together, they enable Patroni to fully support cascading replication.
+
+- Add support for running replication methods to initialize the replica even without a running replication connection (Oleksii).
+
+ This is useful in order to create replicas from the snapshots stored on S3 or FTP.  A replication method that does not require a running replication connection should supply *no_master: true* in the yaml configuration. Those scripts will still be called in order if the replication connection is present.
+
+**Patronictl, API and DCS improvements**
+
+- Implement scheduled failovers (Feike Steenbergen).
+
+  Failovers can be scheduled to happen at a certain time in the future, using either patronictl, or API calls.
+
+- Add support for *dbuser* and *password* parameters in patronictl (Feike).
+
+- Add PostgreSQL version to the health check output (Feike).
+
+- Improve Zookeeper support in patronictl (Oleksandr Shulgin)
+
+- Migrate to python-etcd 0.43 (Alexander Kukushkin)
+
+**Configuration**
+
+- Add a sample systems configuration script for Patroni (Jan Keirse).
+
+- Fix the problem of Patroni ignoring the superuser name specified in the configuration file for DB connections  (Alexander).
+
+- Fix the handling of CTRL-C by creating a separate session ID and process group for the postmaster launched by Patroni (Alexander).
+
+**Tests**
+
+- Add acceptance tests with *behave* in order to check real-world scenarios of running Patroni (Alexander, Oleksii).
+
+  The tests can be launched manually using the *behave* command. They are also launched automatically for pull requests and after commits.
+
+  Releases notes for some older versions can be found on `project's github page <https://github.com/zalando/patroni/releases>`__.

docs/replication_modes.rst

@@ -0,0 +1,68 @@
+.. _replication_modes:
+
+=================
+Replication modes
+=================
+
+Patroni uses PostgreSQL streaming replication. For more information about streaming replication, see the `Postgres documentation <http://www.postgresql.org/docs/current/static/warm-standby.html#STREAMING-REPLICATION>`__. By default Patroni configures PostgreSQL for asynchronous replication. Choosing your replication schema is dependent on your business considerations. Investigate both async and sync replication, as well as other HA solutions, to determine which solution is best for you.
+
+Asynchronous mode durability
+----------------------------
+
+In asynchronous mode the cluster is allowed to lose some committed transactions to ensure availability. When master server fails or becomes unavailable for any other reason Patroni will automatically promote a sufficiently healthy standby to master. Any transactions that have not been replicated to that standby remain in a "forked timeline" on the master, and are effectively unrecoverable [1]_.
+
+The amount of transactions that can be lost is controlled via ``maximum_lag_on_failover`` parameter. Because master transaction log position is not sampled in real time, in reality the amount of lost data on failover is worst case bounded by  ``maximum_lag_on_failover`` bytes of transaction log plus the amount that is written in the last ``ttl`` seconds (``loop_wait``/2 seconds in the average case). However typical steady state replication delay is well under a second.
+
+PostgreSQL synchronous replication
+----------------------------------
+
+You can use Postgres's `synchronous replication <http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION>`__ with Patroni. Synchronous replication ensures consistency across a cluster by confirming that writes are written to a secondary before returning to the connecting client with a success. The cost of synchronous replication: reduced throughput on writes. This throughput will be entirely based on network performance.
+
+In hosted datacenter environments (like AWS, Rackspace, or any network you do not control), synchronous replication significantly increases the variability of write performance. If followers become inaccessible from the leader, the leader effectively becomes read-only.
+
+To enable a simple synchronous replication test, add the follow lines to the ``parameters`` section of your YAML configuration files:
+
+.. code:: YAML
+
+        synchronous_commit: "on"
+        synchronous_standby_names: "*"
+
+When using PostgreSQL synchronous replication, use at least three Postgres data nodes to ensure write availability if one host fails.
+
+Using PostgreSQL synchronous replication does not guarantee zero lost transactions under all circumstances. When master and standby that is currently acting as synchronous fail simultaneously a third node that might not contain all transactions will be promoted.
+
+.. _synchronous_mode:
+
+Synchronous mode
+----------------
+
+For use cases where losing committed transactions is not permissible you can turn on Patronis ``synchronous_mode``. When ``synchronous_mode`` is turned on Patroni will not promote a standby unless it is certain that the standby contains all transactions that may have returned a successful commit status to client [2]_. This means that the system may be unavailable for writes even though some servers are available. System administrators can still use manual failover commmands to promote a standby even if it results in transaction loss.
+
+Turning on ``synchronous_mode`` does not guarantee multi node durability of commits under all circumstances. When no suitable standby is available, master server will still accept writes, but does not guarantee their replication. When the master fails in this mode no standby will be promote.  When the host that used to be master comes back it will get promoted automatically, unless system administrator performed a manual failover. This behavior makes synchronous mode usable with 2 node clusters.
+
+When ``synchronous_mode`` is on and a standby crashes, commits will block until next iteration of Patroni runs and switches master to standalone mode (worst case delay for writes ``ttl`` seconds, average case ``loop_wait``/2 seconds). Manually shutting down or restarting a standby will not cause a commit service interruption. Standby will signal the master to release itself from synchronous standby duties before PostgreSQL shutdown is initiated.
+
+You can ensure that a standby never becomes the synchronous standby by setting ``nosync`` tag to true. This is recommended to set for standbys that are behind slow network connections and would cause performance degradation when becoming a synchronous standby.
+
+Synchronous mode can be switched on and off via Patroni REST interface. See `dynamic configuration <https://github.com/zalando/patroni/blob/master/docs/dynamic_configuration.rst>`__ for instructions.
+
+
+Synchronous mode implementation
+-------------------------------
+
+When in synchronous mode Patroni maintains synchronization state in the DCS, containing the latest master and current synchronous standby. This state is updated with strict ordering constraints to ensure the following invariants:
+
+- A node must be marked as the latest leader whenever it can accept write transactions. Patroni crashing or PostgreSQL not shutting down can cause violations of this invariant.
+
+- A node must be set as the synchronous standby in PostgreSQL as long as it is published as the synchronous standby.
+
+- A node that is not the leader or current synchronous standby is not allowed to promote itself automatically.
+
+Patroni will only ever assign one standby to ``synchronous_standby_names`` because with multiple candidates it is not possible to know which node was acting as synchronous during the failure.
+
+On each HA loop iteration Patroni re-evaluates synchronous standby choice. If the current synchronous standby is connected and has not requested its synchronous status to be removed it remains picked. Otherwise the cluster member avaiable for sync that is furthest ahead in replication is picked.
+
+
+.. [1] The data is still there, but recovering it requires a manual recovery effort by data recovery specialists. When Patroni is allowed to rewind with ``use_pg_rewind`` the forked timeline will be automatically erased to rejoin the failed master with the cluster.
+
+.. [2] Clients can change the behavior per transaction using PostgreSQL's ``synchronous_commit`` setting. Transactions with ``synchronous_commit`` values of ``off`` and ``local`` may be lost on fail over, but will not be blocked by replication delays.

features/basic_replication.feature

@@ -3,15 +3,38 @@ Feature: basic replication
 
   Scenario: check replication of a single table
     Given I start postgres0
-    And postgres0 is a leader after 10 seconds
-    And I start postgres1
-    When I add the table foo to postgres0
+    Then postgres0 is a leader after 10 seconds
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "loop_wait": 2, "synchronous_mode": true}
+    Then I receive a response code 200
+    When I start postgres1
+    And I configure and start postgres2 with a tag replicatefrom postgres0
+    And "sync" key in DCS has leader=postgres0 after 20 seconds
+    And I add the table foo to postgres0
     Then table foo is present on postgres1 after 20 seconds
+    Then table foo is present on postgres2 after 20 seconds
 
-  Scenario: check the basic failover
+  Scenario: check restart of sync replica
+    Given I run patronictl.py restart batman postgres2 --force
+    And "sync" key in DCS has sync_standby=postgres1 after 2 seconds
+    And I run patronictl.py restart batman postgres1 --force
+    Then I receive a response returncode 0
+    And "sync" key in DCS has sync_standby=postgres2 after 10 seconds
+
+  Scenario: check the basic failover in synchronous mode
     When I kill postgres0
-    Then postgres1 role is the primary after 32 seconds
-    When I start postgres0
+    Then postgres2 role is the primary after 22 seconds
+    When I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_mode": null, "master_start_timeout": 0}
+    Then I receive a response code 200
+    When I add the table bar to postgres2
+    Then table bar is present on postgres1 after 20 seconds
+
+  Scenario: check immediate failover when master_start_timeout=0
+    Given I kill postmaster on postgres2
+    Then postgres1 is a leader after 10 seconds
+    And postgres1 role is the primary after 10 seconds
+
+  Scenario: check rejoin of the former master with pg_rewind
+    Given I start postgres0
     Then postgres0 role is the secondary after 20 seconds
-    When I add the table bar to postgres1
-    Then table bar is present on postgres0 after 20 seconds
+    When I add the table buz to postgres1
+    Then table buz is present on postgres0 after 20 seconds

features/cascading_replication.feature

@@ -8,6 +8,7 @@ Scenario: check a base backup and streaming replication from a replica
 	And replication works from postgres0 to postgres1 after 20 seconds
 	And I create label with "postgres0" in postgres0 data directory
 	And I create label with "postgres1" in postgres1 data directory
+	And "members/postgres1" key in DCS has state=running after 12 seconds
 	And I configure and start postgres2 with a tag replicatefrom postgres1
 	Then replication works from postgres0 to postgres2 after 30 seconds
 	And there is a label with "postgres1" in postgres2 data directory

features/environment.py

@@ -16,7 +16,8 @@ import yaml
 @six.add_metaclass(abc.ABCMeta)
 class AbstractController(object):
 
-    def __init__(self, name, work_directory, output_dir):
+    def __init__(self, context, name, work_directory, output_dir):
+        self._context = context
         self._name = name
         self._work_directory = work_directory
         self._output_dir = output_dir
@@ -46,6 +47,7 @@ class AbstractController(object):
 
         assert self._has_started(), "Process {0} is not running after being started".format(self._name)
 
+        max_wait_limit *= self._context.timeout_multiplier
         for _ in range(max_wait_limit):
             if self._is_accessible():
                 break
@@ -54,10 +56,11 @@ class AbstractController(object):
             assert False,\
                 "{0} instance is not available for queries after {1} seconds".format(self._name, max_wait_limit)
 
-    def stop(self, kill=False, timeout=15):
+    def stop(self, kill=False, timeout=15, _=False):
         term = False
         start_time = time.time()
 
+        timeout *= self._context.timeout_multiplier
         while self._handle and self._is_running():
             if kill:
                 self._handle.kill()
@@ -77,12 +80,12 @@ class PatroniController(AbstractController):
     PATRONI_CONFIG = '{}.yml'
     """ starts and stops individual patronis"""
 
-    def __init__(self, dcs, name, work_directory, output_dir, tags=None):
-        super(PatroniController, self).__init__('patroni_' + name, work_directory, output_dir)
+    def __init__(self, context, name, work_directory, output_dir, tags=None):
+        super(PatroniController, self).__init__(context, 'patroni_' + name, work_directory, output_dir)
         PatroniController.__PORT += 1
         self._data_dir = os.path.join(work_directory, 'data', name)
         self._connstring = None
-        self._config = self._make_patroni_test_config(name, dcs, tags)
+        self._config = self._make_patroni_test_config(name, tags)
 
         self._conn = None
         self._curs = None
@@ -109,10 +112,15 @@ class PatroniController(AbstractController):
         return subprocess.Popen(['coverage', 'run', '--source=patroni', '-p', 'patroni.py', self._config],
                                 stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory)
 
+    def stop(self, kill=False, timeout=15, postgres=False):
+        if postgres:
+            return subprocess.call(['pg_ctl', '-D', self._data_dir, 'stop', '-mi', '-w'])
+        super(PatroniController, self).stop(kill, timeout)
+
     def _is_accessible(self):
         return self.query("SELECT 1", fail_ok=True) is not None
 
-    def _make_patroni_test_config(self, name, dcs, tags):
+    def _make_patroni_test_config(self, name, tags):
         patroni_config_name = self.PATRONI_CONFIG.format(name)
         patroni_config_path = os.path.join(self._output_dir, patroni_config_name)
 
@@ -182,12 +190,16 @@ class AbstractDcsController(AbstractController):
 
     _CLUSTER_NODE = '/service/batman'
 
+    def __init__(self, context, mktemp=True):
+        work_directory = mktemp and tempfile.mkdtemp() or None
+        super(AbstractDcsController, self).__init__(context, self.name(), work_directory, context.pctl.output_dir)
+
     def _is_accessible(self):
         return self._is_running()
 
-    def stop_and_remove_work_directory(self, timeout=15):
+    def stop(self, kill=False, timeout=15):
         """ terminate process and wipe out the temp work directory, but only if we actually started it"""
-        self.stop(timeout=timeout)
+        super(AbstractDcsController, self).stop(kill=kill, timeout=timeout)
         if self._work_directory:
             shutil.rmtree(self._work_directory)
 
@@ -206,17 +218,36 @@ class AbstractDcsController(AbstractController):
     def cleanup_service_tree(self):
         """ clean all contents stored in the tree used for the tests """
 
+    @classmethod
+    def get_subclasses(cls):
+        for subclass in cls.__subclasses__():
+            for subsubclass in subclass.get_subclasses():
+                yield subsubclass
+            yield subclass
+
+    @classmethod
+    def name(cls):
+        return cls.__name__[:-10].lower()
+
 
 class ConsulController(AbstractDcsController):
 
-    def __init__(self, output_dir):
-        super(ConsulController, self).__init__('consul', tempfile.mkdtemp(), output_dir)
+    def __init__(self, context):
+        super(ConsulController, self).__init__(context)
         os.environ['PATRONI_CONSUL_HOST'] = 'localhost:8500'
         self._client = consul.Consul()
 
     def _start(self):
-        return subprocess.Popen(['consul', 'agent', '-server', '-bootstrap', '-advertise=127.0.0.1',
-                                 '-data-dir', self._work_directory], stdout=self._log, stderr=subprocess.STDOUT)
+        config_file = self._work_directory + '.json'
+        with open(config_file, 'wb') as f:
+            f.write(b'{"session_ttl_min":"5s","server":true,"bootstrap":true,"advertise_addr":"127.0.0.1"}')
+        return subprocess.Popen(['consul', 'agent', '-config-file', config_file, '-data-dir', self._work_directory],
+                                stdout=self._log, stderr=subprocess.STDOUT)
+
+    def stop(self, kill=False, timeout=15):
+        super(ConsulController, self).stop(kill=kill, timeout=timeout)
+        if self._work_directory:
+            os.unlink(self._work_directory + '.json')
 
     def _is_running(self):
         try:
@@ -237,15 +268,18 @@ class ConsulController(AbstractDcsController):
     def cleanup_service_tree(self):
         self._client.kv.delete(self.path(), recurse=True)
 
+    def start(self, max_wait_limit=15):
+        super(ConsulController, self).start(max_wait_limit)
+
 
 class EtcdController(AbstractDcsController):
 
     """ handles all etcd related tasks, used for the tests setup and cleanup """
 
-    def __init__(self, output_dir):
-        super(EtcdController, self).__init__('etcd', tempfile.mkdtemp(), output_dir)
-        os.environ['PATRONI_ETCD_HOST'] = 'localhost:4001'
-        self._client = etcd.Client()
+    def __init__(self, context):
+        super(EtcdController, self).__init__(context)
+        os.environ['PATRONI_ETCD_HOST'] = 'localhost:2379'
+        self._client = etcd.Client(port=2379)
 
     def _start(self):
         return subprocess.Popen(["etcd", "--debug", "--data-dir", self._work_directory],
@@ -280,8 +314,8 @@ class ZooKeeperController(AbstractDcsController):
 
     """ handles all zookeeper related tasks, used for the tests setup and cleanup """
 
-    def __init__(self, output_dir, export_env=True):
-        super(ZooKeeperController, self).__init__('zookeeper', None, output_dir)
+    def __init__(self, context, export_env=True):
+        super(ZooKeeperController, self).__init__(context, False)
         if export_env:
             os.environ['PATRONI_ZOOKEEPER_HOSTS'] = "'localhost:2181'"
         self._client = kazoo.client.KazooClient()
@@ -318,22 +352,21 @@ class ZooKeeperController(AbstractDcsController):
 
 class ExhibitorController(ZooKeeperController):
 
-    def __init__(self, output_dir):
-        super(ExhibitorController, self).__init__(output_dir, False)
+    def __init__(self, context):
+        super(ExhibitorController, self).__init__(context, False)
         os.environ.update({'PATRONI_EXHIBITOR_HOSTS': 'localhost', 'PATRONI_EXHIBITOR_PORT': '8181'})
 
 
 class PatroniPoolController(object):
 
-    KNOWN_DCS = {'consul': ConsulController, 'etcd': EtcdController,
-                 'zookeeper': ZooKeeperController, 'exhibitor': ExhibitorController}
-
-    def __init__(self):
+    def __init__(self, context):
+        self._context = context
         self._dcs = None
         self._output_dir = None
         self._patroni_path = None
         self._processes = {}
         self.create_and_set_output_directory('')
+        self.known_dcs = {subclass.name(): subclass for subclass in AbstractDcsController.get_subclasses()}
 
     @property
     def patroni_path(self):
@@ -350,17 +383,17 @@ class PatroniPoolController(object):
     def output_dir(self):
         return self._output_dir
 
-    def start(self, pg_name, max_wait_limit=20, tags=None):
-        if pg_name not in self._processes:
-            self._processes[pg_name] = PatroniController(self.dcs, pg_name, self.patroni_path, self._output_dir, tags)
-        self._processes[pg_name].start(max_wait_limit)
+    def start(self, name, max_wait_limit=20, tags=None):
+        if name not in self._processes:
+            self._processes[name] = PatroniController(self._context, name, self.patroni_path, self._output_dir, tags)
+        self._processes[name].start(max_wait_limit)
 
     def __getattr__(self, func):
         if func not in ['stop', 'query', 'write_label', 'read_label', 'check_role_has_changed_to', 'add_tag_to_config']:
             raise AttributeError("PatroniPoolController instance has no attribute '{0}'".format(func))
 
-        def wrapper(pg_name, *args, **kwargs):
-            return getattr(self._processes[pg_name], func)(*args, **kwargs)
+        def wrapper(name, *args, **kwargs):
+            return getattr(self._processes[name], func)(*args, **kwargs)
         return wrapper
 
     def stop_all(self):
@@ -379,24 +412,26 @@ class PatroniPoolController(object):
     def dcs(self):
         if self._dcs is None:
             self._dcs = os.environ.pop('DCS', 'etcd')
-            assert self._dcs in self.KNOWN_DCS, 'Unsupported dcs: ' + self._dcs
+            assert self._dcs in self.known_dcs, 'Unsupported dcs: ' + self._dcs
         return self._dcs
 
 
 # actions to execute on start/stop of the tests and before running invidual features
 def before_all(context):
-    context.pctl = PatroniPoolController()
-    context.dcs_ctl = context.pctl.KNOWN_DCS[context.pctl.dcs](context.pctl.output_dir)
+    context.ci = 'TRAVIS_BUILD_NUMBER' in os.environ or 'BUILD_NUMBER' in os.environ
+    context.timeout_multiplier = 2 if context.ci else 1
+    context.pctl = PatroniPoolController(context)
+    context.dcs_ctl = context.pctl.known_dcs[context.pctl.dcs](context)
     context.dcs_ctl.start()
     try:
         context.dcs_ctl.cleanup_service_tree()
     except AssertionError:  # after_all handlers won't be executed in before_all
-        context.dcs_ctl.stop_and_remove_work_directory()
+        context.dcs_ctl.stop()
         raise
 
 
 def after_all(context):
-    context.dcs_ctl.stop_and_remove_work_directory()
+    context.dcs_ctl.stop()
     subprocess.call(['coverage', 'combine'])
     subprocess.call(['coverage', 'report'])
 

features/patroni_api.feature

@@ -34,7 +34,7 @@ Scenario: check local configuration reload
 	Then I receive a response code 202
 
 Scenario: check dynamic configuration change via DCS
-	Given I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "loop_wait": 2, "postgresql": {"parameters": {"max_connections": 101}}}
+	Given I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 10, "loop_wait": 2, "postgresql": {"parameters": {"max_connections": 101}}}
 	Then I receive a response code 200
 	And I receive a response loop_wait 2
 	And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 11 seconds
@@ -73,12 +73,12 @@ Scenario: check the failover via the API in the pause mode
 	And replication works from postgres1 to postgres0 after 20 seconds
 
 Scenario: check the scheduled failover
-	Given I issue a scheduled failover from postgres1 to postgres0 in 1 seconds
+	Given I issue a scheduled failover from postgres1 to postgres0 in 3 seconds
 	Then I receive a response returncode 1
 	And I receive a response output "Can't schedule failover in the paused state"
 	When I run patronictl.py resume batman
 	Then I receive a response returncode 0
-	Given I issue a scheduled failover from postgres1 to postgres0 in 1 seconds
+	Given I issue a scheduled failover from postgres1 to postgres0 in 3 seconds
 	Then I receive a response returncode 0
 	And postgres0 is a leader after 20 seconds
 	And postgres0 role is the primary after 10 seconds
@@ -89,11 +89,11 @@ Scenario: check the scheduled restart
 	Given I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"superuser_reserved_connections": "6"}}}
 	Then I receive a response code 200
 		And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 5 seconds
-	Given I issue a scheduled restart at http://127.0.0.1:8008 in 1 seconds with {"role": "replica"}
+	Given I issue a scheduled restart at http://127.0.0.1:8008 in 3 seconds with {"role": "replica"}
 	Then I receive a response code 202
-		And I sleep for 2 seconds
+		And I sleep for 4 seconds
 		And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 10 seconds
-	Given I issue a scheduled restart at http://127.0.0.1:8008 in 1 seconds with {"restart_pending": "True"}
+	Given I issue a scheduled restart at http://127.0.0.1:8008 in 3 seconds with {"restart_pending": "True"}
 	Then I receive a response code 202
 		And Response on GET http://127.0.0.1:8008/patroni does not contain pending_restart after 10 seconds
 

features/steps/basic_replication.py

@@ -19,6 +19,11 @@ def kill_patroni(context, name):
     return context.pctl.stop(name, kill=True)
 
 
+@step('I kill postmaster on {name:w}')
+def stop_postgres(context, name):
+    return context.pctl.stop(name, postgres=True)
+
+
 @step('I add the table {table_name:w} to {pg_name:w}')
 def add_table(context, table_name, pg_name):
     # parse the configuration file and get the port
@@ -30,6 +35,7 @@ def add_table(context, table_name, pg_name):
 
 @then('Table {table_name:w} is present on {pg_name:w} after {max_replication_delay:d} seconds')
 def table_is_present_on(context, table_name, pg_name, max_replication_delay):
+    max_replication_delay *= context.timeout_multiplier
     for _ in range(int(max_replication_delay)):
         if context.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
             break
@@ -41,6 +47,7 @@ def table_is_present_on(context, table_name, pg_name, max_replication_delay):
 
 @then('{pg_name:w} role is the {pg_role:w} after {max_promotion_timeout:d} seconds')
 def check_role(context, pg_name, pg_role, max_promotion_timeout):
+    max_promotion_timeout *= context.timeout_multiplier
     assert context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)),\
         "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
 

features/steps/cascading_replication.py

@@ -1,3 +1,6 @@
+import json
+import time
+
 from behave import step, then
 
 
@@ -15,3 +18,18 @@ def check_label(context, content, name):
 @step('I create label with "{content:w}" in {name:w} data directory')
 def write_label(context, content, name):
     context.pctl.write_label(name, content)
+
+
+@step('"{name}" key in DCS has {key:w}={value:w} after {time_limit:d} seconds')
+def check_member(context, name, key, value, time_limit):
+    time_limit *= context.timeout_multiplier
+    max_time = time.time() + int(time_limit)
+    while time.time() < max_time:
+        try:
+            response = json.loads(context.dcs_ctl.query(name))
+            if response.get(key) == value:
+                return
+        except Exception:
+            pass
+        time.sleep(1)
+    assert False, "{0} does not have {1}={2} in dcs after {3} seconds".format(name, key, value, time_limit)

features/steps/patroni_api.py

@@ -1,6 +1,6 @@
 import json
+import os
 import parse
-import pytz
 import requests
 import shlex
 import subprocess
@@ -8,8 +8,11 @@ import time
 import yaml
 
 from behave import register_type, step, then
+from dateutil import tz
 from datetime import datetime, timedelta
 
+tzutc = tz.tzutc()
+
 
 @parse.with_pattern(r'https?://(?:\w|\.|:|/)+')
 def parse_url(text):
@@ -27,6 +30,7 @@ register_type(url=parse_url)
 @step('{name:w} is a leader after {time_limit:d} seconds')
 @then('{name:w} is a leader after {time_limit:d} seconds')
 def is_a_leader(context, name, time_limit):
+    time_limit *= context.timeout_multiplier
     max_time = time.time() + int(time_limit)
     while (context.dcs_ctl.query("leader") != name):
         time.sleep(1)
@@ -90,7 +94,10 @@ def do_request(context, request_method, url, data):
 def do_run(context, cmd):
     cmd = ['coverage', 'run', '--source=patroni', '-p'] + shlex.split(cmd)
     try:
-        response = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+        # XXX: Dirty hack! We need to take name/passwd from the config!
+        env = os.environ.copy()
+        env.update({'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password'})
+        response = subprocess.check_output(cmd, stderr=subprocess.STDOUT, env=env)
         context.status_code = 0
     except subprocess.CalledProcessError as e:
         response = e.output
@@ -118,13 +125,13 @@ def check_response(context, component, data):
 def scheduled_failover(context, from_host, to_host, in_seconds):
     context.execute_steps(u"""
         Given I run patronictl.py failover batman --master {0} --candidate {1} --scheduled "{2}" --force
-    """.format(from_host, to_host, datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds))))
+    """.format(from_host, to_host, datetime.now(tzutc) + timedelta(seconds=int(in_seconds))))
 
 
 @step('I issue a scheduled restart at {url:url} in {in_seconds:d} seconds with {data}')
 def scheduled_restart(context, url, in_seconds, data):
     data = data and json.loads(data) or {}
-    data.update(schedule='{0}'.format((datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds))).isoformat()))
+    data.update(schedule='{0}'.format((datetime.now(tzutc) + timedelta(seconds=int(in_seconds))).isoformat()))
     context.execute_steps(u"""Given I issue a POST request to {0}/restart with {1}""".format(url, json.dumps(data)))
 
 
@@ -135,6 +142,7 @@ def add_tag_to_config(context, tag, value, pg_name):
 
 @then('Response on GET {url} contains {value} after {timeout:d} seconds')
 def check_http_response(context, url, value, timeout, negate=False):
+    timeout *= context.timeout_multiplier
     for _ in range(int(timeout)):
         r = requests.get(url)
         if (value in r.content.decode('utf-8')) != negate:

patroni/__init__.py

@@ -1,23 +1,22 @@
 import logging
+import os
 import signal
 import sys
 import time
 
-from patroni.api import RestApiServer
-from patroni.config import Config
-from patroni.dcs import get_dcs
-from patroni.exceptions import DCSError
-from patroni.ha import Ha
-from patroni.postgresql import Postgresql
-from patroni.utils import reap_children, sigchld_handler
-from patroni.version import __version__
-
 logger = logging.getLogger(__name__)
 
 
 class Patroni(object):
 
     def __init__(self):
+        from patroni.api import RestApiServer
+        from patroni.config import Config
+        from patroni.dcs import get_dcs
+        from patroni.ha import Ha
+        from patroni.postgresql import Postgresql
+        from patroni.version import __version__
+
         self.setup_signal_handlers()
 
         self.version = __version__
@@ -34,6 +33,7 @@ class Patroni(object):
         self.scheduled_restart = {}
 
     def load_dynamic_configuration(self):
+        from patroni.exceptions import DCSError
         while True:
             try:
                 cluster = self.dcs.get_cluster()
@@ -49,12 +49,16 @@ class Patroni(object):
 
     def get_tags(self):
         return {tag: value for tag, value in self.config.get('tags', {}).items()
-                if tag not in ('clonefrom', 'nofailover', 'noloadbalance') or value}
+                if tag not in ('clonefrom', 'nofailover', 'noloadbalance', 'nosync') or value}
 
     @property
     def nofailover(self):
         return bool(self.tags.get('nofailover', False))
 
+    @property
+    def nosync(self):
+        return bool(self.tags.get('nosync', False))
+
     def reload_config(self):
         try:
             self.tags = self.get_tags()
@@ -90,7 +94,7 @@ class Patroni(object):
             time.sleep(0.001)
             # Warn user that Patroni is not keeping up
             logger.warning("Loop time exceeded, rescheduling immediately.")
-        elif self.dcs.watch(nap_time):
+        elif self.ha.watch(nap_time):
             self.next_run = time.time()
 
     def run(self):
@@ -112,7 +116,6 @@ class Patroni(object):
             if not self.postgresql.data_directory_empty():
                 self.config.save_cache()
 
-            reap_children()
             self.schedule_next_run()
 
     def setup_signal_handlers(self):
@@ -120,10 +123,9 @@ class Patroni(object):
         self._received_sigterm = False
         signal.signal(signal.SIGHUP, self.sighup_handler)
         signal.signal(signal.SIGTERM, self.sigterm_handler)
-        signal.signal(signal.SIGCHLD, sigchld_handler)
 
 
-def main():
+def patroni_main():
     logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
     logging.getLogger('requests').setLevel(logging.WARNING)
 
@@ -137,5 +139,62 @@ def main():
         if patroni.ha.is_paused():
             logger.info('Leader key is not deleted and Postgresql is not stopped due paused state')
         else:
-            patroni.postgresql.stop(checkpoint=False)
+            patroni.ha.while_not_sync_standby(lambda: patroni.postgresql.stop(checkpoint=False))
             patroni.dcs.delete_leader()
+
+
+def pg_ctl_start(args):
+    import subprocess
+    postmaster = subprocess.Popen(args)
+    print(postmaster.pid)
+
+
+def call_self(args, **kwargs):
+    """This function executes Patroni once again with provided arguments.
+
+    :args: list of arguments to call Patroni with.
+    :returns: `Popen` object"""
+
+    exe = [sys.executable]
+    if not getattr(sys, 'frozen', False):  # Binary distribution?
+        exe.append(sys.argv[0])
+
+    import subprocess
+    return subprocess.Popen(exe + args, **kwargs)
+
+
+def main():
+    if os.getpid() != 1:
+        if len(sys.argv) > 5 and sys.argv[1] == 'pg_ctl_start':
+            return pg_ctl_start(sys.argv[2:])
+        return patroni_main()
+
+    pid = 0
+
+    # Looks like we are in a docker, so we will act like init
+    def sigchld_handler(signo, stack_frame):
+        try:
+            while True:
+                ret = os.waitpid(-1, os.WNOHANG)
+                if ret == (0, 0):
+                    break
+                elif ret[0] != pid:
+                    logging.info('Reaped pid=%s, exit status=%s', *ret)
+        except OSError:
+            pass
+
+    def passtochild(signo, stack_frame):
+        if pid:
+            os.kill(pid, signo)
+
+    signal.signal(signal.SIGCHLD, sigchld_handler)
+    signal.signal(signal.SIGHUP, passtochild)
+    signal.signal(signal.SIGINT, passtochild)
+    signal.signal(signal.SIGUSR1, passtochild)
+    signal.signal(signal.SIGUSR2, passtochild)
+    signal.signal(signal.SIGQUIT, passtochild)
+    signal.signal(signal.SIGTERM, passtochild)
+
+    patroni = call_self(sys.argv[1:])
+    pid = patroni.pid
+    patroni.wait()

patroni/api.py

@@ -6,10 +6,9 @@ import psycopg2
 import time
 import dateutil.parser
 import datetime
-import pytz
 
 from patroni.exceptions import PostgresConnectionException
-from patroni.utils import deep_compare, patch_config, Retry, RetryFailedError, is_valid_pg_version
+from patroni.utils import deep_compare, patch_config, Retry, RetryFailedError, is_valid_pg_version, parse_int, tzutc
 from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 from six.moves.socketserver import ThreadingMixIn
 from threading import Thread
@@ -57,7 +56,9 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
     def _write_status_response(self, status_code, response):
         patroni = self.server.patroni
-        response.update({'tags': patroni.tags} if patroni.tags else {})
+        tags = patroni.ha.get_effective_tags()
+        if tags:
+            response['tags'] = tags
         if patroni.postgresql.sysid:
             response['database_system_identifier'] = patroni.postgresql.sysid
         if patroni.postgresql.pending_restart:
@@ -140,6 +141,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 value = json.dumps(data, separators=(',', ':'))
                 if not self.server.patroni.dcs.set_config_value(value, cluster.config.index):
                     return self.send_error(409)
+            self.server.patroni.ha.wakeup()
             self._write_json_response(200, data)
 
     @check_auth
@@ -178,7 +180,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
             if scheduled_at.tzinfo is None:
                 error = 'Timezone information is mandatory for the scheduled {0}'.format(action)
                 status_code = 400
-            elif scheduled_at < datetime.datetime.now(pytz.utc):
+            elif scheduled_at < datetime.datetime.now(tzutc):
                 error = 'Cannot schedule {0} in the past'.format(action)
                 status_code = 422
             else:
@@ -221,6 +223,12 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     status_code = 400
                     data = "PostgreSQL version should be in the first.major.minor format"
                     break
+            elif k == 'timeout':
+                request[k] = parse_int(request[k], 's')
+                if request[k] is None or request[k] <= 0:
+                    status_code = 400
+                    data = "Timeout should be a positive number of seconds"
+                    break
             elif k != 'restart_pending':
                 status_code = 400
                 data = "Unknown filter for the scheduled restart: {0}".format(k)
@@ -234,7 +242,6 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     logger.exception('Exception during restart')
                     status_code = 400
             else:
-                request['postmaster_start_time'] = self.server.patroni.ha.state_handler.postmaster_start_time()
                 if self.server.patroni.ha.schedule_future_restart(request):
                     data = "Restart scheduled"
                     status_code = 202
@@ -321,7 +328,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 if _:
                     status_code = _
                 elif self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
-                    self.server.patroni.dcs.event.set()
+                    self.server.patroni.ha.wakeup()
                     data = 'Failover scheduled'
                     status_code = 202
                 else:
@@ -331,7 +338,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 data = self.is_failover_possible(cluster, leader, candidate)
                 if not data:
                     if self.server.patroni.dcs.manual_failover(leader, candidate):
-                        self.server.patroni.dcs.event.set()
+                        self.server.patroni.ha.wakeup()
                         status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, candidate)
                     else:
                         data = 'failed to write failover key into DCS'
@@ -377,11 +384,13 @@ class RestApiHandler(BaseHTTPRequestHandler):
                                             THEN 0
                                             ELSE pg_xlog_location_diff(pg_current_xlog_location(), '0/0')::bigint
                                        END,
-                                       pg_xlog_location_diff(pg_last_xlog_receive_location(), '0/0')::bigint,
+                                       pg_xlog_location_diff(COALESCE(pg_last_xlog_receive_location(),
+                                                                      pg_last_xlog_replay_location()), '0/0')::bigint,
                                        pg_xlog_location_diff(pg_last_xlog_replay_location(), '0/0')::bigint,
                                        to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
                                        pg_is_in_recovery() AND pg_is_xlog_replay_paused(),
-                                       (SELECT json_agg(row_to_json(ri)) FROM replication_info ri)""", retry=retry)[0]
+                                       (SELECT array_to_json(array_agg(row_to_json(ri))) FROM replication_info ri)""",
+                             retry=retry)[0]
 
             result = {
                 'state': self.server.patroni.postgresql.state,

patroni/async_executor.py

@@ -6,7 +6,8 @@ logger = logging.getLogger(__name__)
 
 class AsyncExecutor(object):
 
-    def __init__(self):
+    def __init__(self, ha_wakeup):
+        self._ha_wakeup = ha_wakeup
         self._thread_lock = RLock()
         self._scheduled_action = None
         self._scheduled_action_lock = RLock()
@@ -32,13 +33,18 @@ class AsyncExecutor(object):
             self._scheduled_action = None
 
     def run(self, func, args=()):
+        wakeup = False
         try:
-            return func(*args) if args else func()
+            # if the func returned something (not None) - wake up main HA loop
+            wakeup = func(*args) if args else func()
+            return wakeup
         except:
             logger.exception('Exception during execution of long running task %s', self.scheduled_action)
         finally:
             with self:
                 self.reset_scheduled_action()
+            if wakeup is not None:
+                self._ha_wakeup()
 
     def run_async(self, func, args=()):
         Thread(target=self.run, args=(func, args)).start()

patroni/callback_executor.py

@@ -0,0 +1,37 @@
+import logging
+import subprocess
+from threading import Event, Lock, Thread
+
+logger = logging.getLogger(__name__)
+
+
+class CallbackExecutor(Thread):
+
+    def __init__(self):
+        super(CallbackExecutor, self).__init__()
+        self.daemon = True
+        self._lock = Lock()
+        self._cmd = None
+        self._process = None
+        self._callback_event = Event()
+        self.start()
+
+    def call(self, cmd):
+        with self._lock:
+            if self._process and self._process.poll() is None:
+                self._process.kill()
+                logger.warning('Killed the old callback process because it was still running: %s', self._cmd)
+        self._cmd = cmd
+        self._callback_event.set()
+
+    def run(self):
+        while True:
+            self._callback_event.wait()
+            self._callback_event.clear()
+            with self._lock:
+                try:
+                    self._process = subprocess.Popen(self._cmd, close_fds=True)
+                except Exception:
+                    logger.exception('Failed to execute %s',  self._cmd)
+                    continue
+            self._process.wait()

patroni/config.py

@@ -41,6 +41,8 @@ class Config(object):
     __DEFAULT_CONFIG = {
         'ttl': 30, 'loop_wait': 10, 'retry_timeout': 10,
         'maximum_lag_on_failover': 1048576,
+        'master_start_timeout': 300,
+        'synchronous_mode': False,
         'postgresql': {
             'bin_dir': '',
             'use_slots': True,
@@ -172,7 +174,10 @@ class Config(object):
                     elif name not in ('connect_address', 'listen', 'data_dir', 'pgpass', 'authentication'):
                         config['postgresql'][name] = deepcopy(value)
             elif name in config:  # only variables present in __DEFAULT_CONFIG allowed to be overriden from DCS
-                config[name] = int(value)
+                if name == 'synchronous_mode':
+                    config[name] = value
+                else:
+                    config[name] = int(value)
         return config
 
     @staticmethod
@@ -232,8 +237,9 @@ class Config(object):
             if param.startswith(Config.PATRONI_ENV_PREFIX):
                 name, suffix = (param[8:].rsplit('_', 1) + [''])[:2]
                 if name and suffix:
-                    # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT)
-                    if suffix in ('HOST', 'HOSTS', 'PORT') and '_' not in name:
+                    # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..)
+                    if suffix in ('HOST', 'HOSTS', 'PORT', 'SRV', 'URL', 'PROXY', 'CACERT', 'CERT', 'KEY') \
+                            and '_' not in name:
                         value = os.environ.pop(param)
                         if suffix == 'PORT':
                             value = value and parse_int(value)
@@ -270,9 +276,7 @@ class Config(object):
 
         # restapi server expects to get restapi.auth = 'username:password'
         if 'authentication' in config['restapi']:
-            restapi = config['restapi']
-            auth = restapi['authentication']
-            restapi['auth'] = '{0}:{1}'.format(auth['username'], auth['password'])
+            config['restapi']['auth'] = '{username}:{password}'.format(**config['restapi']['authentication'])
 
         # special treatment for old config
 
@@ -295,7 +299,7 @@ class Config(object):
             config['name'] = pg_config['name']
 
         pg_config.update({p: config[p] for p in ('name', 'scope', 'retry_timeout',
-                          'maximum_lag_on_failover') if p in config})
+                          'synchronous_mode', 'maximum_lag_on_failover') if p in config})
 
         return config
 

patroni/ctl.py

@@ -28,11 +28,10 @@ from six.moves.urllib_parse import urlparse
 
 CONFIG_DIR_PATH = click.get_app_dir('patroni')
 CONFIG_FILE_PATH = os.path.join(CONFIG_DIR_PATH, 'patronictl.yaml')
-LOGLEVEL = 'WARNING'
 DCS_DEFAULTS = {'zookeeper': {'port': 2181, 'template': "zookeeper:\n hosts: ['{host}:{port}']"},
                 'exhibitor': {'port': 8181, 'template': "exhibitor:\n hosts: [{host}]\n port: {port}"},
                 'consul': {'port': 8500, 'template': "consul:\n host: '{host}:{port}'"},
-                'etcd': {'port': 4001, 'template': "etcd:\n host: '{host}:{port}'"}}
+                'etcd': {'port': 2379, 'template': "etcd:\n host: '{host}:{port}'"}}
 
 
 class PatroniCtlException(ClickException):
@@ -88,25 +87,23 @@ def store_config(config, path):
         yaml.dump(config, fd)
 
 
-option_config_file = click.option('--config-file', '-c', help='Configuration file', default=CONFIG_FILE_PATH)
 option_format = click.option('--format', '-f', 'fmt', help='Output format (pretty, json)', default='pretty')
-option_dcs = click.option('--dcs', '-d', help='Use this DCS', envvar='DCS')
 option_watchrefresh = click.option('-w', '--watch', type=float, help='Auto update the screen every X seconds')
 option_watch = click.option('-W', is_flag=True, help='Auto update the screen every 2 seconds')
 option_force = click.option('--force', is_flag=True, help='Do not ask for confirmation at any point')
 
 
 @click.group()
+@click.option('--config-file', '-c', help='Configuration file', default=CONFIG_FILE_PATH)
+@click.option('--dcs', '-d', help='Use this DCS', envvar='DCS')
 @click.pass_context
-def ctl(ctx):
-    global LOGLEVEL
-    LOGLEVEL = os.environ.get('LOGLEVEL', LOGLEVEL)
-
-    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=LOGLEVEL)
+def ctl(ctx, config_file, dcs):
+    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=os.environ.get('LOGLEVEL', 'WARNING'))
+    ctx.obj = load_config(config_file, dcs)
 
 
 def get_dcs(config, scope):
-    config['scope'] = scope
+    config.update({'scope': scope, 'patronictl': True})
     config.setdefault('name', scope)
     try:
         return _get_dcs(config)
@@ -267,16 +264,15 @@ def get_members(cluster, cluster_name, member_names, role, force, action):
 @click.option('--role', '-r', help='Give a dsn of any member with this role', type=click.Choice(['master', 'replica',
               'any']), default=None)
 @click.option('--member', '-m', help='Generate a dsn for this member', type=str)
-@option_dcs
-@option_config_file
 @click.argument('cluster_name')
-def dsn(cluster_name, config_file, dcs, role, member):
+@click.pass_obj
+def dsn(obj, cluster_name, role, member):
     if role is not None and member is not None:
         raise PatroniCtlException('--role and --member are mutually exclusive options')
     if member is None and role is None:
         role = 'master'
 
-    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    cluster = get_dcs(obj, cluster_name).get_cluster()
     m = get_any_member(cluster, role=role, member=member)
     if m is None:
         raise PatroniCtlException('Can not find a suitable member')
@@ -287,13 +283,11 @@ def dsn(cluster_name, config_file, dcs, role, member):
 
 @ctl.command('query', help='Query a Patroni PostgreSQL member')
 @click.argument('cluster_name')
-@option_config_file
 @option_format
 @click.option('--format', 'fmt', help='Output format (pretty, json)', default='tsv')
 @click.option('--file', '-f', 'p_file', help='Execute the SQL commands from this file', type=click.File('rb'))
 @click.option('--password', help='force password prompt', is_flag=True)
 @click.option('-U', '--username', help='database user name', type=str)
-@option_dcs
 @option_watch
 @option_watchrefresh
 @click.option('--role', '-r', help='The role of the query', type=click.Choice(['master', 'replica', 'any']),
@@ -302,10 +296,10 @@ def dsn(cluster_name, config_file, dcs, role, member):
 @click.option('--delimiter', help='The column delimiter', default='\t')
 @click.option('--command', '-c', help='The SQL commands to execute')
 @click.option('-d', '--dbname', help='database name to connect to', type=str)
+@click.pass_obj
 def query(
+    obj,
     cluster_name,
-    config_file,
-    dcs,
     role,
     member,
     w,
@@ -329,7 +323,7 @@ def query(
     if p_file is None and command is None:
         raise PatroniCtlException('You need to specify either --command or --file')
 
-    connect_parameters = dict()
+    connect_parameters = {}
     if username:
         connect_parameters['username'] = username
     if password:
@@ -340,17 +334,16 @@ def query(
     if p_file is not None:
         command = p_file.read()
 
-    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    dcs = get_dcs(obj, cluster_name)
 
     cursor = None
     for _ in watching(w, watch, clear=False):
+        if cursor is None:
+            cluster = dcs.get_cluster()
 
         output, cursor = query_member(cluster, cursor, member, role, command, connect_parameters)
         print_output(None, output, fmt=fmt, delimiter=delimiter)
 
-        if cursor is None:
-            cluster = dcs.get_cluster()
-
 
 def query_member(cluster, cursor, member, role, command, connect_parameters):
     try:
@@ -385,11 +378,11 @@ def query_member(cluster, cursor, member, role, command, connect_parameters):
 
 @ctl.command('remove', help='Remove cluster from DCS')
 @click.argument('cluster_name')
-@option_config_file
 @option_format
-@option_dcs
-def remove(config_file, cluster_name, fmt, dcs):
-    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+@click.pass_obj
+def remove(obj, cluster_name, fmt):
+    dcs = get_dcs(obj, cluster_name)
+    cluster = dcs.get_cluster()
 
     output_members(cluster, cluster_name, fmt=fmt)
 
@@ -412,28 +405,6 @@ def remove(config_file, cluster_name, fmt, dcs):
     dcs.delete_cluster()
 
 
-def wait_for_leader(dcs, timeout=30):
-    t_stop = time.time() + timeout
-    timeout /= 2
-
-    while time.time() < t_stop:
-        dcs.watch(timeout)
-        cluster = dcs.get_cluster()
-
-        if cluster.leader:
-            return cluster
-
-    raise PatroniCtlException('Timeout occured')
-
-
-def ctl_load_config(cluster_name, config_file, dcs):
-    config = load_config(config_file, dcs)
-    dcs = get_dcs(config, cluster_name)
-    cluster = dcs.get_cluster()
-
-    return config, dcs, cluster
-
-
 def check_response(response, member_name, action_name, silent_success=False):
     if response.status_code >= 400:
         click.echo('Failed: {0} for member {1}, status code={2}, ({3})'.format(
@@ -468,11 +439,12 @@ def parse_scheduled(scheduled):
 @click.option('--pg-version', 'version', help='Restart if the PostgreSQL version is less than provided (e.g. 9.5.2)',
               default=None)
 @click.option('--pending', help='Restart if pending', is_flag=True)
-@option_config_file
+@click.option('--timeout',
+              help='Return error and fail over if necessary when restarting takes longer than this.')
 @option_force
-@option_dcs
-def restart(cluster_name, member_names, config_file, dcs, force, role, p_any, scheduled, version, pending):
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+@click.pass_obj
+def restart(obj, cluster_name, member_names, force, role, p_any, scheduled, version, pending, timeout):
+    cluster = get_dcs(obj, cluster_name).get_cluster()
 
     members = get_members(cluster, cluster_name, member_names, role, force, 'restart')
     if p_any:
@@ -503,13 +475,16 @@ def restart(cluster_name, member_names, config_file, dcs, force, role, p_any, sc
             raise PatroniCtlException("Can't schedule restart in the paused state")
         content['schedule'] = scheduled_at.isoformat()
 
+    if timeout is not None:
+        content['timeout'] = timeout
+
     for member in members:
         if 'schedule' in content:
             if force and member.data.get('scheduled_restart'):
-                r = request_patroni(member, 'delete', 'restart', headers=auth_header(config))
+                r = request_patroni(member, 'delete', 'restart', headers=auth_header(obj))
                 check_response(r, member.name, 'flush scheduled restart', True)
 
-        r = request_patroni(member, 'post', 'restart', content, auth_header(config))
+        r = request_patroni(member, 'post', 'restart', content, auth_header(obj))
         if r.status_code == 200:
             click.echo('Success: restart on member {0}'.format(member.name))
         elif r.status_code == 202:
@@ -525,15 +500,14 @@ def restart(cluster_name, member_names, config_file, dcs, force, role, p_any, sc
 @ctl.command('reinit', help='Reinitialize cluster member')
 @click.argument('cluster_name')
 @click.argument('member_names', nargs=-1)
-@option_config_file
 @option_force
-@option_dcs
-def reinit(cluster_name, member_names, config_file, dcs, force):
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+@click.pass_obj
+def reinit(obj, cluster_name, member_names, force):
+    cluster = get_dcs(obj, cluster_name).get_cluster()
     members = get_members(cluster, cluster_name, member_names, None, force, 'reinitialize')
 
     for member in members:
-        r = request_patroni(member, 'post', 'reinitialize', headers=auth_header(config))
+        r = request_patroni(member, 'post', 'reinitialize', headers=auth_header(obj))
         check_response(r, member.name, 'reinitialize')
 
 
@@ -544,9 +518,8 @@ def reinit(cluster_name, member_names, config_file, dcs, force):
 @click.option('--scheduled', help='Timestamp of a scheduled failover in unambiguous format (e.g. ISO 8601)',
               default=None)
 @option_force
-@option_config_file
-@option_dcs
-def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled):
+@click.pass_obj
+def failover(obj, cluster_name, master, candidate, force, scheduled):
     """
         We want to trigger a failover for the specified cluster name.
 
@@ -554,7 +527,8 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
         If so, we trigger a failover and keep the client up to date.
     """
 
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    dcs = get_dcs(obj, cluster_name)
+    cluster = dcs.get_cluster()
 
     if cluster.leader is None and not cluster.is_paused():
         raise PatroniCtlException('This cluster has no master')
@@ -590,12 +564,13 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
 
     scheduled_at = parse_scheduled(scheduled)
 
+    scheduled_at_str = None
     if scheduled_at:
         if cluster.is_paused():
             raise PatroniCtlException("Can't schedule failover in the paused state")
-        scheduled_at = scheduled_at.isoformat()
+        scheduled_at_str = scheduled_at.isoformat()
 
-    failover_value = {'leader': master, 'candidate': candidate, 'scheduled_at': scheduled_at}
+    failover_value = {'leader': master, 'candidate': candidate, 'scheduled_at': scheduled_at_str}
 
     logging.debug(failover_value)
 
@@ -614,7 +589,7 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
     try:
         member = cluster.leader.member if cluster.leader else [m for m in cluster.members if m.name == candidate][0]
 
-        r = request_patroni(member, 'post', 'failover', failover_value, auth_header(config))
+        r = request_patroni(member, 'post', 'failover', failover_value, auth_header(obj))
         if r.status_code in (200, 202):
             logging.debug(r)
             cluster = dcs.get_cluster()
@@ -628,7 +603,7 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
         logging.warning('Failing over to DCS')
         click.echo(timestamp() + ' Could not failover using Patroni api, falling back to DCS')
         click.echo(timestamp() + ' Initializing failover from master {0}'.format(master))
-        dcs.manual_failover(master, candidate, scheduled_at=failover_value)
+        dcs.manual_failover(master, candidate, scheduled_at=scheduled_at)
 
     output_members(cluster, cluster_name)
 
@@ -647,9 +622,11 @@ def output_members(cluster, name, extended=False, fmt='pretty'):
     for m in cluster.members:
         logging.debug(m)
 
-        leader = ''
+        role = ''
         if m.name == leader_name:
-            leader = '*'
+            role = 'Leader'
+        elif m.name == cluster.sync.sync_standby:
+            role = 'Sync standby'
 
         host = m.conn_kwargs()['host']
 
@@ -662,7 +639,7 @@ def output_members(cluster, name, extended=False, fmt='pretty'):
             name,
             m.name,
             host,
-            leader,
+            role,
             m.data.get('state', ''),
             lag,
         ]
@@ -682,7 +659,7 @@ def output_members(cluster, name, extended=False, fmt='pretty'):
         'Cluster',
         'Member',
         'Host',
-        'Leader',
+        'Role',
         'State',
         'Lag in MB',
     ]
@@ -698,19 +675,17 @@ def output_members(cluster, name, extended=False, fmt='pretty'):
 @ctl.command('list', help='List the Patroni members for a given Patroni')
 @click.argument('cluster_names', nargs=-1)
 @click.option('--extended', '-e', help='Show some extra information', is_flag=True)
-@option_config_file
 @option_format
 @option_watch
 @option_watchrefresh
-@option_dcs
-def members(config_file, cluster_names, fmt, watch, w, dcs, extended):
+@click.pass_obj
+def members(obj, cluster_names, fmt, watch, w, extended):
     if not cluster_names:
         logging.warning('Listing members: No cluster names were provided')
         return
 
-    config = load_config(config_file, dcs)
     for cluster_name in cluster_names:
-        dcs = get_dcs(config, cluster_name)
+        dcs = get_dcs(obj, cluster_name)
 
         for _ in watching(w, watch):
             cluster = dcs.get_cluster()
@@ -723,13 +698,10 @@ def timestamp(precision=6):
 
 @ctl.command('configure', help='Create configuration file')
 @click.option('--config-file', '-c', help='Configuration file', prompt='Configuration file', default=CONFIG_FILE_PATH)
-@click.option('--dcs', '-d', help='The DCS connect url', prompt='DCS connect url', default='etcd://localhost:4001')
+@click.option('--dcs', '-d', help='The DCS connect url', prompt='DCS connect url', default='etcd://localhost:2379')
 @click.option('--namespace', '-n', help='The namespace', prompt='Namespace', default='/service/')
 def configure(config_file, dcs, namespace):
-    config = dict()
-    config['dcs_api'] = str(dcs)
-    config['namespace'] = str(namespace)
-    store_config(config, config_file)
+    store_config({'dcs_api': str(dcs), 'namespace': str(namespace)}, config_file)
 
 
 def touch_member(config, dcs):
@@ -766,10 +738,10 @@ def set_defaults(config, cluster_name):
 @ctl.command('scaffold', help='Create a structure for the cluster in DCS')
 @click.argument('cluster_name')
 @click.option('--sysid', '-s', help='System ID of the cluster to put into the initialize key', default="")
-@option_config_file
-@option_dcs
-def scaffold(cluster_name, config_file, dcs, sysid):
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+@click.pass_obj
+def scaffold(obj, cluster_name, sysid):
+    dcs = get_dcs(obj, cluster_name)
+    cluster = dcs.get_cluster()
     if cluster and cluster.initialize is not None:
         raise PatroniCtlException("This cluster is already initialized")
 
@@ -777,10 +749,10 @@ def scaffold(cluster_name, config_file, dcs, sysid):
         # initialize key already exists, don't touch this cluster
         raise PatroniCtlException("Initialize key for cluster {0} already exists".format(cluster_name))
 
-    set_defaults(config, cluster_name)
+    set_defaults(obj, cluster_name)
 
     # make sure the leader keys will never expire
-    if not (touch_member(config, dcs) and dcs.attempt_to_acquire_leader(permanent=True)):
+    if not (touch_member(obj, dcs) and dcs.attempt_to_acquire_leader(permanent=True)):
         # we did initialize this cluster, but failed to write the leader or member keys, wipe it down completely.
         dcs.delete_cluster()
         raise PatroniCtlException("Unable to install permanent leader for cluster {0}".format(cluster_name))
@@ -793,47 +765,57 @@ def scaffold(cluster_name, config_file, dcs, sysid):
 @click.argument('target', type=click.Choice(['restart']))
 @click.option('--role', '-r', help='Flush only members with this role', default='any',
               type=click.Choice(['master', 'replica', 'any']))
-@option_config_file
 @option_force
-@option_dcs
-def flush(cluster_name, member_names, config_file, dcs, force, role, target):
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+@click.pass_obj
+def flush(obj, cluster_name, member_names, force, role, target):
+    cluster = get_dcs(obj, cluster_name).get_cluster()
 
     members = get_members(cluster, cluster_name, member_names, role, force, 'flush')
     for member in members:
         if target == 'restart':
             if member.data.get('scheduled_restart'):
-                r = request_patroni(member, 'delete', 'restart', None, auth_header(config))
+                r = request_patroni(member, 'delete', 'restart', None, auth_header(obj))
                 check_response(r, member.name, 'flush scheduled restart')
             else:
                 click.echo('No scheduled restart for member {0}'.format(member.name))
 
 
-def toggle_pause(config_file, cluster_name, dcs, paused):
-    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+def toggle_pause(config, cluster_name, paused):
+    cluster = get_dcs(config, cluster_name).get_cluster()
     if cluster.is_paused() == paused:
         raise PatroniCtlException('Cluster is {0} paused'.format(paused and 'already' or 'not'))
 
-    r = request_patroni(cluster.leader.member, 'patch', 'config', {'pause': paused or None}, auth_header(config))
+    members = []
+    if cluster.leader:
+        members.append(cluster.leader.member)
+    members.extend([m for m in cluster.members if m.api_url and (not members or members[0].name != m.name)])
 
-    if r.status_code == 200:
-        click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
+    for member in members:
+        try:
+            r = request_patroni(member, 'patch', 'config', {'pause': paused or None}, auth_header(config))
+        except Exception:
+            logging.warning('Member %s is not accessible', member.name)
+            continue
+
+        if r.status_code == 200:
+            click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
+        else:
+            click.echo('Failed: {0} cluster management status code={1}, ({2})'.format(
+                       paused and 'pause' or 'resume', r.status_code, r.text))
+        break
     else:
-        click.echo('Failed: {0} cluster management status code={1}, ({2})'.format(
-                   paused and 'pause' or 'resume', r.status_code, r.text))
+        raise PatroniCtlException('Can not find accessible cluster member')
 
 
 @ctl.command('pause', help='Disable auto failover')
 @click.argument('cluster_name')
-@option_config_file
-@option_dcs
-def pause(config_file, cluster_name, dcs):
-    return toggle_pause(config_file, cluster_name, dcs, True)
+@click.pass_obj
+def pause(obj, cluster_name):
+    return toggle_pause(obj, cluster_name, True)
 
 
 @ctl.command('resume', help='Resume auto failover')
 @click.argument('cluster_name')
-@option_config_file
-@option_dcs
-def resume(config_file, cluster_name, dcs):
-    return toggle_pause(config_file, cluster_name, dcs, False)
+@click.pass_obj
+def resume(obj, cluster_name):
+    return toggle_pause(obj, cluster_name, False)

patroni/dcs/__init__.py

@@ -60,8 +60,8 @@ def get_dcs(config):
                 available_implementations.add(name)
                 if name in config:  # which has configuration section in the config file
                     # propagate some parameters
-                    config[name].update({p: config[p] for p in ('namespace', 'name', 'scope',
-                                         'loop_wait', 'ttl', 'retry_timeout') if p in config})
+                    config[name].update({p: config[p] for p in ('namespace', 'name', 'scope', 'loop_wait',
+                                         'patronictl', 'ttl', 'retry_timeout') if p in config})
                     return value(config[name])
     raise PatroniException("""Can not find suitable configuration of distributed configuration store
 Available implementations: """ + ', '.join(available_implementations))
@@ -142,6 +142,14 @@ class Member(namedtuple('Member', 'index,name,session,data')):
     def clonefrom(self):
         return self.tags.get('clonefrom', False) and bool(self.conn_url)
 
+    @property
+    def state(self):
+        return self.data.get('state', 'unknown')
+
+    @property
+    def is_running(self):
+        return self.state == 'running'
+
 
 class Leader(namedtuple('Leader', 'index,session,member')):
 
@@ -222,7 +230,59 @@ class ClusterConfig(namedtuple('ClusterConfig', 'index,data,modify_index')):
         return ClusterConfig(index, data, modify_index or index)
 
 
-class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operation,members,failover')):
+class SyncState(namedtuple('SyncState', 'index,leader,sync_standby')):
+    """Immutable object (namedtuple) which represents last observed synhcronous replication state
+
+    :param index: modification index of a synchronization key in a Configuration Store
+    :param leader: reference to member that was leader
+    :param sync_standby: standby that was last synchronized to leader
+    """
+
+    @staticmethod
+    def from_node(index, value):
+        """
+        >>> SyncState.from_node(1, None).leader is None
+        True
+        >>> SyncState.from_node(1, '{}').leader is None
+        True
+        >>> SyncState.from_node(1, '{').leader is None
+        True
+        >>> SyncState.from_node(1, '[]').leader is None
+        True
+        >>> SyncState.from_node(1, '{"leader": "leader"}').leader == "leader"
+        True
+        """
+        if value:
+            try:
+                data = json.loads(value)
+                if not isinstance(data, dict):
+                    data = {}
+            except (TypeError, ValueError):
+                data = {}
+        else:
+            data = {}
+        return SyncState(index, data.get('leader'), data.get('sync_standby'))
+
+    def matches(self, name):
+        """
+        Returns if a node name matches one of the nodes in the sync state
+
+        >>> s = SyncState(1, 'foo', 'bar')
+        >>> s.matches('foo')
+        True
+        >>> s.matches('bar')
+        True
+        >>> s.matches('baz')
+        False
+        >>> s.matches(None)
+        False
+        >>> SyncState(1, None, None).matches('foo')
+        False
+        """
+        return name is not None and name in (self.leader, self.sync_standby)
+
+
+class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operation,members,failover,sync')):
 
     """Immutable object (namedtuple) which represents PostgreSQL cluster.
     Consists of the following fields:
@@ -232,7 +292,9 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operat
     :param last_leader_operation: int or long object containing position of last known leader operation.
         This value is stored in `/optime/leader` key
     :param members: list of Member object, all PostgreSQL cluster members including leader
-    :param failover: reference to `Failover` object"""
+    :param failover: reference to `Failover` object
+    :param sync: reference to `SyncState` object, last observed synchronous replication state.
+    """
 
     def is_unlocked(self):
         return not (self.leader and self.leader.name)
@@ -243,8 +305,9 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operat
     def get_member(self, member_name, fallback_to_leader=True):
         return ([m for m in self.members if m.name == member_name] or [self.leader if fallback_to_leader else None])[0]
 
-    def get_clone_member(self):
-        candidates = [m for m in self.members if m.clonefrom and (not self.leader or m.name != self.leader.name)]
+    def get_clone_member(self, exclude):
+        exclude = [exclude] + [self.leader.name] if self.leader else []
+        candidates = [m for m in self.members if m.clonefrom and m.is_running and m.name not in exclude]
         return candidates[randint(0, len(candidates) - 1)] if candidates else self.leader
 
     def is_paused(self):
@@ -261,6 +324,7 @@ class AbstractDCS(object):
     _MEMBERS = 'members/'
     _OPTIME = 'optime'
     _LEADER_OPTIME = _OPTIME + '/' + _LEADER
+    _SYNC = 'sync'
 
     def __init__(self, config):
         """
@@ -272,8 +336,10 @@ class AbstractDCS(object):
         self._base_path = '/'.join([self._namespace, config['scope']])
         self._set_loop_wait(config.get('loop_wait', 10))
 
+        self._ctl = bool(config.get('patronictl', False))
         self._cluster = None
         self._cluster_thread_lock = Lock()
+        self._last_leader_operation = ''
         self.event = Event()
 
     def client_path(self, path):
@@ -307,6 +373,10 @@ class AbstractDCS(object):
     def leader_optime_path(self):
         return self.client_path(self._LEADER_OPTIME)
 
+    @property
+    def sync_path(self):
+        return self.client_path(self._SYNC)
+
     @abc.abstractmethod
     def set_ttl(self, ttl):
         """Set the new ttl value for leader key"""
@@ -356,9 +426,14 @@ class AbstractDCS(object):
             self._cluster = None
 
     @abc.abstractmethod
-    def write_leader_optime(self, last_operation):
+    def _write_leader_optime(self, last_operation):
         """write current xlog location into `/optime/leader` key in DCS
-        :param last_operation: absolute xlog location in bytes"""
+        :param last_operation: absolute xlog location in bytes
+        :returns: `!True` on success."""
+
+    def write_leader_optime(self, last_operation):
+        if self._last_leader_operation != last_operation and self._write_leader_optime(last_operation):
+            self._last_leader_operation = last_operation
 
     @abc.abstractmethod
     def update_leader(self):
@@ -445,10 +520,22 @@ class AbstractDCS(object):
     def delete_cluster(self):
         """Delete cluster from DCS"""
 
-    def watch(self, timeout):
+    def write_sync_state(self, leader, sync_standby, index=None):
+        return self.set_sync_state_value(json.dumps({'leader': leader, 'sync_standby': sync_standby}), index=index)
+
+    @abc.abstractmethod
+    def set_sync_state_value(self, value, index=None):
+        """"""
+
+    @abc.abstractmethod
+    def delete_sync_state(self, index=None):
+        """"""
+
+    def watch(self, leader_index, timeout):
         """If the current node is a master it should just sleep.
         Any other node should watch for changes of leader key with a given timeout
 
+        :param leader_index: index of a leader key
         :param timeout: timeout in seconds
         :returns: `!True` if you would like to reschedule the next run of ha cycle"""
 

patroni/dcs/consul.py

@@ -1,14 +1,17 @@
 from __future__ import absolute_import
 import logging
 import os
+import socket
 import time
-import six
+import urllib3
 
-from consul import ConsulException, NotFound, base, std
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member
+from consul import ConsulException, NotFound, base
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
 from patroni.exceptions import DCSError
-from patroni.utils import sleep
-from requests.exceptions import RequestException
+from patroni.utils import Retry, RetryFailedError
+from urllib3.exceptions import HTTPError
+from six.moves.urllib.parse import urlencode
+from six.moves.http_client import HTTPException
 
 logger = logging.getLogger(__name__)
 
@@ -17,38 +20,60 @@ class ConsulError(DCSError):
     pass
 
 
-class HTTPClient(std.HTTPClient):
+class ConsulInternalError(ConsulException):
+    """An internal Consul server error occurred"""
 
-    def __init__(self, *args, **kwargs):
-        super(HTTPClient, self).__init__(*args, **kwargs)
 
-    def patch_default_timeout(self, timeout):
-        # Set a default timeout for the `request.session.request` method, that is used
-        # internally  by the methods request.session.get, request.session.post and
-        # others. We monkey-patch here to avoid reimplementing each individual method from
-        # `std.HTTPClient`. By default, the timeout is not set. It means that a new
-        # session may hang almost indefinitely waiting for the server to respond,
-        # which is not what we want in Patroni.
+class HTTPClient(object):
 
-        request_func = getattr(self.session.request, '__func__' if six.PY3 else 'im_func')
-        defaults_attr_name = '__defaults__' if six.PY3 else 'func_defaults'
-        defaults = list(getattr(request_func, defaults_attr_name))
-        code = request_func.__code__ if six.PY3 else request_func.func_code
-        defaults[code.co_varnames[code.co_argcount - len(defaults):code.co_argcount].index('timeout')] = timeout
-        setattr(request_func, defaults_attr_name, tuple(defaults))  # monkeypatching
+    def __init__(self, host='127.0.0.1', port=8500, scheme='http', verify=True, timeout=10):
+        self.host = host
+        self.port = port
+        self.scheme = scheme
+        self.verify = verify
+        self.set_read_timeout(timeout)
+        self.base_uri = '{0}://{1}:{2}'.format(self.scheme, self.host, self.port)
+        self.http = urllib3.PoolManager(num_pools=10)
+        self._ttl = None
 
-    def get(self, callback, path, params=None):
-        # The get function is overridden to handle a special case of it being called
-        # with an index and wait parameters. That form indicates that a user needs to
-        # wait for the given key to change its value, with a wait timeout supplied. We
-        # don't want our monkey-patched timeout to be less than the value of the wait
-        # parameter, therefore, we set it to either the value of wait or a default of 5 minutes.
+    def set_read_timeout(self, timeout):
+        self._read_timeout = timeout/3.0
 
-        if isinstance(params, dict) and 'index' in params:
-            timeout = (float(params['wait'][:-1]) if 'wait' in params else 300) + 1
-        else:
-            timeout = None
-        return callback(self.response(self.session.get(self.uri(path, params), verify=self.verify, timeout=timeout)))
+    def set_ttl(self, ttl):
+        ret = self._ttl != ttl
+        self._ttl = ttl
+        return ret
+
+    @staticmethod
+    def response(response):
+        data = response.data.decode('utf-8')
+        if response.status == 500:
+            raise ConsulInternalError('{0} {1}'.format(response.status, data))
+        return base.Response(response.status, response.headers, data)
+
+    def uri(self, path, params=None):
+        return '{0}{1}{2}'.format(self.base_uri, path, params and '?' + urlencode(params) or '')
+
+    def __getattr__(self, method):
+        if method not in ('get', 'post', 'put', 'delete'):
+            raise AttributeError("HTTPClient instance has no attribute '{0}'".format(method))
+
+        def wrapper(callback, path, params=None, data=''):
+            # python-consul doesn't allow to specify ttl smaller then 10 seconds
+            # because session_ttl_min defaults to 10s, so we have to do this ugly dirty hack...
+            if method == 'put' and path == '/v1/session/create':
+                ttl = '"ttl": "{0}s"'.format(self._ttl)
+                if not data or data == '{}':
+                    data = '{' + ttl + '}'
+                else:
+                    data = data[:-1] + ', ' + ttl + '}'
+            kwargs = {'retries': 0, 'preload_content': False, 'body': data}
+            if method == 'get' and isinstance(params, dict) and 'index' in params:
+                kwargs['timeout'] = (float(params['wait'][:-1]) if 'wait' in params else 300) + 1
+            else:
+                kwargs['timeout'] = self._read_timeout
+            return callback(self.response(self.http.request(method.upper(), self.uri(path, params), **kwargs)))
+        return wrapper
 
 
 class ConsulClient(base.Consul):
@@ -62,7 +87,7 @@ def catch_consul_errors(func):
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
-        except (ConsulException, RequestException):
+        except (RetryFailedError, ConsulException, HTTPException, HTTPError, socket.error, socket.timeout):
             return False
     return wrapper
 
@@ -71,16 +96,24 @@ class Consul(AbstractDCS):
 
     def __init__(self, config):
         super(Consul, self).__init__(config)
-        self._ttl = None
+        self._scope = config['scope']
         self._session = None
+        self.__do_not_watch = False
+        self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
+                            retry_exceptions=(ConsulInternalError, HTTPException,
+                                              HTTPError, socket.error, socket.timeout))
+
         self._my_member_data = None
-        self.set_ttl(config.get('ttl') or 30)
         host, port = config.get('host', '127.0.0.1:8500').split(':')
         self._client = ConsulClient(host=host, port=port)
-        self._client.http.patch_default_timeout(config['retry_timeout']/2.0)
-        self._scope = config['scope']
-        self.create_session()
-        self.__do_not_watch = False
+        self.set_retry_timeout(config['retry_timeout'])
+        self.set_ttl(config.get('ttl') or 30)
+        self._last_session_refresh = 0
+        if not self._ctl:
+            self.create_session()
+
+    def retry(self, *args, **kwargs):
+        return self._retry.copy()(*args, **kwargs)
 
     def create_session(self):
         while not self._session:
@@ -88,34 +121,40 @@ class Consul(AbstractDCS):
                 self.refresh_session()
             except ConsulError:
                 logger.info('waiting on consul')
-                sleep(5)
+                time.sleep(5)
 
     def set_ttl(self, ttl):
-        ttl = ttl/2.0  # My experiments have shown that session expires after 2*ttl time
-        if self._ttl != ttl:
+        if self._client.http.set_ttl(ttl/2.0):  # Consul multiplies the TTL by 2x
             self._session = None
             self.__do_not_watch = True
-        self._ttl = ttl
 
     def set_retry_timeout(self, retry_timeout):
-        self._client.http.patch_default_timeout(retry_timeout/2.0)
+        self._retry.deadline = retry_timeout
+        self._client.http.set_read_timeout(retry_timeout)
 
-    def refresh_session(self):
+    def _do_refresh_session(self):
         """:returns: `!True` if it had to create new session"""
+        if self._session and self._last_session_refresh + self._loop_wait > time.time():
+            return False
+
         if self._session:
             try:
-                return self._client.session.renew(self._session) is None
+                self._client.session.renew(self._session)
             except NotFound:
                 self._session = None
-        if not self._session:
-            name = self._scope + '-' + self._name
-            try:
-                self._session = self._client.session.create(name=name, lock_delay=0, behavior='delete', ttl=self._ttl)
-            except (ConsulException, RequestException):
-                logger.exception('session.create')
-        if not self._session:
-            raise ConsulError('Failed to renew/create session')
-        return True
+        ret = not self._session
+        if ret:
+            self._session = self._client.session.create(name=self._scope + '-' + self._name,
+                                                        lock_delay=0.001, behavior='delete')
+        self._last_session_refresh = time.time()
+        return ret
+
+    def refresh_session(self):
+        try:
+            return self.retry(self._do_refresh_session)
+        except (ConsulException, RetryFailedError):
+            logger.exception('refresh_session')
+        raise ConsulError('Failed to renew/create session')
 
     def client_path(self, path):
         return super(Consul, self).client_path(path)[1:]
@@ -127,7 +166,7 @@ class Consul(AbstractDCS):
     def _load_cluster(self):
         try:
             path = self.client_path('/')
-            _, results = self._client.kv.get(path, recurse=True)
+            _, results = self.retry(self._client.kv.get, path, recurse=True)
 
             if results is None:
                 raise NotFound
@@ -154,7 +193,8 @@ class Consul(AbstractDCS):
 
             # get leader
             leader = nodes.get(self._LEADER)
-            if leader and leader['Value'] == self._name and self._session != leader.get('Session', 'x'):
+            if not self._ctl and leader and leader['Value'] == self._name \
+                    and self._session != leader.get('Session', 'x'):
                 logger.info('I am leader but not owner of the session. Removing leader node')
                 self._client.kv.delete(self.leader_path, cas=leader['ModifyIndex'])
                 leader = None
@@ -169,17 +209,22 @@ class Consul(AbstractDCS):
             if failover:
                 failover = Failover.from_node(failover['ModifyIndex'], failover['Value'])
 
-            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover)
+            # get synchronization state
+            sync = nodes.get(self._SYNC)
+            sync = SyncState.from_node(sync and sync['ModifyIndex'], sync and sync['Value'])
+
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
         except NotFound:
-            self._cluster = Cluster(None, None, None, None, [], None)
+            self._cluster = Cluster(None, None, None, None, [], None, None)
         except:
             logger.exception('get_cluster')
             raise ConsulError('Consul is not responding properly')
 
     def touch_member(self, data, **kwargs):
         cluster = self.cluster
-        member = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
+        member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
         create_member = self.refresh_session()
+
         if member and (create_member or member.session != self._session):
             try:
                 self._client.kv.delete(self.member_path)
@@ -201,8 +246,11 @@ class Consul(AbstractDCS):
 
     @catch_consul_errors
     def attempt_to_acquire_leader(self, permanent=False):
+        if not self._session and not permanent:
+            self.refresh_session()
+
         args = {} if permanent else {'acquire': self._session}
-        ret = self._client.kv.put(self.leader_path, self._name, **args)
+        ret = self.retry(self._client.kv.put, self.leader_path, self._name, **args)
         if not ret:
             logger.info('Could not take out TTL lock')
         return ret
@@ -219,25 +267,28 @@ class Consul(AbstractDCS):
         return self._client.kv.put(self.config_path, value, cas=index)
 
     @catch_consul_errors
-    def write_leader_optime(self, last_operation):
+    def _write_leader_optime(self, last_operation):
         return self._client.kv.put(self.leader_optime_path, last_operation)
 
-    @staticmethod
-    def update_leader():
-        return True
+    @catch_consul_errors
+    def update_leader(self):
+        if self._session:
+            self.retry(self._client.session.renew, self._session)
+            self._last_session_refresh = time.time()
+        return bool(self._session)
 
     @catch_consul_errors
     def initialize(self, create_new=True, sysid=''):
         kwargs = {'cas': 0} if create_new else {}
-        return self._client.kv.put(self.initialize_path, sysid, **kwargs)
+        return self.retry(self._client.kv.put, self.initialize_path, sysid, **kwargs)
 
     @catch_consul_errors
     def cancel_initialization(self):
-        return self._client.kv.delete(self.initialize_path)
+        return self.retry(self._client.kv.delete, self.initialize_path)
 
     @catch_consul_errors
     def delete_cluster(self):
-        return self._client.kv.delete(self.client_path(''), recurse=True)
+        return self.retry(self._client.kv.delete, self.client_path(''), recurse=True)
 
     @catch_consul_errors
     def delete_leader(self):
@@ -245,24 +296,31 @@ class Consul(AbstractDCS):
         if cluster and isinstance(cluster.leader, Leader) and cluster.leader.name == self._name:
             return self._client.kv.delete(self.leader_path, cas=cluster.leader.index)
 
-    def watch(self, timeout):
+    @catch_consul_errors
+    def set_sync_state_value(self, value, index=None):
+        return self._client.kv.put(self.sync_path, value, cas=index)
+
+    @catch_consul_errors
+    def delete_sync_state(self, index=None):
+        return self._client.kv.delete(self.sync_path, cas=index)
+
+    def watch(self, leader_index, timeout):
         if self.__do_not_watch:
             self.__do_not_watch = False
             return True
 
-        cluster = self.cluster
-        if cluster and cluster.leader and cluster.leader.name != self._name and cluster.leader.index:
+        if leader_index:
             end_time = time.time() + timeout
             while timeout >= 1:
                 try:
-                    idx, _ = self._client.kv.get(self.leader_path, index=cluster.leader.index, wait=str(timeout) + 's')
-                    return str(idx) != str(cluster.leader.index)
-                except (ConsulException, RequestException):
+                    idx, _ = self._client.kv.get(self.leader_path, index=leader_index, wait=str(timeout) + 's')
+                    return str(idx) != str(leader_index)
+                except (ConsulException, HTTPException, HTTPError, socket.error, socket.timeout):
                     logging.exception('watch')
 
                 timeout = end_time - time.time()
 
         try:
-            return super(Consul, self).watch(timeout)
+            return super(Consul, self).watch(None, timeout)
         finally:
             self.event.clear()

patroni/dcs/etcd.py

@@ -2,6 +2,7 @@ from __future__ import absolute_import
 import etcd
 import logging
 import os
+import urllib3.util.connection
 import random
 import requests
 import socket
@@ -9,12 +10,15 @@ import time
 
 from dns.exception import DNSException
 from dns import resolver
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
 from patroni.exceptions import DCSError
-from patroni.utils import Retry, RetryFailedError, sleep
+from patroni.utils import Retry, RetryFailedError
 from urllib3.exceptions import HTTPError, ReadTimeoutError
 from requests.exceptions import RequestException
+from six.moves.queue import Queue
 from six.moves.http_client import HTTPException
+from six.moves.urllib_parse import urlparse
+from threading import Thread
 
 logger = logging.getLogger(__name__)
 
@@ -23,13 +27,63 @@ class EtcdError(DCSError):
     pass
 
 
+class DnsCachingResolver(Thread):
+
+    def __init__(self, cache_time=600.0, cache_fail_time=30.0):
+        super(DnsCachingResolver, self).__init__()
+        self._cache = {}
+        self._cache_time = cache_time
+        self._cache_fail_time = cache_fail_time
+        self._resolve_queue = Queue()
+        self.daemon = True
+        self.start()
+
+    def run(self):
+        while True:
+            (host, port), attempt = self._resolve_queue.get()
+            response = self._do_resolve(host, port)
+            if response:
+                self._cache[(host, port)] = (time.time(), response)
+            else:
+                if attempt < 10:
+                    self.resolve_async(host, port, attempt + 1)
+                    time.sleep(1)
+
+    def resolve(self, host, port):
+        current_time = time.time()
+        cached_time, response = self._cache.get((host, port), (0, []))
+        time_passed = current_time - cached_time
+        if time_passed > self._cache_time or (not response and time_passed > self._cache_fail_time):
+            new_response = self._do_resolve(host, port)
+            if new_response:
+                self._cache[(host, port)] = (current_time, new_response)
+                response = new_response
+        return response
+
+    def resolve_async(self, host, port, attempt=0):
+        self._resolve_queue.put(((host, port), attempt))
+
+    @staticmethod
+    def _do_resolve(host, port):
+        try:
+            return socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, socket.IPPROTO_TCP)
+        except socket.gaierror:
+            logger.warning('failed to resolve host %s', host)
+            return []
+
+
 class Client(etcd.Client):
 
-    def __init__(self, config):
-        super(Client, self).__init__(read_timeout=config['retry_timeout'])
+    def __init__(self, config, dns_resolver, cache_ttl=300):
+        self._dns_resolver = dns_resolver
+        self.set_machines_cache_ttl(cache_ttl)
+        self._machines_cache_updated = 0
+        args = {p: config.get(p) for p in ('host', 'port', 'protocol', 'use_proxies', 'username', 'password',
+                                           'cert', 'ca_cert') if config.get(p)}
+        super(Client, self).__init__(read_timeout=config['retry_timeout'], **args)
         self._config = config
         self._load_machines_cache()
-        self._allow_reconnect = True
+        self._allow_reconnect = not self._use_proxies
 
     def _build_request_parameters(self):
         kwargs = {'headers': self._get_headers(), 'redirect': self.allow_redirect}
@@ -48,6 +102,9 @@ class Client(etcd.Client):
         kwargs['timeout'] = self.read_timeout/float(kwargs['retries'] + 1)/etcd_nodes
         return kwargs
 
+    def set_machines_cache_ttl(self, cache_ttl):
+        self._machines_cache_ttl = cache_ttl
+
     @property
     def machines(self):
         """Original `machines` method(property) of `etcd.Client` class raise exception
@@ -68,6 +125,10 @@ class Client(etcd.Client):
                 machines = [n.strip() for n in self._handle_server_response(response).data.decode('utf-8').split(',')]
                 logger.debug("Retrieved list of machines: %s", machines)
                 random.shuffle(machines)
+                for url in machines:
+                    r = urlparse(url)
+                    port = r.port or (443 if r.scheme == 'https' else 80)
+                    self._dns_resolver.resolve_async(r.hostname, port)
                 return machines
             except Exception as e:
                 # We can't get the list of machines, if one server is in the
@@ -119,6 +180,11 @@ class Client(etcd.Client):
         # Update machines_cache if previous attempt of update has failed
         if self._update_machines_cache:
             self._load_machines_cache()
+        elif time.time() - self._machines_cache_updated > self._machines_cache_ttl:
+            self._machines_cache = self.machines
+            if self._base_uri in self._machines_cache:
+                self._machines_cache.remove(self._base_uri)
+            self._machines_cache_updated = time.time()
 
         kwargs.update(self._build_request_parameters())
 
@@ -147,40 +213,51 @@ class Client(etcd.Client):
     @staticmethod
     def get_srv_record(host):
         try:
-            return [(str(r.target).rstrip('.'), r.port) for r in resolver.query('_etcd-server._tcp.' + host, 'SRV')]
+            return [(r.target.to_text(True), r.port) for r in resolver.query(host, 'SRV')]
         except DNSException:
-            logger.exception('Can not resolve SRV for %s', host)
-        return []
+            return []
 
-    def _get_machines_cache_from_srv(self, discovery_srv):
+    def _get_machines_cache_from_srv(self, srv):
         """Fetch list of etcd-cluster member by resolving _etcd-server._tcp. SRV record.
         This record should contain list of host and peer ports which could be used to run
         'GET http://{host}:{port}/members' request (peer protocol)"""
 
         ret = []
-        for host, port in self.get_srv_record(discovery_srv):
-            url = '{0}://{1}:{2}/members'.format(self._protocol, host, port)
-            try:
-                response = requests.get(url, timeout=self.read_timeout)
-                if response.ok:
-                    for member in response.json():
-                        ret.extend(member['clientURLs'])
-                    break
-            except RequestException:
-                logger.exception('GET %s', url)
+        for r in ['-client-ssl', '-client', '-ssl', '', '-server-ssl', '-server']:
+            protocol = 'https' if '-ssl' in r else 'http'
+            endpoint = '/members' if '-server' in r else ''
+            for host, port in self.get_srv_record('_etcd{0}._tcp.{1}'.format(r, srv)):
+                url = '{0}://{1}:{2}{3}'.format(protocol, host, port, endpoint)
+                if endpoint:
+                    try:
+                        response = requests.get(url, timeout=self.read_timeout, verify=False)
+                        if response.ok:
+                            for member in response.json():
+                                ret.extend(member['clientURLs'])
+                            break
+                    except RequestException:
+                        logger.exception('GET %s', url)
+                else:
+                    ret.append(url)
+            if ret:
+                self._protocol = protocol
+                break
+        else:
+            logger.warning('Can not resolve SRV for %s', srv)
         return list(set(ret))
 
-    def _get_machines_cache_from_dns(self, addr):
+    def _get_machines_cache_from_dns(self, host, port):
         """One host might be resolved into multiple ip addresses. We will make list out of it"""
-
-        ret = []
-        host, port = addr.split(':')
-        try:
-            for r in set(socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)):
-                ret.append('{0}://{1}:{2}'.format(self._protocol, r[4][0], r[4][1]))
-        except socket.error:
-            logger.exception('Can not resolve %s', host)
-        return list(set(ret)) if ret else ['{0}://{1}:{2}'.format(self._protocol, host, port)]
+        if self.protocol == 'http':
+            ret = []
+            for af, _, _, _, sa in self._dns_resolver.resolve(host, port):
+                host, port = sa[:2]
+                if af == socket.AF_INET6:
+                    host = '[{0}]'.format(host)
+                ret.append('{0}://{1}:{2}'.format(self.protocol, host, port))
+            if ret:
+                return list(set(ret))
+        return ['{0}://{1}:{2}'.format(self.protocol, host, port)]
 
     def _load_machines_cache(self):
         """This method should fill up `_machines_cache` from scratch.
@@ -190,42 +267,33 @@ class Client(etcd.Client):
 
         self._update_machines_cache = True
 
-        if 'discovery_srv' not in self._config and 'host' not in self._config:
-            raise Exception('Neither discovery_srv nor host are defined in etcd section of config')
+        if 'srv' not in self._config and 'host' not in self._config:
+            raise Exception('Neither srv nor host url are defined in etcd section of config')
 
-        self._machines_cache = []
+        if self._use_proxies:
+            self._machines_cache = ['{0}://{1}:{2}'.format(self.protocol, self._config['host'], self._config['port'])]
+        else:
+            self._machines_cache = []
 
-        if 'discovery_srv' in self._config:
-            self._machines_cache = self._get_machines_cache_from_srv(self._config['discovery_srv'])
+            if 'srv' in self._config:
+                self._machines_cache = self._get_machines_cache_from_srv(self._config['srv'])
 
-        if not self._machines_cache and 'host' in self._config:
-            self._machines_cache = self._get_machines_cache_from_dns(self._config['host'])
+            if not self._machines_cache and 'host' in self._config:
+                self._machines_cache = self._get_machines_cache_from_dns(self._config['host'], self._config['port'])
 
         # Can not bootstrap list of etcd-cluster members, giving up
         if not self._machines_cache:
             raise etcd.EtcdException
 
         # After filling up initial list of machines_cache we should ask etcd-cluster about actual list
-        self._base_uri = self._machines_cache.pop(0)
+        self._base_uri = self._next_server()
         self._machines_cache = self.machines
 
         if self._base_uri in self._machines_cache:
             self._machines_cache.remove(self._base_uri)
 
         self._update_machines_cache = False
-
-
-def catch_etcd_errors(func):
-    def wrapper(*args, **kwargs):
-        try:
-            return func(*args, **kwargs) is not None
-        except (RetryFailedError, etcd.EtcdException):
-            return False
-        except:
-            logger.exception("")
-            raise EtcdError("unexpected error")
-
-    return wrapper
+        self._machines_cache_updated = time.time()
 
 
 class Etcd(AbstractDCS):
@@ -239,25 +307,111 @@ class Etcd(AbstractDCS):
                                               etcd.EtcdEventIndexCleared))
         self._client = self.get_etcd_client(config)
         self.__do_not_watch = False
+        self._has_failed = False
 
     def retry(self, *args, **kwargs):
         return self._retry.copy()(*args, **kwargs)
 
+    def _handle_exception(self, e, name='', do_sleep=False, raise_ex=None):
+        if not self._has_failed:
+            logger.exception(name)
+        else:
+            logger.error(e)
+            if do_sleep:
+                time.sleep(1)
+        self._has_failed = True
+        if isinstance(raise_ex, Exception):
+            raise raise_ex
+
+    def catch_etcd_errors(func):
+        def wrapper(self, *args, **kwargs):
+            try:
+                retval = func(self, *args, **kwargs) is not None
+                self._has_failed = False
+                return retval
+            except (RetryFailedError, etcd.EtcdException) as e:
+                self._handle_exception(e)
+                return False
+            except Exception as e:
+                self._handle_exception(e, raise_ex=EtcdError('unexpected error'))
+
+        return wrapper
+
     @staticmethod
     def get_etcd_client(config):
+        if 'proxy' in config:
+            config['use_proxies'] = True
+            config['url'] = config['proxy']
+
+        if 'url' in config:
+            r = urlparse(config['url'])
+            config.update({'protocol': r.scheme, 'host': r.hostname, 'port': r.port or 2379,
+                           'username': r.username, 'password': r.password})
+        elif 'host' in config:
+            host, port = (config['host'] + ':2379').split(':')[:2]
+            config['host'] = host
+            if 'port' not in config:
+                config['port'] = int(port)
+
+        if config.get('cacert'):
+            config['ca_cert'] = config.pop('cacert')
+
+        if config.get('key') and config.get('cert'):
+            config['cert'] = (config['cert'], config['key'])
+
+        for p in ('discovery_srv', 'srv_domain'):
+            if p in config:
+                config['srv'] = config.pop(p)
+
+        dns_resolver = DnsCachingResolver()
+
+        def create_connection_patched(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                                      source_address=None, socket_options=None):
+            host, port = address
+            if host.startswith('['):
+                host = host.strip('[]')
+            err = None
+            for af, socktype, proto, _, sa in dns_resolver.resolve(host, port):
+                sock = None
+                try:
+                    sock = socket.socket(af, socktype, proto)
+                    if socket_options:
+                        for opt in socket_options:
+                            sock.setsockopt(*opt)
+                    if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
+                        sock.settimeout(timeout)
+                    if source_address:
+                        sock.bind(source_address)
+                    sock.connect(sa)
+                    return sock
+
+                except socket.error as e:
+                    err = e
+                    if sock is not None:
+                        sock.close()
+                        sock = None
+
+            if err is not None:
+                raise err
+
+            raise socket.error("getaddrinfo returns an empty list")
+
+        urllib3.util.connection.create_connection = create_connection_patched
+
         client = None
         while not client:
             try:
-                client = Client(config)
+                client = Client(config, dns_resolver)
             except etcd.EtcdException:
                 logger.info('waiting on etcd')
-                sleep(5)
+                time.sleep(5)
         return client
 
     def set_ttl(self, ttl):
         ttl = int(ttl)
         self.__do_not_watch = self._ttl != ttl
         self._ttl = ttl
+        self._client.set_machines_cache_ttl(ttl*10)
 
     def set_retry_timeout(self, retry_timeout):
         self._retry.deadline = retry_timeout
@@ -300,12 +454,16 @@ class Etcd(AbstractDCS):
             if failover:
                 failover = Failover.from_node(failover.modifiedIndex, failover.value)
 
-            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover)
+            # get synchronization state
+            sync = nodes.get(self._SYNC)
+            sync = SyncState.from_node(sync and sync.modifiedIndex, sync and sync.value)
+
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
         except etcd.EtcdKeyNotFound:
-            self._cluster = Cluster(None, None, None, None, [], None)
-        except:
-            logger.exception('get_cluster')
-            raise EtcdError('Etcd is not responding properly')
+            self._cluster = Cluster(None, None, None, None, [], None, None)
+        except Exception as e:
+            self._handle_exception(e, 'get_cluster', raise_ex=EtcdError('Etcd is not responding properly'))
+        self._has_failed = False
 
     @catch_etcd_errors
     def touch_member(self, data, ttl=None, permanent=False):
@@ -337,7 +495,7 @@ class Etcd(AbstractDCS):
         return self._client.write(self.config_path, value, prevIndex=index or 0)
 
     @catch_etcd_errors
-    def write_leader_optime(self, last_operation):
+    def _write_leader_optime(self, last_operation):
         return self._client.set(self.leader_optime_path, last_operation)
 
     @catch_etcd_errors
@@ -360,31 +518,42 @@ class Etcd(AbstractDCS):
     def delete_cluster(self):
         return self.retry(self._client.delete, self.client_path(''), recursive=True)
 
-    def watch(self, timeout):
+    @catch_etcd_errors
+    def set_sync_state_value(self, value, index=None):
+        return self._client.write(self.sync_path, value, prevIndex=index or 0)
+
+    @catch_etcd_errors
+    def delete_sync_state(self, index=None):
+        return self.retry(self._client.delete, self.sync_path, prevIndex=index or 0)
+
+    def watch(self, leader_index, timeout):
         if self.__do_not_watch:
             self.__do_not_watch = False
             return True
 
-        cluster = self.cluster
-        # watch on leader key changes if it is defined and current node is not lock owner
-        if cluster and cluster.leader and cluster.leader.name != self._name and cluster.leader.index:
+        if leader_index:
             end_time = time.time() + timeout
 
             while timeout >= 1:  # when timeout is too small urllib3 doesn't have enough time to connect
                 try:
-                    self._client.watch(self.leader_path, index=cluster.leader.index, timeout=timeout + 0.5)
+                    self._client.watch(self.leader_path, index=leader_index, timeout=timeout + 0.5)
+                    self._has_failed = False
                     # Synchronous work of all cluster members with etcd is less expensive
                     # than reestablishing http connection every time from every replica.
                     return True
                 except etcd.EtcdWatchTimedOut:
                     self._client.http.clear()
+                    self._has_failed = False
                     return False
-                except etcd.EtcdException:
-                    logging.exception('watch')
+                except (etcd.EtcdEventIndexCleared, etcd.EtcdWatcherCleared):  # Watch failed
+                    self._has_failed = False
+                    return True  # leave the loop, because watch with the same parameters will fail anyway
+                except etcd.EtcdException as e:
+                    self._handle_exception(e, 'watch', True)
 
                 timeout = end_time - time.time()
 
         try:
-            return super(Etcd, self).watch(timeout)
+            return super(Etcd, self).watch(None, timeout)
         finally:
             self.event.clear()

patroni/dcs/exhibitor.py

@@ -4,7 +4,6 @@ import requests
 import time
 
 from patroni.dcs.zookeeper import ZooKeeper
-from patroni.utils import sleep
 from requests.exceptions import RequestException
 
 logger = logging.getLogger(__name__)
@@ -24,7 +23,7 @@ class ExhibitorEnsembleProvider(object):
         self._next_poll = None
         while not self.poll():
             logger.info('waiting on exhibitor')
-            sleep(5)
+            time.sleep(5)
 
     def poll(self):
         if self._next_poll and self._next_poll > time.time():

patroni/dcs/zookeeper.py

@@ -3,7 +3,7 @@ import logging
 from kazoo.client import KazooClient, KazooState
 from kazoo.exceptions import NoNodeError, NodeExistsError
 from kazoo.handlers.threading import SequentialThreadingHandler
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
 from patroni.exceptions import DCSError
 
 logger = logging.getLogger(__name__)
@@ -57,7 +57,6 @@ class ZooKeeper(AbstractDCS):
 
         self._my_member_data = None
         self._fetch_cluster = True
-        self._last_leader_operation = 0
 
         self._orig_kazoo_connect = self._client._connection._connect
         self._client._connection._connect = self._kazoo_connect
@@ -65,7 +64,6 @@ class ZooKeeper(AbstractDCS):
         self._client.start()
 
     def _kazoo_connect(self, host, port):
-
         """Kazoo is using Ping's to determine health of connection to zookeeper. If there is no
         response on Ping after Ping interval (1/2 from read_timeout) it will consider current
         connection dead and try to connect to another node. Without this "magic" it was taking
@@ -135,10 +133,11 @@ class ZooKeeper(AbstractDCS):
         except NoNodeError:
             return []
 
-    def load_members(self):
+    def load_members(self, sync_standby):
         members = []
         for member in self.get_children(self.members_path, self.cluster_watcher):
-            data = self.get_node(self.members_path + member)
+            watch = member == sync_standby and self.cluster_watcher or None
+            data = self.get_node(self.members_path + member, watch)
             if data is not None:
                 members.append(self.member(member, *data))
         return members
@@ -157,14 +156,24 @@ class ZooKeeper(AbstractDCS):
         config = self.get_node(self.config_path, watch=self.cluster_watcher) if self._CONFIG in nodes else None
         config = config and ClusterConfig.from_node(config[1].version, config[0], config[1].mzxid)
 
+        # get last leader operation
+        last_leader_operation = self._OPTIME in nodes and self._fetch_cluster and self.get_node(self.leader_optime_path)
+        last_leader_operation = last_leader_operation and int(last_leader_operation[0]) or 0
+
+        # get synchronization state
+        sync = self.get_node(self.sync_path, watch=self.cluster_watcher) if self._SYNC in nodes else None
+        sync = SyncState.from_node(sync and sync[1].version, sync and sync[0])
+
         # get list of members
-        members = self.load_members() if self._MEMBERS[:-1] in nodes else []
+        sync_standby = sync.leader == self._name and sync.sync_standby or None
+        members = self.load_members(sync_standby) if self._MEMBERS[:-1] in nodes else []
 
         # get leader
         leader = self.get_node(self.leader_path) if self._LEADER in nodes else None
         if leader:
             client_id = self._client.client_id
-            if leader[0] == self._name and client_id is not None and client_id[0] != leader[1].ephemeralOwner:
+            if not self._ctl and leader[0] == self._name and client_id is not None \
+                    and client_id[0] != leader[1].ephemeralOwner:
                 logger.info('I am leader but not owner of the session. Removing leader node')
                 self._client.delete(self.leader_path)
                 leader = None
@@ -179,10 +188,7 @@ class ZooKeeper(AbstractDCS):
         failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
         failover = failover and Failover.from_node(failover[1].version, failover[0])
 
-        # get last leader operation
-        optime = self.get_node(self.leader_optime_path) if self._OPTIME in nodes and self._fetch_cluster else None
-        self._last_leader_operation = 0 if optime is None else int(optime[0])
-        self._cluster = Cluster(initialize, config, leader, self._last_leader_operation, members, failover)
+        self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
 
     def _load_cluster(self):
         if self._fetch_cluster or self._cluster is None:
@@ -228,11 +234,11 @@ class ZooKeeper(AbstractDCS):
 
     def initialize(self, create_new=True, sysid=""):
         return self._create(self.initialize_path, sysid, makepath=True) if create_new \
-            else self._client.retry(self._client.set, self.initialize_path,  sysid.encode("utf-8"))
+            else self._client.retry(self._client.set, self.initialize_path, sysid.encode("utf-8"))
 
     def touch_member(self, data, ttl=None, permanent=False):
         cluster = self.cluster
-        member = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
+        member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
         data = data.encode('utf-8')
         if member and self._client.client_id is not None and member.session != self._client.client_id[0]:
             try:
@@ -267,20 +273,20 @@ class ZooKeeper(AbstractDCS):
     def take_leader(self):
         return self.attempt_to_acquire_leader()
 
-    def write_leader_optime(self, last_operation):
+    def _write_leader_optime(self, last_operation):
         last_operation = last_operation.encode('utf-8')
-        if last_operation != self._last_leader_operation:
+        try:
+            self._client.set_async(self.leader_optime_path, last_operation).get(timeout=1)
+            return True
+        except NoNodeError:
             try:
-                self._client.set_async(self.leader_optime_path, last_operation).get(timeout=1)
-                self._last_leader_operation = last_operation
-            except NoNodeError:
-                try:
-                    self._client.create_async(self.leader_optime_path, last_operation, makepath=True).get(timeout=1)
-                    self._last_leader_operation = last_operation
-                except:
-                    logger.exception('Failed to create %s', self.leader_optime_path)
+                self._client.create_async(self.leader_optime_path, last_operation, makepath=True).get(timeout=1)
+                return True
             except:
-                logger.exception('Failed to update %s', self.leader_optime_path)
+                logger.exception('Failed to create %s', self.leader_optime_path)
+        except:
+            logger.exception('Failed to update %s', self.leader_optime_path)
+        return False
 
     def update_leader(self):
         return True
@@ -307,7 +313,20 @@ class ZooKeeper(AbstractDCS):
         except NoNodeError:
             return True
 
-    def watch(self, timeout):
-        if super(ZooKeeper, self).watch(timeout):
+    def set_sync_state_value(self, value, index=None):
+        try:
+            self._client.retry(self._client.set, self.sync_path, value.encode('utf-8'), version=index or -1)
+            return True
+        except NoNodeError:
+            return value == '' or (index is None and self._create(self.sync_path, value))
+        except:
+            logging.exception('set_sync_state_value')
+            return False
+
+    def delete_sync_state(self, index=None):
+        return self.set_sync_state_value("{}", index)
+
+    def watch(self, leader_index, timeout):
+        if super(ZooKeeper, self).watch(leader_index, timeout):
             self._fetch_cluster = True
         return self._fetch_cluster

patroni/ha.py

@@ -1,20 +1,51 @@
+import datetime
+import functools
 import json
 import logging
 import psycopg2
 import requests
 import sys
-import datetime
-import pytz
+import time
 
+from collections import namedtuple
 from multiprocessing.pool import ThreadPool
 from patroni.async_executor import AsyncExecutor
 from patroni.exceptions import DCSError, PostgresConnectionException
 from patroni.postgresql import ACTION_ON_START
-from patroni.utils import sleep
+from patroni.utils import polling_loop, tzutc
+from threading import RLock
 
 logger = logging.getLogger(__name__)
 
 
+class _MemberStatus(namedtuple('_MemberStatus', 'member,reachable,in_recovery,xlog_location,tags')):
+    """Node status distilled from API response:
+
+        member - dcs.Member object of the node
+        reachable - `!False` if the node is not reachable or is not responding with correct JSON
+        in_recovery - `!True` if pg_is_in_recovery() == true
+        xlog_location - value of `replayed_location` or `location` from JSON, dependin on its role.
+        tags - dictionary with values of different tags (i.e. nofailover)
+    """
+    @classmethod
+    def from_api_response(cls, member, json):
+        is_master = json['role'] == 'master'
+        xlog = not is_master and max(json['xlog'].get('received_location', 0), json['xlog'].get('replayed_location', 0))
+        return cls(member, True, not is_master, xlog, json.get('tags', {}))
+
+    @classmethod
+    def unknown(cls, member):
+        return cls(member, False, None, 0, {})
+
+    def failover_limitation(self):
+        """Returns reason why this node can't promote or None if everything is ok."""
+        if not self.reachable:
+            return 'not reachable'
+        if self.tags.get('nofailover', False):
+            return 'not allowed to promote'
+        return None
+
+
 class Ha(object):
 
     def __init__(self, patroni):
@@ -24,7 +55,15 @@ class Ha(object):
         self.cluster = None
         self.old_cluster = None
         self.recovering = False
-        self._async_executor = AsyncExecutor()
+        self._start_timeout = None
+        self._async_executor = AsyncExecutor(self.wakeup)
+
+        # Each member publishes various pieces of information to the DCS using touch_member. This lock protects
+        # the state and publishing procedure to have consistent ordering and avoid publishing stale values.
+        self._member_state_lock = RLock()
+        # Count of concurrent sync disabling requests. Value above zero means that we don't want to be synchronous
+        # standby. Changes protected by _member_state_lock.
+        self._disable_sync = 0
 
     def is_paused(self):
         return self.cluster and self.cluster.is_paused()
@@ -40,9 +79,9 @@ class Ha(object):
     def acquire_lock(self):
         return self.dcs.attempt_to_acquire_leader()
 
-    def update_lock(self):
+    def update_lock(self, write_leader_optime=False):
         ret = self.dcs.update_leader()
-        if ret and not self._async_executor.busy:
+        if ret and write_leader_optime:
             try:
                 self.dcs.write_leader_optime(self.state_handler.last_operation())
             except:
@@ -54,42 +93,52 @@ class Ha(object):
         logger.info('Lock owner: %s; I am %s', lock_owner, self.state_handler.name)
         return lock_owner == self.state_handler.name
 
-    def touch_member(self):
-        data = {
-            'conn_url': self.state_handler.connection_string,
-            'api_url': self.patroni.api.connection_string,
-            'state': self.state_handler.state,
-            'role': self.state_handler.role
-        }
-        if self.patroni.tags:
-            data['tags'] = self.patroni.tags
-        if self.state_handler.pending_restart:
-            data['pending_restart'] = True
-        if not self._async_executor.busy and data['state'] in ['running', 'restarting', 'starting']:
-            try:
-                data['xlog_location'] = self.state_handler.xlog_position()
-            except:
-                pass
-        if self.patroni.scheduled_restart:
-            scheduled_restart_data = self.patroni.scheduled_restart.copy()
-            scheduled_restart_data['schedule'] = scheduled_restart_data['schedule'].isoformat()
-            data['scheduled_restart'] = scheduled_restart_data
+    def get_effective_tags(self):
+        """Return configuration tags merged with dynamically applied tags."""
+        tags = self.patroni.tags.copy()
+        # _disable_sync could be modified concurrently, but we don't care as attribute get and set are atomic.
+        if self._disable_sync > 0:
+            tags['nosync'] = True
+        return tags
 
-        self.dcs.touch_member(json.dumps(data, separators=(',', ':')))
+    def touch_member(self):
+        with self._member_state_lock:
+            data = {
+                'conn_url': self.state_handler.connection_string,
+                'api_url': self.patroni.api.connection_string,
+                'state': self.state_handler.state,
+                'role': self.state_handler.role
+            }
+            tags = self.get_effective_tags()
+            if tags:
+                data['tags'] = tags
+            if self.state_handler.pending_restart:
+                data['pending_restart'] = True
+            if not self._async_executor.busy and data['state'] in ['running', 'restarting', 'starting']:
+                try:
+                    data['xlog_location'] = self.state_handler.xlog_position(retry=False)
+                except:
+                    pass
+            if self.patroni.scheduled_restart:
+                scheduled_restart_data = self.patroni.scheduled_restart.copy()
+                scheduled_restart_data['schedule'] = scheduled_restart_data['schedule'].isoformat()
+                data['scheduled_restart'] = scheduled_restart_data
+
+            return self.dcs.touch_member(json.dumps(data, separators=(',', ':')))
 
     def clone(self, clone_member=None, msg='(without leader)'):
         if self.state_handler.clone(clone_member):
             logger.info('bootstrapped %s', msg)
             cluster = self.dcs.get_cluster()
             node_to_follow = self._get_node_to_follow(cluster)
-            self.state_handler.follow(node_to_follow, cluster.leader, True)
+            return self.state_handler.follow(node_to_follow, cluster.leader, True)
         else:
             logger.error('failed to bootstrap %s', msg)
             self.state_handler.remove_data_directory()
 
     def bootstrap(self):
         if not self.cluster.is_unlocked():  # cluster already has leader
-            clone_member = self.cluster.get_clone_member()
+            clone_member = self.cluster.get_clone_member(self.state_handler.name)
             member_role = 'leader' if clone_member == self.cluster.leader else 'replica'
             msg = "from {0} '{1}'".format(member_role, clone_member.name)
             self._async_executor.schedule('bootstrap {0}'.format(msg))
@@ -121,8 +170,21 @@ class Ha(object):
             return 'waiting for leader to bootstrap'
 
     def recover(self):
+        if self.has_lock() and self.update_lock():
+            timeout = self.patroni.config['master_start_timeout']
+            if timeout == 0:
+                # We are requested to prefer failing over to restarting master. But see first if there
+                # is anyone to fail over to.
+                if self.is_failover_possible(self.cluster.members):
+                    logger.info("Master crashed. Failing over.")
+                    self.demote('immediate')
+                    return 'stopped PostgreSQL to fail over after a crash'
+        else:
+            timeout = None
+
         self.recovering = True
-        return self.follow("starting as readonly because i had the session lock", "starting as a secondary", True, True)
+        return self.follow("starting as readonly because i had the session lock",
+                           "starting as a secondary", True, True, None, timeout)
 
     def _get_node_to_follow(self, cluster):
         # determine the node to follow. If replicatefrom tag is set,
@@ -134,7 +196,7 @@ class Ha(object):
 
         return node_to_follow if node_to_follow and node_to_follow.name != self.state_handler.name else None
 
-    def follow(self, demote_reason, follow_reason, refresh=True, recovery=False, need_rewind=None):
+    def follow(self, demote_reason, follow_reason, refresh=True, recovery=False, need_rewind=None, timeout=None):
         if refresh:
             self.load_cluster_from_dcs()
 
@@ -146,49 +208,134 @@ class Ha(object):
 
         node_to_follow = self._get_node_to_follow(self.cluster)
 
-        if self.is_paused() and not self.state_handler.need_rewind:
+        if self.is_paused() and not (self.state_handler.need_rewind and self.state_handler.can_rewind):
             self.state_handler.set_role('master' if is_leader else 'replica')
             if is_leader:
                 return 'continue to run as master without lock'
             elif not node_to_follow:
                 return 'no action'
 
-        self.state_handler.follow(node_to_follow, self.cluster.leader, recovery, self._async_executor, need_rewind)
+        self.state_handler.follow(node_to_follow, self.cluster.leader, recovery,
+                                  self._async_executor, need_rewind, timeout)
 
         return ret
 
+    def is_synchronous_mode(self):
+        return bool(self.cluster and self.cluster.config and self.cluster.config.data.get('synchronous_mode'))
+
+    def process_sync_replication(self):
+        """Process synchronous standby beahvior.
+
+        Synchronous standbys are registered in two places postgresql.conf and DCS. The order of updating them must
+        be right. The invariant that should be kept is that if a node is master and sync_standby is set in DCS,
+        then that node must have synchronous_standby set to that value. Or more simple, first set in postgresql.conf
+        and then in DCS. When removing, first remove in DCS, then in postgresql.conf. This is so we only consider
+        promoting standbys that were guaranteed to be replicating synchronously.
+        """
+        if self.is_synchronous_mode():
+            current = self.cluster.sync.leader and self.cluster.sync.sync_standby
+            picked, allow_promote = self.state_handler.pick_synchronous_standby(self.cluster)
+            if picked != current:
+                # We need to revoke privilege from current before replacing it in the config
+                if current:
+                    logger.info("Removing synchronous privilege from %s", current)
+                    if not self.dcs.write_sync_state(self.state_handler.name, None, index=self.cluster.sync.index):
+                        logger.info('Synchronous replication key updated by someone else.')
+                        return
+                logger.info("Assigning synchronous standby status to %s", picked)
+                self.state_handler.set_synchronous_standby(picked)
+
+                if picked and not allow_promote:
+                    # Wait for PostgreSQL to enable synchronous mode and see if we can immediately set sync_standby
+                    time.sleep(2)
+                    picked, allow_promote = self.state_handler.pick_synchronous_standby(self.cluster)
+                if allow_promote:
+                    cluster = self.dcs.get_cluster()
+                    if cluster.sync.leader and cluster.sync.leader != self.state_handler.name:
+                        logger.info("Synchronous replication key updated by someone else")
+                        return
+                    if not self.dcs.write_sync_state(self.state_handler.name, picked, index=cluster.sync.index):
+                        logger.info("Synchronous replication key updated by someone else")
+                        return
+                    logger.info("Synchronous standby status assigned to %s", picked)
+        else:
+            if self.cluster.sync.leader and self.dcs.delete_sync_state(index=self.cluster.sync.index):
+                logger.info("Disabled synchronous replication")
+            self.state_handler.set_synchronous_standby(None)
+
+    def is_sync_standby(self, cluster):
+        return cluster.leader and cluster.sync.leader == cluster.leader.name \
+            and cluster.sync.sync_standby == self.state_handler.name
+
+    def while_not_sync_standby(self, func):
+        """Runs specified action while trying to make sure that the node is not assigned synchronous standby status.
+
+        Tags us as not allowed to be a sync standby as we are going to go away, if we currently are wait for
+        leader to notice and pick an alternative one or if the leader changes or goes away we are also free.
+
+        If the connection to DCS fails we run the action anyway, as this is only a hint.
+
+        There is a small race window where this function runs between a master picking us the sync standby and
+        publishing it to the DCS. As the window is rather tiny consequences are holding up commits for one cycle
+        period we don't worry about it here."""
+
+        if not self.is_synchronous_mode() or self.patroni.nosync:
+            return func()
+
+        with self._member_state_lock:
+            self._disable_sync += 1
+        try:
+            if self.touch_member():
+                # Master should notice the updated value during the next cycle. We will wait double that, if master
+                # hasn't noticed the value by then not disabling sync replication is not likely to matter.
+                for _ in polling_loop(timeout=self.dcs.loop_wait*2, interval=2):
+                    try:
+                        if not self.is_sync_standby(self.dcs.get_cluster()):
+                            break
+                    except DCSError:
+                        logger.warning("Could not get cluster state, skipping synchronous standby disable")
+                        break
+                    logger.info("Waiting for master to release us from synchronous standby")
+            else:
+                logger.warning("Updating member state failed, skipping synchronous standby disable")
+
+            return func()
+        finally:
+            with self._member_state_lock:
+                self._disable_sync -= 1
+
     def enforce_master_role(self, message, promote_message):
         if self.state_handler.is_leader() or self.state_handler.role == 'master':
             # Inform the state handler about its master role.
             # It may be unaware of it if postgres is promoted manually.
             self.state_handler.set_role('master')
+            self.process_sync_replication()
             return message
         else:
+            if self.is_synchronous_mode():
+                # Just set ourselves as the authoritative source of truth for now. We don't want to wait for standbys
+                # to connect. We will try finding a synchronous standby in the next cycle.
+                if not self.dcs.write_sync_state(self.state_handler.name, None, index=self.cluster.sync.index):
+                    # Somebody else updated sync state, it may be due to us losing the lock. To be safe, postpone
+                    # promotion until next cycle. TODO: trigger immediate retry of run_cycle
+                    return 'Postponing promotion because synchronous replication state was updated by somebody else'
+                self.state_handler.set_synchronous_standby(None)
             self.state_handler.promote()
-            self.touch_member()
             return promote_message
 
     @staticmethod
     def fetch_node_status(member):
         """This function perform http get request on member.api_url and fetches its status
-        :returns: tuple(`member`, reachable, in_recovery, xlog_location)
-
-        reachable - `!False` if the node is not reachable or is not responding with correct JSON
-        in_recovery - `!True` if pg_is_in_recovery() == true
-        xlog_location - value of `replayed_location` or `location` from JSON, dependin on its role.
-        tags - dictionary with values of different tags (i.e. nofailover)
+        :returns: `_MemberStatus` object
         """
 
         try:
             response = requests.get(member.api_url, timeout=2, verify=False)
             logger.info('Got response from %s %s: %s', member.name, member.api_url, response.content)
-            json = response.json()
-            is_master = json['role'] == 'master'
-            xlog_location = None if is_master else json['xlog']['replayed_location']
-            return (member, True, not is_master, xlog_location, json.get('tags', {}))
+            return _MemberStatus.from_api_response(member, response.json())
         except Exception as e:
             logger.warning("request failed: GET %s (%s)", member.api_url, e)
-        return (member, False, None, 0, {})
+        return _MemberStatus.unknown(member)
 
     def fetch_nodes_statuses(self, members):
         pool = ThreadPool(len(members))
@@ -197,23 +344,32 @@ class Ha(object):
         pool.join()
         return results
 
+    def is_lagging(self, xlog_location):
+        """Returns if instance with an xlog should consider itself unhealthy to be promoted due to replication lag.
+
+        :param xlog_location: Current xlog location.
+        :returns True when node is lagging
+        """
+        lag = (self.cluster.last_leader_operation or 0) - xlog_location
+        return lag > self.state_handler.config.get('maximum_lag_on_failover', 0)
+
     def _is_healthiest_node(self, members, check_replication_lag=True):
         """This method tries to determine whether I am healthy enough to became a new leader candidate or not."""
 
-        if check_replication_lag and not self.state_handler.check_replication_lag(self.cluster.last_leader_operation):
+        my_xlog_location = self.state_handler.xlog_position()
+        if check_replication_lag and self.is_lagging(my_xlog_location):
             return False  # Too far behind last reported xlog location on master
 
         # Prepare list of nodes to run check against
         members = [m for m in members if m.name != self.state_handler.name and not m.nofailover and m.api_url]
 
         if members:
-            my_xlog_location = self.state_handler.xlog_position()
-            for member, reachable, in_recovery, xlog_location, tags in self.fetch_nodes_statuses(members):
-                if reachable and not tags.get('nofailover', False):  # If the node is unreachable it's not healhy
-                    if not in_recovery:
-                        logger.warning('Master (%s) is still alive', member.name)
+            for st in self.fetch_nodes_statuses(members):
+                if st.failover_limitation() is None:
+                    if not st.in_recovery:
+                        logger.warning('Master (%s) is still alive', st.member.name)
                         return False
-                    if my_xlog_location < xlog_location:
+                    if my_xlog_location < st.xlog_location:
                         return False
         return True
 
@@ -221,13 +377,14 @@ class Ha(object):
         ret = False
         members = [m for m in members if m.name != self.state_handler.name and not m.nofailover and m.api_url]
         if members:
-            for member, reachable, _, _, tags in self.fetch_nodes_statuses(members):
-                if reachable and not tags.get('nofailover', False):
-                    ret = True  # TODO: check xlog_location
-                elif not reachable:
-                    logger.info('Member %s is not reachable', member.name)
-                elif tags.get('nofailover', False):
-                    logger.info('Member %s is not allowed to promote', member.name)
+            for st in self.fetch_nodes_statuses(members):
+                not_allowed_reason = st.failover_limitation()
+                if not_allowed_reason:
+                    logger.info('Member %s is %s', st.member.name, not_allowed_reason)
+                elif self.is_lagging(st.xlog_location):
+                    logger.info('Member %s exceeds maximum replication lag', st.member.name)
+                else:
+                    ret = True
         else:
             logger.warning('manual failover: members list is empty')
         return ret
@@ -250,15 +407,13 @@ class Ha(object):
             # find specific node and check that it is healthy
             member = self.cluster.get_member(failover.candidate, fallback_to_leader=False)
             if member:
-                member, reachable, _, _, tags = self.fetch_node_status(member)
-                if reachable and not tags.get('nofailover', False):  # node is healthy
-                    logger.info('manual failover: to %s, i am %s', member.name, self.state_handler.name)
+                st = self.fetch_node_status(member)
+                not_allowed_reason = st.failover_limitation()
+                if not_allowed_reason is None:  # node is healthy
+                    logger.info('manual failover: to %s, i am %s', st.member.name, self.state_handler.name)
                     return False
                 # we wanted to failover to specific member but it is not healthy
-                if not reachable:
-                    logger.warning('manual failover: member %s is unhealthy', member.name)
-                elif tags.get('nofailover', False):
-                    logger.warning('manual failover: member %s is not allowed to promote', member.name)
+                logger.warning('manual failover: member %s is %s', st.member.name, not_allowed_reason)
 
             # at this point we should consider all members as a candidates for failover
             # i.e. we assume that failover.candidate is None
@@ -288,6 +443,9 @@ class Ha(object):
             if ret is not None:  # continue if we just deleted the stale failover key as a master
                 return ret
 
+        if self.state_handler.is_starting():  # postgresql still starting up is unhealthy
+            return False
+
         if self.state_handler.is_leader():  # leader is always the healthiest
             return True
 
@@ -300,23 +458,58 @@ class Ha(object):
         if self.cluster.failover:
             return self.manual_failover_process_no_leader()
 
-        # run usual health check
-        members = {m.name: m for m in self.cluster.members + self.old_cluster.members}
+        # When in sync mode, only last known master and sync standby are allowed to promote automatically.
+        all_known_members = self.cluster.members + self.old_cluster.members
+        if self.is_synchronous_mode() and self.cluster.sync.leader:
+            if not self.cluster.sync.matches(self.state_handler.name):
+                return False
+            # pick between synchronous candidates so we minimize unnecessary failovers/demotions
+            members = {m.name: m for m in all_known_members if self.cluster.sync.matches(m.name)}
+        else:
+            # run usual health check
+            members = {m.name: m for m in all_known_members}
+
         return self._is_healthiest_node(members.values())
 
-    def demote(self, delete_leader=True):
-        if delete_leader:
-            self.state_handler.stop()
+    def release_leader_key_voluntarily(self):
+        self.dcs.delete_leader()
+        self.touch_member()
+        self.dcs.reset_cluster()
+        logger.info("Leader key released")
+
+    def demote(self, mode):
+        """Demote PostgreSQL running as master.
+
+        :param mode: One of offline, graceful or immediate.
+            offline is used when connection to DCS is not available.
+            graceful is used when failing over to another node due to user request. May only be called running async.
+            immediate is used when we determine that we are not suitable for master and want to failover quickly
+                without regard for data durability. May only be called synchronously.
+        """
+        assert mode in ['offline', 'graceful', 'immediate']
+        if mode != 'offline':
+            if mode == 'immediate':
+                self.state_handler.stop('immediate', checkpoint=False)
+            else:
+                self.state_handler.stop()
             self.state_handler.set_role('demoted')
-            self.dcs.delete_leader()
-            self.touch_member()
-            self.dcs.reset_cluster()
-            sleep(2)  # Give a time to somebody to take the leader lock
+            self.release_leader_key_voluntarily()
+            time.sleep(2)  # Give a time to somebody to take the leader lock
             cluster = self.dcs.get_cluster()
             node_to_follow = self._get_node_to_follow(cluster)
-            self.state_handler.follow(node_to_follow, cluster.leader, recovery=True, need_rewind=True)
+            if mode == 'immediate':
+                # We will try to start up as a standby now. If no one takes the leader lock before we finish
+                # recovery we will try to promote ourselves.
+                self._async_executor.schedule('waiting for failover to complete')
+                self._async_executor.run_async(self.state_handler.follow,
+                                               (node_to_follow, cluster.leader, True, None, True))
+            else:
+                return self.state_handler.follow(node_to_follow, cluster.leader, recovery=True, need_rewind=True)
         else:
-            self.state_handler.follow(None, None)
+            # Need to become unavailable as soon as possible, so initiate a stop here. However as we can't release
+            # the leader key we don't care about confirming the shutdown quickly and can use a regular stop.
+            self.state_handler.stop(checkpoint=False)
+            self.state_handler.follow(None, None, recovery=True)
 
     def should_run_scheduled_action(self, action_name, scheduled_at, cleanup_fn):
         if scheduled_at and not self.is_paused():
@@ -326,7 +519,7 @@ class Ha(object):
             # If the value is close to now, we initiate the scheduled action
             # Additionally, if the scheduled action cannot be executed altogether, i.e. there is an error
             # or the action is in the past - we take care of cleaning it up.
-            now = datetime.datetime.now(pytz.utc)
+            now = datetime.datetime.now(tzutc)
             try:
                 delta = (scheduled_at - now).total_seconds()
 
@@ -335,13 +528,16 @@ class Ha(object):
                                 action_name, scheduled_at.isoformat(), delta)
                     return False
                 elif delta < - int(self.dcs.loop_wait * 1.5):
+                    # This means that if run_cycle gets delayed for 2.5x loop_wait we skip the
+                    # scheduled action. Probably not a problem, if things are that bad we don't
+                    # want to be restarting or failing over anyway.
                     logger.warning('Found a stale %s value, cleaning up: %s',
                                    action_name, scheduled_at.isoformat())
                     cleanup_fn()
                     return False
 
                 # The value is very close to now
-                sleep(max(delta, 0))
+                time.sleep(max(delta, 0))
                 logger.info('Manual scheduled {0} at %s'.format(action_name), scheduled_at.isoformat())
                 return True
             except TypeError:
@@ -350,7 +546,14 @@ class Ha(object):
         return False
 
     def process_manual_failover_from_leader(self):
+        """Checks if manual failover is requested and takes action if appropriate.
+
+        Cleans up failover key if failover conditions are not matched.
+
+        :returns: action message if demote was initiated, None if no action was taken"""
         failover = self.cluster.failover
+        if not failover or (self.is_paused() and not self.state_handler.is_leader()):
+            return
 
         if (failover.scheduled_at and not
             self.should_run_scheduled_action("failover", failover.scheduled_at, lambda:
@@ -366,7 +569,7 @@ class Ha(object):
                                if not failover.candidate or m.name == failover.candidate]
                     if self.is_failover_possible(members):  # check that there are healthy members
                         self._async_executor.schedule('manual failover: demote')
-                        self._async_executor.run_async(self.demote)
+                        self._async_executor.run_async(self.demote, ('graceful',))
                         return 'manual failover: demoting myself'
                     else:
                         logger.warning('manual failover: no healthy members found, failover is not possible')
@@ -403,7 +606,7 @@ class Ha(object):
             # node tagged as nofailover can be ahead of the new leader either, but it is always excluded from elections
             need_rewind = bool(self.cluster.failover) or self.patroni.nofailover
             if need_rewind:
-                sleep(2)  # Give a time to somebody to take the leader lock
+                time.sleep(2)  # Give a time to somebody to take the leader lock
 
             if self.patroni.nofailover:
                 return self.follow('demoting self because I am not allowed to become master',
@@ -415,11 +618,6 @@ class Ha(object):
 
     def process_healthy_cluster(self):
         if self.has_lock():
-            if self.cluster.failover and (not self.is_paused() or self.state_handler.is_leader()):
-                msg = self.process_manual_failover_from_leader()
-                if msg is not None:
-                    return msg
-
             if self.is_paused() and not self.state_handler.is_leader():
                 if self.cluster.failover and self.cluster.failover.candidate == self.state_handler.name:
                     return 'waiting to become master after promote...'
@@ -428,19 +626,30 @@ class Ha(object):
                 self.dcs.reset_cluster()
                 return 'removed leader lock because postgres is not running as master'
 
-            if self.update_lock():
+            if self.update_lock(True):
+                msg = self.process_manual_failover_from_leader()
+                if msg is not None:
+                    return msg
+
                 return self.enforce_master_role('no action.  i am the leader with the lock',
                                                 'promoted self to leader because i had the session lock')
             else:
                 # Either there is no connection to DCS or someone else acquired the lock
                 logger.error('failed to update leader lock')
-                self.load_cluster_from_dcs()
+                if self.state_handler.is_leader():
+                    self.demote('offline')
+                    return 'demoted self because failed to update leader lock in DCS'
+                else:
+                    return 'not promoting because failed to update leader lock in DCS'
         else:
             logger.info('does not have lock')
         return self.follow('demoting self because i do not have the lock and i was a leader',
                            'no action.  i am a secondary and i am following a leader', False)
 
     def evaluate_scheduled_restart(self):
+        if self._async_executor.busy:  # Restart already in progress
+            return None
+
         # restart if we need to
         restart_data = self.future_restart_scheduled()
         if restart_data:
@@ -485,6 +694,7 @@ class Ha(object):
 
     def schedule_future_restart(self, restart_data):
         with self._async_executor:
+            restart_data['postmaster_start_time'] = self.state_handler.postmaster_start_time()
             if not self.patroni.scheduled_restart:
                 self.patroni.scheduled_restart = restart_data
                 self.touch_member()
@@ -507,10 +717,11 @@ class Ha(object):
     def restart_scheduled(self):
         return self._async_executor.scheduled_action == 'restart'
 
-    def restart(self, restart_data=None, run_async=False):
+    def restart(self, restart_data, run_async=False):
         """ conditional and unconditional restart """
-        if (restart_data and isinstance(restart_data, dict) and
-            not self.restart_matches(restart_data.get('role'),
+        assert isinstance(restart_data, dict)
+
+        if (not self.restart_matches(restart_data.get('role'),
                                      restart_data.get('postgres_version'),
                                      ('restart_pending' in restart_data))):
             return (False, "restart conditions are not satisfied")
@@ -520,21 +731,40 @@ class Ha(object):
             if prev is not None:
                 return (False, prev + ' already in progress')
 
+            # Make the main loop to think that we were recovering dead postgres. If we fail
+            # to start postgres after a specified timeout (see below), we need to remove
+            # leader key (if it belong to us) rather than trying to start postgres once again.
+            self.recovering = True
+
+        # No that restart is scheduled we can set timeout for startup, it will get reset
+        # once async executor runs and main loop notices PostgreSQL as up.
+        timeout = restart_data.get('timeout', self.patroni.config['master_start_timeout'])
+        self.set_start_timeout(timeout)
+
+        # For non async cases we want to wait for restart to complete or timeout before returning.
+        do_restart = functools.partial(self.state_handler.restart, timeout)
+        if self.is_synchronous_mode() and not self.has_lock():
+            do_restart = functools.partial(self.while_not_sync_standby, do_restart)
+
         if run_async:
-            self._async_executor.run_async(self.state_handler.restart)
+            self._async_executor.run_async(do_restart)
             return (True, 'restart initiated')
-        elif self._async_executor.run(self.state_handler.restart):
-            return (True, 'restarted successfully')
         else:
-            return (False, 'restart failed')
+            res = self._async_executor.run(do_restart)
+            if res:
+                return (True, 'restarted successfully')
+            elif res is None:
+                return (False, 'postgres is still starting')
+            else:
+                return (False, 'restart failed')
 
     def _do_reinitialize(self, cluster):
         self.state_handler.stop('immediate')
         self.state_handler.remove_data_directory()
 
-        clone_member = self.cluster.get_clone_member()
+        clone_member = self.cluster.get_clone_member(self.state_handler.name)
         member_role = 'leader' if clone_member == self.cluster.leader else 'replica'
-        self.clone(clone_member, "from {0} '{1}'".format(member_role, clone_member.name))
+        return self.clone(clone_member, "from {0} '{1}'".format(member_role, clone_member.name))
 
     def reinitialize(self):
         with self._async_executor:
@@ -573,17 +803,63 @@ class Ha(object):
     def post_recover(self):
         if not self.state_handler.is_running():
             if self.has_lock():
+                self.state_handler.set_role('demoted')
                 self.dcs.delete_leader()
                 self.dcs.reset_cluster()
                 return 'removed leader key after trying and failing to start postgres'
             return 'failed to start postgres'
         return None
 
+    def handle_starting_instance(self):
+        """Starting up PostgreSQL may take a long time. In case we are the leader we may want to
+        fail over to."""
+
+        # Check if we are in startup, when paused defer to main loop for manual failovers.
+        if not self.state_handler.check_for_startup() or self.is_paused():
+            self.set_start_timeout(None)
+            return None
+
+        # state_handler.state == 'starting' here
+        if self.has_lock():
+            if not self.update_lock():
+                logger.info("Lost lock while starting up. Demoting self.")
+                self.demote('immediate')
+                return 'stopped PostgreSQL while starting up because leader key was lost'
+
+            timeout = self._start_timeout or self.patroni.config['master_start_timeout']
+            time_left = timeout - self.state_handler.time_in_state()
+
+            if time_left <= 0:
+                if self.is_failover_possible(self.cluster.members):
+                    logger.info("Demoting self because master startup is taking too long")
+                    self.demote('immediate')
+                    return 'stopped PostgreSQL because of startup timeout'
+                else:
+                    return 'master start has timed out, but continuing to wait because failover is not possible'
+            else:
+                msg = self.process_manual_failover_from_leader()
+                if msg is not None:
+                    return msg
+
+                return 'PostgreSQL is still starting up, {0:.0f} seconds until timeout'.format(time_left)
+        else:
+            # Use normal processing for standbys
+            logger.info("Still starting up as a standby.")
+            return None
+
+    def set_start_timeout(self, value):
+        """Sets timeout for starting as master before eligible for failover.
+
+        Must be called when async_executor is busy or in the main thread."""
+        self._start_timeout = value
+
     def _run_cycle(self):
+        dcs_failed = False
         try:
             self.load_cluster_from_dcs()
 
-            self.touch_member()
+            if not self.cluster.has_member(self.state_handler.name):
+                self.touch_member()
 
             # cluster has leader key but not initialize key
             if not (self.cluster.is_unlocked() or self.sysid_valid(self.cluster.initialize)) and self.has_lock():
@@ -591,19 +867,30 @@ class Ha(object):
 
             if not (self.cluster.is_unlocked() or self.cluster.config and self.cluster.config.data) and self.has_lock():
                 self.dcs.set_config_value(json.dumps(self.patroni.config.dynamic_configuration, separators=(',', ':')))
+                self.cluster = self.dcs.get_cluster()
 
             if self._async_executor.busy:
                 return self.handle_long_action_in_progress()
 
-            # we've got here, so any async action has finished. Check if we tried to recover and failed
+            msg = self.handle_starting_instance()
+            if msg is not None:
+                return msg
+
+            # we've got here, so any async action has finished.
             if self.recovering and not self.state_handler.need_rewind:
                 self.recovering = False
+                # Check if we tried to recover and failed
                 msg = self.post_recover()
                 if msg is not None:
                     return msg
 
             # is data directory empty?
             if self.state_handler.data_directory_empty():
+                # is this instance the leader?
+                if self.has_lock():
+                    self.release_leader_key_voluntarily()
+                    return 'released leader key voluntarily as data dir empty and currently leader'
+
                 return self.bootstrap()  # new node
             # "bootstrap", but data directory is not empty
             elif not self.sysid_valid(self.cluster.initialize) and self.cluster.is_unlocked() and not self.is_paused():
@@ -621,7 +908,7 @@ class Ha(object):
                         self.dcs.delete_leader()
                         self.dcs.reset_cluster()
                         return 'removed leader lock because postgres is not running'
-                    elif not self.state_handler.need_rewind:
+                    elif not (self.state_handler.need_rewind and self.state_handler.can_rewind):
                         return 'postgres is not running'
 
                 # try to start dead postgres
@@ -631,28 +918,47 @@ class Ha(object):
                 if self.cluster.is_unlocked():
                     return self.process_unhealthy_cluster()
                 else:
-                    msg = self.evaluate_scheduled_restart()
-                    if msg is not None:
-                        return msg
-                    return self.process_healthy_cluster()
+                    msg = self.process_healthy_cluster()
+                    return self.evaluate_scheduled_restart() or msg
             finally:
                 # we might not have a valid PostgreSQL connection here if another thread
                 # stops PostgreSQL, therefore, we only reload replication slots if no
                 # asynchronous processes are running (should be always the case for the master)
-                if not self._async_executor.busy:
+                if not self._async_executor.busy and not self.state_handler.is_starting():
                     if not self.state_handler.cb_called:
                         self.state_handler.call_nowait(ACTION_ON_START)
                     self.state_handler.sync_replication_slots(self.cluster)
         except DCSError:
+            dcs_failed = True
             logger.error('Error communicating with DCS')
             if not self.is_paused() and self.state_handler.is_running() and self.state_handler.is_leader():
-                self.demote(delete_leader=False)
+                self.demote('offline')
                 return 'demoted self because DCS is not accessible and i was a leader'
             return 'DCS is not accessible'
         except (psycopg2.Error, PostgresConnectionException):
             return 'Error communicating with PostgreSQL. Will try again later'
+        finally:
+            if not dcs_failed:
+                self.touch_member()
 
     def run_cycle(self):
         with self._async_executor:
             info = self._run_cycle()
             return (self.is_paused() and 'PAUSE: ' or '') + info
+
+    def watch(self, timeout):
+        cluster = self.cluster
+        # watch on leader key changes if the postgres is running and leader is known and current node is not lock owner
+        if not self._async_executor.busy and cluster and cluster.leader \
+                and cluster.leader.name != self.state_handler.name:
+            leader_index = cluster.leader.index
+        else:
+            leader_index = None
+
+        return self.dcs.watch(leader_index, timeout)
+
+    def wakeup(self):
+        """Call of this method will trigger the next run of HA loop if there is
+        no "active" leader watch request in progress.
+        This usually happens on the master or if the node is running async action"""
+        self.dcs.event.set()

patroni/postgresql.py

@@ -1,4 +1,3 @@
-from collections import defaultdict
 import logging
 import os
 import psycopg2
@@ -9,10 +8,13 @@ import subprocess
 import tempfile
 import time
 
+from collections import defaultdict
+from patroni import call_self
+from patroni.callback_executor import CallbackExecutor
 from patroni.exceptions import PostgresConnectionException, PostgresException
-from patroni.utils import compare_values, parse_bool, parse_int, Retry, RetryFailedError
+from patroni.utils import compare_values, parse_bool, parse_int, Retry, RetryFailedError, polling_loop
 from six import string_types
-from threading import Lock
+from threading import current_thread, Lock
 
 logger = logging.getLogger(__name__)
 
@@ -22,6 +24,11 @@ ACTION_ON_RESTART = "on_restart"
 ACTION_ON_RELOAD = "on_reload"
 ACTION_ON_ROLE_CHANGE = "on_role_change"
 
+STATE_RUNNING = 'running'
+STATE_REJECT = 'rejecting connections'
+STATE_NO_RESPONSE = 'not responding'
+STATE_UNKNOWN = 'unknown'
+
 
 def slot_name_from_member_name(member_name):
     """Translate member name to valid PostgreSQL slot name.
@@ -59,7 +66,7 @@ class Postgresql(object):
         'listen_addresses': (None, lambda _: False, 9.1),
         'port': (None, lambda _: False, 9.1),
         'cluster_name': (None, lambda _: False, 9.5),
-        'wal_level': ('hot_standby', lambda v: v.lower() in ('hot_standby', 'logical'), 9.1),
+        'wal_level': ('hot_standby', lambda v: v.lower() in ('hot_standby', 'replica', 'logical'), 9.1),
         'hot_standby': ('on', lambda _: False, 9.1),
         'max_connections': (100, lambda v: int(v) >= 100, 9.1),
         'max_wal_senders': (5, lambda v: int(v) >= 5, 9.1),
@@ -80,6 +87,11 @@ class Postgresql(object):
         self._database = config.get('database', 'postgres')
         self._data_dir = config['data_dir']
         self._pending_restart = False
+        self.__thread_ident = current_thread().ident
+
+        self._version_file = os.path.join(self._data_dir, 'PG_VERSION')
+        self._major_version = self.get_major_version()
+        self._synchronous_standby_names = None
         self._server_parameters = self.get_server_parameters(config)
 
         self._connect_address = config.get('connect_address')
@@ -89,12 +101,12 @@ class Postgresql(object):
 
         self._need_rewind = False
         self._use_slots = config.get('use_slots', True)
-        self._version_file = os.path.join(self._data_dir, 'PG_VERSION')
-        self._major_version = self.get_major_version()
         self._schedule_load_slots = self.use_slots
 
         self._pgpass = config.get('pgpass') or os.path.join(os.path.expanduser('~'), 'pgpass')
+        self._callback_executor = CallbackExecutor()
         self.__cb_called = False
+        self.__cb_pending = None
         config_base_name = config.get('config_base_name', 'postgresql')
         self._postgresql_conf = os.path.join(self._data_dir, config_base_name + '.conf')
         self._postgresql_base_conf_name = config_base_name + '.base.conf'
@@ -104,6 +116,7 @@ class Postgresql(object):
         self._trigger_file = config.get('recovery_conf', {}).get('trigger_file') or 'promote'
         self._trigger_file = os.path.abspath(os.path.join(self._data_dir, self._trigger_file))
 
+        self._connection_lock = Lock()
         self._connection = None
         self._cursor_holder = None
         self._sysid = None
@@ -111,11 +124,17 @@ class Postgresql(object):
         self.retry = Retry(max_tries=-1, deadline=config['retry_timeout']/2.0, max_delay=1,
                            retry_exceptions=PostgresConnectionException)
 
+        # Retry 'pg_is_in_recovery()' only once
+        self._is_leader_retry = Retry(max_tries=1, deadline=config['retry_timeout']/2.0, max_delay=1,
+                                      retry_exceptions=PostgresConnectionException)
+
         self._state_lock = Lock()
         self.set_state('stopped')
         self._role_lock = Lock()
         self.set_role(self.get_postgres_role_from_data_directory())
 
+        self._state_entry_timestamp = None
+
         if self.is_running():
             self.set_state('running')
             self.set_role('master' if self.is_leader() else 'replica')
@@ -154,7 +173,15 @@ class Postgresql(object):
         parameters = config['parameters'].copy()
         listen_addresses, port = (config['listen'] + ':5432').split(':')[:2]
         parameters.update({'cluster_name': self.scope, 'listen_addresses': listen_addresses, 'port': port})
-        return parameters
+        if config.get('synchronous_mode', False):
+            if self._synchronous_standby_names is None:
+                parameters.pop('synchronous_standby_names', None)
+            else:
+                parameters['synchronous_standby_names'] = self._synchronous_standby_names
+        if self._major_version >= 9.6 and parameters['wal_level'] == 'hot_standby':
+            parameters['wal_level'] = 'replica'
+        return {k: v for k, v in parameters.items() if not self._major_version or
+                self._major_version >= self.CMDLINE_OPTIONS.get(k, (0, 1, 9.1))[2]}
 
     def resolve_connection_addresses(self):
         self._local_address = self.get_local_address()
@@ -171,7 +198,7 @@ class Postgresql(object):
         :returns: `!True` when return_code == 0, otherwise `!False`"""
 
         pg_ctl = [self._pgcommand('pg_ctl'), cmd]
-        if cmd in ('start', 'stop', 'restart'):
+        if cmd == 'stop':
             pg_ctl += ['-w']
             timeout = self.config.get('pg_ctl_timeout')
             if timeout:
@@ -181,11 +208,31 @@ class Postgresql(object):
                     logger.error('Bad value of pg_ctl_timeout: %s', timeout)
         return subprocess.call(pg_ctl + ['-D', self._data_dir] + list(args), **kwargs) == 0
 
+    def pg_isready(self):
+        """Runs pg_isready to see if PostgreSQL is accepting connections.
+
+        :returns: 'ok' if PostgreSQL is up, 'reject' if starting up, 'no_resopnse' if not up."""
+
+        cmd = [self._pgcommand('pg_isready'),
+               '-h', self._local_address['host'],
+               '-p', self._local_address['port'],
+               '-d', self._database]
+        # We only need the username because pg_isready does not try to authenticate
+        if 'username' in self._superuser:
+            cmd.extend(['-U', self._superuser['username']])
+
+        ret = subprocess.call(cmd)
+        return_codes = {0: STATE_RUNNING,
+                        1: STATE_REJECT,
+                        2: STATE_NO_RESPONSE,
+                        3: STATE_UNKNOWN}
+        return return_codes.get(ret, STATE_UNKNOWN)
+
     def reload_config(self, config):
         server_parameters = self.get_server_parameters(config)
 
         listen_address_changed = pending_reload = pending_restart = False
-        if self.is_healthy():
+        if self.state == 'running':
             changes = {p: v for p, v in server_parameters.items() if '.' not in p}
             changes.update({p: None for p, v in self._server_parameters.items() if not ('.' in p or p in changes)})
             if changes:
@@ -240,7 +287,7 @@ class Postgresql(object):
         if pending_reload:
             self._write_postgresql_conf()
             self.reload()
-        self.retry.deadline = config['retry_timeout']/2.0
+        self._is_leader_retry.deadline = self.retry.deadline = config['retry_timeout']/2.0
 
     @property
     def pending_restart(self):
@@ -305,10 +352,11 @@ class Postgresql(object):
         return ret
 
     def connection(self):
-        if not self._connection or self._connection.closed != 0:
-            self._connection = psycopg2.connect(**self._local_connect_kwargs)
-            self._connection.autocommit = True
-            self.server_version = self._connection.server_version
+        with self._connection_lock:
+            if not self._connection or self._connection.closed != 0:
+                self._connection = psycopg2.connect(**self._local_connect_kwargs)
+                self._connection.autocommit = True
+                self.server_version = self._connection.server_version
         return self._connection
 
     def _cursor(self):
@@ -318,11 +366,15 @@ class Postgresql(object):
         return self._cursor_holder
 
     def close_connection(self):
-        if self._cursor_holder and self._cursor_holder.connection and self._cursor_holder.connection.closed == 0:
-            self._cursor_holder.connection.close()
+        if self._connection and self._connection.closed == 0:
+            self._connection.close()
             logger.info("closed patroni connection to the postgresql cluster")
+        self._cursor_holder = self._connection = None
 
     def _query(self, sql, *params):
+        """We are always using the same cursor, therefore this method is not thread-safe!!!
+        You can call it from different threads only if you are holding explicit `AsyncExecutor` lock,
+        because the main thread is always holding this lock when running HA cycle."""
         cursor = None
         try:
             cursor = self._cursor()
@@ -385,15 +437,46 @@ class Postgresql(object):
         if ret:
             self.write_pg_hba(config.get('pg_hba', []))
             self._major_version = self.get_major_version()
+            self._server_parameters = self.get_server_parameters(self.config)
         else:
             self.set_state('initdb failed')
         return ret
 
+    def run_bootstrap_post_init(self, config):
+        """
+        runs a script after initdb is called and waits until completion.
+        passed: cluster name, parameters
+        """
+        if 'post_init' in config:
+            cmd = config['post_init']
+            r = self._local_connect_kwargs
+            if 'user' in r:
+                connstring = 'postgres://{user}@{host}:{port}/{database}'.format(**r)
+            else:
+                connstring = 'postgres://{host}:{port}/{database}'.format(**r)
+                if 'password' in r:
+                    import getpass
+                    r.setdefault('user', os.environ.get('PGUSER', getpass.getuser()))
+
+            env = self.write_pgpass(r) if 'password' in r else None
+            try:
+                ret = subprocess.call(shlex.split(cmd) + [connstring], env=env)
+            except OSError:
+                logger.error('post_init script %s failed', cmd)
+                return False
+            if ret != 0:
+                logger.error('post_init script %s returned non-zero code %d', cmd, ret)
+                return False
+        return True
+
     def delete_trigger_file(self):
         if os.path.exists(self._trigger_file):
             os.unlink(self._trigger_file)
 
     def write_pgpass(self, record):
+        if 'user' not in record or 'password' not in record:
+            return os.environ.copy()
+
         with open(self._pgpass, 'w') as f:
             os.fchmod(f.fileno(), 0o600)
             f.write('{host}:{port}:*:{user}:{password}\n'.format(**record))
@@ -449,6 +532,9 @@ class Postgresql(object):
                     # if basebackup succeeds, exit with success
                     break
             else:
+                if not self.data_directory_empty():
+                    self.remove_data_directory()
+
                 cmd = replica_method
                 method_config = {}
                 # user-defined method; check for configuration
@@ -458,19 +544,23 @@ class Postgresql(object):
                     # look to see if the user has supplied a full command path
                     # if not, use the method name as the command
                     cmd = method_config.pop('command', cmd)
-                    # add the default parameters
+
+                # add the default parameters
+                method_config.update({"scope": self.scope,
+                                      "role": "replica",
+                                      "datadir": self._data_dir,
+                                      "connstring": connstring})
+                params = ["--{0}={1}".format(arg, val) for arg, val in method_config.items()]
                 try:
-                    method_config.update({"scope": self.scope,
-                                          "role": "replica",
-                                          "datadir": self._data_dir,
-                                          "connstring": connstring})
-                    params = ["--{0}={1}".format(arg, val) for arg, val in method_config.items()]
                     # call script with the full set of parameters
                     ret = subprocess.call(shlex.split(cmd) + params, env=env)
                     # if we succeeded, stop
                     if ret == 0:
                         logger.info('replica has been created using %s', replica_method)
                         break
+                    else:
+                        logger.error('Error creating replica using method %s: %s exited with code=%s',
+                                     replica_method, cmd, ret)
                 except Exception:
                     logger.exception('Error creating replica using method %s', replica_method)
                     ret = 1
@@ -479,17 +569,37 @@ class Postgresql(object):
         return ret
 
     def is_leader(self):
-        return not self.query('SELECT pg_is_in_recovery()').fetchone()[0]
+        try:
+            return not self._is_leader_retry(self._query, 'SELECT pg_is_in_recovery()').fetchone()[0]
+        except RetryFailedError as e:  # SELECT pg_is_in_recovery() failed two times
+            if not self.is_starting() and self.pg_isready() == STATE_REJECT:
+                self.set_state('starting')
+            raise PostgresConnectionException(str(e))
 
     def is_running(self):
         if not (self._version_file_exists() and os.path.isfile(self._postmaster_pid)):
             return False
+        return self.is_pid_running(self.read_pid_file().get('pid', 0))
+
+    def read_pid_file(self):
+        """Reads and parses postmaster.pid from the data directory
+
+        :returns dictionary of values if successful, empty dictionary otherwise
+        """
+        pid_line_names = ['pid', 'data_dir', 'start_time', 'port', 'socket_dir', 'listen_addr', 'shmem_key']
         try:
             with open(self._postmaster_pid) as f:
-                pid = int(f.readline())
-                if pid < 0:
-                    pid = -pid
-                return pid > 0 and pid != os.getpid() and pid != os.getppid() and (os.kill(pid, 0) or True)
+                return {name: line.rstrip("\n") for name, line in zip(pid_line_names, f)}
+        except IOError:
+            return {}
+
+    @staticmethod
+    def is_pid_running(pid):
+        try:
+            pid = int(pid)
+            if pid < 0:
+                pid = -pid
+            return pid > 0 and pid != os.getpid() and pid != os.getppid() and (os.kill(pid, 0) or True)
         except Exception:
             return False
 
@@ -502,15 +612,13 @@ class Postgresql(object):
         if cb_name in (ACTION_ON_START, ACTION_ON_STOP, ACTION_ON_RESTART, ACTION_ON_ROLE_CHANGE):
             self.__cb_called = True
 
-        if not self.callback or cb_name not in self.callback:
-            return False
-        cmd = self.callback[cb_name]
-        try:
-            subprocess.Popen(shlex.split(cmd) + [cb_name, self.role, self.scope])
-        except OSError:
-            logger.exception('callback %s %s %s %s failed', cmd, cb_name, self.role, self.scope)
-            return False
-        return True
+        if self.callback and cb_name in self.callback:
+            cmd = self.callback[cb_name]
+            try:
+                cmd = shlex.split(self.callback[cb_name]) + [cb_name, self.role, self.scope]
+                self._callback_executor.call(cmd)
+            except Exception:
+                logger.exception('callback %s %s %s %s failed', cmd, cb_name, self.role, self.scope)
 
     @property
     def role(self):
@@ -529,8 +637,48 @@ class Postgresql(object):
     def set_state(self, value):
         with self._state_lock:
             self._state = value
+            self._state_entry_timestamp = time.time()
 
-    def start(self, block_callbacks=False):
+    def time_in_state(self):
+        return time.time() - self._state_entry_timestamp
+
+    def is_starting(self):
+        return self.state == 'starting'
+
+    def wait_for_port_open(self, pid, initiated, timeout):
+        """Waits until PostgreSQL opens ports."""
+        for _ in polling_loop(timeout):
+            pid_file = self.read_pid_file()
+            if len(pid_file) > 5:
+                try:
+                    pmpid = int(pid_file['pid'])
+                    pmstart = int(pid_file['start_time'])
+
+                    if pmstart >= initiated - 2 and pmpid == pid:
+                        isready = self.pg_isready()
+                        if isready != STATE_NO_RESPONSE:
+                            if isready not in [STATE_REJECT, STATE_RUNNING]:
+                                logger.warning("Can't determine PostgreSQL startup status, assuming running")
+                            return True
+                except ValueError:
+                    # Garbage in the pid file
+                    pass
+
+            if not self.is_pid_running(pid):
+                logger.error('postmaster is not running')
+                self.set_state('start failed')
+                return False
+
+        logger.warning("Timed out waiting for PostgreSQL to start")
+        return False
+
+    def start(self, timeout=None, block_callbacks=False):
+        """Start PostgreSQL
+
+        Waits for postmaster to open ports or terminate so pg_isready can be used to check startup completion
+        or failure.
+
+        :returns: True if start was initiated and postmaster ports are open, False if start failed"""
         # make sure we close all connections established against
         # the former node, otherwise, we might get a stalled one
         # after kill -9, which would report incorrect data to
@@ -541,37 +689,57 @@ class Postgresql(object):
             logger.error('Cannot start PostgreSQL because one is already running.')
             return True
 
-        self.set_role(self.get_postgres_role_from_data_directory())
-        if os.path.exists(self._postmaster_pid):
-            os.remove(self._postmaster_pid)
-            logger.info('Removed %s', self._postmaster_pid)
-
         if not block_callbacks:
-            self.set_state('starting')
+            self.__cb_pending = ACTION_ON_START
 
-        env = {'PATH': os.environ.get('PATH')}
-        # pg_ctl will write a FATAL if the username is incorrect. exporting PGUSER if necessary
-        if 'username' in self._superuser and self._superuser['username'] != os.environ.get('USER'):
-            env['PGUSER'] = self._superuser['username']
+        self.set_role(self.get_postgres_role_from_data_directory())
+
+        self.set_state('starting')
+        self._pending_restart = False
 
         self._write_postgresql_conf()
         self.resolve_connection_addresses()
 
-        options = ' '.join("--{0}='{1}'".format(p, self._server_parameters[p]) for p, v in self.CMDLINE_OPTIONS.items()
-                           if self._major_version >= v[2])
+        opts = {p: self._server_parameters[p] for p in self.CMDLINE_OPTIONS if p in self._server_parameters}
+        options = ['--{0}={1}'.format(p, v) for p, v in opts.items()]
 
-        ret = self.pg_ctl('start', '-o', options, env=env, preexec_fn=os.setsid)
-        self._pending_restart = False
+        start_initiated = time.time()
 
-        self.set_state('running' if ret else 'start failed')
+        # Unfortunately `pg_ctl start` does not return postmaster pid to us. Without this information
+        # it is hard to know the current state of postgres startup, so we had to reimplement pg_ctl start
+        # in python. It will start postgres, wait for port to be open and wait until postgres will start
+        # accepting connections.
+        # Important!!! We can't just start postgres using subprocess.Popen, because in this case it
+        # will be our child for the rest of our live and we will have to take care of it (`waitpid`).
+        # So we will use the same approach as pg_ctl uses: start a new process, which will start postgres.
+        # This process will write postmaster pid to stdout and exit immediately. Now it's responsibility
+        # of init process to take care about postmaster.
+        # In order to make everything portable we can't use fork&exec approach here, so  we will call
+        # ourselves and pass list of arguments which must be used to start postgres.
+        proc = call_self(['pg_ctl_start', self._pgcommand('postgres'), '-D', self._data_dir] + options, close_fds=True,
+                         preexec_fn=os.setsid, stdout=subprocess.PIPE, env={'PATH': os.environ.get('PATH')})
+        pid = int(proc.stdout.readline().strip())
+        proc.wait()
+        logger.info('postmaster pid=%s', pid)
 
-        self._schedule_load_slots = ret and self.use_slots
-        self.save_configuration_files()
-        # block_callbacks is used during restart to avoid
-        # running start/stop callbacks in addition to restart ones
-        if ret and not block_callbacks:
-            self.call_nowait(ACTION_ON_START)
-        return ret
+        start_timeout = timeout
+        if not start_timeout:
+            try:
+                start_timeout = float(self.config.get('pg_ctl_timeout', 60))
+            except ValueError:
+                start_timeout = 60
+
+        # We want postmaster to open ports before we continue
+        if not self.wait_for_port_open(pid, start_initiated, start_timeout):
+            return False
+
+        ret = self.wait_for_startup(start_timeout)
+        if ret is not None:
+            return ret
+        elif timeout is not None:
+            return False
+        else:
+            return None
 
     def checkpoint(self, connect_kwargs=None):
         check_not_is_in_recovery = connect_kwargs is not None
@@ -598,7 +766,7 @@ class Postgresql(object):
                 self.set_state('stopped')
             return True
 
-        if checkpoint:
+        if checkpoint and not self.is_starting():
             self.checkpoint()
 
         if not block_callbacks:
@@ -621,12 +789,70 @@ class Postgresql(object):
             self.call_nowait(ACTION_ON_RELOAD)
         return ret
 
-    def restart(self):
-        self.set_state('restarting')
-        ret = self.stop(block_callbacks=True) and self.start(block_callbacks=True)
-        if ret:
-            self.call_nowait(ACTION_ON_RESTART)
+    def check_for_startup(self):
+        """Checks PostgreSQL status and returns if PostgreSQL is in the middle of startup."""
+        return self.is_starting() and not self.check_startup_state_changed()
+
+    def check_startup_state_changed(self):
+        """Checks if PostgreSQL has completed starting up or failed or still starting.
+
+        Should only be called when state == 'starting'
+
+        :returns: True iff state was changed from 'starting'
+        """
+        ready = self.pg_isready()
+
+        if ready == STATE_REJECT:
+            return False
+        elif ready == STATE_NO_RESPONSE:
+            self.set_state('start failed')
+            self._schedule_load_slots = False  # TODO: can remove this?
+            self.save_configuration_files()  # TODO: maybe remove this?
+            return True
         else:
+            if ready != STATE_RUNNING:
+                # Bad configuration or unexpected OS error. No idea of PostgreSQL status.
+                # Let the main loop of run cycle clean up the mess.
+                logger.warning("%s status returned from pg_isready",
+                               "Unknown" if ready == STATE_UNKNOWN else "Invalid")
+            self.set_state('running')
+            self._schedule_load_slots = self.use_slots
+            self.save_configuration_files()
+            # TODO: __cb_pending can be None here after PostgreSQL restarts on its own. Do we want to call the callback?
+            # Previously we didn't even notice.
+            action = self.__cb_pending or ACTION_ON_START
+            self.call_nowait(action)
+            self.__cb_pending = None
+
+            return True
+
+    def wait_for_startup(self, timeout=None):
+        """Waits for PostgreSQL startup to complete or fail.
+
+        :returns: True if start was successful, False otherwise"""
+        if not self.is_starting():
+            # Should not happen
+            logger.warning("wait_for_startup() called when not in starting state")
+
+        while not self.check_startup_state_changed():
+            if timeout and self.time_in_state() > timeout:
+                return None
+            time.sleep(1)
+
+        return self.state == 'running'
+
+    def restart(self, timeout=None):
+        """Restarts PostgreSQL.
+
+        When timeout parameter is set the call will block either until PostgreSQL has started, failed to start or
+        timeout arrives.
+
+        :returns: True when restart was successful and timeout did not expire when waiting.
+        """
+        self.set_state('restarting')
+        self.__cb_pending = ACTION_ON_RESTART
+        ret = self.stop(block_callbacks=True) and self.start(timeout=timeout, block_callbacks=True)
+        if not ret and not self.is_starting():
             self.set_state('restart failed ({0})'.format(self.state))
         return ret
 
@@ -639,8 +865,7 @@ class Postgresql(object):
             f.write('# Do not edit this file manually!\n# It will be overwritten by Patroni!\n')
             f.write("include '{0}'\n\n".format(self.config.get('custom_conf') or self._postgresql_base_conf_name))
             for name, value in sorted(self._server_parameters.items()):
-                if name not in self.CMDLINE_OPTIONS:
-                    f.write("{0} = '{1}'\n".format(name, value))
+                f.write("{0} = '{1}'\n".format(name, value))
 
     def is_healthy(self):
         if not self.is_running():
@@ -648,9 +873,6 @@ class Postgresql(object):
             return False
         return True
 
-    def check_replication_lag(self, last_leader_operation):
-        return (last_leader_operation or 0) - self.xlog_position() <= self.config.get('maximum_lag_on_failover', 0)
-
     def write_pg_hba(self, config):
         with open(os.path.join(self._data_dir, 'pg_hba.conf'), 'a') as f:
             f.write('\n{}\n'.format('\n'.join(config)))
@@ -687,7 +909,15 @@ class Postgresql(object):
     def rewind(self, r):
         # prepare pg_rewind connection
         env = self.write_pgpass(r)
-        dsn = 'user={user} host={host} port={port} dbname={database} sslmode=prefer sslcompression=1'.format(**r)
+        dsn_attrs = [
+            ('user', r.get('user')),
+            ('host', r.get('host')),
+            ('port', r.get('port')),
+            ('dbname', r.get('database')),
+            ('sslmode', 'prefer'),
+            ('sslcompression', '1'),
+        ]
+        dsn = " ".join("{0}={1}".format(k, v) for k, v in dsn_attrs if v is not None)
         logger.info('running pg_rewind from %s', dsn)
         try:
             return subprocess.call([self._pgcommand('pg_rewind'),
@@ -761,7 +991,7 @@ class Postgresql(object):
     def need_rewind(self):
         return self._need_rewind
 
-    def follow(self, member, leader, recovery=False, async_executor=None, need_rewind=None):
+    def follow(self, member, leader, recovery=False, async_executor=None, need_rewind=None, timeout=None):
         if need_rewind is not None:
             self._need_rewind = need_rewind
 
@@ -772,11 +1002,11 @@ class Postgresql(object):
 
         if async_executor:
             async_executor.schedule('changing primary_conninfo and restarting')
-            async_executor.run_async(self._do_follow, (primary_conninfo, leader, recovery))
+            async_executor.run_async(self._do_follow, (primary_conninfo, leader, recovery, timeout))
         else:
-            self._do_follow(primary_conninfo, leader, recovery)
+            return self._do_follow(primary_conninfo, leader, recovery, timeout)
 
-    def _do_follow(self, primary_conninfo, leader, recovery=False):
+    def _do_follow(self, primary_conninfo, leader, recovery=False, timeout=None):
         change_role = self.role in ('master', 'demoted')
 
         if leader and leader.name == self.name:
@@ -787,12 +1017,13 @@ class Postgresql(object):
         elif change_role:
             self._need_rewind = True
 
-        self._need_rewind &= bool(leader and leader.conn_url) and self.can_rewind
+        if self._need_rewind and not self.can_rewind:
+            logger.warning("Data directory may be out of sync master, rewind may be needed.")
 
-        if self._need_rewind:
+        if self._need_rewind and leader and leader.conn_url and self.can_rewind:
             logger.info("rewind flag is set")
 
-            if self.is_running() and not self.stop():
+            if self.is_running() and not self.stop(checkpoint=False):
                 return logger.warning('Can not run pg_rewind because postgres is still running')
 
             # prepare pg_rewind connection
@@ -823,20 +1054,23 @@ class Postgresql(object):
 
             if self.rewind(r) or not self.config.get('remove_data_directory_on_rewind_failure', False):
                 self.write_recovery_conf(primary_conninfo)
-                ret = self.start()
+                self.start()
             else:
                 logger.error('unable to rewind the former master')
                 self.remove_data_directory()
-                ret = True
             self._need_rewind = False
         else:
             self.write_recovery_conf(primary_conninfo)
-            ret = self.start() if recovery else self.restart()
+            if recovery:
+                self.start(timeout=timeout)
+            else:
+                self.restart()
             self.set_role('replica')
 
         if change_role:
+            # TODO: postpone this until start completes, or maybe do even earlier
             self.call_nowait(ACTION_ON_ROLE_CHANGE)
-        return ret
+        return True
 
     def save_configuration_files(self):
         """
@@ -888,11 +1122,22 @@ BEGIN
 END;
 $$""".format(name, ' '.join(options)), name, password, password)
 
-    def xlog_position(self):
-        return self.query("""SELECT pg_xlog_location_diff(CASE WHEN pg_is_in_recovery()
-                                                               THEN pg_last_xlog_replay_location()
-                                                               ELSE pg_current_xlog_location()
-                                                          END, '0/0')::bigint""").fetchone()[0]
+    def xlog_position(self, retry=True):
+        stmt = """SELECT CASE WHEN pg_is_in_recovery()
+                              THEN GREATEST(pg_xlog_location_diff(COALESCE(pg_last_xlog_receive_location(), '0/0'),
+                                                                  '0/0')::bigint,
+                                            pg_xlog_location_diff(pg_last_xlog_replay_location(), '0/0')::bigint)
+                              ELSE pg_xlog_location_diff(pg_current_xlog_location(), '0/0')::bigint
+                          END"""
+
+        # This method could be called from different threads (simultaneously with some other `_query` calls).
+        # If it is called not from main thread we will create a new cursor to execute statement.
+        if current_thread().ident == self.__thread_ident:
+            return (self.query(stmt) if retry else self._query(stmt)).fetchone()[0]
+
+        with self.connection().cursor() as cursor:
+            cursor.execute(stmt)
+            return cursor.fetchone()[0]
 
     def load_replication_slots(self):
         if self.use_slots and self._schedule_load_slots:
@@ -936,9 +1181,12 @@ $$""".format(name, ' '.join(options)), name, password, password)
 
                 # drop unused slots
                 for slot in set(self._replication_slots) - slots:
-                    self._query("""SELECT pg_drop_replication_slot(%s)
-                                    WHERE EXISTS(SELECT 1 FROM pg_replication_slots
-                                    WHERE slot_name = %s AND NOT active)""", slot, slot)
+                    cursor = self._query("""SELECT pg_drop_replication_slot(%s)
+                                             WHERE EXISTS(SELECT 1 FROM pg_replication_slots
+                                             WHERE slot_name = %s AND NOT active)""", slot, slot)
+
+                    if cursor.rowcount != 1:  # Either slot doesn't exists or it is still active
+                        self._schedule_load_slots = True  # schedule load_replication_slots on the next iteration
 
                 # create new slots
                 for slot in slots - set(self._replication_slots):
@@ -965,14 +1213,15 @@ $$""".format(name, ' '.join(options)), name, password, password)
         ret = self.create_replica(clone_member) == 0
         if ret:
             self._major_version = self.get_major_version()
+            self._server_parameters = self.get_server_parameters(self.config)
             self.delete_trigger_file()
             self.restore_configuration_files()
         return ret
 
     def bootstrap(self, config):
         """ Initialize a new node from scratch and start it. """
-        if self._initialize(config) and self.start():
-            for name, value in config['users'].items():
+        if self._initialize(config) and self.start() and self.run_bootstrap_post_init(config):
+            for name, value in (config.get('users') or {}).items():
                 if name not in (self._superuser.get('username'), self._replication['username']):
                     self.create_or_update_role(name, value['password'], value.get('options', []))
             self.create_or_update_role(self._replication['username'], self._replication['password'], ['REPLICATION'])
@@ -1012,21 +1261,69 @@ $$""".format(name, ' '.join(options)), name, password, password)
         maxfailures = 2
         ret = 1
         for bbfailures in range(0, maxfailures):
+            if not self.data_directory_empty():
+                self.remove_data_directory()
+
             try:
                 ret = subprocess.call([self._pgcommand('pg_basebackup'), '--pgdata=' + self._data_dir,
                                        '--xlog-method=stream', "--dbname=" + conn_url], env=env)
                 if ret == 0:
                     break
+                else:
+                    logger.error('Error when fetching backup: pg_basebackup exited with code=%s', ret)
 
             except Exception as e:
-                logger.error('Error when fetching backup with pg_basebackup: {0}'.format(e))
+                logger.error('Error when fetching backup with pg_basebackup: %s', e)
 
             if bbfailures < maxfailures - 1:
-                logger.error('Trying again in 5 seconds')
+                logger.warning('Trying again in 5 seconds')
                 time.sleep(5)
 
         return ret
 
+    def pick_synchronous_standby(self, cluster):
+        """Finds the best candidate to be the synchronous standby.
+
+        Current synchronous standby is always preferred, unless it has disconnected or does not want to be a
+        synchronous standby any longer.
+
+        :returns tuple of candidate name or None, and bool showing if the member is the active synchronous standby.
+        """
+        current = cluster.sync.sync_standby
+        members = {m.name: m for m in cluster.members}
+        candidates = []
+        # Pick candidates based on who has flushed WAL farthest.
+        # TODO: for synchronous_commit = remote_write we actually want to order on write_location
+        for app_name, state, sync_state in self.query(
+                """SELECT application_name, state, sync_state
+                     FROM pg_stat_replication
+                    ORDER BY flush_location DESC"""):
+            member = members.get(app_name)
+            if state != 'streaming' or not member or member.tags.get('nosync', False):
+                continue
+            if sync_state == 'sync':
+                return app_name, True
+            if sync_state == 'potential' and app_name == current:
+                # Prefer current even if not the best one any more to avoid indecisivness and spurious swaps.
+                return current, False
+            if sync_state == 'async':
+                candidates.append(app_name)
+
+        if candidates:
+            return candidates[0], False
+        return None, False
+
+    def set_synchronous_standby(self, name):
+        """Sets a node to be synchronous standby and if changed does a reload for PostgreSQL."""
+        if name != self._synchronous_standby_names:
+            if name is None:
+                self._server_parameters.pop('synchronous_standby_names', None)
+            else:
+                self._server_parameters['synchronous_standby_names'] = name
+            self._synchronous_standby_names = name
+            self._write_postgresql_conf()
+            self.reload()
+
     @staticmethod
     def postgres_version_to_int(pg_version):
         """ Convert the server_version to integer

patroni/scripts/aws.py

@@ -6,13 +6,18 @@ from requests.exceptions import RequestException
 import sys
 import boto.ec2
 
+from patroni.utils import Retry, RetryFailedError
+
 logger = logging.getLogger(__name__)
 
+retry_timeout = 15
+
 
 class AWSConnection(object):
     def __init__(self, cluster_name):
         self.available = False
         self.cluster_name = cluster_name if cluster_name is not None else 'unknown'
+        self._retry = Retry(deadline=retry_timeout, max_delay=5, max_tries=-1, retry_exceptions=(boto.exception,))
         try:
             # get the instance id
             r = requests.get('http://169.254.169.254/latest/dynamic/instance-identity/document', timeout=0.1)
@@ -29,40 +34,36 @@ class AWSConnection(object):
                 return
             self.available = True
 
+    def retry(self, *args, **kwargs):
+        return self._retry.copy()(*args, **kwargs)
+
     def aws_available(self):
         return self.available
 
-    def _tag_ebs(self, role):
+    def _tag_ebs(self, conn, role):
         """ set tags, carrying the cluster name, instance role and instance id for the EBS storage """
-        if not self.available:
-            return False
-
         tags = {'Name': 'spilo_' + self.cluster_name, 'Role': role, 'Instance': self.instance_id}
-        try:
-            conn = boto.ec2.connect_to_region(self.region)
-            volumes = conn.get_all_volumes(filters={'attachment.instance-id': self.instance_id})
-            conn.create_tags([v.id for v in volumes], tags)
-        except Exception as e:
-            logger.info('could not set tags for EBS storage devices attached: {}'.format(e))
-            return False
-        return True
+        volumes = conn.get_all_volumes(filters={'attachment.instance-id': self.instance_id})
+        conn.create_tags([v.id for v in volumes], tags)
 
-    def _tag_ec2(self, role):
+    def _tag_ec2(self, conn, role):
         """ tag the current EC2 instance with a cluster role """
-        if not self.available:
-            return False
         tags = {'Role': role}
-        try:
-            conn = boto.ec2.connect_to_region(self.region)
-            conn.create_tags([self.instance_id], tags)
-        except Exception as e:
-            logger.info("could not set tags for EC2 instance %s: %s", self.instance_id, e)
-            return False
-        return True
+        conn.create_tags([self.instance_id], tags)
 
     def on_role_change(self, new_role):
-        ret = self._tag_ec2(new_role)
-        return self._tag_ebs(new_role) and ret
+        if not self.available:
+            return False
+        try:
+            conn = self.retry(boto.ec2.connect_to_region, self.region)
+            self.retry(self._tag_ec2, conn, new_role)
+            self.retry(self._tag_ebs, conn, new_role)
+        except RetryFailedError:
+            logger.warning("Unable to communicate to AWS "
+                           "when setting tags for the EC2 instance {0} "
+                           "and attached EBS volumes".format(self.instance_id))
+            return False
+        return True
 
 
 def main():
@@ -72,5 +73,6 @@ def main():
     else:
         sys.exit("Usage: {0} action role name".format(sys.argv[0]))
 
+
 if __name__ == '__main__':
     main()

patroni/scripts/wale_restore.py

@@ -30,18 +30,28 @@ import os
 import psycopg2
 import subprocess
 import sys
+import time
 import argparse
 
-
-if sys.hexversion >= 0x0300000:
-    long = int
-
 logger = logging.getLogger(__name__)
 
+RETRY_SLEEP_INTERVAL = 1
+
+
+# We need to know the current PG version in order to figure out the correct WAL directory name
+def get_major_version(data_dir):
+    version_file = os.path.join(data_dir, 'PG_VERSION')
+    if os.path.isfile(version_file):  # version file exists
+        try:
+            with open(version_file) as f:
+                return float(f.read())
+        except Exception:
+            logger.exception('Failed to read PG_VERSION from %s', data_dir)
+    return 0.0
+
 
 class WALERestore(object):
-
-    def __init__(self, scope, datadir, connstring, env_dir, threshold_mb, threshold_pct, use_iam, no_master):
+    def __init__(self, scope, datadir, connstring, env_dir, threshold_mb, threshold_pct, use_iam, no_master, retries):
         self.scope = scope
         self.master_connection = connstring
         self.data_dir = datadir
@@ -53,11 +63,19 @@ class WALERestore(object):
         self.no_master = no_master
         self.wal_e.cmd = 'envdir {0} wal-e {1} '.format(self.wal_e.dir, self.wal_e.iam_string)
         self.init_error = (not os.path.exists(self.wal_e.dir))
+        self.retries = retries
 
     def run(self):
         """ creates a new replica using WAL-E """
-        if not self.init_error and self.should_use_s3_to_create_replica():
-            return self.create_replica_with_s3()
+        if not self.init_error:
+            try:
+                ret = self.should_use_s3_to_create_replica()
+                if ret:
+                    return self.create_replica_with_s3()
+                elif ret is None:  # caught an exception, need to retry
+                    return 1
+            except Exception:
+                logger.exception("Exception when running WAL-E restore")
         return 2
 
     def should_use_s3_to_create_replica(self):
@@ -83,17 +101,17 @@ class WALERestore(object):
                 return False
 
             backup_info = dict(zip(names, vals))
-        except subprocess.CalledProcessError as e:
-            logger.error("could not query wal-e latest backup: {}".format(e))
-            return False
+        except subprocess.CalledProcessError:
+            logger.exception("could not query wal-e latest backup")
+            return None
 
         try:
             backup_size = backup_info['expanded_size_bytes']
             backup_start_segment = backup_info['wal_segment_backup_start']
             backup_start_offset = backup_info['wal_segment_offset_backup_start']
-        except Exception as e:
-            logger.error("unable to get some of WALE backup parameters: {}".format(e))
-            return False
+        except Exception:
+            logger.exception("unable to get some of WALE backup parameters")
+            return None
 
         # WAL filename is XXXXXXXXYYYYYYYY000000ZZ, where X - timeline, Y - LSN logical log file,
         # ZZ - 2 high digits of LSN offset. The rest of the offset is the provided decimal offset,
@@ -101,31 +119,70 @@ class WALERestore(object):
 
         lsn_segment = backup_start_segment[8:16]
         # first 2 characters of the result are 0x and the last one is L
-        lsn_offset = hex((long(backup_start_segment[16:32], 16) << 24) + long(backup_start_offset))[2:-1]
+        lsn_offset = hex((int(backup_start_segment[16:32], 16) << 24) + int(backup_start_offset))[2:-1]
 
         # construct the LSN from the segment and offset
         backup_start_lsn = '{0}/{1}'.format(lsn_segment, lsn_offset)
 
-        diff_in_bytes = long(backup_size)
-        if not self.no_master:
-            try:
-                # get the difference in bytes between the current WAL location and the backup start offset
-                with psycopg2.connect(self.master_connection) as con:
-                    con.autocommit = True
-                    with con.cursor() as cur:
-                        cur.execute("SELECT pg_xlog_location_diff(pg_current_xlog_location(), %s)", (backup_start_lsn,))
-                        diff_in_bytes = long(cur.fetchone()[0])
-            except psycopg2.Error as e:
-                logger.error('could not determine difference with the master location: %s', e)
-                return False
-        else:
-            # always try to use WAL-E if base backup is available
-            diff_in_bytes = 0
+        diff_in_bytes = int(backup_size)
+        attempts_no = 0
+        while True:
+            if self.master_connection:
+                try:
+                    # get the difference in bytes between the current WAL location and the backup start offset
+                    with psycopg2.connect(self.master_connection) as con:
+                        con.autocommit = True
+                        with con.cursor() as cur:
+                            cur.execute("""SELECT CASE WHEN pg_is_in_recovery()
+                                                       THEN GREATEST(
+                                                                pg_xlog_location_diff(COALESCE(
+                                                                  pg_last_xlog_receive_location(), '0/0'), %s)::bigint,
+                                                                pg_xlog_location_diff(
+                                                                  pg_last_xlog_replay_location(), %s)::bigint)
+                                                       ELSE pg_xlog_location_diff(
+                                                                pg_current_xlog_location(), %s)::bigint
+                                                   END""", (backup_start_lsn, backup_start_lsn, backup_start_lsn))
+
+                            diff_in_bytes = int(cur.fetchone()[0])
+                except psycopg2.Error:
+                    logger.exception('could not determine difference with the master location')
+                    if attempts_no < self.retries:  # retry in case of a temporarily connection issue
+                        attempts_no = attempts_no + 1
+                        time.sleep(RETRY_SLEEP_INTERVAL)
+                        continue
+                    else:
+                        if not self.no_master:
+                            return False  # do no more retries on the outer level
+                        logger.info("continue with base backup from S3 since master is not available")
+                        diff_in_bytes = 0
+                        break
+            else:
+                # always try to use WAL-E if master connection string is not available
+                diff_in_bytes = 0
+            break
 
         # if the size of the accumulated WAL segments is more than a certan percentage of the backup size
         # or exceeds the pre-determined size - pg_basebackup is chosen instead.
-        return (diff_in_bytes < long(threshold_megabytes) * 1048576) and\
-            (diff_in_bytes < long(backup_size) * float(threshold_backup_size_percentage) / 100)
+        return (diff_in_bytes < int(threshold_megabytes) * 1048576) and\
+            (diff_in_bytes < int(backup_size) * float(threshold_backup_size_percentage) / 100)
+
+    def fix_subdirectory_path_if_broken(self, dirname):
+        # in case it is a symlink pointing to a non-existing location, remove it and create the actual directory
+        path = os.path.join(self.data_dir, dirname)
+        if not os.path.exists(path):
+            if os.path.islink(path):  # broken xlog symlink, to remove
+                try:
+                    os.remove(path)
+                except OSError:
+                    logger.exception("could not remove broken %s symlink pointing to %s",
+                                     dirname, os.readlink(path))
+                    return False
+            try:
+                os.mkdir(path)
+            except OSError:
+                logger.exception("coud not create missing %s directory path", dirname)
+                return False
+        return True
 
     def create_replica_with_s3(self):
         # if we're set up, restore the replica using fetch latest
@@ -135,6 +192,9 @@ class WALERestore(object):
             logger.error('Error when fetching backup with WAL-E: {0}'.format(e))
             return 1
 
+        if (ret == 0 and not
+           self.fix_subdirectory_path_if_broken('pg_xlog' if get_major_version(self.data_dir) < 10.0 else 'pg_wal')):
+            return 2
         return ret
 
 
@@ -153,17 +213,23 @@ def main():
     parser.add_argument('--no_master', type=int, default=0)
     args = parser.parse_args()
 
-    # retry cloning in a loop
+    # Retry cloning in a loop. We do separate retries for the master
+    # connection attempt inside should_use_s3_to_create_replica,
+    # because we need to differentiate between the last attempt and
+    # the rest and make a decision when the last attempt fails on
+    # whether to use WAL-E or not depending on the no_master flag.
     for _ in range(0, args.retries + 1):
         restore = WALERestore(scope=args.scope, datadir=args.datadir, connstring=args.connstring,
                               env_dir=args.envdir, threshold_mb=args.threshold_megabytes,
                               threshold_pct=args.threshold_backup_size_percentage, use_iam=args.use_iam,
-                              no_master=args.no_master)
+                              no_master=args.no_master, retries=args.retries)
         ret = restore.run()
-        if ret == 0:
+        if ret != 1:  # only WAL-E failures lead to the retry
             break
+        time.sleep(RETRY_SLEEP_INTERVAL)
+
+    return ret
 
-    sys.exit(ret)
 
 if __name__ == '__main__':
-    main()
+    sys.exit(main())

patroni/utils.py

@@ -1,16 +1,11 @@
-import os
 import random
-import sys
 import time
 import re
 
+from dateutil import tz
 from patroni.exceptions import PatroniException
 
-if sys.hexversion >= 0x0300000:
-    long = int
-
-__interrupted_sleep = False
-__reap_children = False
+tzutc = tz.tzutc()
 
 
 def deep_compare(obj1, obj2):
@@ -122,7 +117,7 @@ def strtol(value, strict=True):
         while i <= l:
             try:  # try to find maximally long number
                 i += 1  # by giving to `int` longer and longer strings
-                ret = long(value[:i], base)
+                ret = int(value[:i], base)
             except ValueError:  # until we will not get an exception or end of the string
                 i -= 1
                 break
@@ -195,36 +190,8 @@ def compare_values(vartype, unit, old_value, new_value):
     return old_value is not None and new_value is not None and old_value == new_value
 
 
-def sigchld_handler(signo, stack_frame):
-    global __interrupted_sleep, __reap_children
-    __reap_children = __interrupted_sleep = True
-
-
-def sleep(interval):
-    global __interrupted_sleep
-    current_time = time.time()
-    end_time = current_time + interval
-    while current_time < end_time:
-        __interrupted_sleep = False
-        time.sleep(end_time - current_time)
-        if not __interrupted_sleep:  # we will ignore only sigchld
-            break
-        current_time = time.time()
-    __interrupted_sleep = False
-
-
-def reap_children():
-    global __reap_children
-    if __reap_children:
-        try:
-            while True:
-                ret = os.waitpid(-1, os.WNOHANG)
-                if ret == (0, 0):
-                    break
-        except OSError:
-            pass
-        finally:
-            __reap_children = False
+def _sleep(interval):
+    time.sleep(interval)
 
 
 def is_valid_pg_version(version):
@@ -241,7 +208,7 @@ class Retry(object):
     """Helper for retrying a method in the face of retry-able exceptions"""
 
     def __init__(self, max_tries=1, delay=0.1, backoff=2, max_jitter=0.8, max_delay=3600,
-                 sleep_func=sleep, deadline=None, retry_exceptions=PatroniException):
+                 sleep_func=_sleep, deadline=None, retry_exceptions=PatroniException):
         """Create a :class:`Retry` instance for retrying function calls
 
         :param max_tries: How many times to retry the command. -1 means infinite tries.
@@ -302,3 +269,14 @@ class Retry(object):
                 else:
                     self.sleep_func(sleeptime)
                 self._cur_delay = min(self._cur_delay * self.backoff, self.max_delay)
+
+
+def polling_loop(timeout, interval=1):
+    """Returns an iterator that returns values until timeout has passed. Timeout is measured from start of iteration."""
+    start_time = time.time()
+    iteration = 0
+    end_time = start_time + timeout
+    while time.time() < end_time:
+        yield iteration
+        iteration += 1
+        time.sleep(interval)

patroni/version.py

@@ -1 +1 @@
-__version__ = '1.1'
+__version__ = '1.2.5'

postgres0.yml

@@ -12,7 +12,7 @@ restapi:
 #    password: password
 
 etcd:
-  host: 127.0.0.1:4001
+  host: 127.0.0.1:2379
 
 bootstrap:
   # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
@@ -22,6 +22,8 @@ bootstrap:
     loop_wait: 10
     retry_timeout: 10
     maximum_lag_on_failover: 1048576
+#    master_start_timeout: 300
+#    synchronous_mode: false
     postgresql:
       use_pg_rewind: true
 #      use_slots: true
@@ -48,6 +50,9 @@ bootstrap:
   - host all all 0.0.0.0/0 md5
 #  - hostssl all all 0.0.0.0/0 md5
 
+  # Additional script to be launched after initial cluster creation (will be passed the connection URL as parameter)
+# post_init: /usr/local/bin/setup_cluster.sh
+
   # Some additional users users which needs to be created after initializing new cluster
   users:
     admin:
@@ -75,3 +80,4 @@ tags:
     nofailover: false
     noloadbalance: false
     clonefrom: false
+    nosync: false

postgres1.yml

@@ -12,7 +12,7 @@ restapi:
 #    password: password
 
 etcd:
-  host: 127.0.0.1:4001
+  host: 127.0.0.1:2379
 
 bootstrap:
   # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
@@ -48,6 +48,9 @@ bootstrap:
   - host all all 0.0.0.0/0 md5
 #  - hostssl all all 0.0.0.0/0 md5
 
+  # Additional script to be launched after initial cluster creation (will be passed the connection URL as parameter)
+# post_init: /usr/local/bin/setup_cluster.sh
+
   # Some additional users users which needs to be created after initializing new cluster
   users:
     admin:

postgres2.yml

@@ -12,7 +12,7 @@ restapi:
     password: password
 
 etcd:
-  host: 127.0.0.1:4001
+  host: 127.0.0.1:2379
 
 bootstrap:
   # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster

requirements.txt

@@ -1,3 +1,4 @@
+urllib3>=1.9
 boto
 psycopg2>=2.6.1
 PyYAML
@@ -5,8 +6,8 @@ requests
 six >= 1.7
 kazoo==2.2.1
 python-etcd==0.4.3
-python-consul==0.6.0
+python-consul==0.7.0
 click>=4.1
 prettytable>=0.7
 tzlocal
-python-dateutil
+python-dateutil

setup.py

@@ -24,6 +24,7 @@ def read_version(package):
         exec(fd.read(), data)
     return data['__version__']
 
+
 NAME = 'patroni'
 MAIN_PACKAGE = NAME
 SCRIPTS = 'scripts'

tests/test_api.py

@@ -1,19 +1,19 @@
 import datetime
 import json
 import psycopg2
-import pytz
 import unittest
 
 from mock import Mock, patch
 from patroni.api import RestApiHandler, RestApiServer
 from patroni.dcs import ClusterConfig, Member
+from patroni.utils import tzutc
 from six import BytesIO as IO
 from six.moves import BaseHTTPServer
 from test_postgresql import psycopg2_connect, MockCursor
 
 
-future_restart_time = datetime.datetime.now(pytz.utc) + datetime.timedelta(days=5)
-postmaster_start_time = datetime.datetime.now(pytz.utc)
+future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5)
+postmaster_start_time = datetime.datetime.now(tzutc)
 
 
 class MockPostgresql(object):
@@ -63,6 +63,18 @@ class MockHa(object):
     def schedule_future_restart(data):
         return True
 
+    @staticmethod
+    def is_lagging(xlog):
+        return False
+
+    @staticmethod
+    def get_effective_tags():
+        return {'nosync': True}
+
+    @staticmethod
+    def wakeup():
+        pass
+
 
 class MockPatroni(object):
 
@@ -89,6 +101,8 @@ class MockRequest(object):
     def makefile(self, *args, **kwargs):
         return IO(self.request)
 
+    def sendall(self, *args, **kwargs):
+        pass
 
 class MockRestApiServer(RestApiServer):
 
@@ -225,6 +239,10 @@ class TestRestApiHandler(unittest.TestCase):
 
         mock_dcs.get_cluster.return_value.is_paused.return_value = True
         MockRestApiServer(RestApiHandler, make_request(schedule='2016-08-42 12:45TZ+1', role='master'))
+        # Valid timeout
+        MockRestApiServer(RestApiHandler, make_request(timeout='60s'))
+        # Invalid timeout
+        MockRestApiServer(RestApiHandler, make_request(timeout='42towels'))
 
     def test_do_DELETE_restart(self):
         for retval in (True, False):

tests/test_async_executor.py

@@ -8,7 +8,7 @@ from threading import Thread
 class TestAsyncExecutor(unittest.TestCase):
 
     def setUp(self):
-        self.a = AsyncExecutor()
+        self.a = AsyncExecutor(Mock())
 
     @patch.object(Thread, 'start', Mock())
     def test_run_async(self):

tests/test_aws.py

@@ -6,6 +6,7 @@ import unittest
 from mock import Mock, patch
 from collections import namedtuple
 from patroni.scripts.aws import AWSConnection, main as _main
+from patroni.utils import RetryFailedError
 from requests.exceptions import RequestException
 
 
@@ -16,13 +17,13 @@ class MockEc2Connection(object):
 
     def get_all_volumes(self, filters):
         if self.error:
-            raise Exception("get_all_volumes")
+            raise boto.exception("get_all_volumes")
         oid = namedtuple('Volume', 'id')
         return [oid(id='a'), oid(id='b')]
 
     def create_tags(self, objects, tags):
         if self.error or len(objects) == 0:
-            raise Exception("create_tags")
+            raise boto.exception("create_tags")
         return True
 
 
@@ -63,30 +64,20 @@ class TestAWSConnection(unittest.TestCase):
         self.assertTrue(self.conn.aws_available())
 
     def test_on_role_change(self):
-        self.assertTrue(self.conn._tag_ebs('master'))
-        self.assertTrue(self.conn._tag_ec2('master'))
         self.assertTrue(self.conn.on_role_change('master'))
+        self.conn.retry = Mock(side_effect=RetryFailedError("retry failed"))
+        self.assertFalse(self.conn.on_role_change('master'))
 
     def test_non_aws(self):
         self.error = True
         conn = AWSConnection('test')
-        self.assertFalse(conn.aws_available())
-        self.assertFalse(conn._tag_ebs('master'))
-        self.assertFalse(conn._tag_ec2('master'))
+        self.assertFalse(conn.on_role_change("master"))
 
     def test_aws_bizare_response(self):
         self.json_error = True
         conn = AWSConnection('test')
         self.assertFalse(conn.aws_available())
 
-    def test_aws_tag_ebs_error(self):
-        self.error = True
-        self.assertFalse(self.conn._tag_ebs("master"))
-
-    def test_aws_tag_ec2_error(self):
-        self.error = True
-        self.assertFalse(self.conn._tag_ec2("master"))
-
     @patch('sys.exit', Mock())
     def test_main(self):
         self.assertIsNone(_main())

tests/test_callback_executor.py

@@ -0,0 +1,24 @@
+import unittest
+
+from mock import Mock, patch
+from patroni.callback_executor import CallbackExecutor
+
+
+class TestCallbackExecutor(unittest.TestCase):
+
+    @patch('subprocess.Popen')
+    def test_callback_executor(self, mock_popen):
+        mock_popen.return_value.wait.side_effect = Exception
+        mock_popen.return_value.poll.return_value = None
+
+        ce = CallbackExecutor()
+        self.assertIsNone(ce.call([]))
+        ce.join()
+
+        self.assertIsNone(ce.call([]))
+
+        mock_popen.side_effect = Exception
+        ce = CallbackExecutor()
+        ce._callback_event.wait = Mock(side_effect=[None, Exception])
+        self.assertIsNone(ce.call([]))
+        ce.join()

tests/test_config.py

@@ -20,9 +20,10 @@ class TestConfig(unittest.TestCase):
     def test_no_config(self):
         self.assertRaises(SystemExit, Config)
 
-    @patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception))
     def test_set_dynamic_configuration(self):
-        self.assertIsNone(self.config.set_dynamic_configuration({'foo': 'bar'}))
+        with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
+            self.assertIsNone(self.config.set_dynamic_configuration({'foo': 'bar'}))
+        self.assertTrue(self.config.set_dynamic_configuration({'synchronous_mode': True}))
 
     def test_reload_local_configuration(self):
         os.environ.update({
@@ -40,6 +41,12 @@ class TestConfig(unittest.TestCase):
             'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0',
             'PATRONI_POSTGRESQL_PGPASS': '/tmp/pgpass0',
             'PATRONI_ETCD_HOST': '127.0.0.1:2379',
+            'PATRONI_ETCD_URL': 'https://127.0.0.1:2379',
+            'PATRONI_ETCD_PROXY': 'http://127.0.0.1:2379',
+            'PATRONI_ETCD_SRV': 'test',
+            'PATRONI_ETCD_CACERT': '/cacert',
+            'PATRONI_ETCD_CERT': '/cert',
+            'PATRONI_ETCD_KEY': '/key',
             'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
             'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
             'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',

tests/test_consul.py

@@ -1,8 +1,9 @@
 import consul
 import unittest
 
+from consul import ConsulException, NotFound
 from mock import Mock, patch
-from patroni.dcs.consul import AbstractDCS, Cluster, Consul, ConsulError, ConsulException, HTTPClient, NotFound
+from patroni.dcs.consul import AbstractDCS, Cluster, Consul, ConsulInternalError, ConsulError, HTTPClient
 from test_etcd import SleepException
 
 
@@ -30,17 +31,34 @@ def kv_get(self, key, **kwargs):
                   'Value': ('postgres://replicator:[email protected]:5433/postgres' +
                             '?application_name=http://127.0.0.1:8009/patroni').encode('utf-8')},
                  {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'optime/leader', 'LockIndex': 0,
-                  'ModifyIndex': 6429, 'Value': b'4496294792'}])
+                  'ModifyIndex': 6429, 'Value': b'4496294792'},
+                 {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'sync', 'LockIndex': 0,
+                  'ModifyIndex': 6429, 'Value': b'{"leader": "leader", "sync_standby": null}'}])
     raise ConsulException
 
 
 class TestHTTPClient(unittest.TestCase):
 
-    def test_get(self):
+    def setUp(self):
         self.client = HTTPClient('127.0.0.1', '8500', 'http', False)
-        self.client.session.get = Mock()
+        self.client.http.request = Mock()
+
+    def test_get(self):
         self.client.get(Mock(), '')
         self.client.get(Mock(), '', {'wait': '1s', 'index': 1})
+        self.client.http.request.return_value.status = 500
+        self.assertRaises(ConsulInternalError, self.client.get, Mock(), '')
+
+    def test_unknown_method(self):
+        try:
+            self.client.bla(Mock(), '')
+            self.assertFail()
+        except Exception as e:
+            self.assertTrue(isinstance(e, AttributeError))
+
+    def test_put(self):
+        self.client.put(Mock(), '/v1/session/create')
+        self.client.put(Mock(), '/v1/session/create', data='{"foo": "bar"}')
 
 
 @patch.object(consul.Consul.KV, 'get', kv_get)
@@ -65,7 +83,8 @@ class TestConsul(unittest.TestCase):
     @patch.object(consul.Consul.Session, 'create', Mock(side_effect=ConsulException))
     def test_referesh_session(self):
         self.c._session = '1'
-        self.c._name = ''
+        self.assertFalse(self.c.refresh_session())
+        self.c._last_session_refresh = 0
         self.assertRaises(ConsulError, self.c.refresh_session)
 
     @patch.object(consul.Consul.KV, 'delete', Mock())
@@ -91,6 +110,8 @@ class TestConsul(unittest.TestCase):
 
     @patch.object(consul.Consul.KV, 'put', Mock(return_value=False))
     def test_take_leader(self):
+        self.c.set_ttl(20)
+        self.c.refresh_session = Mock()
         self.c.take_leader()
 
     @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
@@ -103,8 +124,9 @@ class TestConsul(unittest.TestCase):
 
     @patch.object(consul.Consul.KV, 'put', Mock(side_effect=ConsulException))
     def test_write_leader_optime(self):
-        self.c.write_leader_optime('')
+        self.c.write_leader_optime('1')
 
+    @patch.object(consul.Consul.Session, 'renew', Mock())
     def test_update_leader(self):
         self.c.update_leader()
 
@@ -126,15 +148,17 @@ class TestConsul(unittest.TestCase):
 
     @patch.object(AbstractDCS, 'watch', Mock())
     def test_watch(self):
+        self.c.watch(None, 1)
         self.c._name = ''
-        self.c.watch(1)
+        self.c.watch(6429, 1)
         with patch.object(consul.Consul.KV, 'get', Mock(side_effect=ConsulException)):
-            self.c.watch(1)
-
-    @patch.object(consul.Consul.Session, 'destroy', Mock(side_effect=ConsulException))
-    def test_set_ttl(self):
-        self.c.set_ttl(20)
-        self.assertTrue(self.c.watch(1))
+            self.c.watch(6429, 1)
 
     def test_set_retry_timeout(self):
         self.c.set_retry_timeout(10)
+
+    @patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
+    @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
+    def test_sync_state(self):
+        self.assertTrue(self.c.set_sync_state_value('{}'))
+        self.assertTrue(self.c.delete_sync_state())

tests/test_ctl.py

@@ -7,7 +7,7 @@ import unittest
 from click.testing import CliRunner
 from mock import patch, Mock
 from patroni.ctl import ctl, members, store_config, load_config, output_members, request_patroni, get_dcs, parse_dcs, \
-    wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException
+    get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException
 from patroni.dcs.etcd import Client
 from psycopg2 import OperationalError
 from test_etcd import etcd_read, requests_get, socket_getaddrinfo, MockResponse
@@ -31,7 +31,7 @@ def test_rw_config():
 
 @patch('patroni.ctl.load_config',
        Mock(return_value={'postgresql': {'data_dir': '.', 'parameters': {}, 'retry_timeout': 5},
-                          'restapi': {'auth': 'u:p', 'listen': ''}, 'etcd': {'host': 'localhost:4001'}}))
+                          'restapi': {'auth': 'u:p', 'listen': ''}, 'etcd': {'host': 'localhost:2379'}}))
 class TestCtl(unittest.TestCase):
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
@@ -54,8 +54,8 @@ class TestCtl(unittest.TestCase):
 
     def test_parse_dcs(self):
         assert parse_dcs(None) is None
-        assert parse_dcs('localhost') == {'etcd': {'host': 'localhost:4001'}}
-        assert parse_dcs('') == {'etcd': {'host': 'localhost:4001'}}
+        assert parse_dcs('localhost') == {'etcd': {'host': 'localhost:2379'}}
+        assert parse_dcs('') == {'etcd': {'host': 'localhost:2379'}}
         assert parse_dcs('localhost:8500') == {'consul': {'host': 'localhost:8500'}}
         assert parse_dcs('zookeeper://localhost') == {'zookeeper': {'hosts': ['localhost:2181']}}
         assert parse_dcs('exhibitor://dummy') == {'exhibitor': {'hosts': ['dummy'], 'port': 8181}}
@@ -73,18 +73,15 @@ class TestCtl(unittest.TestCase):
     def test_failover(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        mock_get_dcs.return_value.set_failover_value = Mock()
         result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
         assert 'leader' in result.output
 
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2100-01-01T12:23:00\ny')
-        assert result.exit_code == 0
-
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2030-01-01T12:23:00\ny')
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
         assert result.exit_code == 0
 
         with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
-            result = self.runner.invoke(ctl,
-                                        ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
+            result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00'])
             assert result.exit_code == 1
 
         # Aborting failover,as we anser NO to the confirmation
@@ -119,7 +116,7 @@ class TestCtl(unittest.TestCase):
 
         with patch('patroni.ctl.request_patroni', Mock(side_effect=Exception)):
             # Non-responding patroni
-            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
+            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
             assert 'falling back to DCS' in result.output
 
         with patch('patroni.ctl.request_patroni') as mocked:
@@ -240,6 +237,9 @@ class TestCtl(unittest.TestCase):
         assert 'Error: PostgreSQL version' in result.output
         assert result.exit_code == 1
 
+        result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force', '--timeout', '10min'])
+        assert result.exit_code == 0
+
         with patch('requests.delete', Mock(return_value=MockResponse(500))):
             # normal restart, the schedule is actually parsed, but not validated in patronictl
             result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force',
@@ -308,14 +308,6 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['remove', 'alpha'], input='alpha\nYes I am aware\nleader')
         assert result.exit_code == 0
 
-    @patch('patroni.dcs.AbstractDCS.watch', Mock(return_value=None))
-    @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
-    def test_wait_for_leader(self):
-        self.assertRaises(PatroniCtlException, wait_for_leader, self.e, 0)
-
-        cluster = wait_for_leader(self.e, timeout=2)
-        assert cluster.leader.member.name == 'leader'
-
     @patch('requests.post', Mock(side_effect=requests.exceptions.ConnectionError('foo')))
     def test_request_patroni(self):
         member = get_cluster_initialized_with_leader().leader.member
@@ -364,6 +356,7 @@ class TestCtl(unittest.TestCase):
         mock_get_dcs.return_value.initialize = Mock(return_value=True)
         mock_get_dcs.return_value.touch_member = Mock(return_value=True)
         mock_get_dcs.return_value.attempt_to_acquire_leader = Mock(return_value=True)
+        mock_get_dcs.return_value.delete_cluster = Mock()
 
         with patch.object(self.e, 'initialize', return_value=False):
             result = self.runner.invoke(ctl, ['scaffold', 'alpha'])
@@ -383,7 +376,8 @@ class TestCtl(unittest.TestCase):
     @patch('patroni.ctl.get_dcs')
     def test_list_extended(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
+        mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster)
 
         result = self.runner.invoke(ctl, ['list', 'dummy', '--extended'])
         assert '2100' in result.output
@@ -440,3 +434,7 @@ class TestCtl(unittest.TestCase):
                     patch('patroni.dcs.Cluster.is_paused', Mock(return_value=False)):
                 result = self.runner.invoke(ctl, ['resume', 'dummy'])
                 assert 'Cluster is not paused' in result.output
+
+            with patch('requests.patch', Mock(side_effect=Exception)):
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Can not find accessible cluster member' in result.output

tests/test_etcd.py

@@ -1,12 +1,13 @@
 import etcd
 import json
+import urllib3.util.connection
 import requests
 import socket
 import unittest
 
 from dns.exception import DNSException
 from mock import Mock, patch
-from patroni.dcs.etcd import AbstractDCS, Client, Cluster, Etcd, EtcdError
+from patroni.dcs.etcd import AbstractDCS, Client, Cluster, Etcd, EtcdError, DnsCachingResolver
 from patroni.exceptions import DCSError
 from urllib3.exceptions import ReadTimeoutError
 
@@ -42,7 +43,7 @@ def requests_get(url, **kwargs):
     if url.startswith('http://local'):
         raise requests.exceptions.RequestException()
     elif ':8011/patroni' in url:
-        response.content = '{"role": "replica", "xlog": {"replayed_location": 0}, "tags": {}}'
+        response.content = '{"role": "replica", "xlog": {"received_location": 0}, "tags": {}}'
     elif url.endswith('/members'):
         response.content = '[{}]' if url.startswith('http://error') else members
     elif url.startswith('http://exhibitor'):
@@ -58,8 +59,10 @@ def etcd_watch(self, key, index=None, timeout=None, recursive=None):
         raise etcd.EtcdWatchTimedOut
     elif timeout == 5.0:
         return etcd.EtcdResult('delete', {})
-    elif timeout == 10.0:
+    elif 5 < timeout <= 10.0:
         raise etcd.EtcdException
+    elif timeout == 20.0:
+        raise etcd.EtcdEventIndexCleared
 
 
 def etcd_write(self, key, value, **kwargs):
@@ -91,6 +94,8 @@ def etcd_read(self, key, **kwargs):
                     {"key": "/service/batman5/optime/leader", "value": "2164261704",
                      "modifiedIndex": 20729, "createdIndex": 20729}],
                  "modifiedIndex": 20437, "createdIndex": 20437},
+                {"key": "/service/batman5/sync", "value": '{"leader": "leader"}',
+                 "modifiedIndex": 1582, "createdIndex": 1582},
                 {"key": "/service/batman5/members", "dir": True, "nodes": [
                     {"key": "/service/batman5/members/postgresql1",
                      "value": "postgres://replicator:[email protected]:5434/postgres" +
@@ -112,23 +117,23 @@ class SleepException(Exception):
     pass
 
 
-class MockSRV(object):
-    port = 2380
-    target = '127.0.0.1'
-
-
 def dns_query(name, _):
+    if '-server' not in name or '-ssl' in name:
+        return []
     if name == '_etcd-server._tcp.blabla':
         return []
     elif name == '_etcd-server._tcp.exception':
         raise DNSException()
-    return [MockSRV()]
+    srv = Mock()
+    srv.port = 2380
+    srv.target.to_text.return_value = 'localhost' if name == '_etcd-server._tcp.foobar' else '127.0.0.1'
+    return [srv]
 
 
 def socket_getaddrinfo(*args):
-    if args[0] == 'ok':
-        return [(2, 1, 6, '', ('127.0.0.1', 2379)), (2, 1, 6, '', ('127.0.0.1', 2379))]
-    raise socket.error
+    if args[0] in ('ok', 'localhost', '127.0.0.1'):
+        return [(socket.AF_INET, 1, 6, '', ('127.0.0.1', 0)), (socket.AF_INET6, 1, 6, '', ('::1', 0))]
+    raise socket.gaierror
 
 
 def http_request(method, url, **kwargs):
@@ -143,17 +148,28 @@ def http_request(method, url, **kwargs):
     raise socket.error
 
 
+class TestDnsCachingResolver(unittest.TestCase):
+
+    @patch('time.sleep', Mock(side_effect=SleepException))
+    @patch('socket.getaddrinfo', Mock(side_effect=socket.gaierror))
+    def test_run(self):
+        r = DnsCachingResolver()
+        self.assertIsNone(r.resolve_async('', 0))
+        r.join()
+
+
 @patch('dns.resolver.query', dns_query)
 @patch('socket.getaddrinfo', socket_getaddrinfo)
 @patch('requests.get', requests_get)
 class TestClient(unittest.TestCase):
 
     @patch('dns.resolver.query', dns_query)
+    @patch('socket.getaddrinfo', socket_getaddrinfo)
     @patch('requests.get', requests_get)
     def setUp(self):
         with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
-            self.client = Client({'discovery_srv': 'test', 'retry_timeout': 3})
+            self.client = Client({'srv': 'test', 'retry_timeout': 3}, DnsCachingResolver())
             self.client.http.request = http_request
             self.client.http.request_encode_body = http_request
 
@@ -179,6 +195,9 @@ class TestClient(unittest.TestCase):
         self.client._base_uri = 'http://localhost:4001'
         self.client._machines_cache = ['http://localhost:2379']
         self.client.api_execute('/', 'POST', timeout=0)
+        mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
+        self.client._machines_cache_updated = 0
+        self.client.api_execute('/', 'POST', timeout=0)
         self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
         self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '')
         self.client._update_machines_cache = True
@@ -186,31 +205,43 @@ class TestClient(unittest.TestCase):
             self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'GET')
 
     def test_get_srv_record(self):
-        self.assertEquals(self.client.get_srv_record('blabla'), [])
-        self.assertEquals(self.client.get_srv_record('exception'), [])
+        self.assertEquals(self.client.get_srv_record('_etcd-server._tcp.blabla'), [])
+        self.assertEquals(self.client.get_srv_record('_etcd-server._tcp.exception'), [])
 
     def test__get_machines_cache_from_srv(self):
+        self.client._get_machines_cache_from_srv('foobar')
         self.client.get_srv_record = Mock(return_value=[('localhost', 2380)])
         self.client._get_machines_cache_from_srv('blabla')
 
     def test__get_machines_cache_from_dns(self):
-        self.client._get_machines_cache_from_dns('error:2379')
+        self.client._get_machines_cache_from_dns('error', 2379)
 
     @patch.object(Client, 'machines')
     def test__load_machines_cache(self, mock_machines):
         mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
         self.client._config = {}
         self.assertRaises(Exception, self.client._load_machines_cache)
-        self.client._config = {'discovery_srv': 'blabla'}
+        self.client._config = {'srv': 'blabla'}
         self.assertRaises(etcd.EtcdException, self.client._load_machines_cache)
 
+    @patch.object(socket.socket, 'connect')
+    def test_create_connection_patched(self, mock_connect):
+        self.assertRaises(socket.error, urllib3.util.connection.create_connection, ('fail', 2379))
+        urllib3.util.connection.create_connection(('[localhost]', 2379))
+        mock_connect.side_effect = socket.error
+        self.assertRaises(socket.error, urllib3.util.connection.create_connection, ('[localhost]', 2379),
+                          timeout=1, source_address=('localhost', 53333),
+                          socket_options=[(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)])
+
 
 @patch('requests.get', requests_get)
+@patch('socket.getaddrinfo', socket_getaddrinfo)
 @patch.object(etcd.Client, 'write', etcd_write)
 @patch.object(etcd.Client, 'read', etcd_read)
 @patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
 class TestEtcd(unittest.TestCase):
 
+    @patch('socket.getaddrinfo', socket_getaddrinfo)
     def setUp(self):
         with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
@@ -226,7 +257,11 @@ class TestEtcd(unittest.TestCase):
             mock_machines.__get__ = Mock(side_effect=etcd.EtcdException)
             with patch('time.sleep', Mock(side_effect=SleepException)):
                 self.assertRaises(SleepException, self.etcd.get_etcd_client,
-                                  {'discovery_srv': 'test', 'retry_timeout': 10})
+                                  {'discovery_srv': 'test', 'retry_timeout': 10, 'cacert': '1', 'key': '1', 'cert': 1})
+                self.assertRaises(SleepException, self.etcd.get_etcd_client,
+                                  {'url': 'https://test:2379', 'retry_timeout': 10})
+                self.assertRaises(SleepException, self.etcd.get_etcd_client,
+                                  {'proxy': 'https://user:password@test:2379', 'retry_timeout': 10})
 
     def test_get_cluster(self):
         self.assertIsInstance(self.etcd.get_cluster(), Cluster)
@@ -267,14 +302,16 @@ class TestEtcd(unittest.TestCase):
     def test_delete_cluster(self):
         self.assertFalse(self.etcd.delete_cluster())
 
+    @patch('time.sleep', Mock(side_effect=SleepException))
     @patch.object(etcd.Client, 'watch', etcd_watch)
     def test_watch(self):
-        self.etcd.watch(0)
+        self.etcd.watch(None, 0)
         self.etcd.get_cluster()
-        self.etcd.watch(1.5)
-        self.etcd.watch(4.5)
+        self.etcd.watch(20729, 1.5)
+        self.etcd.watch(20729, 4.5)
         with patch.object(AbstractDCS, 'watch', Mock()):
-            self.etcd.watch(9.5)
+            self.assertTrue(self.etcd.watch(20729, 19.5))
+            self.assertRaises(SleepException, self.etcd.watch, 20729, 9.5)
 
     def test_other_exceptions(self):
         self.etcd.retry = Mock(side_effect=AttributeError('foo'))
@@ -282,4 +319,8 @@ class TestEtcd(unittest.TestCase):
 
     def test_set_ttl(self):
         self.etcd.set_ttl(20)
-        self.assertTrue(self.etcd.watch(1))
+        self.assertTrue(self.etcd.watch(None, 1))
+
+    def test_sync_state(self):
+        self.assertFalse(self.etcd.write_sync_state('leader', None))
+        self.assertFalse(self.etcd.delete_sync_state())

tests/test_ha.py

@@ -1,17 +1,18 @@
 import datetime
 import etcd
 import os
-import pytz
 import unittest
 
 from mock import Mock, MagicMock, PropertyMock, patch
 from patroni.config import Config
-from patroni.dcs import Cluster, Failover, Leader, Member, get_dcs
+from patroni.dcs import Cluster, ClusterConfig, Failover, Leader, Member, get_dcs, SyncState
 from patroni.dcs.etcd import Client
 from patroni.exceptions import DCSError, PostgresException
-from patroni.ha import Ha
+from patroni.ha import Ha, _MemberStatus
 from patroni.postgresql import Postgresql
+from patroni.utils import tzutc
 from test_etcd import socket_getaddrinfo, etcd_read, etcd_write, requests_get
+from test_postgresql import psycopg2_connect
 
 
 def true(*args, **kwargs):
@@ -22,36 +23,47 @@ def false(*args, **kwargs):
     return False
 
 
-def get_cluster(initialize, leader, members, failover):
-    return Cluster(initialize, None, leader, 10, members, failover)
+def get_cluster(initialize, leader, members, failover, sync):
+    return Cluster(initialize, ClusterConfig(1, {1: 2}, 1), leader, 10, members, failover, sync)
 
 
 def get_cluster_not_initialized_without_leader():
-    return get_cluster(None, None, [], None)
+    return get_cluster(None, None, [], None, SyncState(None, None, None))
 
 
-def get_cluster_initialized_without_leader(leader=False, failover=None):
+def get_cluster_initialized_without_leader(leader=False, failover=None, sync=None):
     m1 = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres',
                                   'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location': 4})
     l = Leader(0, 0, m1) if leader else None
     m2 = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
                                  'api_url': 'http://127.0.0.1:8011/patroni',
+                                 'state': 'running',
                                  'tags': {'clonefrom': True},
                                  'scheduled_restart': {'schedule': "2100-01-01 10:53:07.560445+00:00",
                                                        'postgres_version': '99.0.0'}})
-    return get_cluster(True, l, [m1, m2], failover)
+    syncstate = SyncState(0 if sync else None, sync and sync[0], sync and sync[1])
+    return get_cluster(True, l, [m1, m2], failover, syncstate)
 
 
-def get_cluster_initialized_with_leader(failover=None):
-    return get_cluster_initialized_without_leader(leader=True, failover=failover)
+def get_cluster_initialized_with_leader(failover=None, sync=None):
+    return get_cluster_initialized_without_leader(leader=True, failover=failover, sync=sync)
 
 
 def get_cluster_initialized_with_only_leader(failover=None):
     l = get_cluster_initialized_without_leader(leader=True, failover=failover).leader
-    return get_cluster(True, l, [l], failover)
+    return get_cluster(True, l, [l], failover, None)
 
-future_restart_time = datetime.datetime.now(pytz.utc) + datetime.timedelta(days=5)
-postmaster_start_time = datetime.datetime.now(pytz.utc)
+
+def get_node_status(reachable=True, in_recovery=True, xlog_location=10, nofailover=False):
+    def fetch_node_status(e):
+        tags = {}
+        if nofailover:
+            tags['nofailover'] = True
+        return _MemberStatus(e, reachable, in_recovery, xlog_location, tags)
+    return fetch_node_status
+
+future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5)
+postmaster_start_time = datetime.datetime.now(tzutc)
 
 
 class MockPatroni(object):
@@ -86,6 +98,7 @@ zookeeper:
         self.replicatefrom = None
         self.api.connection_string = 'http://127.0.0.1:8008'
         self.clonefrom = None
+        self.nosync = False
         self.scheduled_restart = {'schedule': future_restart_time,
                                   'postmaster_start_time': str(postmaster_start_time)}
 
@@ -96,7 +109,7 @@ def run_async(self, func, args=()):
 
 @patch.object(Postgresql, 'is_running', Mock(return_value=True))
 @patch.object(Postgresql, 'is_leader', Mock(return_value=True))
-@patch.object(Postgresql, 'xlog_position', Mock(return_value=0))
+@patch.object(Postgresql, 'xlog_position', Mock(return_value=10))
 @patch.object(Postgresql, 'call_nowait', Mock(return_value=True))
 @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
 @patch.object(Postgresql, 'controldata', Mock(return_value={'Database system identifier': '1234567890'}))
@@ -106,6 +119,7 @@ def run_async(self, func, args=()):
 @patch.object(Postgresql, 'write_recovery_conf', Mock())
 @patch.object(Postgresql, 'query', Mock())
 @patch.object(Postgresql, 'checkpoint', Mock())
+@patch.object(Postgresql, 'call_nowait', Mock())
 @patch.object(etcd.Client, 'write', etcd_write)
 @patch.object(etcd.Client, 'read', etcd_read)
 @patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
@@ -115,12 +129,14 @@ def run_async(self, func, args=()):
 class TestHa(unittest.TestCase):
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
+    @patch('psycopg2.connect', psycopg2_connect)
     @patch.object(etcd.Client, 'read', etcd_read)
     def setUp(self):
         with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
             self.p = Postgresql({'name': 'postgresql0', 'scope': 'dummy', 'listen': '127.0.0.1:5432',
                                  'data_dir': 'data/postgresql0', 'retry_timeout': 10,
+                                 'maximum_lag_on_failover': 5,
                                  'authentication': {'superuser': {'username': 'foo', 'password': 'bar'},
                                                     'replication': {'username': '', 'password': ''}},
                                  'parameters': {'wal_level': 'hot_standby', 'max_replication_slots': 5, 'foo': 'bar',
@@ -128,7 +144,6 @@ class TestHa(unittest.TestCase):
             self.p.set_state('running')
             self.p.set_role('replica')
             self.p.postmaster_start_time = MagicMock(return_value=str(postmaster_start_time))
-            self.p.check_replication_lag = true
             self.p.can_create_replica_without_replication_connection = MagicMock(return_value=False)
             self.e = get_dcs({'etcd': {'ttl': 30, 'host': 'ok:2379', 'scope': 'test',
                                        'name': 'foo', 'retry_timeout': 10}})
@@ -136,10 +151,11 @@ class TestHa(unittest.TestCase):
             self.ha.old_cluster = self.e.get_cluster()
             self.ha.cluster = get_cluster_not_initialized_without_leader()
             self.ha.load_cluster_from_dcs = Mock()
+            self.ha.is_synchronous_mode = false
 
     def test_update_lock(self):
         self.p.last_operation = Mock(side_effect=PostgresException(''))
-        self.assertTrue(self.ha.update_lock())
+        self.assertTrue(self.ha.update_lock(True))
 
     def test_touch_member(self):
         self.p.xlog_position = Mock(side_effect=Exception)
@@ -167,9 +183,6 @@ class TestHa(unittest.TestCase):
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.assertEquals(self.ha.run_cycle(), 'starting as readonly because i had the session lock')
 
-    def test_do_not_recover_in_pause(self):
-        pass
-
     @patch('sys.exit', return_value=1)
     @patch('patroni.ha.Ha.sysid_valid', MagicMock(return_value=True))
     def test_sysid_no_match(self, exit_mock):
@@ -229,7 +242,9 @@ class TestHa(unittest.TestCase):
         self.ha.cluster.is_unlocked = false
         self.ha.has_lock = true
         self.ha.update_lock = false
-        self.assertEquals(self.ha.run_cycle(), 'demoting self because i do not have the lock and i was a leader')
+        self.assertEquals(self.ha.run_cycle(), 'demoted self because failed to update leader lock in DCS')
+        self.p.is_leader = false
+        self.assertEquals(self.ha.run_cycle(), 'not promoting because failed to update leader lock in DCS')
 
     def test_follow(self):
         self.ha.cluster.is_unlocked = false
@@ -289,18 +304,20 @@ class TestHa(unittest.TestCase):
         self.assertIsNotNone(self.ha.reinitialize())
 
     def test_restart(self):
-        self.assertEquals(self.ha.restart(), (True, 'restarted successfully'))
+        self.assertEquals(self.ha.restart({}), (True, 'restarted successfully'))
+        self.p.restart = Mock(return_value=None)
+        self.assertEquals(self.ha.restart({}), (False, 'postgres is still starting'))
         self.p.restart = false
-        self.assertEquals(self.ha.restart(), (False, 'restart failed'))
+        self.assertEquals(self.ha.restart({}), (False, 'restart failed'))
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.ha.reinitialize()
-        self.assertEquals(self.ha.restart(), (False, 'reinitialize already in progress'))
+        self.assertEquals(self.ha.restart({}), (False, 'reinitialize already in progress'))
         with patch.object(self.ha, "restart_matches", return_value=False):
             self.assertEquals(self.ha.restart({'foo': 'bar'}), (False, "restart conditions are not satisfied"))
 
     def test_restart_in_progress(self):
         with patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=True)):
-            self.ha.restart(run_async=True)
+            self.ha.restart({}, run_async=True)
             self.assertTrue(self.ha.restart_scheduled())
             self.assertEquals(self.ha.run_cycle(), 'not healthy enough for leader race')
 
@@ -316,6 +333,7 @@ class TestHa(unittest.TestCase):
     @patch('requests.get', requests_get)
     @patch('time.sleep', Mock())
     def test_manual_failover_from_leader(self):
+        self.ha.fetch_node_status = get_node_status()
         self.ha.has_lock = true
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', '', None))
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
@@ -326,7 +344,9 @@ class TestHa(unittest.TestCase):
         f = Failover(0, self.p.name, '', None)
         self.ha.cluster = get_cluster_initialized_with_leader(f)
         self.assertEquals(self.ha.run_cycle(), 'manual failover: demoting myself')
-        self.ha.fetch_node_status = lambda e: (e, True, True, 0, {'nofailover': 'True'})
+        self.ha.fetch_node_status = get_node_status(nofailover=True)
+        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
+        self.ha.fetch_node_status = get_node_status(xlog_location=1)
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
         # manual failover from the previous leader to us won't happen if we hold the nofailover flag
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, None))
@@ -337,7 +357,7 @@ class TestHa(unittest.TestCase):
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
         self.ha.run_cycle()
 
-        scheduled = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
+        scheduled = datetime.datetime.utcnow().replace(tzinfo=tzutc)
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
         self.assertEquals('no action.  i am the leader with the lock', self.ha.run_cycle())
 
@@ -372,17 +392,17 @@ class TestHa(unittest.TestCase):
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'leader', None))
         self.p.set_role('replica')
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
-        self.ha.fetch_node_status = lambda e: (e, True, True, 0, {})  # accessible, in_recovery
+        self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
         self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, self.p.name, '', None))
         self.assertEquals(self.ha.run_cycle(), 'following a different leader because i am not the healthiest node')
-        self.ha.fetch_node_status = lambda e: (e, False, True, 0, {})  # inaccessible, in_recovery
+        self.ha.fetch_node_status = get_node_status(reachable=False)  # inaccessible, in_recovery
         self.p.set_role('replica')
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
         # set failover flag to True for all members of the cluster
         # this should elect the current member, as we are not going to call the API for it.
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
-        self.ha.fetch_node_status = lambda e: (e, True, True, 0, {'nofailover': 'True'})  # accessible, in_recovery
+        self.ha.fetch_node_status = get_node_status(nofailover=True)  # accessible, in_recovery
         self.p.set_role('replica')
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
         # same as previous, but set the current member to nofailover. In no case it should be elected as a leader
@@ -406,22 +426,24 @@ class TestHa(unittest.TestCase):
     def test_is_healthiest_node(self):
         self.ha.state_handler.is_leader = false
         self.ha.patroni.nofailover = False
-        self.ha.fetch_node_status = lambda e: (e, True, True, 0, {})
+        self.ha.fetch_node_status = get_node_status()
         self.assertTrue(self.ha.is_healthiest_node())
+        with patch('patroni.postgresql.Postgresql.is_starting', return_value=True):
+            self.assertFalse(self.ha.is_healthiest_node())
         self.ha.is_paused = true
         self.assertFalse(self.ha.is_healthiest_node())
 
     def test__is_healthiest_node(self):
         self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.p.is_leader = false
-        self.ha.fetch_node_status = lambda e: (e, True, True, 0, {})  # accessible, in_recovery
+        self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
         self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
-        self.ha.fetch_node_status = lambda e: (e, True, False, 0, {})  # accessible, not in_recovery
+        self.ha.fetch_node_status = get_node_status(in_recovery=False)  # accessible, not in_recovery
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
-        self.ha.fetch_node_status = lambda e: (e, True, True, 1, {})  # accessible, in_recovery, xlog location ahead
-        self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
-        self.p.check_replication_lag = false
+        self.ha.fetch_node_status = get_node_status(xlog_location=11)  # accessible, in_recovery, xlog location ahead
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
+        with patch('patroni.postgresql.Postgresql.xlog_position', return_value=1):
+            self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.patroni.nofailover = True
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.patroni.nofailover = False
@@ -453,6 +475,9 @@ class TestHa(unittest.TestCase):
 
     def test_evaluate_scheduled_restart(self):
         self.p.postmaster_start_time = Mock(return_value=str(postmaster_start_time))
+        # restart already in progres
+        with patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=True)):
+            self.assertIsNone(self.ha.evaluate_scheduled_restart())
         # restart while the postmaster has been already restarted, fails
         with patch.object(self.ha,
                           'future_restart_scheduled',
@@ -507,3 +532,253 @@ class TestHa(unittest.TestCase):
         self.ha.is_paused = true
         self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
         self.assertEquals(self.ha.run_cycle(), 'PAUSE: DCS is not accessible')
+
+    @patch('patroni.ha.Ha.update_lock', return_value=True)
+    @patch('patroni.ha.Ha.demote')
+    def test_starting_timeout(self, demote, update_lock):
+        def check_calls(seq):
+            for mock, called in seq:
+                if called:
+                    mock.assert_called_once()
+                else:
+                    mock.assert_not_called()
+                mock.reset_mock()
+        self.ha.has_lock = true
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.p.check_for_startup = true
+        self.p.time_in_state = lambda: 30
+        self.assertEquals(self.ha.run_cycle(), 'PostgreSQL is still starting up, 270 seconds until timeout')
+        check_calls([(update_lock, True), (demote, False)])
+
+        self.p.time_in_state = lambda: 350
+        self.ha.fetch_node_status = get_node_status(reachable=False)  # inaccessible, in_recovery
+        self.assertEquals(self.ha.run_cycle(),
+                          'master start has timed out, but continuing to wait because failover is not possible')
+        check_calls([(update_lock, True), (demote, False)])
+
+        self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
+        self.assertEquals(self.ha.run_cycle(), 'stopped PostgreSQL because of startup timeout')
+        check_calls([(update_lock, True), (demote, True)])
+
+        update_lock.return_value = False
+        self.assertEquals(self.ha.run_cycle(), 'stopped PostgreSQL while starting up because leader key was lost')
+        check_calls([(update_lock, True), (demote, True)])
+
+        self.ha.has_lock = false
+        self.p.is_leader = false
+        self.assertEquals(self.ha.run_cycle(), 'no action.  i am a secondary and i am following a leader')
+        check_calls([(update_lock, False), (demote, False)])
+
+    @patch('time.sleep', Mock())
+    def test_manual_failover_while_starting(self):
+        self.ha.has_lock = true
+        self.p.check_for_startup = true
+        f = Failover(0, self.p.name, '', None)
+        self.ha.cluster = get_cluster_initialized_with_leader(f)
+        self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
+        self.assertEquals(self.ha.run_cycle(), 'manual failover: demoting myself')
+
+    @patch('patroni.ha.Ha.demote')
+    def test_failover_immediately_on_zero_master_start_timeout(self, demote):
+        self.p.is_running = false
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.ha.patroni.config.set_dynamic_configuration({'master_start_timeout': 0})
+        self.ha.has_lock = true
+        self.ha.update_lock = true
+        self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
+        self.assertEquals(self.ha.run_cycle(), 'stopped PostgreSQL to fail over after a crash')
+        demote.assert_called_once()
+
+    @patch('time.sleep', Mock())
+    @patch('patroni.postgresql.Postgresql.follow')
+    def test_demote_immediate(self, follow):
+        self.ha.has_lock = true
+        self.e.get_cluster = Mock(return_value=get_cluster_initialized_without_leader())
+        self.ha.demote('immediate')
+        follow.assert_called_once_with(None, None, True, None, True)
+
+    @patch('time.sleep', Mock())
+    def test_process_sync_replication(self):
+        self.ha.has_lock = true
+        mock_set_sync = self.p.set_synchronous_standby = Mock()
+        self.p.name = 'leader'
+
+        # Test sync key removed when sync mode disabled
+        self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
+        with patch.object(self.ha.dcs, 'delete_sync_state') as mock_delete_sync:
+            self.ha.run_cycle()
+            mock_delete_sync.assert_called_once()
+            mock_set_sync.assert_called_once_with(None)
+
+        mock_set_sync.reset_mock()
+        # Test sync key not touched when not there
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        with patch.object(self.ha.dcs, 'delete_sync_state') as mock_delete_sync:
+            self.ha.run_cycle()
+            mock_delete_sync.assert_not_called()
+            mock_set_sync.assert_called_once_with(None)
+
+        mock_set_sync.reset_mock()
+
+        self.ha.is_synchronous_mode = true
+
+        # Test sync standby not touched when picking the same node
+        self.p.pick_synchronous_standby = Mock(return_value=('other', True))
+        self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
+        self.ha.run_cycle()
+        mock_set_sync.assert_not_called()
+
+        mock_set_sync.reset_mock()
+
+        # Test sync standby is replaced when switching standbys
+        self.p.pick_synchronous_standby = Mock(return_value=('other2', False))
+        self.ha.dcs.write_sync_state = Mock(return_value=True)
+        self.ha.run_cycle()
+        mock_set_sync.assert_called_once_with('other2')
+
+        mock_set_sync.reset_mock()
+        # Test sync standby is not disabled when updating dcs fails
+        self.ha.dcs.write_sync_state = Mock(return_value=False)
+        self.ha.run_cycle()
+        mock_set_sync.assert_not_called()
+
+        mock_set_sync.reset_mock()
+        # Test changing sync standby
+        self.ha.dcs.write_sync_state = Mock(return_value=True)
+        self.ha.dcs.get_cluster = Mock(return_value=get_cluster_initialized_with_leader(sync=('leader', 'other')))
+        # self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
+        self.p.pick_synchronous_standby = Mock(return_value=('other2', True))
+        self.ha.run_cycle()
+        self.ha.dcs.get_cluster.assert_called_once()
+        self.assertEquals(self.ha.dcs.write_sync_state.call_count, 2)
+
+        # Test updating sync standby key failed due to race
+        self.ha.dcs.write_sync_state = Mock(side_effect=[True, False])
+        self.ha.run_cycle()
+        self.assertEquals(self.ha.dcs.write_sync_state.call_count, 2)
+
+        # Test changing sync standby failed due to race
+        self.ha.dcs.write_sync_state = Mock(return_value=True)
+        self.ha.dcs.get_cluster = Mock(return_value=get_cluster_initialized_with_leader(sync=('somebodyelse', None)))
+        self.ha.run_cycle()
+        self.assertEquals(self.ha.dcs.write_sync_state.call_count, 1)
+
+    def test_sync_replication_become_master(self):
+        self.ha.is_synchronous_mode = true
+
+        mock_set_sync = self.p.set_synchronous_standby = Mock()
+        self.p.is_leader = false
+        self.p.set_role('replica')
+        self.ha.has_lock = true
+        mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=True)
+        self.p.name = 'leader'
+        self.ha.cluster = get_cluster_initialized_with_leader(sync=('other', None))
+
+        # When we just became master nobody is sync
+        self.assertEquals(self.ha.enforce_master_role('msg', 'promote msg'), 'promote msg')
+        mock_set_sync.assert_called_once_with(None)
+        mock_write_sync.assert_called_once_with('leader', None, index=0)
+
+        mock_set_sync.reset_mock()
+
+        # When we just became master nobody is sync
+        self.p.set_role('replica')
+        mock_write_sync.return_value = False
+        self.assertTrue(self.ha.enforce_master_role('msg', 'promote msg') != 'promote msg')
+        mock_set_sync.assert_not_called()
+
+    def test_unhealthy_sync_mode(self):
+        self.ha.is_synchronous_mode = true
+
+        self.p.is_leader = false
+        self.p.set_role('replica')
+        self.p.name = 'other'
+        self.ha.cluster = get_cluster_initialized_without_leader(sync=('leader', 'other2'))
+        mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=True)
+        mock_acquire = self.ha.acquire_lock = Mock(return_value=True)
+        mock_follow = self.p.follow = Mock()
+        mock_promote = self.p.promote = Mock()
+
+        # If we don't match the sync replica we are not allowed to acquire lock
+        self.ha.run_cycle()
+        mock_acquire.assert_not_called()
+        mock_follow.assert_called_once()
+        self.assertEquals(mock_follow.call_args[0][0], None)
+        mock_write_sync.assert_not_called()
+
+        mock_follow.reset_mock()
+        # If we do match we will try to promote
+        self.ha._is_healthiest_node = true
+
+        self.ha.cluster = get_cluster_initialized_without_leader(sync=('leader', 'other'))
+        self.ha.run_cycle()
+        mock_acquire.assert_called_once()
+        mock_follow.assert_not_called()
+        mock_promote.assert_called_once()
+        mock_write_sync.assert_called_once_with('other', None, index=0)
+
+    @patch('time.sleep')
+    def test_disable_sync_when_restarting(self, mock_sleep):
+        self.ha.is_synchronous_mode = true
+
+        self.p.name = 'other'
+        self.p.is_leader = false
+        self.p.set_role('replica')
+        mock_restart = self.p.restart = Mock(return_value=True)
+        self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
+        self.ha.touch_member = Mock(return_value=True)
+        self.ha.dcs.get_cluster = Mock(side_effect=[
+            get_cluster_initialized_with_leader(sync=('leader', syncstandby))
+            for syncstandby in ['other', None]])
+
+        self.ha.restart({})
+
+        mock_restart.assert_called_once()
+        mock_sleep.assert_called()
+
+        # Restart is still called when DCS connection fails
+        mock_restart.reset_mock()
+        self.ha.dcs.get_cluster = Mock(side_effect=DCSError("foo"))
+        self.ha.restart({})
+
+        mock_restart.assert_called_once()
+
+        # We don't try to fetch the cluster state when touch_member fails
+        mock_restart.reset_mock()
+        self.ha.dcs.get_cluster.reset_mock()
+        self.ha.touch_member = Mock(return_value=False)
+
+        self.ha.restart({})
+
+        mock_restart.assert_called_once()
+        self.ha.dcs.get_cluster.assert_not_called()
+
+    def test_effective_tags(self):
+        self.ha._disable_sync = True
+        self.assertEquals(self.ha.get_effective_tags(), {'foo': 'bar', 'nosync': True})
+        self.ha._disable_sync = False
+        self.assertEquals(self.ha.get_effective_tags(), {'foo': 'bar'})
+
+    def test_restore_cluster_config(self):
+        self.ha.cluster.config.data.clear()
+        self.ha.has_lock = true
+        self.ha.cluster.is_unlocked = false
+        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
+
+    def test_watch(self):
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.ha.watch(0)
+
+    def test_wakup(self):
+        self.ha.wakeup()
+
+    def test_leader_with_empty_directory(self):
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.ha.has_lock = true
+        self.p.data_directory_empty = true
+        self.assertEquals(self.ha.run_cycle(), 'released leader key voluntarily as data dir empty and currently leader')
+
+        # as has_lock is mocked out, we need to fake the leader key release
+        self.ha.has_lock = false
+        # will not say bootstrap from leader as replica can't self elect
+        self.assertEquals(self.ha.run_cycle(), "trying to bootstrap from replica 'other'")

tests/test_patroni.py

@@ -1,14 +1,15 @@
 import etcd
+import signal
 import sys
 import time
 import unittest
 
-from mock import Mock, patch
+from mock import Mock, PropertyMock, patch
 from patroni.api import RestApiServer
 from patroni.async_executor import AsyncExecutor
 from patroni.dcs.etcd import Client
 from patroni.exceptions import DCSError
-from patroni import Patroni, main as _main
+from patroni import Patroni, main as _main, patroni_main
 from six.moves import BaseHTTPServer
 from test_etcd import SleepException, etcd_read, etcd_write
 from test_postgresql import Postgresql, psycopg2_connect
@@ -26,6 +27,7 @@ class MockFrozenImporter(object):
 @patch.object(Postgresql, '_write_postgresql_conf', Mock())
 @patch.object(Postgresql, 'write_recovery_conf', Mock())
 @patch.object(Postgresql, 'is_running', Mock(return_value=True))
+@patch.object(Postgresql, 'call_nowait', Mock())
 @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
 @patch.object(AsyncExecutor, 'run', Mock())
 @patch.object(etcd.Client, 'write', etcd_write)
@@ -53,19 +55,55 @@ class TestPatroni(unittest.TestCase):
     @patch('time.sleep', Mock(side_effect=SleepException))
     @patch.object(etcd.Client, 'delete', Mock())
     @patch.object(Client, 'machines')
-    def test_patroni_main(self, mock_machines):
+    def test_patroni_patroni_main(self, mock_machines):
         with patch('subprocess.call', Mock(return_value=1)):
             sys.argv = ['patroni.py', 'postgres0.yml']
 
             mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
             with patch.object(Patroni, 'run', Mock(side_effect=SleepException)):
-                self.assertRaises(SleepException, _main)
+                self.assertRaises(SleepException, patroni_main)
             with patch.object(Patroni, 'run', Mock(side_effect=KeyboardInterrupt())):
                 with patch('patroni.ha.Ha.is_paused', Mock(return_value=True)):
-                    _main()
+                    patroni_main()
+
+    @patch('os.getpid')
+    @patch('subprocess.Popen', )
+    @patch('patroni.patroni_main', Mock())
+    def test_patroni_main(self, mock_popen, mock_getpid):
+        mock_getpid.return_value = 2
+        _main()
+
+        with patch('sys.frozen', Mock(return_value=True), create=True):
+            sys.argv = ['/patroni', 'pg_ctl_start', 'postgres', '-D', '/data', '--max_connections=100']
+            _main()
+
+        mock_getpid.return_value = 1
+
+        def mock_signal(signo, handler):
+            handler(signo, None)
+
+        with patch('signal.signal', mock_signal):
+            with patch('os.waitpid', Mock(side_effect=[(1, 0), (0, 0)])):
+                _main()
+            with patch('os.waitpid', Mock(side_effect=OSError)):
+                _main()
+
+        ref = {'passtochild': lambda signo, stack_frame: 0}
+
+        def mock_sighup(signo, handler):
+            if signo == signal.SIGHUP:
+                ref['passtochild'] = handler
+
+        def mock_wait():
+            ref['passtochild'](0, None)
+
+        mock_popen.return_value.wait = mock_wait
+        with patch('signal.signal', mock_sighup), patch('os.kill', Mock()):
+            self.assertIsNone(_main())
 
     @patch('patroni.config.Config.save_cache', Mock())
     @patch('patroni.config.Config.reload_local_configuration', Mock(return_value=True))
+    @patch.object(Postgresql, 'state', PropertyMock(return_value='running'))
     def test_run(self):
         self.p.sighup_handler()
         self.p.ha.dcs.watch = Mock(side_effect=SleepException)
@@ -105,3 +143,9 @@ class TestPatroni(unittest.TestCase):
         self.p.reload_config()
         self.p.get_tags = Mock(side_effect=Exception)
         self.p.reload_config()
+
+    def test_nosync(self):
+        self.p.tags['nosync'] = True
+        self.assertTrue(self.p.nosync)
+        self.p.tags['nosync'] = None
+        self.assertFalse(self.p.nosync)

tests/test_postgresql.py

@@ -6,12 +6,12 @@ import subprocess
 import unittest
 
 from mock import Mock, MagicMock, PropertyMock, patch, mock_open
-from patroni.dcs import Cluster, Leader, Member
+from patroni.dcs import Cluster, Leader, Member, SyncState
 from patroni.exceptions import PostgresException, PostgresConnectionException
-from patroni.postgresql import Postgresql
+from patroni.postgresql import Postgresql, STATE_REJECT, STATE_NO_RESPONSE
 from patroni.utils import RetryFailedError
 from six.moves import builtins
-from test_ha import false
+from threading import Thread
 
 
 class MockCursor(object):
@@ -19,6 +19,7 @@ class MockCursor(object):
     def __init__(self, connection):
         self.connection = connection
         self.closed = False
+        self.rowcount = 0
         self.results = []
 
     def execute(self, sql, *params):
@@ -28,7 +29,7 @@ class MockCursor(object):
             raise RetryFailedError('retry')
         elif sql.startswith('SELECT slot_name'):
             self.results = [('blabla',), ('foobar',)]
-        elif sql.startswith('SELECT pg_xlog_location_diff'):
+        elif sql.startswith('SELECT CASE WHEN pg_is_in_recovery()'):
             self.results = [(0,)]
         elif sql == 'SELECT pg_is_in_recovery()':
             self.results = [(False, )]
@@ -158,12 +159,13 @@ class TestPostgresql(unittest.TestCase):
     _PARAMETERS = {'wal_level': 'hot_standby', 'max_replication_slots': 5, 'f.oo': 'bar',
                    'search_path': 'public', 'hot_standby': 'on', 'max_wal_senders': 5,
                    'wal_keep_segments': 8, 'wal_log_hints': 'on', 'max_locks_per_transaction': 64,
-                   'max_worker_processes': 8, 'max_connections': 100, 'max_prepared_transactions': 0}
+                   'max_worker_processes': 8, 'max_connections': 100, 'max_prepared_transactions': 0,
+                   'track_commit_timestamp': 'off'}
 
     @patch('subprocess.call', Mock(return_value=0))
     @patch('psycopg2.connect', psycopg2_connect)
     @patch('os.rename', Mock())
-    @patch.object(Postgresql, 'get_major_version', Mock(return_value=9.4))
+    @patch.object(Postgresql, 'get_major_version', Mock(return_value=9.6))
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def setUp(self):
         self.data_dir = 'data/test0'
@@ -182,6 +184,7 @@ class TestPostgresql(unittest.TestCase):
                                            'on_reload': 'true'
                                            },
                              'restore': 'true'})
+        self.p._callback_executor = Mock()
         self.leadermem = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres'})
         self.leader = Leader(-1, 28, self.leadermem)
         self.other = Member(0, 'test-1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres',
@@ -203,19 +206,64 @@ class TestPostgresql(unittest.TestCase):
     def test_delete_trigger_file(self):
         self.p.delete_trigger_file()
 
+    @patch('subprocess.Popen')
+    @patch.object(Postgresql, 'wait_for_startup')
+    @patch.object(Postgresql, 'wait_for_port_open')
     @patch.object(Postgresql, 'is_running')
-    def test_start(self, mock_is_running):
+    def test_start(self, mock_is_running, mock_wait_for_port_open, mock_wait_for_startup, mock_popen):
         mock_is_running.return_value = True
+        mock_wait_for_port_open.return_value = True
+        mock_wait_for_startup.return_value = False
+        mock_popen.stdout.readline.return_value = '123'
         self.assertTrue(self.p.start())
         mock_is_running.return_value = False
         open(os.path.join(self.data_dir, 'postmaster.pid'), 'w').close()
         pg_conf = os.path.join(self.data_dir, 'postgresql.conf')
         open(pg_conf, 'w').close()
-        self.assertTrue(self.p.start())
+        self.assertFalse(self.p.start())
         with open(pg_conf) as f:
             lines = f.readlines()
             self.assertTrue("f.oo = 'bar'\n" in lines)
 
+        mock_wait_for_startup.return_value = None
+        self.assertFalse(self.p.start(10))
+        self.assertIsNone(self.p.start())
+
+        mock_wait_for_port_open.return_value = False
+        self.assertFalse(self.p.start())
+
+    @patch.object(Postgresql, 'pg_isready')
+    @patch.object(Postgresql, 'read_pid_file')
+    @patch.object(Postgresql, 'is_pid_running')
+    @patch('patroni.postgresql.polling_loop', Mock(return_value=range(1)))
+    def test_wait_for_port_open(self, mock_is_pid_running, mock_read_pid_file, mock_pg_isready):
+        mock_is_pid_running.return_value = False
+        mock_pg_isready.return_value = STATE_NO_RESPONSE
+
+        # No pid file and postmaster death
+        mock_read_pid_file.return_value = {}
+        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
+
+        mock_is_pid_running.return_value = True
+
+        # timeout
+        mock_read_pid_file.return_value = {'pid', 1}
+        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
+
+        # Garbage pid
+        mock_read_pid_file.return_value = {'pid': 'garbage', 'start_time': '101', 'data_dir': '',
+                                           'socket_dir': '', 'port': '', 'listen_addr': ''}
+        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
+
+        # Not ready
+        mock_read_pid_file.return_value = {'pid': '42', 'start_time': '101', 'data_dir': '',
+                                           'socket_dir': '', 'port': '', 'listen_addr': ''}
+        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
+
+        # pg_isready failure
+        mock_pg_isready.return_value = 'garbage'
+        self.assertTrue(self.p.wait_for_port_open(42, 100., 1))
+
     @patch.object(Postgresql, 'is_running')
     def test_stop(self, mock_is_running):
         mock_is_running.return_value = True
@@ -225,12 +273,13 @@ class TestPostgresql(unittest.TestCase):
             self.assertTrue(self.p.stop())
 
     def test_restart(self):
-        self.p.start = false
+        self.p.start = Mock(return_value=False)
         self.assertFalse(self.p.restart())
         self.assertEquals(self.p.state, 'restart failed (restarting)')
 
     @patch.object(builtins, 'open', MagicMock())
     def test_write_pgpass(self):
+        self.p.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo'})
         self.p.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo', 'password': 'bar'})
 
     def test_checkpoint(self):
@@ -290,6 +339,7 @@ class TestPostgresql(unittest.TestCase):
         self.assertFalse(self.p.can_rewind)
 
     @patch('time.sleep', Mock())
+    @patch.object(Postgresql, 'remove_data_directory', Mock(return_value=True))
     def test_create_replica(self):
         self.p.delete_trigger_file = Mock(side_effect=OSError)
         with patch('subprocess.call', Mock(side_effect=[1, 0])):
@@ -307,10 +357,13 @@ class TestPostgresql(unittest.TestCase):
         with patch('subprocess.call', Mock(side_effect=Exception("foo"))):
             self.assertEquals(self.p.create_replica(self.leader), 1)
 
+        with patch('subprocess.call', Mock(return_value=1)):
+            self.assertEquals(self.p.create_replica(self.leader), 1)
+
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def test_sync_replication_slots(self):
         self.p.start()
-        cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None)
+        cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None, None)
         with mock.patch('patroni.postgresql.Postgresql._query', Mock(side_effect=psycopg2.OperationalError)):
             self.p.sync_replication_slots(cluster)
         self.p.sync_replication_slots(cluster)
@@ -337,8 +390,11 @@ class TestPostgresql(unittest.TestCase):
         self.assertRaises(PostgresConnectionException, self.p.query, 'RetryFailedError')
         self.assertRaises(psycopg2.OperationalError, self.p.query, 'blabla')
 
+    @patch.object(Postgresql, 'pg_isready', Mock(return_value=STATE_REJECT))
     def test_is_leader(self):
         self.assertTrue(self.p.is_leader())
+        with patch.object(Postgresql, '_query', Mock(side_effect=RetryFailedError(''))):
+            self.assertRaises(PostgresConnectionException, self.p.is_leader)
 
     def test_reload(self):
         self.assertTrue(self.p.reload())
@@ -357,6 +413,7 @@ class TestPostgresql(unittest.TestCase):
 
     def test_last_operation(self):
         self.assertEquals(self.p.last_operation(), '0')
+        Thread(target=self.p.last_operation).start()
 
     @patch('os.path.isfile', Mock(return_value=True))
     @patch('os.kill', Mock(side_effect=Exception))
@@ -367,9 +424,9 @@ class TestPostgresql(unittest.TestCase):
     def test_is_running(self):
         self.assertFalse(self.p.is_running())
 
-    @patch('subprocess.Popen', Mock(side_effect=OSError))
+    @patch('shlex.split', Mock(side_effect=OSError))
     def test_call_nowait(self):
-        self.assertFalse(self.p.call_nowait('on_start'))
+        self.assertIsNone(self.p.call_nowait('on_start'))
 
     def test_non_existing_callback(self):
         self.assertFalse(self.p.call_nowait('foobar'))
@@ -380,9 +437,6 @@ class TestPostgresql(unittest.TestCase):
         self.p.query = Mock(side_effect=psycopg2.OperationalError("not supported"))
         self.assertTrue(self.p.stop())
 
-    def test_check_replication_lag(self):
-        self.assertTrue(self.p.check_replication_lag(0))
-
     @patch('os.rename', Mock())
     @patch('os.path.isdir', Mock(return_value=True))
     def test_move_data_directory(self):
@@ -395,15 +449,35 @@ class TestPostgresql(unittest.TestCase):
         with patch('subprocess.call', Mock(return_value=1)):
             self.assertRaises(PostgresException, self.p.bootstrap, {})
 
+        with patch.object(Postgresql, 'run_bootstrap_post_init', Mock(return_value=False)):
+            self.assertRaises(PostgresException, self.p.bootstrap, {})
+
         self.p.bootstrap({'users': {'replicator': {'password': 'rep-pass', 'options': ['replication']}},
                           'pg_hba': ['host replication replicator 127.0.0.1/32 md5',
                                      'hostssl all all 0.0.0.0/0 md5',
-                                     'host all all 0.0.0.0/0 md5']})
+                                     'host all all 0.0.0.0/0 md5'],
+                          'post_init': '/bin/false'})
         with open(os.path.join(self.data_dir, 'pg_hba.conf')) as f:
             lines = f.readlines()
             assert 'host replication replicator 127.0.0.1/32 md5\n' in lines
             assert 'host all all 0.0.0.0/0 md5\n' in lines
 
+    def test_run_bootstrap_post_init(self):
+        with patch('subprocess.call', Mock(return_value=1)):
+            self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+
+        with patch('subprocess.call', Mock(side_effect=OSError)):
+            self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+
+        with patch('subprocess.call', Mock(return_value=0)) as mock_method:
+            self.p._superuser.pop('username')
+            self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+
+        mock_method.assert_called()
+        args, kwargs = mock_method.call_args
+        assert 'PGPASSFILE' in kwargs['env'].keys()
+        self.assertEquals(args[0], ['/bin/false', 'postgres://localhost:5432/postgres'])
+
     @patch('patroni.postgresql.Postgresql.create_replica', Mock(return_value=0))
     def test_clone(self):
         self.p.clone(self.leader)
@@ -548,3 +622,151 @@ class TestPostgresql(unittest.TestCase):
             self.assertEqual(self.p.postmaster_start_time(), 'foo')
         with patch.object(MockCursor, "execute", side_effect=psycopg2.Error):
             self.assertIsNone(self.p.postmaster_start_time())
+
+    def test_check_for_startup(self):
+        with patch('subprocess.call', return_value=0):
+            self.p._state = 'starting'
+            self.assertFalse(self.p.check_for_startup())
+            self.assertEquals(self.p.state, 'running')
+
+        with patch('subprocess.call', return_value=1):
+            self.p._state = 'starting'
+            self.assertTrue(self.p.check_for_startup())
+            self.assertEquals(self.p.state, 'starting')
+
+        with patch('subprocess.call', return_value=2):
+            self.p._state = 'starting'
+            self.assertFalse(self.p.check_for_startup())
+            self.assertEquals(self.p.state, 'start failed')
+
+        with patch('subprocess.call', return_value=0):
+            self.p._state = 'running'
+            self.assertFalse(self.p.check_for_startup())
+            self.assertEquals(self.p.state, 'running')
+
+        with patch('subprocess.call', return_value=127):
+            self.p._state = 'running'
+            self.assertFalse(self.p.check_for_startup())
+            self.assertEquals(self.p.state, 'running')
+
+            self.p._state = 'starting'
+            self.assertFalse(self.p.check_for_startup())
+            self.assertEquals(self.p.state, 'running')
+
+    def test_wait_for_startup(self):
+        state = {'sleeps': 0, 'num_rejects': 0, 'final_return': 0}
+
+        def increment_sleeps(*args):
+            print("Sleep")
+            state['sleeps'] += 1
+
+        def isready_return(*args):
+            ret = 1 if state['sleeps'] < state['num_rejects'] else state['final_return']
+            print("Isready {0} {1}".format(ret, state))
+            return ret
+
+        def time_in_state(*args):
+            return state['sleeps']
+
+        with patch('subprocess.call', side_effect=isready_return):
+            with patch('time.sleep', side_effect=increment_sleeps):
+                self.p.time_in_state = Mock(side_effect=time_in_state)
+
+                self.p._state = 'stopped'
+                self.assertTrue(self.p.wait_for_startup())
+                self.assertEquals(state['sleeps'], 0)
+
+                self.p._state = 'starting'
+                state['num_rejects'] = 5
+                self.assertTrue(self.p.wait_for_startup())
+                self.assertEquals(state['sleeps'], 5)
+
+                self.p._state = 'starting'
+                state['sleeps'] = 0
+                state['final_return'] = 2
+                self.assertFalse(self.p.wait_for_startup())
+
+                self.p._state = 'starting'
+                state['sleeps'] = 0
+                state['final_return'] = 0
+                self.assertFalse(self.p.wait_for_startup(timeout=2))
+                self.assertEquals(state['sleeps'], 3)
+
+    def test_read_pid_file(self):
+        pidfile = os.path.join(self.data_dir, 'postmaster.pid')
+        if os.path.exists(pidfile):
+            os.remove(pidfile)
+        self.assertEquals(self.p.read_pid_file(), {})
+
+    @patch('os.kill')
+    def test_is_pid_running(self, mock_kill):
+        mock_kill.return_value = True
+        self.assertTrue(self.p.is_pid_running(-100))
+        self.assertFalse(self.p.is_pid_running(0))
+        self.assertFalse(self.p.is_pid_running(None))
+
+    def test_pick_sync_standby(self):
+        cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None,
+                          SyncState(0, self.me.name, self.leadermem.name))
+
+        with patch.object(Postgresql, "query", return_value=[
+                    (self.leadermem.name, 'streaming', 'sync'),
+                    (self.me.name, 'streaming', 'async'),
+                    (self.other.name, 'streaming', 'async'),
+                ]):
+            self.assertEquals(self.p.pick_synchronous_standby(cluster), (self.leadermem.name, True))
+
+        with patch.object(Postgresql, "query", return_value=[
+                    (self.me.name, 'streaming', 'async'),
+                    (self.leadermem.name, 'streaming', 'potential'),
+                    (self.other.name, 'streaming', 'async'),
+                ]):
+            self.assertEquals(self.p.pick_synchronous_standby(cluster), (self.leadermem.name, False))
+
+        with patch.object(Postgresql, "query", return_value=[
+                    (self.me.name, 'streaming', 'async'),
+                    (self.other.name, 'streaming', 'async'),
+                ]):
+            self.assertEquals(self.p.pick_synchronous_standby(cluster), (self.me.name, False))
+
+        with patch.object(Postgresql, "query", return_value=[
+                    ('missing', 'streaming', 'sync'),
+                    (self.me.name, 'streaming', 'async'),
+                    (self.other.name, 'streaming', 'async'),
+                ]):
+            self.assertEquals(self.p.pick_synchronous_standby(cluster), (self.me.name, False))
+
+        with patch.object(Postgresql, "query", return_value=[]):
+            self.assertEquals(self.p.pick_synchronous_standby(cluster), (None, False))
+
+    def test_set_sync_standby(self):
+        def value_in_conf():
+            with open(os.path.join(self.data_dir, 'postgresql.conf')) as f:
+                for line in f:
+                    if line.startswith('synchronous_standby_names'):
+                        return line.strip()
+
+        mock_reload = self.p.reload = Mock()
+        self.p.set_synchronous_standby('n1')
+        self.assertEquals(value_in_conf(), "synchronous_standby_names = 'n1'")
+        mock_reload.assert_called()
+
+        mock_reload.reset_mock()
+        self.p.set_synchronous_standby('n1')
+        mock_reload.assert_not_called()
+        self.assertEquals(value_in_conf(), "synchronous_standby_names = 'n1'")
+
+        self.p.set_synchronous_standby('n2')
+        mock_reload.assert_called()
+        self.assertEquals(value_in_conf(), "synchronous_standby_names = 'n2'")
+
+        mock_reload.reset_mock()
+        self.p.set_synchronous_standby(None)
+        mock_reload.assert_called()
+        self.assertEquals(value_in_conf(), None)
+
+    def test_get_server_parameters(self):
+        config = {'synchronous_mode': True, 'parameters': {'wal_level': 'hot_standby'}, 'listen': '0'}
+        self.p.get_server_parameters(config)
+        self.p.set_synchronous_standby('foo')
+        self.p.get_server_parameters(config)

tests/test_utils.py

@@ -2,25 +2,13 @@ import unittest
 
 from mock import Mock, patch
 from patroni.exceptions import PatroniException
-from patroni.utils import reap_children, Retry, RetryFailedError, sigchld_handler, sleep
-
-
-def time_sleep(_):
-    sigchld_handler(None, None)
+from patroni.utils import Retry, RetryFailedError, polling_loop
 
 
 class TestUtils(unittest.TestCase):
 
-    @patch('time.sleep', Mock())
-    def test_reap_children(self):
-        self.assertIsNone(reap_children())
-        with patch('os.waitpid', Mock(return_value=(0, 0))):
-            sigchld_handler(None, None)
-            self.assertIsNone(reap_children())
-
-    @patch('time.sleep', time_sleep)
-    def test_sleep(self):
-        self.assertIsNone(sleep(0.01))
+    def test_polling_loop(self):
+        self.assertEquals(list(polling_loop(0.001, interval=0.001)), [0])
 
 
 @patch('time.sleep', Mock())

tests/test_wale_restore.py

@@ -2,8 +2,9 @@ import psycopg2
 import subprocess
 import unittest
 
-from mock import MagicMock, patch, PropertyMock
-from patroni.scripts.wale_restore import WALERestore, main as _main
+from mock import Mock, MagicMock, patch, mock_open
+from patroni.scripts.wale_restore import WALERestore, main as _main, get_major_version
+from six.moves import builtins
 
 
 wale_output = b'name last_modified expanded_size_bytes wal_segment_backup_start ' +\
@@ -12,51 +13,89 @@ wale_output = b'name last_modified expanded_size_bytes wal_segment_backup_start
               b'00000001000000000000007F 00000040 00000001000000000000007F 00000240\n'
 
 
-@patch('os.access', MagicMock(return_value=True))
-@patch('os.makedirs', MagicMock(return_value=True))
-@patch('os.path.exists', MagicMock(return_value=True))
-@patch('os.path.isdir', MagicMock(return_value=True))
-@patch('psycopg2.extensions.cursor', MagicMock(autospec=True))
-@patch('psycopg2.extensions.connection', MagicMock(autospec=True))
+@patch('os.access', Mock(return_value=True))
+@patch('os.makedirs', Mock(return_value=True))
+@patch('os.path.exists', Mock(return_value=True))
+@patch('os.path.isdir', Mock(return_value=True))
+@patch('psycopg2.extensions.cursor', Mock(autospec=True))
+@patch('psycopg2.extensions.connection', Mock(autospec=True))
 @patch('psycopg2.connect', MagicMock(autospec=True))
-@patch('subprocess.check_output', MagicMock(return_value=wale_output))
+@patch('subprocess.check_output', Mock(return_value=wale_output))
 class TestWALERestore(unittest.TestCase):
 
     def setUp(self):
-        self.wale_restore = WALERestore("batman", "/data", "host=batman port=5432 user=batman", "/etc", 100, 100, 1, 0)
+        self.wale_restore = WALERestore("batman", "/data", "host=batman port=5432 user=batman",
+                                        "/etc", 100, 100, 1, 0, 1)
 
     def test_should_use_s3_to_create_replica(self):
-        with patch('psycopg2.connect', MagicMock(side_effect=psycopg2.Error("foo"))):
-            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output', MagicMock(side_effect=subprocess.CalledProcessError(1, "cmd", "foo"))):
-            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output', MagicMock(return_value=wale_output.split(b'\n')[0])):
-            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output',
-                   MagicMock(return_value=wale_output.replace(b' wal_segment_offset_backup_stop', b''))):
-            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output',
-                   MagicMock(return_value=wale_output.replace(b'expanded_size_bytes', b'expanded_size_foo'))):
-            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-
-        self.wale_restore.should_use_s3_to_create_replica()
-        self.wale_restore.no_master = 1
         self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
 
+        with patch('psycopg2.connect', Mock(side_effect=psycopg2.Error("foo"))):
+            save_no_master = self.wale_restore.no_master
+            save_master_connection = self.wale_restore.master_connection
+
+            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
+            self.wale_restore.no_master = 1
+            self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())  # this would do 2 retries 1 sec each
+            self.wale_restore.master_connection = ''
+            self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
+
+            self.wale_restore.no_master = save_no_master
+            self.wale_restore.master_connection = save_master_connection
+
+        with patch('subprocess.check_output', Mock(side_effect=subprocess.CalledProcessError(1, "cmd", "foo"))):
+            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
+        with patch('subprocess.check_output', Mock(return_value=wale_output.split(b'\n')[0])):
+            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
+        with patch('subprocess.check_output',
+                   Mock(return_value=wale_output.replace(b' wal_segment_offset_backup_stop', b''))):
+            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
+        with patch('subprocess.check_output',
+                   Mock(return_value=wale_output.replace(b'expanded_size_bytes', b'expanded_size_foo'))):
+            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
+
     def test_create_replica_with_s3(self):
-        with patch('subprocess.call', MagicMock(return_value=0)):
+        with patch('subprocess.call', Mock(return_value=0)):
             self.assertEqual(self.wale_restore.create_replica_with_s3(), 0)
-        with patch('subprocess.call', MagicMock(side_effect=Exception("foo"))):
+            with patch.object(self.wale_restore, 'fix_subdirectory_path_if_broken', Mock(return_value=False)):
+                self.assertEqual(self.wale_restore.create_replica_with_s3(), 2)
+
+        with patch('subprocess.call', Mock(side_effect=Exception("foo"))):
             self.assertEqual(self.wale_restore.create_replica_with_s3(), 1)
 
     def test_run(self):
-        with patch.object(self.wale_restore, 'init_error', PropertyMock(return_value=True)):
-            self.assertEqual(self.wale_restore.run(), 2)
-        with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', MagicMock(return_value=True)):
-            with patch.object(self.wale_restore, 'create_replica_with_s3', MagicMock(return_value=0)):
+        self.wale_restore.init_error = True
+        self.assertEqual(self.wale_restore.run(), 2)  # this would do 2 retries 1 sec each
+        self.wale_restore.init_error = False
+        with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(return_value=True)):
+            with patch.object(self.wale_restore, 'create_replica_with_s3', Mock(return_value=0)):
                 self.assertEqual(self.wale_restore.run(), 0)
+            with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(return_value=None)):
+                self.assertEqual(self.wale_restore.run(), 1)
+            with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(side_effect=Exception)):
+                self.assertEqual(self.wale_restore.run(), 2)
 
-    @patch('sys.exit', MagicMock())
-    @patch.object(WALERestore, 'run', MagicMock(return_value=0))
+    @patch('sys.exit', Mock())
     def test_main(self):
-        self.assertEqual(_main(), None)
+        with patch.object(WALERestore, 'run', Mock(return_value=0)):
+            self.assertEqual(_main(), 0)
+        with patch.object(WALERestore, 'run', Mock(return_value=1)):
+            self.assertEqual(_main(), 1)
+
+    @patch('os.path.isfile', Mock(return_value=True))
+    def test_get_major_version(self):
+        with patch.object(builtins, 'open', mock_open(read_data='9.4')):
+            self.assertEqual(get_major_version("data"), 9.4)
+        with patch.object(builtins, 'open', side_effect=OSError):
+            self.assertEqual(get_major_version("data"), 0.0)
+
+    @patch('os.path.islink', Mock(return_value=True))
+    @patch('os.readlink', Mock(return_value="foo"))
+    @patch('os.remove', Mock())
+    @patch('os.mkdir', Mock())
+    def test_fix_subdirectory_path_if_broken(self):
+        with patch('os.path.exists', Mock(return_value=False)):  # overriding the class-wide mock
+            self.assertTrue(self.wale_restore.fix_subdirectory_path_if_broken("data1"))
+            for fn in ('os.remove', 'os.mkdir'):
+                with patch(fn, side_effect=OSError):
+                    self.assertFalse(self.wale_restore.fix_subdirectory_path_if_broken("data3"))

tests/test_zookeeper.py

@@ -50,7 +50,7 @@ class MockKazooClient(Mock):
         if path.startswith('/no_node'):
             raise NoNodeError
         elif path in ['/service/bla/', '/service/test/']:
-            return ['initialize', 'leader', 'members', 'optime', 'failover']
+            return ['initialize', 'leader', 'members', 'optime', 'failover', 'sync']
         return ['foo', 'bar', 'buzz']
 
     def create(self, path, value=b"", acl=None, ephemeral=False, sequence=False, makepath=False):
@@ -78,7 +78,7 @@ class MockKazooClient(Mock):
             raise Exception
         if path == '/service/test/members/bar' and value == b'retry':
             return
-        if path in ('/service/test/failover', '/service/test/config'):
+        if path in ('/service/test/failover', '/service/test/config', '/service/test/sync'):
             if value == b'Exception':
                 raise Exception
             elif value == b'ok':
@@ -203,11 +203,17 @@ class TestZooKeeper(unittest.TestCase):
         self.assertTrue(self.zk.delete_cluster())
 
     def test_watch(self):
-        self.zk.watch(0)
-        self.zk.event.isSet = lambda: True
-        self.zk.watch(0)
+        self.zk.watch(None, 0)
+        self.zk.event.isSet = Mock(return_value=True)
+        self.zk.watch(None, 0)
 
     def test__kazoo_connect(self):
         self.zk._client._retry.deadline = 1
         self.zk._orig_kazoo_connect = Mock(return_value=(0, 0))
         self.zk._kazoo_connect(None, None)
+
+    def test_sync_state(self):
+        self.zk.set_sync_state_value('')
+        self.zk.set_sync_state_value('ok')
+        self.zk.set_sync_state_value('Exception')
+        self.zk.delete_sync_state()

tox.ini

@@ -1,8 +1,2 @@
 [flake8]
 max-line-length=120
-
-[testenv:py27]
-deps = -rrequirements-py2.txt
-
-[testenv:py33]
-deps = -rrequirements-py3.txt