Small bugfixes (#499)

* Short after promote synchronous replication was disabled even is synchronous_mode_strict is set
* Create empty pg_ident.conf if it is missing after restoring from backup
* Bump version

.travis.yml

@@ -65,6 +65,7 @@ install:
                 echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \
                     | nc -l 8181 &> /dev/null
             done&
+            ZK_PID=$!
         }
 
         attempt_num=1
@@ -110,5 +111,8 @@ script:
 
     set +e
 after_success:
+  # before_cache is executed earlier than after_success, so we need to restore one of virtualenv directories
+  - fpv=$(basename $(readlink $HOME/virtualenv/python3.5)) && mv $HOME/mycache/${fpv} $HOME/virtualenv/${fpv}
   - coveralls
   - if [[ $TEST_SUITE != "behave" ]]; then python-codacy-coverage -r coverage.xml; fi
+  - if [[ $DCS == "exhibitor" ]]; then ~/mycache/zookeeper-${ZKVERSION}/bin/zkServer.sh stop; kill -9 $ZK_PID; fi

Dockerfile

@@ -1,33 +1,22 @@
 ## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine
 ## It has all the necessary components to play/debug with a single node appliance, running etcd
-FROM ubuntu:16.04
+FROM postgres:9.6
 MAINTAINER Alexander Kukushkin <[email protected]>
 
-RUN echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/01norecommend \
-    && echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/01norecommend
-
-ENV PGVERSION 9.6
-ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:$PATH
-RUN apt-get update -y \
+RUN export DEBIAN_FRONTEND=noninteractive \
+    && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
+    && apt-get update -y \
     && apt-get upgrade -y \
-    && apt-get install -y curl jq haproxy zookeeper python-psycopg2 python-yaml python-requests \
-        python-six python-click python-dateutil python-tzlocal python-urllib3 python-dnspython \
-        python-pip python-setuptools python-kazoo python-prettytable python-wheel python \
-
-    && export DISTRIB_CODENAME=$(sed -n 's/DISTRIB_CODENAME=//p' /etc/lsb-release) \
-    && echo "deb http://apt.postgresql.org/pub/repos/apt/ ${DISTRIB_CODENAME}-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
-    && echo "deb-src http://apt.postgresql.org/pub/repos/apt/ ${DISTRIB_CODENAME}-pgdg main" >> /etc/apt/sources.list.d/pgdg.list \
-    && curl -s -o - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
+    && apt-get install -y curl jq haproxy python-psycopg2 python-yaml python-requests python-six python-pysocks \
+        python-dateutil python-pip python-setuptools python-prettytable python-wheel python-psutil python locales \
 
     ## Make sure we have a en_US.UTF-8 locale available
     && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
 
-    && apt-get update -y \
-    && apt-get install -y postgresql-contrib-${PGVERSION} \
+    && pip install 'python-etcd>=0.4.3,<0.5' click tzlocal cdiff \
 
-    # Remove the default cluster, which Debian stupidly starts right after installation of the packages
-    && pg_dropcluster --stop ${PGVERSION} main \
-    && pip install python-etcd==0.4.3 python-consul==0.6.1 --upgrade \
+    && mkdir -p /home/postgres \
+    && chown postgres:postgres /home/postgres \
 
     # Clean up
     && apt-get remove -y python-pip python-setuptools \
@@ -35,7 +24,7 @@ RUN apt-get update -y \
     && apt-get clean -y \
     && rm -rf /var/lib/apt/lists/* /root/.cache
 
-ENV ETCDVERSION 3.0.15
+ENV ETCDVERSION 3.2.3
 RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
     | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl
 
@@ -50,10 +39,10 @@ RUN ln -s /patronictl.py /usr/local/bin/patronictl
 
 ### Setting up a simple script that will serve as an entrypoint
 RUN mkdir /data/ && touch /pgpass /patroni.yml \
-    && chown postgres:postgres -R /patroni/ /data/ /pgpass /patroni.yml /etc/haproxy /var/run/ /var/lib/ /var/log/ \
-    && echo 1 > /etc/zookeeper/conf/myid
+    && chown postgres:postgres -R /patroni/ /data/ /pgpass /patroni.yml /etc/haproxy /var/run/ /var/lib/ /var/log/
 
 EXPOSE 2379 5432 8008
 
+ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
 ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
 USER postgres

README.rst

@@ -29,6 +29,7 @@ For an example of a Docker-based deployment with Patroni, see `Spilo <https://gi
 
 For additional background info, see:
 
+* `Elephants on Automatic: HA Clustered PostgreSQL with Helm <https://www.youtube.com/watch?v=CftcVhFMGSY>`_, talk by Josh Berkus and Oleksii Kliukin at KubeCon Berlin 2017 
 * `PostgreSQL HA with Kubernetes and Patroni <https://www.youtube.com/watch?v=iruaCgeG7qs>`__, talk by Josh Berkus at KubeCon 2016 (video)
 * `Feb. 2016 Zalando Tech blog post <https://tech.zalando.de/blog/zalandos-patroni-a-template-for-high-availability-postgresql/>`__
 

docker-compose.yml

@@ -50,8 +50,8 @@ haproxy:
     links:
         - patroni_etcd:patroni_etcd
     ports:
-        - "5000"
-        - "5001"
+        - "5000:5000"
+        - "5001:5001"
     environment:
         PATRONI_ETCD_HOST: patroni_etcd:2379
         PATRONI_SCOPE: testcluster

docker/entrypoint.sh

@@ -22,7 +22,7 @@ __EOF__
 
 DOCKER_IP=$(hostname --ip-address)
 PATRONI_SCOPE=${PATRONI_SCOPE:-batman}
-ETCD_ARGS="--data-dir /tmp/etcd.data -advertise-client-urls=http://${DOCKER_IP}:2379 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://0.0.0.0:2380"
+ETCD_ARGS="--data-dir /tmp/etcd.data -advertise-client-urls=http://${DOCKER_IP}:2379 -listen-client-urls=http://0.0.0.0:2379"
 
 optspec=":vh-:"
 while getopts "$optspec" optchar; do

docs/ENVIRONMENT.rst

@@ -46,6 +46,7 @@ PostgreSQL
 -  **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
 -  **PATRONI\_POSTGRESQL\_CONNECT\_ADDRESS**: IP address + port through which Postgres is accessible from other nodes and applications.
 -  **PATRONI\_POSTGRESQL\_DATA\_DIR**: The location of the Postgres data directory, either existing or to be initialized by Patroni.
+-  **PATRONI\_POSTGRESQL\_CONFIG\_DIR**: The location of the Postgres configuration directory, defaults to the data directory. Must be writable by Patroni.
 -  **PATRONI\_POSTGRESQL\_BIN_DIR**: Path to PostgreSQL binaries. (pg_ctl, pg_rewind, pg_basebackup, postgres) The  default value is an empty string meaning that PATH environment variable will be used to find the executables.
 -  **PATRONI\_POSTGRESQL\_PGPASS**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni.
 -  **PATRONI\_REPLICATION\_USERNAME**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication

docs/SETTINGS.rst

@@ -15,6 +15,7 @@ Bootstrap configuration
 -  **dcs**: This section will be written into `/<namespace>/<scope>/config` of a given configuration store after initializing of new cluster. This is the global configuration for the cluster. If you want to change some parameters for all cluster nodes - just do it in DCS (or via Patroni API) and all nodes will apply this configuration.
     -  **loop\_wait**: the number of seconds the loop will sleep. Default value: 10
     -  **ttl**: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process. Default value: 30
+    -  **retry\_timeout**: timeout for DCS and PostgreSQL operation retries. DCS or network issues shorter than this will not cause Patroni to demote the leader. Default value: 10
     -  **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election.
     -  **master\_start\_timeout**: the amount of time a master is allowed to recover from failures before failover is triggered. Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Best worst case failover time for master failure is: loop\_wait + master\_start\_timeout + loop\_wait, unless master\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff.
     -  **synchronous\_mode**: turns on synchronous replication mode. In this mode a replica will be chosen as synchronous and only the latest leader and synchronous replica are able to participate in leader election. Synchronous mode makes sure that succesfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See `replication modes documentation <https://github.com/zalando/patroni/blob/master/docs/replication_modes.rst>`__ for details.
@@ -23,6 +24,10 @@ Bootstrap configuration
         -  **use\_slots**: whether or not to use replication_slots. Must be False for PostgreSQL 9.3. You should comment out max_replication_slots before it becomes ineligible for leader status.
         -  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower. 
         -  **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work.
+-  **method**: custom script to use for bootstrpapping this cluster.
+   See :ref:`custom bootstrap methods documentation <custom_bootstrap>` for details.
+   When ``initdb`` is specified revert to the default ``initdb`` command. ``initdb`` is also triggered when no ``method``
+   parameter is present in the configuration file.
 -  **initdb**: List options to be passed on to initdb.
         -  **- data-checksums**: Must be enabled when pg_rewind is needed on 9.3.
         -  **- encoding: UTF8**: default encoding for new databases.
@@ -36,7 +41,7 @@ Bootstrap configuration
         -  **options**: list of options for CREATE USER statement
             -  **- createrole**
             -  **- createdb**
--  **post_init**: An additional script that will be executed after initializing the cluster. The script receives a connection string URL (with the cluster superuser as a user name). The PGPASSFILE variable is set to the location of pgpass file.
+-  **post\_bootstrap** or **post\_init**: An additional script that will be executed after initializing the cluster. The script receives a connection string URL (with the cluster superuser as a user name). The PGPASSFILE variable is set to the location of pgpass file.
 
 Consul
 ------
@@ -80,14 +85,21 @@ PostgreSQL
         -  **on\_start**: run this script when the cluster starts.
         -  **on\_stop**: run this script when the cluster stops.
 -  **connect\_address**: IP address + port through which Postgres is accessible from other nodes and applications.
--  **create\_replica\_methods**: an ordered list of the create methods for turning a Patroni node into a new replica. "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured as its own config item.
+-  **create\_replica\_method**: an ordered list of the create methods for turning a Patroni node into a new replica.
+   "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured as its
+   own config item. See :ref:`custom replica creation methods documentation <custom_replica_creation>` for further explanation.
 -  **data\_dir**: The location of the Postgres data directory, either existing or to be initialized by Patroni.
+-  **config\_dir**: The location of the Postgres configuration directory, defaults to the data directory. Must be writable by Patroni.
 -  **bin\_dir**: Path to PostgreSQL binaries. (pg_ctl, pg_rewind, pg_basebackup, postgres) The  default value is an empty string meaning that PATH environment variable will be used to find the executables.
 -  **listen**: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
+-  **use\_unix\_socket**: specifies that Patroni should prefer to use unix sockets to connect to the cluster. Default value is ``false``. If ``unix_socket_directories`` is definded, Patroni will use first suitable value from it to connect to the cluster and fallback to tcp if nothing is suitable. If ``unix_socket_directories`` is not specified in ``postgresql.parameters``, Patroni will assume that default value should be used and omit ``host`` from connection parameters.
 -  **pgpass**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup, the post_init script and under some other circumstances. The location must be writable by Patroni.
 -  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower.
--  **custom_conf** : path to an optional custom ``postgresql.conf`` file, that will be used in place of ``postgresql.base.conf``. The file must exist on all cluster nodes, be readable by PostgreSQL and will be included from its location on the real ``postgresql.conf``. Note that Patroni will not monitor this file for changes, nor backup it. However, its settings can still be overriden by Patroni's own configuration facilities - see `dynamic configuration <https://github.com/zalando/patroni/blob/master/docs/dynamic_configuration.rst>`__ for details.
+-  **custom\_conf** : path to an optional custom ``postgresql.conf`` file, that will be used in place of ``postgresql.base.conf``. The file must exist on all cluster nodes, be readable by PostgreSQL and will be included from its location on the real ``postgresql.conf``. Note that Patroni will not monitor this file for changes, nor backup it. However, its settings can still be overriden by Patroni's own configuration facilities - see `dynamic configuration <https://github.com/zalando/patroni/blob/master/docs/dynamic_configuration.rst>`__ for details.
 -  **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work.
+-  **pg\_hba**: list of lines that Patroni will use to generate ``pg_hba.conf``. This parameter has higher priority than ``bootstrap.pg_hba``. Together with :ref:`dynamic configuration <dynamic_configuration>` it simplifies management of ``pg_hba.conf``.
+        -  **- host all all 0.0.0.0/0 md5**.
+        -  **- host replication replicator 127.0.0.1/32 md5**: A line like this is required for replication.
 -  **pg\_ctl\_timeout**: How long should pg_ctl wait when doing ``start``, ``stop`` or ``restart``. Default value is 60 seconds.
 -  **use\_pg\_rewind**: try to use pg\_rewind on the former leader when it joins cluster as a replica.
 -  **remove\_data\_directory\_on\_rewind\_failure**: If this option is enabled, Patroni will remove postgres data directory and recreate replica. Otherwise it will try to follow the new leader. Default value is **false**.
@@ -108,3 +120,9 @@ REST API
 ZooKeeper
 ----------
 -  **hosts**: list of ZooKeeper cluster members in format: ['host1:port1', 'host2:port2', 'etc...'].
+
+Watchdog
+--------
+- **mode**: ``off``, ``automatic`` or ``required``. When ``off`` watchdog is disabled. When ``automatic`` watchdog will be used if available, but ignored if it is not. When ``required`` the node will not become a leader unless watchdog can be succesfully enabled.
+- **device**: Path to watchdog device. Defaults to ``/dev/watchdog``.
+- **safety_margin**: Number of seconds of safety margin between watchdog triggering and leader key expiration.

docs/index.rst

@@ -21,6 +21,7 @@ We call Patroni a "template" because it is far from being a one-size-fits-all or
    dynamic_configuration
    ENVIRONMENT
    SETTINGS
+   replica_bootstrap
    replication_modes
    pause
    releases

docs/releases.rst

@@ -3,6 +3,134 @@
 Release notes
 =============
 
+Version 1.3
+-----------
+
+Version 1.3 adds custom bootstrap possibility, significantly improves support for pg_rewind, enhances the
+synchronous mode support, adds configuration editing to patronictl and implements watchdog support on Linux.
+In addition, this is the first version to work correctly with PostgreSQL 10.
+
+**Upgrade notice**
+
+There are no known compatibility issues with the new version of Patroni. Configuration from version 1.2 should work
+without any changes. It is possible to upgrade by installing new packages and either  restarting Patroni (will cause
+PostgreSQL restart), or by putting Patroni into a :ref:`pause mode <pause>` first and then restarting Patroni on all
+nodes in the cluster (Patroni in a pause mode will not attempt to stop/start PostgreSQL), resuming from the pause mode
+at the end.
+
+**Custom bootstrap**
+
+- Make the process of bootstrapping the cluster configurable (Alexander Kukushkin)
+
+  Allow custom bootstrap scripts instead of ``initdb`` when initializing the very first node in the cluster.
+  The bootstrap command receives the name of the cluster and the path to the data directory. The resulting cluster can
+  be configured to perform recovery, making it possible to bootstrap from a backup and do point in time recovery. Refer
+  to the :ref:`documentaton page <custom_bootstrap>` for more detailed description of this feature.
+
+**Smarter pg_rewind support**
+
+-  Decide on whether to run pg_rewind by looking at the timeline differences from the current master (Alexander)
+
+   Previously, Patroni had a fixed set of conditions to trigger pg_rewind, namely when starting a former master, when
+   doing a switchover to the designated node for every other node in the cluster or when there is a replica with the
+   nofailover tag. All those cases have in common a chance that some replica may be ahead of the new master. In some cases,
+   pg_rewind did nothing, in some other ones it was not running when necessary. Instead of relying on this limited list
+   of rules make Patroni compare the master and the replica WAL positions (using the streaming replication protocol)
+   in order to reliably decide if rewind is necessary for the replica.
+
+**Synchronous replication mode strict**
+
+-  Enhance synchronous replication support by adding the strict mode (James Sewell, Alexander)
+
+   Normally, when ``synchronous_mode`` is enabled and there are no replicas attached to the master, Patroni will disable
+   synchronous replication in order to keep the master available for writes. The ``synchronous_mode_strict`` option
+   changes that, when it is set Patroni will not disable the synchronous replication in a lack of replicas, effectively
+   blocking all clients writing data to the master. In addition to the synchronous mode guarantee of preventing any data
+   loss due to automatic failover, the strict mode ensures that each write is either durably stored on two nodes or not
+   happening altogether if there is only one node in the cluster.
+
+**Configuration editing with patronictl**
+
+- Add configuration editing to patronictl (Ants Aasma, Alexander)
+
+  Add the ability to patronictl of editing dynamic cluster configuration stored in DCS. Support either specifying the
+  parameter/values from the command-line, invoking the $EDITOR, or applying configuration from the yaml file.
+
+**Linux watchdog support**
+
+- Implement watchdog support for Linux (Ants)
+
+  Support Linux software watchdog in order to reboot the node where Patroni is not running or not responding (e.g because
+  of the high load) The Linux software watchdog reboots the non-responsive node. It is possible to configure the watchdog
+  device to use (`/dev/watchdog` by default) and the mode (on, automatic, off) from the watchdog section of the Patroni
+  configuration. You can get more information from the :ref:`watchdog documentation <watchdog>`.
+
+**Add support for PostgreSQL 10**
+
+- Patroni is compatible with all beta versions of PostgreSQL 10 released so far and we expect it to be compatible with
+  the PostgreSQL 10 when it will be released.
+
+**PostgreSQL-related minor improvements**
+
+- Define pg_hba.conf via the Patroni configuration file or the dynamic configuration in DCS (Alexander)
+
+  Allow to define the contents of ``pg_hba.conf`` in the ``pg_hba`` sub-section of the ``postgresql`` section of the
+  configuration. This simplifies managing ``pg_hba.conf`` on multiple nodes, as one needs to define it only ones in DCS
+  instead of logging to every node, changing it manually and reload the configuration.
+
+  When defined, the contents of this section will replace the current ``pg_hba.conf`` completely. Patroni ignores it
+  if ``hba_file`` PostgreSQL parameter is set.
+
+- Support connecting via a UNIX socket to the local PostgreSQL cluster (Alexander)
+
+  Add the ``use_unix_socket`` option to the ``postgresql`` section of Patroni configuration. When set to true and the
+  PostgreSQL ``unix_socket_directories`` option is not empty, enables Patroni to use the first value from it to connect
+  to the local PostgreSQL cluster. If ``unix_socket_directories`` is not defined, Patroni will assume its default value
+  and omit the ``host`` parameter in the PostgreSQL connection string altogether.
+
+- Support change of superuser and replication credentials on reload (Alexander)
+
+- Support storing of configuration files outside of PostgreSQL data directory (@jouir)
+
+  Add the new configuration ``postgresql`` configuration directive ``config_dir``.
+  It defaults to the data directory and must be writable by Patroni.
+
+**Bug fixes and stability improvements**
+
+- Handle EtcdEventIndexCleared and EtcdWatcherCleared exceptions (Alexander)
+
+  Faster recovery when the watch operation is ended by Etcd by avoiding useless retries.
+
+- Remove error spinning on Etcd failure and reduce log spam (Ants)
+
+  Avoid immediate retrying and emitting stack traces in the log on the second and subsequent Etcd connection failures.
+
+- Export locale variables when forking PostgreSQL processes (Oleksii Kliukin)
+
+  Avoid the `postmaster became multithreaded during startup` fatal error on non-English locales for PostgreSQL built with NLS.
+
+- Extra checks when dropping the replication slot (Alexander)
+
+  In some cases Patroni is prevented from dropping the replication slot by the WAL sender.
+
+- Truncate the replication slot name to 63  (NAMEDATALEN - 1) characters to comply with PostgreSQL naming rules (Nick Scott)
+
+- Fix a race condition resulting in extra connections being opened to the PostgreSQL cluster from Patroni (Alexander)
+
+- Release the leader key when the node restarts with an empty data directory (Alex Kerney)
+
+- Set asynchronous executor busy when running bootstrap without a leader (Alexander)
+
+  Failure to do so could have resulted in errors stating the node belonged to a different cluster, as Patroni proceeded with
+  the normal business while being bootstrapped by a bootstrap method that doesn't require a leader to be present in the
+  cluster.
+
+- Improve WAL-E replica creation method (Joar Wandborg, Alexander).
+
+  - Use csv.DictReader when parsing WAL-E base backup, accepting ISO dates with space-delimited date and time.
+  - Support fetching current WAL position from the replica to estimate the amount of WAL to restore. Previously, the code used to call system information functions that were available only on the master node.
+
+
 Version 1.2
 -----------
 
@@ -391,4 +519,4 @@ This release adds support for *cascading replication* and simplifies Patroni man
 
   The tests can be launched manually using the *behave* command. They are also launched automatically for pull requests and after commits.
 
-  Releases notes for some older versions can be found on `project's github page <https://github.com/zalando/patroni/releases>`__.
+  Release notes for some older versions can be found on `project's github page <https://github.com/zalando/patroni/releases>`__.

docs/replica_bootstrap.rst

@@ -0,0 +1,98 @@
+Replica imaging and bootstrap
+=============================
+
+Patroni allows customizing creation of a new replica. It also supports defining what happens when the new empty cluster
+is being bootstrapped. The distinction between two is well defined: Patroni creates replicas only if the ``initialize``
+key is present in Etcd for the cluster. If there is no ``initialize`` key - Patroni calls bootstrap exclusively on the
+first node that takes the initialize key lock.
+
+.. _custom_bootstrap:
+
+Bootstrap
+---------
+
+PostgreSQL provides ``initdb`` command to initialize a new cluster and Patroni calls it by default. In certain cases,
+particularly when creating a new cluster as a copy of an existing one, it is necessary to replace a built-in method with
+custom actions. Patroni supports executing user-defined scripts to bootstrap new clusters, supplying some required
+arguments to them, i.e. the name of the cluster and the path to the data directory. This is configured in the
+``bootstrap`` section of the Patroni configuration. For example:
+
+.. code:: YAML
+
+    bootstrap:
+        method: <custom_bootstrap_method_name>
+        <custom_bootstrap_method_name>:
+            command: <path_to_custom_bootstrap_script> [param1 [, ...]]
+            recovery_conf:
+                recovery_target_action: promote
+                recovery_target_timeline: latest
+                restore_command: <method_specific_restore_command>
+
+
+Each bootstrap method must define at least a ``name`` and a ``command``. A special ``initdb`` method is available to trigger
+the default behavior, in which case ``method`` parameter can be omitted altogether. The ``command`` can be specified using either
+an absolute path, or the one relative to the ``patroni`` command location. In addition to the fixed parameters defined
+in the configuration files, Patroni supplies two cluster-specific ones:
+
+--scope
+    Name of the cluster to be bootstrapped
+--datadir
+    Path to the data directory of the cluster instance to be bootstrapped
+
+If the bootstrap script returns 0, Patroni tries to configure and start the PostgreSQL instance produced by it. If any
+of the intermediate steps fail, or the script returns a non-zero value, Patroni assumes that the bootstrap has failed,
+cleans up after itself and releases the initialize lock to give another node the opportunity to bootstrap.
+
+If a ``recovery_conf`` block is defined in the same section as the custom bootstrap method, Patroni will generate a
+``recovery.conf`` before starting the newly bootstrapped instance. Typically, such recovery.conf should contain at least
+one of the ``recovery_target_*`` parameters, together with the ``recovery_target_timeline`` set to ``promote``.
+
+ .. note:: Bootstrap methods are neither chained, nor fallen-back to the default one in case the primary one fails
+
+
+.. _custom_replica_creation:
+
+Building replicas
+-----------------
+
+Patroni uses tried and proven ``pg_basebackup`` in order to create new replicas. One downside of it is that it requires
+a running master node. Another one is the lack of 'on-the-fly' compression for the backup data and no built-in cleanup
+for outdated backup files. Some people prefer other backup solutions, such as ``WAL-E``, ``pgBackRest``, ``Barman`` and
+others, or simply roll their own scripts. In order to accommodate all those use-cases Patroni supports running custom
+scripts to clone a new replica. Those are configured in the ``postgresql`` configuration block:
+
+.. code:: YAML
+
+    postgresql:
+        create_replica_method:
+            - wal_e
+            - basebackup
+        wal_e:
+            command: patroni_wale_restore
+            no_master: 1
+            envdir: {{WALE_ENV_DIR}}
+            use_iam: 1
+
+
+The ``create_replica_method`` defines available replica creation methods and the order of executing them. Patroni will
+stop on the first one that returns 0. The basebackup is the built-in method and doesn't require any configuration. The
+rest of the methods should define a separate section in the configuration file, listing the command to execute and any
+custom parameters that should be passed to that command. All parameters will be passed in a ``--name=value`` format.
+Besides user-defined parameters, Patroni supplies a couple of cluster-specific ones:
+
+--scope
+    Which cluster this replica belongs to
+--datadir
+    Path to the data directory of the replica
+--role
+    Always 'replica'
+--connstring
+    Connection string to connect to the cluster member to clone from (master or other replica). The user in the
+    connection string can execute SQL and replication protocol commands.
+
+A special ``no_master`` parameter, if defined, allows Patroni to call the replica creation method even if there is no
+running master or replicas. In that case, an empty string will be passed in a connection string. This is useful for
+restoring the formerly running cluster from the binary backup.
+
+If all replica creation methods fail, Patroni will try again all methods in order during the next event loop cycle.
+

docs/replication_modes.rst

@@ -44,7 +44,7 @@ When ``synchronous_mode`` is on and a standby crashes, commits will block until
 
 You can ensure that a standby never becomes the synchronous standby by setting ``nosync`` tag to true. This is recommended to set for standbys that are behind slow network connections and would cause performance degradation when becoming a synchronous standby.
 
-Synchronous mode can be switched on and off via Patroni REST interface. See `dynamic configuration <https://github.com/zalando/patroni/blob/master/docs/dynamic_configuration.rst>`__ for instructions.
+Synchronous mode can be switched on and off via Patroni REST interface. See :ref:`dynamic configuration <dynamic_configuration>` for instructions.
 
 
 Synchronous mode implementation

docs/watchdog.rst

@@ -0,0 +1,40 @@
+.. _watchdog:
+
+================
+Watchdog support
+================
+
+Having multiple PostgreSQL servers running as master can result in transactions lost due to diverging timelines. This situation is also called a split-brain problem. To avoid split-brain Patroni needs to ensure PostgreSQL will not accept any transaction commits after leader key expires in the DCS. Under normal circumstances Patroni will try to achieve this by stopping PostgreSQL when leader lock update fails for any reason. However, this may fail to happen due to various reasons:
+
+- Patroni has crashed due to a bug, out-of-memory condition or by being accidentally killed by a system administrator.
+
+- Shutting down PostgreSQL is too slow.
+
+- Patroni does not get to run due to high load on the system, th VM being paused by the hypervisor, or other infrastructure issues.
+
+To guarantee correct behavior under these conditions Patroni supports watchdog devices. Watchdog devices are software or hardware mechanisms that will reset the whole system when they do not get a keepalive heartbeat within a specified timeframe. This adds an additional layer of fail safe in case usual Patroni split-brain protection mechanisms fail.
+
+Patroni will try to activate the watchdog before promoting PostgreSQL to master. If watchdog activation fails and watchdog mode is ``required`` then the node will refuse to become master. When deciding to participate in leader election Patroni will also check that watchdog configuration will allow it to become leader at all. After demoting PostgreSQL (for example due to a manual failover) Patroni will disable the watchdog again. Watchdog will also be disabled while Patroni is in paused state.
+
+By default Patroni will set up the watchdog to expire 5 seconds before TTL expires. With the default setup of ``loop_wait=10`` and ``ttl=30`` this gives HA loop at least 15 seconds (``ttl`` - ``safety_margin`` - ``loop_wait``) to complete before the system gets forcefully reset. By default accessing DCS is configured to time out after 10 seconds. This means that when DCS is unavailable, for example due to network issues, Patroni and PostgreSQL will have at least 5 seconds (``ttl`` - ``safety_margin`` - ``loop_wait`` - ``retry_timeout``) to come to a state where all client connections are terminated.
+
+Safety margin is the amount of time that Patroni reserves for time between leader key update and watchdog keepalive. Patroni will try to send a keepalive immediately after confirmation of leader key update. If Patroni process is suspended for extended amount of time at exactly the right moment the keepalive may be delayed for more than the safety margin without triggering the watchdog. This results in a window of time where watchdog will not trigger before leader key expiration, invalidating the guarantee. To be absolutely sure that watchdog will trigger under all circumstances set up the watchdog to expire after half of TTL by setting ``safety_margin`` to -1 to set watchdog timeout to ``ttl // 2``. If you need this guarantee you probably should increase ``ttl`` and/or reduce ``loop_wait`` and ``retry_timeout``.
+
+Currently watchdogs are only supported using Linux watchdog device interface.
+
+Setting up software watchdog on Linux
+-------------------------------------
+
+Default Patroni configuration will try to use ``/dev/watchdog`` on Linux if it is accessible to Patroni. For most use cases using software watchdog built into the Linux kernel is secure enough.
+
+To enable software watchdog issue the following commands as root before starting Patroni:
+
+.. code-block:: bash
+
+    modprobe softdog
+    # Replace postgres with the user you will be running patroni under
+    chown postgres /dev/watchdog
+
+For testing it may be helpful to disable rebooting by adding ``soft_noboot=1`` to the modprobe command line. In this case the watchdog will just log a line in kernel ring buffer, visible via `dmesg`.
+
+Patroni will log information about the watchdog when it is successfully enabled.

extras/startup-scripts/README.md

@@ -11,3 +11,12 @@ Upstart job for Ubuntu 12.04 or 14.04.  Requires Upstart > 1.4. Intended for sys
 
 ### patroni.service
 Systemd service file, to be copied to /etc/systemd/system/patroni.service, tested on Centos 7.1 with Patroni installed from pip. 
+
+### patroni
+Init.d service file for Debian-like distributions. Copy it to /etc/init.d/, make executable:
+```chmod 755 /etc/init.d/patroni``` and run with ```service patroni start```, or make it starting on boot with ```update-rc.d patroni defaults```. Also you might edit some configuration variables in it:
+PATRONI for patroni.py location
+CONF for configuration file
+LOGFILE for log (script creates it if does not exist)
+
+Note. If you have several versions of Postgres installed, please add to POSTGRES_VERSION the release number which you wish to run. Script uses this value to append PATH environment with correct path to Postgres bin.

extras/startup-scripts/patroni

@@ -0,0 +1,144 @@
+#!/bin/sh
+#
+### BEGIN INIT INFO
+# Provides:          patroni
+# Required-Start:    $remote_fs $syslog
+# Required-Stop:     $remote_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Patroni init script
+# Description:       Runners to orchestrate a high-availability PostgreSQL
+### END INIT INFO
+
+### BEGIN USER CONFIGURATION
+
+CONF="/etc/patroni/postgres.yml"
+LOGFILE="/var/log/patroni.log"
+USER="postgres"
+GROUP="postgres"
+
+NAME=patroni
+PATRONI="/opt/patroni/$NAME.py"
+PIDFILE="/var/run/$NAME.pid"
+
+# Set this parameter, if you have several Postgres versions installed
+# POSTGRES_VERSION="9.4"
+POSTGRES_VERSION=""
+
+### END USER CONFIGURATION
+
+. /lib/lsb/init-functions
+
+# Loading this library for get_versions() function
+if test ! -e /usr/share/postgresql-common/init.d-functions; then
+  log_failure_msg "Probably postgresql-common does not installed."
+  exit 1
+else
+  . /usr/share/postgresql-common/init.d-functions
+fi
+
+# Is there Patroni executable?
+if test ! -e $PATRONI; then
+  log_failure_msg "Patroni executable $PATRONI does not exist."
+  exit 1
+fi
+
+# Is there Patroni configuration file?
+if test ! -e $CONF; then
+  log_failure_msg "Patroni configuration file $CONF does not exist."
+  exit 1
+fi
+
+# Create logfile if doesn't exist
+if test ! -e $LOGFILE; then
+  log_action_msg "Creating logfile for Patroni..."
+  touch $LOGFILE
+  chown $USER:$GROUP $LOGFILE
+fi
+
+prepare_pgpath() {
+  if [ "$POSTGRES_VERSION" != "" ]; then
+    if [ -x /usr/lib/postgresql/$POSTGRES_VERSION/bin/pg_ctl ]; then
+      PGPATH="/usr/lib/postgresql/$POSTGRES_VERSION/bin"
+    else
+      log_failure_msg "Postgres version incorrect, check POSTGRES_VERSION variable."
+      exit 0
+    fi
+  else
+    get_versions
+    if echo $versions | grep -q -e "\s"; then
+      log_warning_msg "You have several Postgres versions installed. Please, use POSTGRES_VERSION to define correct environment."
+    else
+      versions=`echo $versions | sed -e 's/^[ \t]*//'`
+      PGPATH="/usr/lib/postgresql/$versions/bin"
+    fi
+  fi
+}
+
+get_pid() {
+  if test -e $PIDFILE; then
+    PID=`cat $PIDFILE`
+    CHILDPID=`ps --ppid $PID -o %p --no-headers`
+  else
+    log_failure_msg "Could not find PID file. Patroni probably down."
+    exit 1
+  fi
+}
+
+
+case "$1" in
+  start)
+    prepare_pgpath
+    PGPATH=$PATH:$PGPATH
+    log_success_msg "Starting Patroni\n"
+    exec start-stop-daemon --start --quiet \
+        --background \
+        --pidfile $PIDFILE --make-pidfile \
+        --chuid $USER:$GROUP \
+        --chdir `eval echo ~$USER` \
+        --exec $PATRONI \
+        --startas /bin/sh -- \
+        -c "/usr/bin/env PATH=$PGPATH /usr/bin/python $PATRONI $CONF >> $LOGFILE 2>&1"
+    ;;
+
+  stop)
+    log_success_msg "Stopping Patroni"
+    get_pid
+    start-stop-daemon --stop --pid $CHILDPID
+    start-stop-daemon --stop --pidfile $PIDFILE --remove-pidfile --quiet
+    ;;
+
+  reload)
+    log_success_msg "Reloading Patroni configuration"
+    get_pid
+    kill -HUP $CHILDPID
+    ;;
+
+  status)
+    get_pid
+    if start-stop-daemon -T --pid $CHILDPID; then
+      log_success_msg "Patroni is running\n"
+      exit 0
+    else
+      log_warning_msg "Patroni in not running\n"
+    fi
+    ;;
+
+  restart)
+    $0 stop
+    $0 start
+    ;;
+
+  *)
+    echo "Usage: /etc/init.d/$NAME {start|stop|restart|reload|status}"
+    exit 1
+    ;;
+esac
+
+if [ $? -eq 0 ]; then
+        echo .
+        exit 0
+else
+        echo " failed"
+        exit 1
+fi

extras/startup-scripts/patroni.service

@@ -21,7 +21,7 @@ ExecStart=/bin/patroni /etc/patroni.yml
 KillMode=process
 
 # Give a reasonable amount of time for the server to start up/shut down
-TimeoutSec=10
+TimeoutSec=30
 
 # Do not restart the service if it crashes, we want to manually inspect database on failure
 Restart=no

features/backup_create.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+while getopts ":-:" optchar; do
+    [[ "${optchar}" == "-" ]] || continue
+    case "${OPTARG}" in
+        datadir=* )
+            PGDATA=${OPTARG#*=}
+            ;;
+        dbname=* )
+            DBNAME=${OPTARG#*=}
+            ;;
+        walmethod=* )
+            WALMETHOD=${OPTARG#*=}
+            ;;
+    esac
+done
+
+[[ -z $PGDATA || -z $DBNAME || -z $WALMETHOD ]] && exit 1
+
+[[ $WALMETHOD != "none" ]] && WALMETHOD="-X $WALMETHOD" || WALMETHOD=""
+
+exec pg_basebackup -D $PGDATA $WALMETHOD -c fast -d $DBNAME

features/backup_restore.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -x
+
+while getopts ":-:" optchar; do
+    [[ "${optchar}" == "-" ]] || continue
+    case "${OPTARG}" in
+        datadir=* )
+            PGDATA=${OPTARG#*=}
+            ;;
+        sourcedir=* )
+            SOURCE=${OPTARG#*=}
+            ;;
+    esac
+done
+
+[[ -z $PGDATA || -z $SOURCE ]] && exit 1
+
+mkdir -p $(dirname $PGDATA)
+
+exec cp -af $SOURCE $PGDATA

features/basic_replication.feature

@@ -34,7 +34,8 @@ Feature: basic replication
     And postgres1 role is the primary after 10 seconds
 
   Scenario: check rejoin of the former master with pg_rewind
-    Given I start postgres0
+    Given I add the table splitbrain to postgres0
+    And I start postgres0
     Then postgres0 role is the secondary after 20 seconds
     When I add the table buz to postgres1
     Then table buz is present on postgres0 after 20 seconds

features/custom_bootstrap.feature

@@ -0,0 +1,17 @@
+Feature: custom bootstrap
+    We should check that patroni can bootstrap a new cluster from a backup
+
+Scenario: clone existing cluster using pg_basebackup
+    Given I start postgres0
+    Then postgres0 is a leader after 10 seconds
+    When I add the table foo to postgres0
+    And I start postgres1 in a cluster batman1 as a clone of postgres0
+    Then postgres1 is a leader of batman1 after 10 seconds
+    Then table foo is present on postgres1 after 10 seconds
+
+Scenario: make a backup and do a restore into a new cluster
+    Given I add the table bar to postgres1
+    And I do a backup of postgres1
+    When I start postgres2 in a cluster batman2 from backup
+    Then postgres2 is a leader of batman2 after 10 seconds
+    And table bar is present on postgres2 after 10 seconds

features/environment.py

@@ -1,14 +1,18 @@
 import abc
 import consul
+import datetime
 import etcd
 import kazoo.client
 import kazoo.exceptions
 import os
+import psutil
 import psycopg2
 import shutil
+import signal
 import six
 import subprocess
 import tempfile
+import threading
 import time
 import yaml
 
@@ -74,18 +78,28 @@ class AbstractController(object):
         if self._log:
             self._log.close()
 
+    def cancel_background(self):
+        pass
+
 
 class PatroniController(AbstractController):
     __PORT = 5440
     PATRONI_CONFIG = '{}.yml'
     """ starts and stops individual patronis"""
 
-    def __init__(self, context, name, work_directory, output_dir, tags=None):
+    def __init__(self, context, name, work_directory, output_dir, custom_config=None):
         super(PatroniController, self).__init__(context, 'patroni_' + name, work_directory, output_dir)
         PatroniController.__PORT += 1
         self._data_dir = os.path.join(work_directory, 'data', name)
         self._connstring = None
-        self._config = self._make_patroni_test_config(name, tags)
+        if custom_config and 'watchdog' in custom_config:
+            self.watchdog = WatchdogMonitor(name, work_directory, output_dir)
+            custom_config['watchdog'] = {'driver': 'testing', 'device': self.watchdog.fifo_path, 'mode': 'required'}
+        else:
+            self.watchdog = None
+
+        self._config = self._make_patroni_test_config(name, custom_config)
+        self._closables = []
 
         self._conn = None
         self._curs = None
@@ -109,6 +123,8 @@ class PatroniController(AbstractController):
                 yaml.safe_dump(config, w, default_flow_style=False)
 
     def _start(self):
+        if self.watchdog:
+            self.watchdog.start()
         return subprocess.Popen(['coverage', 'run', '--source=patroni', '-p', 'patroni.py', self._config],
                                 stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory)
 
@@ -116,11 +132,16 @@ class PatroniController(AbstractController):
         if postgres:
             return subprocess.call(['pg_ctl', '-D', self._data_dir, 'stop', '-mi', '-w'])
         super(PatroniController, self).stop(kill, timeout)
+        if self.watchdog:
+            self.watchdog.stop()
 
     def _is_accessible(self):
-        return self.query("SELECT 1", fail_ok=True) is not None
+        cursor = self.query("SELECT 1", fail_ok=True)
+        if cursor is not None:
+            cursor.execute("SET synchronous_commit TO 'local'")
+            return True
 
-    def _make_patroni_test_config(self, name, tags):
+    def _make_patroni_test_config(self, name, custom_config):
         patroni_config_name = self.PATRONI_CONFIG.format(name)
         patroni_config_path = os.path.join(self._output_dir, patroni_config_name)
 
@@ -132,24 +153,37 @@ class PatroniController(AbstractController):
 
         config['postgresql']['listen'] = config['postgresql']['connect_address'] = '{0}:{1}'.format(host, self.__PORT)
 
+        config['name'] = name
+        config['postgresql']['data_dir'] = self._data_dir
+        config['postgresql']['use_unix_socket'] = True
+        config['postgresql']['parameters'].update({
+            'logging_collector': 'on', 'log_destination': 'csvlog', 'log_directory': self._output_dir,
+            'log_filename': name + '.log', 'log_statement': 'all', 'log_min_messages': 'debug1',
+            'unix_socket_directories': self._data_dir})
+
+        if 'bootstrap' in config:
+            config['bootstrap']['post_bootstrap'] = 'psql -w -c "SELECT 1"'
+            if 'initdb' in config['bootstrap']:
+                config['bootstrap']['initdb'].extend([{'auth': 'md5'}, {'auth-host': 'md5'}])
+
+        if custom_config is not None:
+            def recursive_update(dst, src):
+                for k, v in src.items():
+                    if k in dst and isinstance(dst[k], dict):
+                        recursive_update(dst[k], v)
+                    else:
+                        dst[k] = v
+            recursive_update(config, custom_config)
+
+        with open(patroni_config_path, 'w') as f:
+            yaml.safe_dump(config, f, default_flow_style=False)
+
         user = config['postgresql'].get('authentication', config['postgresql']).get('superuser', {})
         self._connkwargs = {k: user[n] for n, k in [('username', 'user'), ('password', 'password')] if n in user}
         self._connkwargs.update({'host': host, 'port': self.__PORT, 'database': 'postgres'})
 
-        config['name'] = name
-        config['postgresql']['data_dir'] = self._data_dir
-        config['postgresql']['parameters'].update({
-            'logging_collector': 'on', 'log_destination': 'csvlog', 'log_directory': self._output_dir,
-            'log_filename': name + '.log', 'log_statement': 'all', 'log_min_messages': 'debug1'})
-
-        if 'bootstrap' in config and 'initdb' in config['bootstrap']:
-            config['bootstrap']['initdb'].extend([{'auth': 'md5'}, {'auth-host': 'md5'}])
-
-        if tags:
-            config['tags'] = tags
-
-        with open(patroni_config_path, 'w') as f:
-            yaml.safe_dump(config, f, default_flow_style=False)
+        self._replication = config['postgresql'].get('authentication', config['postgresql']).get('replication', {})
+        self._replication.update({'host': host, 'port': self.__PORT, 'database': 'postgres'})
 
         return patroni_config_path
 
@@ -185,10 +219,101 @@ class PatroniController(AbstractController):
             time.sleep(1)
         return False
 
+    def get_watchdog(self):
+        return self.watchdog
+
+    def _get_pid(self):
+        try:
+            pidfile = os.path.join(self._data_dir, 'postmaster.pid')
+            if not os.path.exists(pidfile):
+                return None
+            return int(open(pidfile).readline().strip())
+        except:
+            return None
+
+    def database_is_running(self):
+        pid = self._get_pid()
+        if not pid:
+            return False
+        try:
+            os.kill(pid, 0)
+        except OSError:
+            return False
+        return True
+
+    def patroni_hang(self, timeout):
+        hang = ProcessHang(self._handle.pid, timeout)
+        self._closables.append(hang)
+        hang.start()
+
+    def checkpoint_hang(self, timeout):
+        pid = self._get_pid()
+        if not pid:
+            return False
+        proc = psutil.Process(pid)
+        for child in proc.children():
+            if 'checkpoint' in child.cmdline()[0]:
+                checkpointer = child
+                break
+        else:
+            return False
+        hang = ProcessHang(checkpointer.pid, timeout)
+        self._closables.append(hang)
+        hang.start()
+        return True
+
+    def cancel_background(self):
+        for obj in self._closables:
+            obj.close()
+        self._closables = []
+
+    def terminate_backends(self):
+        pid = self._get_pid()
+        if not pid:
+            return False
+        proc = psutil.Process(pid)
+        for p in proc.children():
+            if 'process' not in p.cmdline()[0]:
+                p.terminate()
+
+    @property
+    def backup_source(self):
+        return 'postgres://{username}:{password}@{host}:{port}/{database}'.format(**self._replication)
+
+    def backup(self, dest='basebackup'):
+        subprocess.call([PatroniPoolController.BACKUP_SCRIPT, '--walmethod=none',
+                         '--datadir=' + os.path.join(self._output_dir, dest),
+                         '--dbname=' + self.backup_source])
+
+
+class ProcessHang(object):
+
+    """A background thread implementing a cancelable process hang via SIGSTOP."""
+
+    def __init__(self, pid, timeout):
+        self._cancelled = threading.Event()
+        self._thread = threading.Thread(target=self.run)
+        self.pid = pid
+        self.timeout = timeout
+
+    def start(self):
+        self._thread.start()
+
+    def run(self):
+        os.kill(self.pid, signal.SIGSTOP)
+        try:
+            self._cancelled.wait(self.timeout)
+        finally:
+            os.kill(self.pid, signal.SIGCONT)
+
+    def close(self):
+        self._cancelled.set()
+        self._thread.join()
+
 
 class AbstractDcsController(AbstractController):
 
-    _CLUSTER_NODE = '/service/batman'
+    _CLUSTER_NODE = '/service/{0}'
 
     def __init__(self, context, mktemp=True):
         work_directory = mktemp and tempfile.mkdtemp() or None
@@ -203,11 +328,11 @@ class AbstractDcsController(AbstractController):
         if self._work_directory:
             shutil.rmtree(self._work_directory)
 
-    def path(self, key=None):
-        return self._CLUSTER_NODE + (key and '/' + key or '')
+    def path(self, key=None, scope='batman'):
+        return self._CLUSTER_NODE.format(scope) + (key and '/' + key or '')
 
     @abc.abstractmethod
-    def query(self, key):
+    def query(self, key, scope='batman'):
         """ query for a value of a given key """
 
     @abc.abstractmethod
@@ -236,18 +361,19 @@ class ConsulController(AbstractDcsController):
         super(ConsulController, self).__init__(context)
         os.environ['PATRONI_CONSUL_HOST'] = 'localhost:8500'
         self._client = consul.Consul()
+        self._config_file = None
 
     def _start(self):
-        config_file = self._work_directory + '.json'
-        with open(config_file, 'wb') as f:
+        self._config_file = self._work_directory + '.json'
+        with open(self._config_file, 'wb') as f:
             f.write(b'{"session_ttl_min":"5s","server":true,"bootstrap":true,"advertise_addr":"127.0.0.1"}')
-        return subprocess.Popen(['consul', 'agent', '-config-file', config_file, '-data-dir', self._work_directory],
-                                stdout=self._log, stderr=subprocess.STDOUT)
+        return subprocess.Popen(['consul', 'agent', '-config-file', self._config_file, '-data-dir',
+                                 self._work_directory], stdout=self._log, stderr=subprocess.STDOUT)
 
     def stop(self, kill=False, timeout=15):
         super(ConsulController, self).stop(kill=kill, timeout=timeout)
-        if self._work_directory:
-            os.unlink(self._work_directory + '.json')
+        if self._config_file:
+            os.unlink(self._config_file)
 
     def _is_running(self):
         try:
@@ -255,18 +381,18 @@ class ConsulController(AbstractDcsController):
         except Exception:
             return False
 
-    def path(self, key=None):
-        return super(ConsulController, self).path(key)[1:]
+    def path(self, key=None, scope='batman'):
+        return super(ConsulController, self).path(key, scope)[1:]
 
-    def query(self, key):
-        _, value = self._client.kv.get(self.path(key))
+    def query(self, key, scope='batman'):
+        _, value = self._client.kv.get(self.path(key, scope))
         return value and value['Value'].decode('utf-8')
 
     def set(self, key, value):
         self._client.kv.put(self.path(key), value)
 
     def cleanup_service_tree(self):
-        self._client.kv.delete(self.path(), recurse=True)
+        self._client.kv.delete(self.path(scope=''), recurse=True)
 
     def start(self, max_wait_limit=15):
         super(ConsulController, self).start(max_wait_limit)
@@ -285,9 +411,9 @@ class EtcdController(AbstractDcsController):
         return subprocess.Popen(["etcd", "--debug", "--data-dir", self._work_directory],
                                 stdout=self._log, stderr=subprocess.STDOUT)
 
-    def query(self, key):
+    def query(self, key, scope='batman'):
         try:
-            return self._client.get(self.path(key)).value
+            return self._client.get(self.path(key, scope)).value
         except etcd.EtcdKeyNotFound:
             return None
 
@@ -296,7 +422,7 @@ class EtcdController(AbstractDcsController):
 
     def cleanup_service_tree(self):
         try:
-            self._client.delete(self.path(), recursive=True)
+            self._client.delete(self.path(scope=''), recursive=True)
         except (etcd.EtcdKeyNotFound, etcd.EtcdConnectionFailed):
             return
         except Exception as e:
@@ -323,9 +449,9 @@ class ZooKeeperController(AbstractDcsController):
     def _start(self):
         pass  # TODO: implement later
 
-    def query(self, key):
+    def query(self, key, scope='batman'):
         try:
-            return self._client.get(self.path(key))[0].decode('utf-8')
+            return self._client.get(self.path(key, scope))[0].decode('utf-8')
         except kazoo.exceptions.NoNodeError:
             return None
 
@@ -334,7 +460,7 @@ class ZooKeeperController(AbstractDcsController):
 
     def cleanup_service_tree(self):
         try:
-            self._client.delete(self.path(), recursive=True)
+            self._client.delete(self.path(scope=''), recursive=True)
         except (kazoo.exceptions.NoNodeError):
             return
         except Exception as e:
@@ -359,6 +485,8 @@ class ExhibitorController(ZooKeeperController):
 
 class PatroniPoolController(object):
 
+    BACKUP_SCRIPT = 'features/backup_create.sh'
+
     def __init__(self, context):
         self._context = context
         self._dcs = None
@@ -383,13 +511,16 @@ class PatroniPoolController(object):
     def output_dir(self):
         return self._output_dir
 
-    def start(self, name, max_wait_limit=20, tags=None):
+    def start(self, name, max_wait_limit=20, custom_config=None):
         if name not in self._processes:
-            self._processes[name] = PatroniController(self._context, name, self.patroni_path, self._output_dir, tags)
+            self._processes[name] = PatroniController(self._context, name, self.patroni_path,
+                                                      self._output_dir, custom_config)
         self._processes[name].start(max_wait_limit)
 
     def __getattr__(self, func):
-        if func not in ['stop', 'query', 'write_label', 'read_label', 'check_role_has_changed_to', 'add_tag_to_config']:
+        if func not in ['stop', 'query', 'write_label', 'read_label', 'check_role_has_changed_to', 'add_tag_to_config',
+                        'get_watchdog', 'database_is_running', 'checkpoint_hang', 'patroni_hang',
+                        'terminate_backends', 'backup']:
             raise AttributeError("PatroniPoolController instance has no attribute '{0}'".format(func))
 
         def wrapper(name, *args, **kwargs):
@@ -398,6 +529,7 @@ class PatroniPoolController(object):
 
     def stop_all(self):
         for ctl in self._processes.values():
+            ctl.cancel_background()
             ctl.stop()
         self._processes.clear()
 
@@ -408,6 +540,53 @@ class PatroniPoolController(object):
         os.makedirs(feature_dir)
         self._output_dir = feature_dir
 
+    def clone(self, from_name, cluster_name, to_name):
+        f = self._processes[from_name]
+        custom_config = {
+            'scope': cluster_name,
+            'bootstrap': {
+                'method': 'pg_basebackup',
+                'pg_basebackup': {
+                    'command': self.BACKUP_SCRIPT + ' --walmethod=stream --dbname=' + f.backup_source
+                }
+            },
+            'postgresql': {
+                'parameters': {
+                    'archive_mode': 'on',
+                    'archive_command': 'mkdir -p {0} && test ! -f {0}/%f && cp %p {0}/%f'.format(
+                            os.path.join(self._output_dir, 'wal_archive'))
+                },
+                'authentication': {
+                    'superuser': {'password': 'zalando1'},
+                    'replication': {'password': 'rep-pass1'}
+                }
+            }
+        }
+        self.start(to_name, custom_config=custom_config)
+
+    def bootstrap_from_backup(self, name, cluster_name):
+        custom_config = {
+            'scope': cluster_name,
+            'bootstrap': {
+                'method': 'backup_restore',
+                'backup_restore': {
+                    'command': 'features/backup_restore.sh --sourcedir=' + os.path.join(self._output_dir, 'basebackup'),
+                    'recovery_conf': {
+                        'recovery_target_action': 'promote',
+                        'recovery_target_timeline': 'latest',
+                        'restore_command': 'cp {0}/wal_archive/%f %p'.format(self._output_dir)
+                    }
+                }
+            },
+            'postgresql': {
+                'authentication': {
+                    'superuser': {'password': 'zalando2'},
+                    'replication': {'password': 'rep-pass2'}
+                }
+            }
+        }
+        self.start(name, custom_config=custom_config)
+
     @property
     def dcs(self):
         if self._dcs is None:
@@ -416,6 +595,122 @@ class PatroniPoolController(object):
         return self._dcs
 
 
+class WatchdogMonitor(object):
+    """Testing harness for emulating a watchdog device as a named pipe. Because we can't easily emulate ioctl's we
+    require a custom driver on Patroni side. The device takes no action, only notes if it was pinged and/or triggered.
+    """
+    def __init__(self, name, work_directory, output_dir):
+        self.fifo_path = os.path.join(work_directory, 'data', 'watchdog.{0}.fifo'.format(name))
+        self.fifo_file = None
+        self._stop_requested = False  # Relying on bool setting being atomic
+        self._thread = None
+        self.last_ping = None
+        self.was_pinged = False
+        self.was_closed = False
+        self._was_triggered = False
+        self.timeout = 60
+        self._log_file = open(os.path.join(output_dir, 'watchdog.{0}.log'.format(name)), 'w')
+        self._log("watchdog {0} initialized".format(name))
+
+    def _log(self, msg):
+        tstamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")
+        self._log_file.write("{0}: {1}\n".format(tstamp, msg))
+
+    def start(self):
+        assert self._thread is None
+        self._stop_requested = False
+        self._log("starting fifo {0}".format(self.fifo_path))
+        fifo_dir = os.path.dirname(self.fifo_path)
+        if os.path.exists(self.fifo_path):
+            os.unlink(self.fifo_path)
+        elif not os.path.exists(fifo_dir):
+            os.mkdir(fifo_dir)
+        os.mkfifo(self.fifo_path)
+        self.last_ping = time.time()
+
+        self._thread = threading.Thread(target=self.run)
+        self._thread.start()
+
+    def run(self):
+        try:
+            while not self._stop_requested:
+                self._log("opening")
+                self.fifo_file = os.open(self.fifo_path, os.O_RDONLY)
+                try:
+                    self._log("Fifo {0} connected".format(self.fifo_path))
+                    self.was_closed = False
+                    while not self._stop_requested:
+                        c = os.read(self.fifo_file, 1)
+
+                        if c == b'X':
+                            self._log("Stop requested")
+                            return
+                        elif c == b'':
+                            self._log("Pipe closed")
+                            break
+                        elif c == b'C':
+                            command = b''
+                            c = os.read(self.fifo_file, 1)
+                            while c != b'\n' and c != b'':
+                                command += c
+                                c = os.read(self.fifo_file, 1)
+                            command = command.decode('utf8')
+
+                            if command.startswith('timeout='):
+                                self.timeout = int(command.split('=')[1])
+                                self._log("timeout={0}".format(self.timeout))
+                        elif c in [b'V', b'1']:
+                            cur_time = time.time()
+                            if cur_time - self.last_ping > self.timeout:
+                                self._log("Triggered")
+                                self._was_triggered = True
+                            if c == b'V':
+                                self._log("magic close")
+                                self.was_closed = True
+                            elif c == b'1':
+                                self.was_pinged = True
+                                self._log("ping after {0} seconds".format(cur_time - (self.last_ping or cur_time)))
+                                self.last_ping = cur_time
+                        else:
+                            self._log('Unknown command {0} received from fifo'.format(c))
+                finally:
+                    self.was_closed = True
+                    self._log("closing")
+                    os.close(self.fifo_file)
+        except Exception as e:
+            self._log("Error {0}".format(e))
+        finally:
+            self._log("stopping")
+            self._log_file.flush()
+            if os.path.exists(self.fifo_path):
+                os.unlink(self.fifo_path)
+
+    def stop(self):
+        self._log("Monitor stop")
+        self._stop_requested = True
+        try:
+            if os.path.exists(self.fifo_path):
+                fd = os.open(self.fifo_path, os.O_WRONLY)
+                os.write(fd, b'X')
+                os.close(fd)
+        except Exception as e:
+            self._log("err while closing: {0}".format(str(e)))
+        if self._thread:
+            self._thread.join()
+            self._thread = None
+
+    def reset(self):
+        self._log("reset")
+        self.was_pinged = self.was_closed = self._was_triggered = False
+
+    @property
+    def was_triggered(self):
+        delta = time.time() - self.last_ping
+        triggered = self._was_triggered or not self.was_closed and delta > self.timeout
+        self._log("triggered={0}, {1}s left".format(triggered, self.timeout - delta))
+        return triggered
+
+
 # actions to execute on start/stop of the tests and before running invidual features
 def before_all(context):
     context.ci = 'TRAVIS_BUILD_NUMBER' in os.environ or 'BUILD_NUMBER' in os.environ

features/patroni_api.feature

@@ -34,9 +34,9 @@ Scenario: check local configuration reload
 	Then I receive a response code 202
 
 Scenario: check dynamic configuration change via DCS
-	Given I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 10, "loop_wait": 2, "postgresql": {"parameters": {"max_connections": 101}}}
-	Then I receive a response code 200
-	And I receive a response loop_wait 2
+	Given I run patronictl.py edit-config -s 'ttl=10' -s 'loop_wait=2' -p 'max_connections=101' --force batman
+	Then I receive a response returncode 0
+	And I receive a response output "+loop_wait: 2"
 	And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 11 seconds
 	When I issue a GET request to http://127.0.0.1:8008/config
 	Then I receive a response code 200
@@ -65,8 +65,8 @@ Scenario: check API requests for the primary-replica pair in the pause mode
 	Then postgres1 role is the secondary after 15 seconds
 
 Scenario: check the failover via the API in the pause mode
-	Given I run patronictl.py failover batman --master postgres0 --candidate postgres1 --force
-	Then I receive a response returncode 0
+	Given I issue a POST request to http://127.0.0.1:8008/failover with {"leader": "postgres0", "candidate": "postgres1"}
+	Then I receive a response code 200
 	And postgres1 is a leader after 5 seconds
 	And postgres1 role is the primary after 10 seconds
 	And postgres0 role is the secondary after 10 seconds

features/steps/basic_replication.py

@@ -11,7 +11,7 @@ def start_patroni(context, name):
 
 @step('I shut down {name:w}')
 def stop_patroni(context, name):
-    return context.pctl.stop(name)
+    return context.pctl.stop(name, timeout=60)
 
 
 @step('I kill {name:w}')

features/steps/cascading_replication.py

@@ -6,7 +6,7 @@ from behave import step, then
 
 @step('I configure and start {name:w} with a tag {tag_name:w} {tag_value:w}')
 def start_patroni_with_a_name_value_tag(context, name, tag_name, tag_value):
-    return context.pctl.start(name, tags={tag_name: tag_value})
+    return context.pctl.start(name, custom_config={'tags': {tag_name: tag_value}})
 
 
 @then('There is a label with "{content:w}" in {name:w} data directory')

features/steps/custom_bootstrap.py

@@ -0,0 +1,27 @@
+import time
+
+from behave import step, then
+
+
+@step('I start {name:w} in a cluster {cluster_name:w} as a clone of {name2:w}')
+def start_cluster_clone(context, name, cluster_name, name2):
+    context.pctl.clone(name2, cluster_name, name)
+
+
+@step('I start {name:w} in a cluster {cluster_name:w} from backup')
+def start_cluster_from_backup(context, name, cluster_name):
+    context.pctl.bootstrap_from_backup(name, cluster_name)
+
+
+@then('{name:w} is a leader of {cluster_name:w} after {time_limit:d} seconds')
+def is_a_leader(context, name, cluster_name, time_limit):
+    time_limit *= context.timeout_multiplier
+    max_time = time.time() + int(time_limit)
+    while (context.dcs_ctl.query("leader", scope=cluster_name) != name):
+        time.sleep(1)
+        assert time.time() < max_time, "{0} is not a leader in dcs after {1} seconds".format(name, time_limit)
+
+
+@step('I do a backup of {name:w}')
+def do_backup(context, name):
+    context.pctl.backup(name)

features/steps/patroni_api.py

@@ -111,7 +111,8 @@ def check_response(context, component, data):
         assert context.status_code == int(data),\
             "status code {0} != {1}, response: {2}".format(context.status_code, data, context.response)
     elif component == 'returncode':
-        assert context.status_code == int(data), "return code {0} != {1}".format(context.status_code, data)
+        assert context.status_code == int(data), "return code {0} != {1}, {2}".format(context.status_code,
+                                                                                      data, context.response)
     elif component == 'text':
         assert context.response == data.strip('"'), "response {0} does not contain {1}".format(context.response, data)
     elif component == 'output':

features/steps/watchdog.py

@@ -0,0 +1,74 @@
+from behave import step, then
+import time
+
+
+def polling_loop(timeout, interval=1):
+    """Returns an iterator that returns values until timeout has passed. Timeout is measured from start of iteration."""
+    start_time = time.time()
+    iteration = 0
+    end_time = start_time + timeout
+    while time.time() < end_time:
+        yield iteration
+        iteration += 1
+        time.sleep(interval)
+
+
+@step('I start {name:w} with watchdog')
+def start_patroni_with_watchdog(context, name):
+    return context.pctl.start(name, custom_config={'watchdog': True})
+
+
+@step('{name:w} watchdog has been pinged after {timeout:d} seconds')
+def watchdog_was_pinged(context, name, timeout):
+    for _ in polling_loop(timeout):
+        if context.pctl.get_watchdog(name).was_pinged:
+            return True
+    return False
+
+
+@then('{name:w} watchdog has been closed')
+def watchdog_was_closed(context, name):
+    assert context.pctl.get_watchdog(name).was_closed
+
+
+@step('I wait for next {name:w} watchdog ping')
+def watchdog_reset_pinged(context, name):
+    context.pctl.get_watchdog(name).reset()
+
+
+@then('{name:w} watchdog is triggered after {timeout:d} seconds')
+def watchdog_was_triggered(context, name, timeout):
+    for _ in polling_loop(timeout):
+        if context.pctl.get_watchdog(name).was_triggered:
+            return True
+    assert False
+
+
+@then('{name:w} watchdog was not triggered')
+def watchdog_was_not_triggered(context, name):
+    assert not context.pctl.get_watchdog(name).was_triggered
+
+
+@step('{name:w} checkpoint takes {timeout:d} seconds')
+def checkpoint_hang(context, name, timeout):
+    assert context.pctl.checkpoint_hang(name, timeout)
+
+
+@step('{name:w} hangs for {timeout:d} seconds')
+def patroni_hang(context, name, timeout):
+    return context.pctl.patroni_hang(name, timeout)
+
+
+@step('I terminate {name:w} user processes')
+def terminate_backends(context, name):
+    return context.pctl.terminate_backends(name)
+
+
+@step('Sleep for {timeout:d} seconds')
+def dcs_connection_lost(context, timeout):
+    time.sleep(timeout)
+
+
+@then('{name:w} database is running')
+def database_is_running(context, name):
+    assert context.pctl.database_is_running(name)

features/watchdog.feature

@@ -0,0 +1,19 @@
+Feature: watchdog
+  Verify that watchdog gets pinged and triggered under appropriate circumstances.
+
+  Scenario: watchdog is opened, pinged and closed
+    Given I start postgres0 with watchdog
+    Then postgres0 is a leader after 10 seconds
+    And postgres0 role is the primary after 10 seconds
+    And postgres0 watchdog has been pinged after 10 seconds
+    When I shut down postgres0
+    Then postgres0 watchdog has been closed
+
+  #TODO: test watchdog is disabled during pause
+  #TODO: test watchdog is disabled properly when shutting down
+
+  Scenario: watchdog is triggered if patroni stops responding
+    Given I start postgres0 with watchdog
+    Then postgres0 role is the primary after 10 seconds
+    When postgres0 hangs for 30 seconds
+    Then postgres0 watchdog is triggered after 30 seconds

haproxy.cfg

@@ -1,21 +1,25 @@
 global
-	maxconn 100
+    maxconn 100
 
 defaults
-	log	global
-	mode	tcp
-	retries 2
-	timeout client 30m
-	timeout connect 4s
-	timeout server 30m
-	timeout check 5s
+    log global
+    mode tcp
+    retries 2
+    timeout client 30m
+    timeout connect 4s
+    timeout server 30m
+    timeout check 5s
 
-frontend ft_postgresql
-	bind *:5000
-	default_backend bk_db
-
-backend bk_db
-	option httpchk
+listen stats
+    mode http
+    bind *:7000
+    stats enable
+    stats uri /
 
+listen batman
+    bind *:5000
+    option httpchk
+    http-check expect status 200
+    default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
     server postgresql_127.0.0.1_5432 127.0.0.1:5432 maxconn 100 check port 8008
     server postgresql_127.0.0.1_5433 127.0.0.1:5433 maxconn 100 check port 8009

patroni/__init__.py

@@ -16,12 +16,14 @@ class Patroni(object):
         from patroni.ha import Ha
         from patroni.postgresql import Postgresql
         from patroni.version import __version__
+        from patroni.watchdog import Watchdog
 
         self.setup_signal_handlers()
 
         self.version = __version__
         self.config = Config()
         self.dcs = get_dcs(self.config)
+        self.watchdog = Watchdog(self.config)
         self.load_dynamic_configuration()
 
         self.postgresql = Postgresql(self.config['postgresql'])
@@ -40,6 +42,7 @@ class Patroni(object):
                 if cluster and cluster.config:
                     if self.config.set_dynamic_configuration(cluster.config):
                         self.dcs.reload_config(self.config)
+                        self.watchdog.reload_config(self.config)
                 elif not self.config.dynamic_configuration and 'bootstrap' in self.config:
                     if self.config.set_dynamic_configuration(self.config['bootstrap']['dcs']):
                         self.dcs.reload_config(self.config)
@@ -63,6 +66,7 @@ class Patroni(object):
         try:
             self.tags = self.get_tags()
             self.dcs.reload_config(self.config)
+            self.watchdog.reload_config(self.config)
             self.api.reload_config(self.config['restapi'])
             self.postgresql.reload_config(self.config['postgresql'])
         except Exception:
@@ -113,7 +117,7 @@ class Patroni(object):
             if cluster and cluster.config and self.config.set_dynamic_configuration(cluster.config):
                 self.reload_config()
 
-            if not self.postgresql.data_directory_empty():
+            if self.postgresql.role != 'uninitialized':
                 self.config.save_cache()
 
             self.schedule_next_run()
@@ -124,6 +128,10 @@ class Patroni(object):
         signal.signal(signal.SIGHUP, self.sighup_handler)
         signal.signal(signal.SIGTERM, self.sigterm_handler)
 
+    def shutdown(self):
+        self.api.shutdown()
+        self.ha.shutdown()
+
 
 def patroni_main():
     logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
@@ -135,12 +143,7 @@ def patroni_main():
     except KeyboardInterrupt:
         pass
     finally:
-        patroni.api.shutdown()
-        if patroni.ha.is_paused():
-            logger.info('Leader key is not deleted and Postgresql is not stopped due paused state')
-        else:
-            patroni.ha.while_not_sync_standby(lambda: patroni.postgresql.stop(checkpoint=False))
-            patroni.dcs.delete_leader()
+        patroni.shutdown()
 
 
 def pg_ctl_start(args):

patroni/api.py

@@ -68,6 +68,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
             response['scheduled_restart'] = patroni.scheduled_restart.copy()
             del response['scheduled_restart']['postmaster_start_time']
             response['scheduled_restart']['schedule'] = (response['scheduled_restart']['schedule']).isoformat()
+        if not patroni.ha.watchdog.is_healthy:
+            response['watchdog_failed'] = True
         self._write_json_response(status_code, response)
 
     def do_GET(self, write_status_code_only=False):
@@ -298,8 +300,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
             members = [m for m in cluster.members if m.name != cluster.leader.name and m.api_url]
             if not members:
                 return 'failover is not possible: cluster does not have members except leader'
-        for _, reachable, _, _, tags in self.server.patroni.ha.fetch_nodes_statuses(members):
-            if reachable and not tags.get('nofailover', False):
+        for st in self.server.patroni.ha.fetch_nodes_statuses(members):
+            if st.failover_limitation() is None:
                 return None
         return 'failover is not possible: no good candidates have been found'
 
@@ -382,14 +384,16 @@ class RestApiHandler(BaseHTTPRequestHandler):
                                        pg_is_in_recovery(),
                                        CASE WHEN pg_is_in_recovery()
                                             THEN 0
-                                            ELSE pg_xlog_location_diff(pg_current_xlog_location(), '0/0')::bigint
+                                            ELSE pg_{0}_{1}_diff(pg_current_{0}_{1}(), '0/0')::bigint
                                        END,
-                                       pg_xlog_location_diff(COALESCE(pg_last_xlog_receive_location(),
-                                                                      pg_last_xlog_replay_location()), '0/0')::bigint,
-                                       pg_xlog_location_diff(pg_last_xlog_replay_location(), '0/0')::bigint,
+                                       pg_{0}_{1}_diff(COALESCE(pg_last_{0}_receive_{1}(),
+                                                                      pg_last_{0}_replay_{1}()), '0/0')::bigint,
+                                       pg_{0}_{1}_diff(pg_last_{0}_replay_{1}(), '0/0')::bigint,
                                        to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
-                                       pg_is_in_recovery() AND pg_is_xlog_replay_paused(),
-                                       (SELECT array_to_json(array_agg(row_to_json(ri))) FROM replication_info ri)""",
+                                       pg_is_in_recovery() AND pg_is_{0}_replay_paused(),
+                                       (SELECT array_to_json(array_agg(row_to_json(ri)))
+                                          FROM replication_info ri)""".format(self.server.patroni.postgresql.wal_name,
+                                                                              self.server.patroni.postgresql.lsn_name),
                              retry=retry)[0]
 
             result = {

patroni/async_executor.py

@@ -1,9 +1,55 @@
 import logging
-from threading import RLock, Thread
+from threading import Lock, RLock, Thread
 
 logger = logging.getLogger(__name__)
 
 
+class CriticalTask(object):
+    """Represents a critical task in a background process that we either need to cancel or get the result of.
+
+    Fields of this object may be accessed only when holding a lock on it. To perform the critical task the background
+    thread must, while holding lock on this object, check `is_cancelled` flag, run the task and mark the task as
+    complete using `complete()`.
+
+    The main thread must hold async lock to prevent the task from completing, hold lock on critical task object,
+    call cancel. If the task has completed `cancel()` will return False and `result` field will contain the result of
+    the task. When cancel returns True it is guaranteed that the background task will notice the `is_cancelled` flag.
+    """
+    def __init__(self):
+        self._lock = Lock()
+        self.is_cancelled = False
+        self.result = None
+
+    def reset(self):
+        """Must be called every time the background task is finished.
+
+        Must be called from async thread. Caller must hold lock on async executor when calling."""
+        self.is_cancelled = False
+        self.result = None
+
+    def cancel(self):
+        """Tries to cancel the task, returns True if the task has already run.
+
+        Caller must hold lock on async executor and the task when calling."""
+        if self.result is not None:
+            return False
+        self.is_cancelled = True
+        return True
+
+    def complete(self, result):
+        """Mark task as completed along with a result.
+
+        Must be called from async thread. Caller must hold lock on task when calling."""
+        self.result = result
+
+    def __enter__(self):
+        self._lock.acquire()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._lock.release()
+
+
 class AsyncExecutor(object):
 
     def __init__(self, ha_wakeup):
@@ -11,6 +57,7 @@ class AsyncExecutor(object):
         self._thread_lock = RLock()
         self._scheduled_action = None
         self._scheduled_action_lock = RLock()
+        self.critical_task = CriticalTask()
 
     @property
     def busy(self):
@@ -43,6 +90,8 @@ class AsyncExecutor(object):
         finally:
             with self:
                 self.reset_scheduled_action()
+                with self.critical_task:
+                    self.critical_task.reset()
             if wakeup is not None:
                 self._ha_wakeup()
 

patroni/config.py

@@ -43,10 +43,14 @@ class Config(object):
         'maximum_lag_on_failover': 1048576,
         'master_start_timeout': 300,
         'synchronous_mode': False,
+        'synchronous_mode_strict': False,
         'postgresql': {
             'bin_dir': '',
             'use_slots': True,
             'parameters': {p: v[0] for p, v in Postgresql.CMDLINE_OPTIONS.items()}
+        },
+        'watchdog': {
+            'mode': 'automatic',
         }
     }
 
@@ -174,7 +178,7 @@ class Config(object):
                     elif name not in ('connect_address', 'listen', 'data_dir', 'pgpass', 'authentication'):
                         config['postgresql'][name] = deepcopy(value)
             elif name in config:  # only variables present in __DEFAULT_CONFIG allowed to be overriden from DCS
-                if name == 'synchronous_mode':
+                if name in ('synchronous_mode', 'synchronous_mode_strict'):
                     config[name] = value
                 else:
                     config[name] = int(value)
@@ -271,7 +275,7 @@ class Config(object):
                         config['postgresql'][name].update(self._process_postgresql_parameters(value, True))
                     elif name != 'use_slots':  # replication slots must be enabled/disabled globally
                         config['postgresql'][name] = deepcopy(value)
-            elif name not in config:
+            elif name not in config or name in ['watchdog']:
                 config[name] = deepcopy(value) if value else {}
 
         # restapi server expects to get restapi.auth = 'username:password'

patroni/ctl.py

@@ -4,27 +4,36 @@ Patroni Control
 
 import base64
 import click
+import codecs
 import datetime
 import dateutil.parser
+import cdiff
+import copy
+import difflib
+import io
 import json
 import logging
 import os
 import psycopg2
 import random
 import requests
+import subprocess
 import sys
+import tempfile
 import time
 import tzlocal
 import yaml
 
 from click import ClickException
+from contextlib import contextmanager
 from patroni.config import Config
 from patroni.dcs import get_dcs as _get_dcs
 from patroni.exceptions import PatroniException
 from patroni.postgresql import Postgresql
-from patroni.utils import is_valid_pg_version
+from patroni.utils import is_valid_pg_version, patch_config
 from prettytable import PrettyTable
 from six.moves.urllib_parse import urlparse
+from six import text_type
 
 CONFIG_DIR_PATH = click.get_app_dir('patroni')
 CONFIG_FILE_PATH = os.path.join(CONFIG_DIR_PATH, 'patronictl.yaml')
@@ -819,3 +828,204 @@ def pause(obj, cluster_name):
 @click.pass_obj
 def resume(obj, cluster_name):
     return toggle_pause(obj, cluster_name, False)
+
+
+@contextmanager
+def temporary_file(contents, suffix='', prefix='tmp'):
+    """Creates a temporary file with specified contents that persists for the context.
+
+    :param contents: binary string that will be written to the file.
+    :param prefix: will be prefixed to the filename.
+    :param suffix: will be appended to the filename.
+    :returns path of the created file.
+    """
+    tmp = tempfile.NamedTemporaryFile(suffix=suffix, prefix=prefix, delete=False)
+    with tmp:
+        tmp.write(contents)
+
+    try:
+        yield tmp.name
+    finally:
+        os.unlink(tmp.name)
+
+
+def show_diff(before_editing, after_editing):
+    """Shows a diff between two strings.
+
+    If the output is to a tty the diff will be colored. Inputs are expected to be unicode strings.
+    """
+    def listify(string):
+        return [l+'\n' for l in string.rstrip('\n').split('\n')]
+
+    unified_diff = difflib.unified_diff(listify(before_editing), listify(after_editing))
+
+    if sys.stdout.isatty():
+        buf = io.StringIO()
+        for line in unified_diff:
+            # Force cast to unicode as difflib on Python 2.7 returns a mix of unicode and str.
+            buf.write(text_type(line))
+        buf.seek(0)
+
+        class opts:
+            side_by_side = False
+            width = 80
+            tab_width = 8
+        cdiff.markup_to_pager(cdiff.PatchStream(buf), opts)
+    else:
+        for line in unified_diff:
+            click.echo(line.rstrip('\n'))
+
+
+def format_config_for_editing(data):
+    """Formats configuration as YAML for human consumption.
+
+    :param data: configuration as nested dictionaries
+    :returns unicode YAML of the configuration"""
+    return yaml.safe_dump(data, default_flow_style=False, encoding=None, allow_unicode=True)
+
+
+def apply_config_changes(before_editing, data, kvpairs):
+    """Applies config changes specified as a list of key-value pairs.
+
+    Keys are interpreted as dotted paths into the configuration data structure. Except for paths beginning with
+    `postgresql.parameters` where rest of the path is used directly to allow for PostgreSQL GUCs containing dots.
+    Values are interpreted as YAML values.
+
+    :param before_editing: human representation before editing
+    :param data: configuration datastructure
+    :param kvpairs: list of strings containing key value pairs separated by =
+    :returns tuple of human readable and parsed datastructure after changes
+    """
+    changed_data = copy.deepcopy(data)
+
+    def set_path_value(config, path, value, prefix=()):
+        # Postgresql GUCs can't be nested, but can contain dots so we re-flatten the structure for this case
+        if prefix == ('postgresql', 'parameters'):
+            path = ['.'.join(path)]
+
+        key = path[0]
+        if len(path) == 1:
+            if value is None:
+                config.pop(key, None)
+            else:
+                config[key] = value
+        else:
+            if not isinstance(config.get(key), dict):
+                config[key] = {}
+            set_path_value(config[key], path[1:], value, prefix + (key,))
+            if config[key] == {}:
+                del config[key]
+
+    for pair in kvpairs:
+        if not pair or "=" not in pair:
+            raise PatroniCtlException("Invalid parameter setting {0}".format(pair))
+        key_path, value = pair.split("=", 1)
+        set_path_value(changed_data, key_path.strip().split("."), yaml.safe_load(value))
+
+    return format_config_for_editing(changed_data), changed_data
+
+
+def apply_yaml_file(data, filename):
+    """Applies changes from a YAML file to configuration
+
+    :param data: configuration datastructure
+    :param filename: name of the YAML file, - is taken to mean standard input
+    :returns tuple of human readable and parsed datastructure after changes
+    """
+    changed_data = copy.deepcopy(data)
+
+    if filename == '-':
+        new_options = yaml.safe_load(sys.stdin)
+    else:
+        with open(filename) as fd:
+            new_options = yaml.safe_load(fd)
+
+    patch_config(changed_data, new_options)
+
+    return format_config_for_editing(changed_data), changed_data
+
+
+def invoke_editor(before_editing, cluster_name):
+    """Starts editor command to edit configuration in human readable format
+
+    :param before_editing: human representation before editing
+    :returns tuple of human readable and parsed datastructure after changes
+    """
+    editor_cmd = os.environ.get('EDITOR')
+    if not editor_cmd:
+        raise PatroniCtlException('EDITOR environment variable is not set')
+
+    with temporary_file(contents=before_editing.encode('utf-8'),
+                        suffix='.yaml',
+                        prefix='{0}-config-'.format(cluster_name)) as tmpfile:
+        ret = subprocess.call([editor_cmd, tmpfile])
+        if ret:
+            raise PatroniCtlException("Editor exited with return code {0}".format(ret))
+
+        with codecs.open(tmpfile, encoding='utf-8') as fd:
+            after_editing = fd.read()
+
+        return after_editing, yaml.safe_load(after_editing)
+
+
+@ctl.command('edit-config', help="Edit cluster configuration")
+@click.argument('cluster_name')
+@click.option('--quiet', '-q', is_flag=True, help='Do not show changes')
+@click.option('--set', '-s', 'kvpairs', multiple=True,
+              help='Set specific configuration value. Can be specified multiple times')
+@click.option('--pg', '-p', 'pgkvpairs', multiple=True,
+              help='Set specific PostgreSQL parameter value. Shorthand for -s postgresql.parameters. '
+                   'Can be specified multiple times')
+@click.option('--apply', 'apply_filename', help='Apply configuration from file. Use - for stdin.')
+@click.option('--replace', 'replace_filename', help='Apply configuration from file, replacing existing configuration.'
+              ' Use - for stdin.')
+@option_force
+@click.pass_obj
+def edit_config(obj, cluster_name, force, quiet, kvpairs, pgkvpairs, apply_filename, replace_filename):
+    dcs = get_dcs(obj, cluster_name)
+    cluster = dcs.get_cluster()
+
+    before_editing = format_config_for_editing(cluster.config.data)
+
+    after_editing = None  # Serves as a flag if any changes were requested
+    changed_data = cluster.config.data
+
+    if replace_filename:
+        after_editing, changed_data = apply_yaml_file({}, replace_filename)
+
+    if apply_filename:
+        after_editing, changed_data = apply_yaml_file(changed_data, apply_filename)
+
+    if kvpairs or pgkvpairs:
+        all_pairs = list(kvpairs) + ['postgresql.parameters.'+v.lstrip() for v in pgkvpairs]
+        after_editing, changed_data = apply_config_changes(before_editing, changed_data, all_pairs)
+
+    # If no changes were specified on the command line invoke editor
+    if after_editing is None:
+        after_editing, changed_data = invoke_editor(before_editing, cluster_name)
+
+    if cluster.config.data == changed_data:
+        if not quiet:
+            click.echo("Not changed")
+        return
+
+    if not quiet:
+        show_diff(before_editing, after_editing)
+
+    if (apply_filename == '-' or replace_filename == '-') and not force:
+        click.echo("Use --force option to apply changes")
+        return
+
+    if force or click.confirm('Apply these changes?'):
+        if not dcs.set_config_value(json.dumps(changed_data), cluster.config.index):
+            raise PatroniCtlException("Config modification aborted due to concurrent changes")
+        click.echo("Configuration changed")
+
+
+@ctl.command('show-config', help="Show cluster configuration")
+@click.argument('cluster_name')
+@click.pass_obj
+def show_config(obj, cluster_name):
+    cluster = get_dcs(obj, cluster_name).get_cluster()
+
+    click.echo(format_config_for_editing(cluster.config.data))

patroni/dcs/__init__.py

@@ -3,6 +3,7 @@ import dateutil
 import importlib
 import inspect
 import json
+import logging
 import os
 import pkgutil
 import six
@@ -14,6 +15,8 @@ from random import randint
 from six.moves.urllib_parse import urlparse, urlunparse, parse_qsl
 from threading import Event, Lock
 
+logger = logging.getLogger(__name__)
+
 
 def parse_connection_string(value):
     """Original Governor stores connection strings for each cluster members if a following format:
@@ -51,18 +54,22 @@ def dcs_modules():
 def get_dcs(config):
     available_implementations = set()
     for module_name in dcs_modules():
-        module = importlib.import_module(module_name)
-        for name in filter(lambda name: not name.startswith('__'), dir(module)):  # iterate through module content
-            value = getattr(module, name)
-            name = name.lower()
-            # try to find implementation of AbstractDCS interface, class name must match with module_name
-            if inspect.isclass(value) and issubclass(value, AbstractDCS) and __package__ + '.' + name == module_name:
-                available_implementations.add(name)
-                if name in config:  # which has configuration section in the config file
-                    # propagate some parameters
-                    config[name].update({p: config[p] for p in ('namespace', 'name', 'scope', 'loop_wait',
-                                         'patronictl', 'ttl', 'retry_timeout') if p in config})
-                    return value(config[name])
+        try:
+            module = importlib.import_module(module_name)
+            for name in filter(lambda name: not name.startswith('__'), dir(module)):  # iterate through module content
+                item = getattr(module, name)
+                name = name.lower()
+                # try to find implementation of AbstractDCS interface, class name must match with module_name
+                if inspect.isclass(item) and issubclass(item, AbstractDCS) and __package__ + '.' + name == module_name:
+                    available_implementations.add(name)
+                    if name in config:  # which has configuration section in the config file
+                        # propagate some parameters
+                        config[name].update({p: config[p] for p in ('namespace', 'name', 'scope', 'loop_wait',
+                                             'patronictl', 'ttl', 'retry_timeout') if p in config})
+                        return item(config[name])
+        except ImportError:
+            if not config.get('patronictl'):
+                logger.info('Failed to import %s', module_name)
     raise PatroniException("""Can not find suitable configuration of distributed configuration store
 Available implementations: """ + ', '.join(available_implementations))
 

patroni/dcs/etcd.py

@@ -296,19 +296,6 @@ class Client(etcd.Client):
         self._machines_cache_updated = time.time()
 
 
-def catch_etcd_errors(func):
-    def wrapper(*args, **kwargs):
-        try:
-            return func(*args, **kwargs) is not None
-        except (RetryFailedError, etcd.EtcdException):
-            return False
-        except:
-            logger.exception("")
-            raise EtcdError("unexpected error")
-
-    return wrapper
-
-
 class Etcd(AbstractDCS):
 
     def __init__(self, config):
@@ -320,10 +307,36 @@ class Etcd(AbstractDCS):
                                               etcd.EtcdEventIndexCleared))
         self._client = self.get_etcd_client(config)
         self.__do_not_watch = False
+        self._has_failed = False
 
     def retry(self, *args, **kwargs):
         return self._retry.copy()(*args, **kwargs)
 
+    def _handle_exception(self, e, name='', do_sleep=False, raise_ex=None):
+        if not self._has_failed:
+            logger.exception(name)
+        else:
+            logger.error(e)
+            if do_sleep:
+                time.sleep(1)
+        self._has_failed = True
+        if isinstance(raise_ex, Exception):
+            raise raise_ex
+
+    def catch_etcd_errors(func):
+        def wrapper(self, *args, **kwargs):
+            try:
+                retval = func(self, *args, **kwargs) is not None
+                self._has_failed = False
+                return retval
+            except (RetryFailedError, etcd.EtcdException) as e:
+                self._handle_exception(e)
+                return False
+            except Exception as e:
+                self._handle_exception(e, raise_ex=EtcdError('unexpected error'))
+
+        return wrapper
+
     @staticmethod
     def get_etcd_client(config):
         if 'proxy' in config:
@@ -448,9 +461,9 @@ class Etcd(AbstractDCS):
             self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
         except etcd.EtcdKeyNotFound:
             self._cluster = Cluster(None, None, None, None, [], None, None)
-        except:
-            logger.exception('get_cluster')
-            raise EtcdError('Etcd is not responding properly')
+        except Exception as e:
+            self._handle_exception(e, 'get_cluster', raise_ex=EtcdError('Etcd is not responding properly'))
+        self._has_failed = False
 
     @catch_etcd_errors
     def touch_member(self, data, ttl=None, permanent=False):
@@ -524,16 +537,19 @@ class Etcd(AbstractDCS):
             while timeout >= 1:  # when timeout is too small urllib3 doesn't have enough time to connect
                 try:
                     self._client.watch(self.leader_path, index=leader_index, timeout=timeout + 0.5)
+                    self._has_failed = False
                     # Synchronous work of all cluster members with etcd is less expensive
                     # than reestablishing http connection every time from every replica.
                     return True
                 except etcd.EtcdWatchTimedOut:
                     self._client.http.clear()
+                    self._has_failed = False
                     return False
                 except (etcd.EtcdEventIndexCleared, etcd.EtcdWatcherCleared):  # Watch failed
+                    self._has_failed = False
                     return True  # leave the loop, because watch with the same parameters will fail anyway
-                except etcd.EtcdException:
-                    logger.exception('watch')
+                except etcd.EtcdException as e:
+                    self._handle_exception(e, 'watch', True)
 
                 timeout = end_time - time.time()
 

patroni/dcs/zookeeper.py

@@ -1,6 +1,7 @@
 import logging
+import time
 
-from kazoo.client import KazooClient, KazooState
+from kazoo.client import KazooClient, KazooState, KazooRetry
 from kazoo.exceptions import NoNodeError, NodeExistsError
 from kazoo.handlers.threading import SequentialThreadingHandler
 from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
@@ -51,8 +52,9 @@ class ZooKeeper(AbstractDCS):
             hosts = ','.join(hosts)
 
         self._client = KazooClient(hosts, handler=PatroniSequentialThreadingHandler(config['retry_timeout']),
-                                   timeout=config['ttl'], connection_retry={'max_delay': 1, 'max_tries': -1},
-                                   command_retry={'deadline': config['retry_timeout'], 'max_delay': 1, 'max_tries': -1})
+                                   timeout=config['ttl'], connection_retry=KazooRetry(max_delay=1, max_tries=-1,
+                                   sleep_func=time.sleep), command_retry=KazooRetry(deadline=config['retry_timeout'],
+                                   max_delay=1, max_tries=-1, sleep_func=time.sleep))
         self._client.add_listener(self.session_listener)
 
         self._my_member_data = None
@@ -114,7 +116,8 @@ class ZooKeeper(AbstractDCS):
             return True
 
     def set_retry_timeout(self, retry_timeout):
-        self._client._retry.deadline = retry_timeout
+        retry = self._client.retry if isinstance(self._client.retry, KazooRetry) else self._client._retry
+        retry.deadline = retry_timeout
 
     def get_node(self, key, watch=None):
         try:

patroni/exceptions.py

@@ -23,3 +23,7 @@ class DCSError(PatroniException):
 
 class PostgresConnectionException(PostgresException):
     pass
+
+
+class WatchdogError(PatroniException):
+    pass

patroni/ha.py

@@ -9,8 +9,8 @@ import time
 
 from collections import namedtuple
 from multiprocessing.pool import ThreadPool
-from patroni.async_executor import AsyncExecutor
-from patroni.exceptions import DCSError, PostgresConnectionException
+from patroni.async_executor import AsyncExecutor, CriticalTask
+from patroni.exceptions import DCSError, PostgresConnectionException, PatroniException
 from patroni.postgresql import ACTION_ON_START
 from patroni.utils import polling_loop, tzutc
 from threading import RLock
@@ -18,24 +18,25 @@ from threading import RLock
 logger = logging.getLogger(__name__)
 
 
-class _MemberStatus(namedtuple('_MemberStatus', 'member,reachable,in_recovery,xlog_location,tags')):
+class _MemberStatus(namedtuple('_MemberStatus', 'member,reachable,in_recovery,wal_position,tags,watchdog_failed')):
     """Node status distilled from API response:
 
         member - dcs.Member object of the node
         reachable - `!False` if the node is not reachable or is not responding with correct JSON
         in_recovery - `!True` if pg_is_in_recovery() == true
-        xlog_location - value of `replayed_location` or `location` from JSON, dependin on its role.
+        wal_position - value of `replayed_location` or `location` from JSON, dependin on its role.
         tags - dictionary with values of different tags (i.e. nofailover)
+        watchdog_failed - indicates that watchdog is required by configuration but not available or failed
     """
     @classmethod
     def from_api_response(cls, member, json):
         is_master = json['role'] == 'master'
-        xlog = not is_master and max(json['xlog'].get('received_location', 0), json['xlog'].get('replayed_location', 0))
-        return cls(member, True, not is_master, xlog, json.get('tags', {}))
+        wal = not is_master and max(json['xlog'].get('received_location', 0), json['xlog'].get('replayed_location', 0))
+        return cls(member, True, not is_master, wal, json.get('tags', {}), json.get('watchdog_failed', False))
 
     @classmethod
     def unknown(cls, member):
-        return cls(member, False, None, 0, {})
+        return cls(member, False, None, 0, {}, False)
 
     def failover_limitation(self):
         """Returns reason why this node can't promote or None if everything is ok."""
@@ -43,6 +44,8 @@ class _MemberStatus(namedtuple('_MemberStatus', 'member,reachable,in_recovery,xl
             return 'not reachable'
         if self.tags.get('nofailover', False):
             return 'not allowed to promote'
+        if self.watchdog_failed:
+            return 'not watchdog capable'
         return None
 
 
@@ -55,8 +58,10 @@ class Ha(object):
         self.cluster = None
         self.old_cluster = None
         self.recovering = False
+        self._post_bootstrap_task = None
         self._start_timeout = None
         self._async_executor = AsyncExecutor(self.wakeup)
+        self.watchdog = patroni.watchdog
 
         # Each member publishes various pieces of information to the DCS using touch_member. This lock protects
         # the state and publishing procedure to have consistent ordering and avoid publishing stale values.
@@ -81,11 +86,13 @@ class Ha(object):
 
     def update_lock(self, write_leader_optime=False):
         ret = self.dcs.update_leader()
-        if ret and write_leader_optime:
-            try:
-                self.dcs.write_leader_optime(self.state_handler.last_operation())
-            except:
-                pass
+        if ret:
+            self.watchdog.keepalive()
+            if write_leader_optime:
+                try:
+                    self.dcs.write_leader_optime(self.state_handler.last_operation())
+                except:
+                    pass
         return ret
 
     def has_lock(self):
@@ -116,7 +123,7 @@ class Ha(object):
                 data['pending_restart'] = True
             if not self._async_executor.busy and data['state'] in ['running', 'restarting', 'starting']:
                 try:
-                    data['xlog_location'] = self.state_handler.xlog_position(retry=False)
+                    data['xlog_location'] = self.state_handler.wal_position(retry=False)
                 except:
                     pass
             if self.patroni.scheduled_restart:
@@ -131,7 +138,7 @@ class Ha(object):
             logger.info('bootstrapped %s', msg)
             cluster = self.dcs.get_cluster()
             node_to_follow = self._get_node_to_follow(cluster)
-            return self.state_handler.follow(node_to_follow, cluster.leader, True)
+            return self.state_handler.follow(node_to_follow)
         else:
             logger.error('failed to bootstrap %s', msg)
             self.state_handler.remove_data_directory()
@@ -147,29 +154,31 @@ class Ha(object):
         # no initialize key and node is allowed to be master and has 'bootstrap' section in a configuration file
         elif self.cluster.initialize is None and not self.patroni.nofailover and 'bootstrap' in self.patroni.config:
             if self.dcs.initialize(create_new=True):  # race for initialization
-                try:
-                    self.state_handler.bootstrap(self.patroni.config['bootstrap'])
-                    self.dcs.initialize(create_new=False, sysid=self.state_handler.sysid)
-                except:  # initdb or start failed
-                    # remove initialization key and give a chance to other members
-                    logger.info("removing initialize key after failed attempt to initialize the cluster")
-                    self.dcs.cancel_initialization()
-                    self.state_handler.stop('immediate')
-                    self.state_handler.move_data_directory()
-                    raise
-                self.dcs.set_config_value(json.dumps(self.patroni.config.dynamic_configuration, separators=(',', ':')))
-                self.dcs.take_leader()
-                self.load_cluster_from_dcs()
-                return 'initialized a new cluster'
+                self.state_handler.bootstrapping = True
+                self._post_bootstrap_task = CriticalTask()
+                self._async_executor.schedule('bootstrap')
+                self._async_executor.run_async(self.state_handler.bootstrap, args=(self.patroni.config['bootstrap'],))
+                return 'trying to bootstrap a new cluster'
             else:
                 return 'failed to acquire initialize lock'
         else:
             if self.state_handler.can_create_replica_without_replication_connection():
+                msg = 'bootstrap (without leader)'
+                self._async_executor.schedule(msg)
                 self._async_executor.run_async(self.clone)
-                return "trying to bootstrap (without leader)"
+                return 'trying to ' + msg
             return 'waiting for leader to bootstrap'
 
+    def _handle_rewind(self):
+        if self.state_handler.rewind_needed_and_possible(self.cluster.leader):
+            self._async_executor.schedule('running pg_rewind from ' + self.cluster.leader.name)
+            self._async_executor.run_async(self.state_handler.rewind, (self.cluster.leader,))
+            return True
+
     def recover(self):
+        # Postgres is not running and we will restart in standby mode. Watchdog is not needed until we promote.
+        self.watchdog.disable()
+
         if self.has_lock() and self.update_lock():
             timeout = self.patroni.config['master_start_timeout']
             if timeout == 0:
@@ -182,9 +191,24 @@ class Ha(object):
         else:
             timeout = None
 
+        self.load_cluster_from_dcs()
+
+        if self.has_lock():
+            msg = "starting as readonly because i had the session lock"
+            node_to_follow = None
+        else:
+            if not self.state_handler.rewind_executed:
+                self.state_handler.trigger_check_diverged_lsn()
+            if self._handle_rewind():
+                return self._async_executor.scheduled_action
+            msg = "starting as a secondary"
+            node_to_follow = self._get_node_to_follow(self.cluster)
+
         self.recovering = True
-        return self.follow("starting as readonly because i had the session lock",
-                           "starting as a secondary", True, True, None, timeout)
+
+        self._async_executor.schedule('restarting after failure')
+        self._async_executor.run_async(self.state_handler.follow, (node_to_follow, timeout))
+        return msg
 
     def _get_node_to_follow(self, cluster):
         # determine the node to follow. If replicatefrom tag is set,
@@ -196,33 +220,40 @@ class Ha(object):
 
         return node_to_follow if node_to_follow and node_to_follow.name != self.state_handler.name else None
 
-    def follow(self, demote_reason, follow_reason, refresh=True, recovery=False, need_rewind=None, timeout=None):
+    def follow(self, demote_reason, follow_reason, refresh=True):
         if refresh:
             self.load_cluster_from_dcs()
 
-        if recovery:
-            ret = demote_reason if self.has_lock() else follow_reason
-        else:
-            is_leader = self.state_handler.is_leader()
-            ret = demote_reason if is_leader else follow_reason
+        is_leader = self.state_handler.is_leader()
 
         node_to_follow = self._get_node_to_follow(self.cluster)
 
-        if self.is_paused() and not (self.state_handler.need_rewind and self.state_handler.can_rewind):
-            self.state_handler.set_role('master' if is_leader else 'replica')
-            if is_leader:
-                return 'continue to run as master without lock'
-            elif not node_to_follow:
-                return 'no action'
+        if self.is_paused():
+            if not (self.state_handler.need_rewind and self.state_handler.can_rewind) or self.cluster.is_unlocked():
+                self.state_handler.set_role('master' if is_leader else 'replica')
+                if is_leader:
+                    return 'continue to run as master without lock'
+                elif not node_to_follow:
+                    return 'no action'
+        elif is_leader:
+            self.demote('immediate-nolock')
+            return demote_reason
 
-        self.state_handler.follow(node_to_follow, self.cluster.leader, recovery,
-                                  self._async_executor, need_rewind, timeout)
+        if self._handle_rewind():
+            return self._async_executor.scheduled_action
 
-        return ret
+        if not self.state_handler.check_recovery_conf(node_to_follow):
+            self._async_executor.schedule('changing primary_conninfo and restarting')
+            self._async_executor.run_async(self.state_handler.follow, (node_to_follow,))
+
+        return follow_reason
 
     def is_synchronous_mode(self):
         return bool(self.cluster and self.cluster.config and self.cluster.config.data.get('synchronous_mode'))
 
+    def is_synchronous_mode_strict(self):
+        return bool(self.cluster and self.cluster.config and self.cluster.config.data.get('synchronous_mode_strict'))
+
     def process_sync_replication(self):
         """Process synchronous standby beahvior.
 
@@ -242,10 +273,15 @@ class Ha(object):
                     if not self.dcs.write_sync_state(self.state_handler.name, None, index=self.cluster.sync.index):
                         logger.info('Synchronous replication key updated by someone else.')
                         return
+
+                if self.is_synchronous_mode_strict() and picked is None:
+                    picked = '*'
+                    logger.warning("No standbys available!")
+
                 logger.info("Assigning synchronous standby status to %s", picked)
                 self.state_handler.set_synchronous_standby(picked)
 
-                if picked and not allow_promote:
+                if picked and picked != '*' and not allow_promote:
                     # Wait for PostgreSQL to enable synchronous mode and see if we can immediately set sync_standby
                     time.sleep(2)
                     picked, allow_promote = self.state_handler.pick_synchronous_standby(self.cluster)
@@ -305,6 +341,15 @@ class Ha(object):
                 self._disable_sync -= 1
 
     def enforce_master_role(self, message, promote_message):
+        if not self.watchdog.is_running:
+            if not self.watchdog.activate():
+                if self.state_handler.is_leader():
+                    self.demote('immediate')
+                    return 'Demoting self because watchdog could not be activated'
+                else:
+                    self.release_leader_key_voluntarily()
+                    return 'Not promoting self because watchdog could not be actived'
+
         if self.state_handler.is_leader() or self.state_handler.role == 'master':
             # Inform the state handler about its master role.
             # It may be unaware of it if postgres is promoted manually.
@@ -319,7 +364,7 @@ class Ha(object):
                     # Somebody else updated sync state, it may be due to us losing the lock. To be safe, postpone
                     # promotion until next cycle. TODO: trigger immediate retry of run_cycle
                     return 'Postponing promotion because synchronous replication state was updated by somebody else'
-                self.state_handler.set_synchronous_standby(None)
+                self.state_handler.set_synchronous_standby('*' if self.is_synchronous_mode_strict() else None)
             self.state_handler.promote()
             return promote_message
 
@@ -344,21 +389,21 @@ class Ha(object):
         pool.join()
         return results
 
-    def is_lagging(self, xlog_location):
-        """Returns if instance with an xlog should consider itself unhealthy to be promoted due to replication lag.
+    def is_lagging(self, wal_position):
+        """Returns if instance with an wal should consider itself unhealthy to be promoted due to replication lag.
 
-        :param xlog_location: Current xlog location.
+        :param wal_position: Current wal position.
         :returns True when node is lagging
         """
-        lag = (self.cluster.last_leader_operation or 0) - xlog_location
+        lag = (self.cluster.last_leader_operation or 0) - wal_position
         return lag > self.state_handler.config.get('maximum_lag_on_failover', 0)
 
     def _is_healthiest_node(self, members, check_replication_lag=True):
         """This method tries to determine whether I am healthy enough to became a new leader candidate or not."""
 
-        my_xlog_location = self.state_handler.xlog_position()
-        if check_replication_lag and self.is_lagging(my_xlog_location):
-            return False  # Too far behind last reported xlog location on master
+        my_wal_position = self.state_handler.wal_position()
+        if check_replication_lag and self.is_lagging(my_wal_position):
+            return False  # Too far behind last reported wal position on master
 
         # Prepare list of nodes to run check against
         members = [m for m in members if m.name != self.state_handler.name and not m.nofailover and m.api_url]
@@ -369,7 +414,7 @@ class Ha(object):
                     if not st.in_recovery:
                         logger.warning('Master (%s) is still alive', st.member.name)
                         return False
-                    if my_xlog_location < st.xlog_location:
+                    if my_wal_position < st.wal_position:
                         return False
         return True
 
@@ -381,7 +426,7 @@ class Ha(object):
                 not_allowed_reason = st.failover_limitation()
                 if not_allowed_reason:
                     logger.info('Member %s is %s', st.member.name, not_allowed_reason)
-                elif self.is_lagging(st.xlog_location):
+                elif self.is_lagging(st.wal_position):
                     logger.info('Member %s exceeds maximum replication lag', st.member.name)
                 else:
                     ret = True
@@ -458,6 +503,9 @@ class Ha(object):
         if self.cluster.failover:
             return self.manual_failover_process_no_leader()
 
+        if not self.watchdog.is_healthy:
+            return False
+
         # When in sync mode, only last known master and sync standby are allowed to promote automatically.
         all_known_members = self.cluster.members + self.old_cluster.members
         if self.is_synchronous_mode() and self.cluster.sync.leader:
@@ -485,31 +533,40 @@ class Ha(object):
             graceful is used when failing over to another node due to user request. May only be called running async.
             immediate is used when we determine that we are not suitable for master and want to failover quickly
                 without regard for data durability. May only be called synchronously.
+            immediate-nolock is used when find out that we have lost the lock to be master. Need to bring down
+                PostgreSQL as quickly as possible without regard for data durability. May only be called synchronously.
         """
-        assert mode in ['offline', 'graceful', 'immediate']
-        if mode != 'offline':
-            if mode == 'immediate':
-                self.state_handler.stop('immediate', checkpoint=False)
-            else:
-                self.state_handler.stop()
-            self.state_handler.set_role('demoted')
-            self.release_leader_key_voluntarily()
-            time.sleep(2)  # Give a time to somebody to take the leader lock
-            cluster = self.dcs.get_cluster()
-            node_to_follow = self._get_node_to_follow(cluster)
-            if mode == 'immediate':
-                # We will try to start up as a standby now. If no one takes the leader lock before we finish
-                # recovery we will try to promote ourselves.
-                self._async_executor.schedule('waiting for failover to complete')
-                self._async_executor.run_async(self.state_handler.follow,
-                                               (node_to_follow, cluster.leader, True, None, True))
-            else:
-                return self.state_handler.follow(node_to_follow, cluster.leader, recovery=True, need_rewind=True)
+        mode_control = {
+            'offline':          dict(stop='fast', checkpoint=False, release=False, offline=True, async=False),
+            'graceful':         dict(stop='fast', checkpoint=True, release=True, offline=False, async=False),
+            'immediate':        dict(stop='immediate', checkpoint=False, release=True, offline=False, async=True),
+            'immediate-nolock': dict(stop='immediate', checkpoint=False, release=False, offline=False, async=True),
+        }[mode]
+
+        self.state_handler.trigger_check_diverged_lsn()
+        self.state_handler.stop(mode_control['stop'], checkpoint=mode_control['checkpoint'],
+                                on_safepoint=self.watchdog.disable if self.watchdog.is_running else None)
+        self.state_handler.set_role('demoted')
+
+        if mode_control['release']:
+                self.release_leader_key_voluntarily()
+                time.sleep(2)  # Give a time to somebody to take the leader lock
+        if mode_control['offline']:
+            node_to_follow, leader = None, None
         else:
-            # Need to become unavailable as soon as possible, so initiate a stop here. However as we can't release
-            # the leader key we don't care about confirming the shutdown quickly and can use a regular stop.
-            self.state_handler.stop(checkpoint=False)
-            self.state_handler.follow(None, None, recovery=True)
+            cluster = self.dcs.get_cluster()
+            node_to_follow, leader = self._get_node_to_follow(cluster), cluster.leader
+
+        # FIXME: with mode offline called from DCS exception handler and handle_long_action_in_progress
+        # there could be an async action already running, calling follow from here will lead
+        # to racy state handler state updates.
+        if mode_control['async']:
+            self._async_executor.schedule('starting after demotion')
+            self._async_executor.run_async(self.state_handler.follow, (node_to_follow,))
+        else:
+            if self.state_handler.rewind_needed_and_possible(leader):
+                return False  # do not start postgres, but run pg_rewind on the next iteration
+            self.state_handler.follow(node_to_follow)
 
     def should_run_scheduled_action(self, action_name, scheduled_at, cleanup_fn):
         if scheduled_at and not self.is_paused():
@@ -604,17 +661,15 @@ class Ha(object):
         else:
             # when we are doing manual failover there is no guaranty that new leader is ahead of any other node
             # node tagged as nofailover can be ahead of the new leader either, but it is always excluded from elections
-            need_rewind = bool(self.cluster.failover) or self.patroni.nofailover
-            if need_rewind:
+            if bool(self.cluster.failover) or self.patroni.nofailover:
+                self.state_handler.trigger_check_diverged_lsn()
                 time.sleep(2)  # Give a time to somebody to take the leader lock
 
             if self.patroni.nofailover:
                 return self.follow('demoting self because I am not allowed to become master',
-                                   'following a different leader because I am not allowed to promote',
-                                   need_rewind=need_rewind)
+                                   'following a different leader because I am not allowed to promote')
             return self.follow('demoting self because i am not the healthiest node',
-                               'following a different leader because i am not the healthiest node',
-                               need_rewind=need_rewind)
+                               'following a different leader because i am not the healthiest node')
 
     def process_healthy_cluster(self):
         if self.has_lock():
@@ -637,14 +692,14 @@ class Ha(object):
                 # Either there is no connection to DCS or someone else acquired the lock
                 logger.error('failed to update leader lock')
                 if self.state_handler.is_leader():
-                    self.demote('offline')
+                    self.demote('immediate-nolock')
                     return 'demoted self because failed to update leader lock in DCS'
                 else:
                     return 'not promoting because failed to update leader lock in DCS'
         else:
             logger.info('does not have lock')
         return self.follow('demoting self because i do not have the lock and i was a leader',
-                           'no action.  i am a secondary and i am following a leader', False)
+                           'no action.  i am a secondary and i am following a leader', refresh=False)
 
     def evaluate_scheduled_restart(self):
         if self._async_executor.busy:  # Restart already in progress
@@ -736,13 +791,13 @@ class Ha(object):
             # leader key (if it belong to us) rather than trying to start postgres once again.
             self.recovering = True
 
-        # No that restart is scheduled we can set timeout for startup, it will get reset
+        # Now that restart is scheduled we can set timeout for startup, it will get reset
         # once async executor runs and main loop notices PostgreSQL as up.
         timeout = restart_data.get('timeout', self.patroni.config['master_start_timeout'])
         self.set_start_timeout(timeout)
 
         # For non async cases we want to wait for restart to complete or timeout before returning.
-        do_restart = functools.partial(self.state_handler.restart, timeout)
+        do_restart = functools.partial(self.state_handler.restart, timeout, self._async_executor.critical_task)
         if self.is_synchronous_mode() and not self.has_lock():
             do_restart = functools.partial(self.while_not_sync_standby, do_restart)
 
@@ -783,15 +838,25 @@ class Ha(object):
         self._async_executor.run_async(self._do_reinitialize, args=(self.cluster, ))
 
     def handle_long_action_in_progress(self):
-        if self.has_lock():
-            if self.update_lock():
-                return 'updated leader lock during ' + self._async_executor.scheduled_action
-            else:
-                return 'failed to update leader lock during ' + self._async_executor.scheduled_action
-        elif self.cluster.is_unlocked():
-            return 'not healthy enough for leader race'
-        else:
-            return self._async_executor.scheduled_action + ' in progress'
+        if self.has_lock() and self.update_lock():
+            return 'updated leader lock during ' + self._async_executor.scheduled_action
+        elif not self.state_handler.bootstrapping:
+            # Don't have lock, make sure we are not starting up a master in the background
+            if self.state_handler.role == 'master':
+                logger.info("Demoting master during " + self._async_executor.scheduled_action)
+                if self._async_executor.scheduled_action == 'restart':
+                    # Restart needs a special interlocking cancel because postmaster may be just started in a
+                    # background thread and has not even written a pid file yet.
+                    with self._async_executor.critical_task as task:
+                        if not task.cancel():
+                            self.state_handler.terminate_starting_postmaster(pid=task.result)
+                self.demote('immediate-nolock')
+                return 'lost leader lock during ' + self._async_executor.scheduled_action
+
+        if self.cluster.is_unlocked():
+            logger.info('not healthy enough for leader race')
+
+        return self._async_executor.scheduled_action + ' in progress'
 
     @staticmethod
     def sysid_valid(sysid):
@@ -802,6 +867,7 @@ class Ha(object):
 
     def post_recover(self):
         if not self.state_handler.is_running():
+            self.watchdog.disable()
             if self.has_lock():
                 self.state_handler.set_role('demoted')
                 self.dcs.delete_leader()
@@ -810,6 +876,38 @@ class Ha(object):
             return 'failed to start postgres'
         return None
 
+    def cancel_initialization(self):
+        logger.info('removing initialize key after failed attempt to bootstrap the cluster')
+        self.dcs.cancel_initialization()
+        self.state_handler.stop('immediate')
+        self.state_handler.move_data_directory()
+        raise PatroniException('Failed to bootstrap cluster')
+
+    def post_bootstrap(self):
+        # bootstrap has failed if postgres is not running
+        if not self.state_handler.is_running() or self._post_bootstrap_task.result is False:
+            self.cancel_initialization()
+
+        if self._post_bootstrap_task.result is None:
+            if not self.state_handler.is_leader():
+                return 'waiting for end of recovery after bootstrap'
+
+            self.state_handler.set_role('master')
+            self._async_executor.schedule('post_bootstrap')
+            self._async_executor.run_async(self.state_handler.post_bootstrap,
+                                           args=(self.patroni.config['bootstrap'], self._post_bootstrap_task))
+            return 'running post_bootstrap'
+
+        self.state_handler.bootstrapping = False
+        self.dcs.set_config_value(json.dumps(self.patroni.config.dynamic_configuration, separators=(',', ':')))
+        if not self.watchdog.activate():
+            logger.error('Cancelling bootstrap because watchdog activation failed')
+            self.cancel_initialization()
+        self.dcs.take_leader()
+        self.state_handler.call_nowait(ACTION_ON_START)
+        self.load_cluster_from_dcs()
+        return 'initialized a new cluster'
+
     def handle_starting_instance(self):
         """Starting up PostgreSQL may take a long time. In case we are the leader we may want to
         fail over to."""
@@ -823,7 +921,7 @@ class Ha(object):
         if self.has_lock():
             if not self.update_lock():
                 logger.info("Lost lock while starting up. Demoting self.")
-                self.demote('immediate')
+                self.demote('immediate-nolock')
                 return 'stopped PostgreSQL while starting up because leader key was lost'
 
             timeout = self._start_timeout or self.patroni.config['master_start_timeout']
@@ -858,6 +956,9 @@ class Ha(object):
         try:
             self.load_cluster_from_dcs()
 
+            if self.is_paused():
+                self.watchdog.disable()
+
             if not self.cluster.has_member(self.state_handler.name):
                 self.touch_member()
 
@@ -877,6 +978,9 @@ class Ha(object):
                 return msg
 
             # we've got here, so any async action has finished.
+            if self.state_handler.bootstrapping:
+                return self.post_bootstrap()
+
             if self.recovering and not self.state_handler.need_rewind:
                 self.recovering = False
                 # Check if we tried to recover and failed
@@ -886,6 +990,14 @@ class Ha(object):
 
             # is data directory empty?
             if self.state_handler.data_directory_empty():
+                # In case datadir went away while we were master. TODO: check for this and try to stop postgresql.
+                self.watchdog.disable()
+
+                # is this instance the leader?
+                if self.has_lock():
+                    self.release_leader_key_voluntarily()
+                    return 'released leader key voluntarily as data dir empty and currently leader'
+
                 return self.bootstrap()  # new node
             # "bootstrap", but data directory is not empty
             elif not self.sysid_valid(self.cluster.initialize) and self.cluster.is_unlocked() and not self.is_paused():
@@ -903,7 +1015,8 @@ class Ha(object):
                         self.dcs.delete_leader()
                         self.dcs.reset_cluster()
                         return 'removed leader lock because postgres is not running'
-                    elif not (self.state_handler.need_rewind and self.state_handler.can_rewind):
+                    elif not (self.state_handler.rewind_executed or
+                              self.state_handler.need_rewind and self.state_handler.can_rewind):
                         return 'postgres is not running'
 
                 # try to start dead postgres
@@ -921,6 +1034,8 @@ class Ha(object):
                 # asynchronous processes are running (should be always the case for the master)
                 if not self._async_executor.busy and not self.state_handler.is_starting():
                     if not self.state_handler.cb_called:
+                        if not self.state_handler.is_leader():
+                            self.state_handler.trigger_check_diverged_lsn()
                         self.state_handler.call_nowait(ACTION_ON_START)
                     self.state_handler.sync_replication_slots(self.cluster)
         except DCSError:
@@ -941,6 +1056,26 @@ class Ha(object):
             info = self._run_cycle()
             return (self.is_paused() and 'PAUSE: ' or '') + info
 
+    def shutdown(self):
+        if self.is_paused():
+            logger.info('Leader key is not deleted and Postgresql is not stopped due paused state')
+            self.watchdog.disable()
+        else:
+            # FIXME: If stop doesn't reach safepoint quickly enough keepalive is triggered. If shutdown checkpoint
+            # takes longer than ttl, then leader key is lost and replication might not have sent out all xlog.
+            # This might not be the desired behavior of users, as a graceful shutdown of the host can mean lost data.
+            # We probably need to something smarter here.
+            disable_wd = self.watchdog.disable if self.watchdog.is_running else None
+            self.while_not_sync_standby(lambda: self.state_handler.stop(checkpoint=False, on_safepoint=disable_wd))
+            if not self.state_handler.is_running():
+                self.dcs.delete_leader()
+            else:
+                # XXX: what about when Patroni is started as the wrong user that has access to the watchdog device
+                # but cannot shut down PostgreSQL. Root would be the obvious example. Would be nice to not kill the
+                # system due to a bad config.
+                logger.error("PostgreSQL shutdown failed, leader key not removed." +
+                             (" Leaving watchdog running." if self.watchdog.is_running else ""))
+
     def watch(self, timeout):
         cluster = self.cluster
         # watch on leader key changes if the postgres is running and leader is known and current node is not lock owner

patroni/scripts/aws.py

@@ -6,63 +6,64 @@ from requests.exceptions import RequestException
 import sys
 import boto.ec2
 
+from patroni.utils import Retry, RetryFailedError
+
 logger = logging.getLogger(__name__)
 
 
 class AWSConnection(object):
+
     def __init__(self, cluster_name):
         self.available = False
         self.cluster_name = cluster_name if cluster_name is not None else 'unknown'
+        self._retry = Retry(deadline=300, max_delay=30, max_tries=-1, retry_exceptions=(boto.exception.StandardError,))
         try:
             # get the instance id
-            r = requests.get('http://169.254.169.254/latest/dynamic/instance-identity/document', timeout=0.1)
+            r = requests.get('http://169.254.169.254/latest/dynamic/instance-identity/document', timeout=2.1)
         except RequestException:
-            logger.info("cannot query AWS meta-data")
+            logger.error('cannot query AWS meta-data')
             return
+
         if r.ok:
             try:
                 content = r.json()
                 self.instance_id = content['instanceId']
                 self.region = content['region']
-            except Exception as e:
-                logger.info('unable to fetch instance id and region from AWS meta-data: {}'.format(e))
+            except Exception:
+                logger.exception('unable to fetch instance id and region from AWS meta-data')
                 return
             self.available = True
 
+    def retry(self, *args, **kwargs):
+        return self._retry.copy()(*args, **kwargs)
+
     def aws_available(self):
         return self.available
 
-    def _tag_ebs(self, role):
+    def _tag_ebs(self, conn, role):
         """ set tags, carrying the cluster name, instance role and instance id for the EBS storage """
-        if not self.available:
-            return False
-
         tags = {'Name': 'spilo_' + self.cluster_name, 'Role': role, 'Instance': self.instance_id}
-        try:
-            conn = boto.ec2.connect_to_region(self.region)
-            volumes = conn.get_all_volumes(filters={'attachment.instance-id': self.instance_id})
-            conn.create_tags([v.id for v in volumes], tags)
-        except Exception as e:
-            logger.info('could not set tags for EBS storage devices attached: {}'.format(e))
-            return False
-        return True
+        volumes = conn.get_all_volumes(filters={'attachment.instance-id': self.instance_id})
+        conn.create_tags([v.id for v in volumes], tags)
 
-    def _tag_ec2(self, role):
+    def _tag_ec2(self, conn, role):
         """ tag the current EC2 instance with a cluster role """
-        if not self.available:
-            return False
         tags = {'Role': role}
-        try:
-            conn = boto.ec2.connect_to_region(self.region)
-            conn.create_tags([self.instance_id], tags)
-        except Exception as e:
-            logger.info("could not set tags for EC2 instance %s: %s", self.instance_id, e)
-            return False
-        return True
+        conn.create_tags([self.instance_id], tags)
 
     def on_role_change(self, new_role):
-        ret = self._tag_ec2(new_role)
-        return self._tag_ebs(new_role) and ret
+        if not self.available:
+            return False
+        try:
+            conn = self.retry(boto.ec2.connect_to_region, self.region)
+            self.retry(self._tag_ec2, conn, new_role)
+            self.retry(self._tag_ebs, conn, new_role)
+        except RetryFailedError:
+            logger.warning("Unable to communicate to AWS "
+                           "when setting tags for the EC2 instance {0} "
+                           "and attached EBS volumes".format(self.instance_id))
+            return False
+        return True
 
 
 def main():

patroni/scripts/wale_restore.py

@@ -23,19 +23,29 @@
 # currently also requires that you configure the restore_command to use wal_e, example:
 #       recovery_conf:
 #               restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1
-
-from collections import namedtuple
+import argparse
+import csv
 import logging
 import os
 import psycopg2
 import subprocess
 import sys
 import time
-import argparse
+
+from collections import namedtuple
 
 logger = logging.getLogger(__name__)
 
 RETRY_SLEEP_INTERVAL = 1
+si_prefixes = ['K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']
+
+
+# Meaningful names to the exit codes used by WALERestore
+ExitCode = type('Enum', (), {
+    'SUCCESS': 0,  #: Succeeded
+    'RETRY_LATER': 1,  #: External issue, retry later
+    'FAIL': 2  #: Don't try again unless configuration changes
+})
 
 
 # We need to know the current PG version in order to figure out the correct WAL directory name
@@ -50,66 +60,141 @@ def get_major_version(data_dir):
     return 0.0
 
 
+def repr_size(n_bytes):
+    """
+    >>> repr_size(1000)
+    '1000 Bytes'
+    >>> repr_size(8257332324597)
+    '7.5 TiB'
+    """
+    if n_bytes < 1024:
+        return '{0} Bytes'.format(n_bytes)
+    i = -1
+    while n_bytes > 1023:
+        n_bytes /= 1024.0
+        i += 1
+    return '{0} {1}iB'.format(round(n_bytes, 1), si_prefixes[i])
+
+
+def size_as_bytes(size_, prefix):
+    """
+    >>> size_as_bytes(7.5, 'T')
+    8246337208320
+    """
+    prefix = prefix.upper()
+
+    assert prefix in si_prefixes
+
+    exponent = si_prefixes.index(prefix) + 1
+
+    return int(size_ * (1024.0 ** exponent))
+
+
+WALEConfig = namedtuple(
+    'WALEConfig',
+    [
+        'env_dir',
+        'threshold_mb',
+        'threshold_pct',
+        'cmd',
+    ]
+)
+
+
 class WALERestore(object):
-    def __init__(self, scope, datadir, connstring, env_dir, threshold_mb, threshold_pct, use_iam, no_master, retries):
+    def __init__(self, scope, datadir, connstring, env_dir, threshold_mb,
+                 threshold_pct, use_iam, no_master, retries):
         self.scope = scope
         self.master_connection = connstring
         self.data_dir = datadir
-        self.wal_e = namedtuple('wale', 'dir,threshold_mb,threshold_pct,iam_string,cmd')
-        self.wal_e.dir = env_dir
-        self.wal_e.threshold_mb = threshold_mb
-        self.wal_e.threshold_pct = threshold_pct
-        self.wal_e.iam_string = ' --aws-instance-profile ' if use_iam == 1 else ''
         self.no_master = no_master
-        self.wal_e.cmd = 'envdir {0} wal-e {1} '.format(self.wal_e.dir, self.wal_e.iam_string)
-        self.init_error = (not os.path.exists(self.wal_e.dir))
+
+        wale_cmd = [
+            'envdir',
+            env_dir,
+            'wal-e',
+        ]
+
+        if use_iam == 1:
+            wale_cmd += ['--aws-instance-profile']
+
+        self.wal_e = WALEConfig(
+            env_dir=env_dir,
+            threshold_mb=threshold_mb,
+            threshold_pct=threshold_pct,
+            cmd=wale_cmd,
+        )
+
+        self.init_error = (not os.path.exists(self.wal_e.env_dir))
         self.retries = retries
 
     def run(self):
-        """ creates a new replica using WAL-E """
-        if not self.init_error:
-            try:
-                ret = self.should_use_s3_to_create_replica()
-                if ret:
-                    return self.create_replica_with_s3()
-                elif ret is None:  # caught an exception, need to retry
-                    return 1
-            except Exception:
-                logger.exception("Exception when running WAL-E restore")
-        return 2
+        """
+        Creates a new replica using WAL-E
+
+        Returns
+        -------
+        ExitCode
+            0 = Success
+            1 = Error, try again
+            2 = Error, don't try again
+
+        """
+        if self.init_error:
+            logger.error('init error: %r did not exist at initialization time',
+                         self.wal_e.env_dir)
+            return ExitCode.FAIL
+
+        try:
+            should_use_s3 = self.should_use_s3_to_create_replica()
+            if should_use_s3 is None:  # Need to retry
+                return ExitCode.RETRY_LATER
+            elif should_use_s3:
+                return self.create_replica_with_s3()
+            elif not should_use_s3:
+                return ExitCode.FAIL
+        except Exception:
+            logger.exception("Unhandled exception when running WAL-E restore")
+        return ExitCode.FAIL
 
     def should_use_s3_to_create_replica(self):
         """ determine whether it makes sense to use S3 and not pg_basebackup """
 
         threshold_megabytes = self.wal_e.threshold_mb
-        threshold_backup_size_percentage = self.wal_e.threshold_pct
+        threshold_percent = self.wal_e.threshold_pct
 
         try:
-            latest_backup = subprocess.check_output(self.wal_e.cmd.split() + ['backup-list', '--detail', 'LATEST'])
-            # name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start
-            #                                                   wal_segment_backup_stop wal_segment_offset_backup_stop
-            # base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z
-            # 20310671    00000001000000000000007F    00000040
-            # 00000001000000000000007F    00000240
-            backup_strings = latest_backup.decode('utf-8').splitlines() if latest_backup else ()
-            if len(backup_strings) != 2:
+            cmd = self.wal_e.cmd + ['backup-list', '--detail', 'LATEST']
+
+            logger.debug('calling %r', cmd)
+            wale_output = subprocess.check_output(cmd)
+
+            reader = csv.DictReader(wale_output.decode('utf-8').splitlines(),
+                                    dialect='excel-tab')
+            rows = list(reader)
+            if not len(rows):
+                logger.warning('wal-e did not find any backups')
                 return False
 
-            names = backup_strings[0].split()
-            vals = backup_strings[1].split()
-            if (len(names) != len(vals)) or (len(names) != 7):
+            # This check might not add much, it was performed in the previous
+            # version of this code. since the old version rolled CSV parsing the
+            # check may have been part of the CSV parsing.
+            if len(rows) > 1:
+                logger.warning(
+                    'wal-e returned more than one row of backups: %r',
+                    rows)
                 return False
 
-            backup_info = dict(zip(names, vals))
+            backup_info = rows[0]
         except subprocess.CalledProcessError:
             logger.exception("could not query wal-e latest backup")
             return None
 
         try:
-            backup_size = backup_info['expanded_size_bytes']
+            backup_size = int(backup_info['expanded_size_bytes'])
             backup_start_segment = backup_info['wal_segment_backup_start']
             backup_start_offset = backup_info['wal_segment_offset_backup_start']
-        except Exception:
+        except KeyError:
             logger.exception("unable to get some of WALE backup parameters")
             return None
 
@@ -124,24 +209,29 @@ class WALERestore(object):
         # construct the LSN from the segment and offset
         backup_start_lsn = '{0}/{1}'.format(lsn_segment, lsn_offset)
 
-        diff_in_bytes = int(backup_size)
+        diff_in_bytes = backup_size
         attempts_no = 0
         while True:
             if self.master_connection:
                 try:
                     # get the difference in bytes between the current WAL location and the backup start offset
                     with psycopg2.connect(self.master_connection) as con:
+                        if con.server_version >= 100000:
+                            wal_name = 'wal'
+                            lsn_name = 'lsn'
+                        else:
+                            wal_name = 'xlog'
+                            lsn_name = 'location'
                         con.autocommit = True
                         with con.cursor() as cur:
                             cur.execute("""SELECT CASE WHEN pg_is_in_recovery()
                                                        THEN GREATEST(
-                                                                pg_xlog_location_diff(COALESCE(
-                                                                  pg_last_xlog_receive_location(), '0/0'), %s)::bigint,
-                                                                pg_xlog_location_diff(
-                                                                  pg_last_xlog_replay_location(), %s)::bigint)
-                                                       ELSE pg_xlog_location_diff(
-                                                                pg_current_xlog_location(), %s)::bigint
-                                                   END""", (backup_start_lsn, backup_start_lsn, backup_start_lsn))
+                                                                pg_{0}_{1}_diff(COALESCE(
+                                                                  pg_last_{0}_receive_{1}(), '0/0'), %s)::bigint,
+                                                                pg_{0}_{1}_diff(pg_last_{0}_replay_{1}(), %s)::bigint)
+                                                       ELSE pg_{0}_{1}_diff(pg_current_{0}_{1}(), %s)::bigint
+                                                   END""".format(wal_name, lsn_name),
+                                        (backup_start_lsn, backup_start_lsn, backup_start_lsn))
 
                             diff_in_bytes = int(cur.fetchone()[0])
                 except psycopg2.Error:
@@ -163,8 +253,47 @@ class WALERestore(object):
 
         # if the size of the accumulated WAL segments is more than a certan percentage of the backup size
         # or exceeds the pre-determined size - pg_basebackup is chosen instead.
-        return (diff_in_bytes < int(threshold_megabytes) * 1048576) and\
-            (diff_in_bytes < int(backup_size) * float(threshold_backup_size_percentage) / 100)
+        is_size_thresh_ok = diff_in_bytes < int(threshold_megabytes) * 1048576
+        threshold_pct_bytes = backup_size * threshold_percent / 100.0
+        is_percentage_thresh_ok = float(diff_in_bytes) < int(threshold_pct_bytes)
+        are_thresholds_ok = is_size_thresh_ok and is_percentage_thresh_ok
+
+        class Size(object):
+            def __init__(self, n_bytes, prefix=None):
+                self.n_bytes = n_bytes
+                self.prefix = prefix
+
+            def __repr__(self):
+                if self.prefix is not None:
+                    n_bytes = size_as_bytes(self.n_bytes, self.prefix)
+                else:
+                    n_bytes = self.n_bytes
+                return repr_size(n_bytes)
+
+        class HumanContext(object):
+            def __init__(self, items):
+                self.items = items
+
+            def __repr__(self):
+                return ', '.join('{}={!r}'.format(key, value)
+                                 for key, value in self.items)
+
+        human_context = repr(HumanContext([
+            ('threshold_size', Size(threshold_megabytes, 'M')),
+            ('threshold_percent', threshold_percent),
+            ('threshold_percent_size', Size(threshold_pct_bytes)),
+            ('backup_size', Size(backup_size)),
+            ('backup_diff', Size(diff_in_bytes)),
+            ('is_size_thresh_ok', is_size_thresh_ok),
+            ('is_percentage_thresh_ok', is_percentage_thresh_ok),
+        ]))
+
+        if not are_thresholds_ok:
+            logger.info('wal-e backup size diff is over threshold, falling back '
+                        'to other means of restore: %s', human_context)
+        else:
+            logger.info('Thresholds are OK, using wal-e basebackup: %s', human_context)
+        return are_thresholds_ok
 
     def fix_subdirectory_path_if_broken(self, dirname):
         # in case it is a symlink pointing to a non-existing location, remove it and create the actual directory
@@ -187,15 +316,19 @@ class WALERestore(object):
     def create_replica_with_s3(self):
         # if we're set up, restore the replica using fetch latest
         try:
-            ret = subprocess.call(self.wal_e.cmd.split() + ['backup-fetch', '{}'.format(self.data_dir), 'LATEST'])
+            cmd = self.wal_e.cmd + ['backup-fetch',
+                                    '{}'.format(self.data_dir),
+                                    'LATEST']
+            logger.debug('calling: %r', cmd)
+            exit_code = subprocess.call(cmd)
         except Exception as e:
             logger.error('Error when fetching backup with WAL-E: {0}'.format(e))
-            return 1
+            return ExitCode.RETRY_LATER
 
-        if (ret == 0 and not
-           self.fix_subdirectory_path_if_broken('pg_xlog' if get_major_version(self.data_dir) < 10.0 else 'pg_wal')):
-            return 2
-        return ret
+        if (exit_code == 0 and not
+           self.fix_subdirectory_path_if_broken('pg_xlog' if get_major_version(self.data_dir) < 10 else 'pg_wal')):
+            return ExitCode.FAIL
+        return exit_code
 
 
 def main():
@@ -213,6 +346,9 @@ def main():
     parser.add_argument('--no_master', type=int, default=0)
     args = parser.parse_args()
 
+    exit_code = None
+    assert args.retries >= 0
+
     # Retry cloning in a loop. We do separate retries for the master
     # connection attempt inside should_use_s3_to_create_replica,
     # because we need to differentiate between the last attempt and
@@ -223,12 +359,13 @@ def main():
                               env_dir=args.envdir, threshold_mb=args.threshold_megabytes,
                               threshold_pct=args.threshold_backup_size_percentage, use_iam=args.use_iam,
                               no_master=args.no_master, retries=args.retries)
-        ret = restore.run()
-        if ret != 1:  # only WAL-E failures lead to the retry
+        exit_code = restore.run()
+        if not exit_code == ExitCode.RETRY_LATER:  # only WAL-E failures lead to the retry
+            logger.debug('exit_code is %r, not retrying', exit_code)
             break
         time.sleep(RETRY_SLEEP_INTERVAL)
 
-    return ret
+    return exit_code
 
 
 if __name__ == '__main__':

patroni/utils.py

@@ -1,3 +1,4 @@
+import contextlib
 import random
 import time
 import re
@@ -280,3 +281,8 @@ def polling_loop(timeout, interval=1):
         yield iteration
         iteration += 1
         time.sleep(interval)
+
+
+@contextlib.contextmanager
+def null_context():
+    yield

patroni/version.py

@@ -1 +1 @@
-__version__ = '1.2.4'
+__version__ = '1.3.3'

patroni/watchdog/__init__.py

@@ -0,0 +1,2 @@
+from patroni.watchdog.base import WatchdogError, Watchdog
+__all__ = ['WatchdogError', 'Watchdog']

patroni/watchdog/base.py

@@ -0,0 +1,313 @@
+import abc
+import logging
+import platform
+import six
+import sys
+from threading import RLock
+
+from patroni.exceptions import WatchdogError
+
+__all__ = ['WatchdogError', 'Watchdog']
+
+logger = logging.getLogger(__name__)
+
+MODE_REQUIRED = 'required'    # Will not run if a watchdog is not available
+MODE_AUTOMATIC = 'automatic'  # Will use a watchdog if one is available
+MODE_OFF = 'off'              # Will not try to use a watchdog
+
+
+def parse_mode(mode):
+    if mode is False:
+        return MODE_OFF
+    mode = mode.lower()
+    if mode in ['require', 'required']:
+        return MODE_REQUIRED
+    elif mode in ['auto', 'automatic']:
+        return MODE_AUTOMATIC
+    else:
+        if mode not in ['off', 'disable', 'disabled']:
+            logger.warning("Watchdog mode {0} not recognized, disabling watchdog".format(mode))
+        return MODE_OFF
+
+
+def synchronized(func):
+    def wrapped(self, *args, **kwargs):
+        with self._lock:
+            return func(self, *args, **kwargs)
+    return wrapped
+
+
+class WatchdogConfig(object):
+    """Helper to contain a snapshot of configuration"""
+    def __init__(self, config):
+        self.mode = parse_mode(config['watchdog'].get('mode', 'automatic'))
+        self.ttl = config['ttl']
+        self.loop_wait = config['loop_wait']
+        self.safety_margin = config['watchdog'].get('safety_margin', 5)
+        self.driver = config['watchdog'].get('driver', 'default')
+        self.driver_config = dict((k, v) for k, v in config['watchdog'].items()
+                                  if k not in ['mode', 'safety_margin', 'driver'])
+
+    def __eq__(self, other):
+        return isinstance(other, WatchdogConfig) and \
+            all(getattr(self, attr) == getattr(other, attr) for attr in
+                ['mode', 'ttl', 'loop_wait', 'safety_margin', 'driver', 'driver_config'])
+
+    def __ne__(self, other):
+        return not self == other
+
+    def get_impl(self):
+        if self.driver == 'testing':
+            from patroni.watchdog.linux import TestingWatchdogDevice
+            return TestingWatchdogDevice.from_config(self.driver_config)
+        elif platform.system() == 'Linux' and self.driver == 'default':
+            from patroni.watchdog.linux import LinuxWatchdogDevice
+            return LinuxWatchdogDevice.from_config(self.driver_config)
+        else:
+            return NullWatchdog()
+
+    @property
+    def timeout(self):
+        if self.safety_margin == -1:
+            return int(self.ttl // 2)
+        else:
+            return self.ttl - self.safety_margin
+
+    @property
+    def timing_slack(self):
+        return self.timeout - self.loop_wait
+
+
+class Watchdog(object):
+    """Facade to dynamically manage watchdog implementations and handle config changes.
+
+    When activation fails underlying implementation will be switched to a Null implementation. To avoid log spam
+    activation will only be retried when watchdog configuration is changed."""
+    def __init__(self, config):
+        self.active_config = self.config = WatchdogConfig(config)
+        self._lock = RLock()
+        self.active = False
+
+        if self.config.mode == MODE_OFF:
+            self.impl = NullWatchdog()
+        else:
+            self.impl = self.config.get_impl()
+            if self.config.mode == MODE_REQUIRED and self.impl.is_null:
+                logger.error("Configuration requires a watchdog, but watchdog is not supported on this platform.")
+                sys.exit(1)
+
+    @synchronized
+    def reload_config(self, config):
+        self.config = WatchdogConfig(config)
+        # Turning a watchdog off can always be done immediately
+        if self.config.mode == MODE_OFF:
+            if self.active:
+                self._disable()
+            self.active_config = self.config
+            self.impl = NullWatchdog()
+        # If watchdog is not active we can apply config immediately to show any warnings early. Otherwise we need to
+        # delay until next time a keepalive is sent so timeout matches up with leader key update.
+        if not self.active:
+            if self.config.driver != self.active_config.driver or \
+               self.config.driver_config != self.active_config.driver_config:
+                self.impl = self.config.get_impl()
+            self.active_config = self.config
+
+    @synchronized
+    def activate(self):
+        """Activates the watchdog device with suitable timeouts. While watchdog is active keepalive needs
+        to be called every time loop_wait expires.
+
+        :returns False if a safe watchdog could not be configured, but is required.
+        """
+        self.active = True
+        return self._activate()
+
+    def _activate(self):
+        self.active_config = self.config
+
+        if self.config.timing_slack < 0:
+            logger.warning('Watchdog not supported because leader TTL {0} is less than 2x loop_wait {1}'
+                           .format(self.config.ttl, self.config.loop_wait))
+            self.impl = NullWatchdog()
+
+        try:
+            self.impl.open()
+        except WatchdogError as e:
+            logger.warning("Could not activate %s: %s", self.impl.describe(), e)
+            self.impl = NullWatchdog()
+
+        if self.impl.is_running and not self.impl.can_be_disabled:
+            logger.warning("Watchdog implementation can't be disabled."
+                           " Watchdog will trigger after Patroni loses leader key.")
+
+        actual_timeout = self._set_timeout()
+
+        if not self.impl.is_running or actual_timeout > self.config.timeout:
+            if self.config.mode == MODE_REQUIRED:
+                if self.impl.is_null:
+                    logger.error("Configuration requires watchdog, but watchdog could not be configured.")
+                else:
+                    logger.error("Configuration requires watchdog, but a safe watchdog timeout {0} could"
+                                 " not be configured. Watchdog timeout is {1}.".format(
+                                     self.config.timeout, actual_timeout))
+                return False
+            else:
+                if not self.impl.is_null:
+                    logger.warning("Watchdog timeout {0} seconds does not ensure safe termination within {1} seconds"
+                                   .format(actual_timeout, self.config.timeout))
+
+        if self.is_running:
+            logger.info("{0} activated with {1} second timeout, timing slack {2} seconds"
+                        .format(self.impl.describe(), actual_timeout, self.config.timing_slack))
+        else:
+            if self.config.mode == MODE_REQUIRED:
+                logger.error("Configuration requires watchdog, but watchdog could not be activated")
+                return False
+
+        return True
+
+    def _set_timeout(self):
+        if self.impl.has_set_timeout():
+            self.impl.set_timeout(self.config.timeout)
+
+        # Safety checks for watchdog implementations that don't support configurable timeouts
+        actual_timeout = self.impl.get_timeout()
+        if self.impl.is_running and actual_timeout < self.config.loop_wait:
+            logger.error('loop_wait of {0} seconds is too long for watchdog {1} second timeout'
+                         .format(self.config.loop_wait, actual_timeout))
+            if self.impl.can_be_disabled:
+                logger.info('Disabling watchdog due to unsafe timeout.')
+                self.impl.close()
+                self.impl = NullWatchdog()
+                return None
+        return actual_timeout
+
+    @synchronized
+    def disable(self):
+        self._disable()
+        self.active = False
+
+    def _disable(self):
+        try:
+            if self.impl.is_running and not self.impl.can_be_disabled:
+                # Give sysadmin some extra time to clean stuff up.
+                self.impl.keepalive()
+                logger.warning("Watchdog implementation can't be disabled. System will reboot after "
+                               "{0} seconds when watchdog times out.".format(self.impl.get_timeout()))
+            self.impl.close()
+        except WatchdogError as e:
+            logger.error("Error while disabling watchdog: %s", e)
+
+    @synchronized
+    def keepalive(self):
+        try:
+            self.impl.keepalive()
+            # In case there are any pending configuration changes apply them now.
+            if self.active and self.config != self.active_config:
+                if self.config.mode != MODE_OFF and self.active_config.mode == MODE_OFF:
+                    self.impl = self.config.get_impl()
+                    self._activate()
+                if self.config.driver != self.active_config.driver \
+                   or self.config.driver_config != self.active_config.driver_config:
+                    self._disable()
+                    self.impl = self.config.get_impl()
+                    self._activate()
+                if self.config.timeout != self.active_config.timeout:
+                    self.impl.set_timeout(self.config.timeout)
+        except WatchdogError as e:
+            logger.error("Error while sending keepalive: %s", e)
+
+    @property
+    @synchronized
+    def is_running(self):
+        return self.impl.is_running
+
+    @property
+    @synchronized
+    def is_healthy(self):
+        if self.config.mode != MODE_REQUIRED:
+            return True
+        return self.config.timing_slack >= 0 and self.impl.is_healthy
+
+
+@six.add_metaclass(abc.ABCMeta)
+class WatchdogBase(object):
+    """A watchdog object when opened requires periodic calls to keepalive.
+    When keepalive is not called within a timeout the system will be terminated."""
+    is_null = False
+
+    @property
+    def is_running(self):
+        """Returns True when watchdog is activated and capable of performing it's task."""
+        return False
+
+    @property
+    def is_healthy(self):
+        """Returns False when calling open() is known to fail."""
+        return False
+
+    @property
+    def can_be_disabled(self):
+        """Returns True when watchdog will be disabled by calling close(). Some watchdog devices
+        will keep running no matter what once activated. May raise WatchdogError if called without
+        calling open() first."""
+        return True
+
+    @abc.abstractmethod
+    def open(self):
+        """Open watchdog device.
+
+        When watchdog is opened keepalive must be called. Returns nothing on success
+        or raises WatchdogError if the device could not be opened."""
+
+    @abc.abstractmethod
+    def close(self):
+        """Gracefully close watchdog device."""
+
+    @abc.abstractmethod
+    def keepalive(self):
+        """Resets the watchdog timer.
+
+        Watchdog must be open when keepalive is called."""
+
+    @abc.abstractmethod
+    def get_timeout(self):
+        """Returns the current keepalive timeout in effect."""
+
+    @staticmethod
+    def has_set_timeout():
+        """Returns True if setting a timeout is supported."""
+        return False
+
+    def set_timeout(self, timeout):
+        """Set the watchdog timer timeout.
+
+        :param timeout: watchdog timeout in seconds"""
+        raise WatchdogError("Setting timeout is not supported on {0}".format(self.describe()))
+
+    def describe(self):
+        """Human readable name for this device"""
+        return self.__class__.__name__
+
+    @classmethod
+    def from_config(cls, config):
+        return cls()
+
+
+class NullWatchdog(WatchdogBase):
+    """Null implementation when watchdog is not supported."""
+    is_null = True
+
+    def open(self):
+        return
+
+    def close(self):
+        return
+
+    def keepalive(self):
+        return
+
+    def get_timeout(self):
+        # A big enough number to not matter
+        return 1000000000

patroni/watchdog/linux.py

@@ -0,0 +1,224 @@
+import collections
+import ctypes
+import fcntl
+import os
+import platform
+from patroni.watchdog.base import WatchdogBase, WatchdogError
+
+# Pythonification of linux/ioctl.h
+IOC_NONE = 0
+IOC_WRITE = 1
+IOC_READ = 2
+
+IOC_NRBITS = 8
+IOC_TYPEBITS = 8
+IOC_SIZEBITS = 14
+IOC_DIRBITS = 2
+
+# Non-generic platform special cases
+machine = platform.machine()
+if machine in ['mips', 'sparc', 'powerpc', 'ppc64']:
+    IOC_SIZEBITS = 13
+    IOC_DIRBITS = 3
+    IOC_NONE, IOC_WRITE, IOC_READ = 1, 2, 4
+elif machine == 'parisc':
+    IOC_WRITE, IOC_READ = 2, 1
+
+IOC_NRSHIFT = 0
+IOC_TYPESHIFT = IOC_NRSHIFT + IOC_NRBITS
+IOC_SIZESHIFT = IOC_TYPESHIFT + IOC_TYPEBITS
+IOC_DIRSHIFT = IOC_SIZESHIFT + IOC_SIZEBITS
+
+
+def IOW(type_, nr, size):
+    return IOC(IOC_WRITE, type_, nr, size)
+
+
+def IOR(type_, nr, size):
+    return IOC(IOC_READ, type_, nr, size)
+
+
+def IOWR(type_, nr, size):
+    return IOC(IOC_READ | IOC_WRITE, type_, nr, size)
+
+
+def IOC(dir_, type_, nr, size):
+    return (dir_ << IOC_DIRSHIFT) \
+        | (ord(type_) << IOC_TYPESHIFT) \
+        | (nr << IOC_NRSHIFT) \
+        | (size << IOC_SIZESHIFT)
+
+
+# Pythonification of linux/watchdog.h
+
+WATCHDOG_IOCTL_BASE = 'W'
+
+
+class watchdog_info(ctypes.Structure):
+    _fields_ = [
+        ('options', ctypes.c_uint32),           # Options the card/driver supports
+        ('firmware_version', ctypes.c_uint32),  # Firmware version of the card
+        ('identity', ctypes.c_uint8 * 32),      # Identity of the board
+    ]
+
+
+struct_watchdog_info_size = ctypes.sizeof(watchdog_info)
+int_size = ctypes.sizeof(ctypes.c_int)
+
+WDIOC_GETSUPPORT = IOR(WATCHDOG_IOCTL_BASE, 0, struct_watchdog_info_size)
+WDIOC_GETSTATUS = IOR(WATCHDOG_IOCTL_BASE, 1, int_size)
+WDIOC_GETBOOTSTATUS = IOR(WATCHDOG_IOCTL_BASE, 2, int_size)
+WDIOC_GETTEMP = IOR(WATCHDOG_IOCTL_BASE, 3, int_size)
+WDIOC_SETOPTIONS = IOR(WATCHDOG_IOCTL_BASE, 4, int_size)
+WDIOC_KEEPALIVE = IOR(WATCHDOG_IOCTL_BASE, 5, int_size)
+WDIOC_SETTIMEOUT = IOWR(WATCHDOG_IOCTL_BASE, 6, int_size)
+WDIOC_GETTIMEOUT = IOR(WATCHDOG_IOCTL_BASE, 7, int_size)
+WDIOC_SETPRETIMEOUT = IOWR(WATCHDOG_IOCTL_BASE, 8, int_size)
+WDIOC_GETPRETIMEOUT = IOR(WATCHDOG_IOCTL_BASE, 9, int_size)
+WDIOC_GETTIMELEFT = IOR(WATCHDOG_IOCTL_BASE, 10, int_size)
+
+
+WDIOF_UNKNOWN = -1  # Unknown flag error
+WDIOS_UNKNOWN = -1  # Unknown status error
+
+WDIOF = {
+    "OVERHEAT": 0x0001,       # Reset due to CPU overheat
+    "FANFAULT": 0x0002,       # Fan failed
+    "EXTERN1": 0x0004,        # External relay 1
+    "EXTERN2": 0x0008,        # External relay 2
+    "POWERUNDER": 0x0010,     # Power bad/power fault
+    "CARDRESET": 0x0020,      # Card previously reset the CPU
+    "POWEROVER": 0x0040,      # Power over voltage
+    "SETTIMEOUT": 0x0080,     # Set timeout (in seconds)
+    "MAGICCLOSE": 0x0100,     # Supports magic close char
+    "PRETIMEOUT": 0x0200,     # Pretimeout (in seconds), get/set
+    "ALARMONLY": 0x0400,      # Watchdog triggers a management or other external alarm not a reboot
+    "KEEPALIVEPING": 0x8000,  # Keep alive ping reply
+}
+
+WDIOS = {
+    "DISABLECARD": 0x0001,    # Turn off the watchdog timer
+    "ENABLECARD": 0x0002,     # Turn on the watchdog timer
+    "TEMPPANIC": 0x0004,      # Kernel panic on temperature trip
+}
+
+# Implementation
+
+
+class WatchdogInfo(collections.namedtuple('WatchdogInfo', 'options,version,identity')):
+    """Watchdog descriptor from the kernel"""
+    def __getattr__(self, name):
+        """Convenience has_XYZ attributes for checking WDIOF bits in options"""
+        if name.startswith('has_') and name[4:] in WDIOF:
+            return bool(self.options & WDIOF[name[4:]])
+
+        raise AttributeError("WatchdogInfo instance has no attribute '{0}'".format(name))
+
+
+class LinuxWatchdogDevice(WatchdogBase):
+    DEFAULT_DEVICE = '/dev/watchdog'
+
+    def __init__(self, device):
+        self.device = device
+        self._support_cache = None
+        self._fd = None
+
+    @classmethod
+    def from_config(cls, config):
+        device = config.get('device', cls.DEFAULT_DEVICE)
+        return cls(device)
+
+    @property
+    def is_running(self):
+        return self._fd is not None
+
+    @property
+    def is_healthy(self):
+        return os.path.exists(self.device) and os.access(self.device, os.W_OK)
+
+    def open(self):
+        try:
+            self._fd = os.open(self.device, os.O_WRONLY)
+        except OSError as e:
+            raise WatchdogError("Can't open watchdog device: {0}".format(e))
+
+    def close(self):
+        if self.is_running:
+            try:
+                os.write(self._fd, b'V')
+                os.close(self._fd)
+                self._fd = None
+            except OSError as e:
+                raise WatchdogError("Error while closing {0}: {1}".format(self.describe(), e))
+
+    @property
+    def can_be_disabled(self):
+        return self.get_support().has_MAGICCLOSE
+
+    def _ioctl(self, func, arg, mutate_arg=False):
+        if self._fd is None:
+            raise WatchdogError("Watchdog device is closed")
+
+        result = fcntl.ioctl(self._fd, func, arg, mutate_arg)
+        if result < 0:
+            raise IOError(result)
+
+    def get_support(self):
+        if self._support_cache is None:
+            info = watchdog_info()
+            self._ioctl(WDIOC_GETSUPPORT, info, True)
+            self._support_cache = WatchdogInfo(info.options,
+                                               info.firmware_version,
+                                               str(bytearray(info.identity)).rstrip('\x00'))
+        return self._support_cache
+
+    def describe(self):
+        dev_str = " at {0}".format(self.device) if self.device != self.DEFAULT_DEVICE else ""
+        ver_str = ""
+        identity = "Linux watchdog device"
+        if self._fd:
+            try:
+                _, version, identity = self.get_support()
+                ver_str = " (firmware {0})".format(version) if version else ""
+            except WatchdogError:  # XXX: Can it really be raise when self._fd is not None?
+                pass
+
+        return identity + ver_str + dev_str
+
+    def keepalive(self):
+        try:
+            os.write(self._fd, b'1')
+        except OSError as e:
+            raise WatchdogError("Could not send watchdog keepalive: {0}".format(e))
+
+    def has_set_timeout(self):
+        """Returns True if setting a timeout is supported."""
+        return self.get_support().has_SETTIMEOUT
+
+    def set_timeout(self, timeout):
+        timeout = int(timeout)
+        if not 0 < timeout < 0xFFFF:
+            raise WatchdogError("Invalid timeout {0}. Supported values are between 1 and 65535".format(timeout))
+        self._ioctl(WDIOC_SETTIMEOUT, ctypes.c_int(timeout))
+
+    def get_timeout(self):
+        timeout = ctypes.c_int()
+        self._ioctl(WDIOC_GETTIMEOUT, timeout, True)
+        return timeout.value
+
+
+class TestingWatchdogDevice(LinuxWatchdogDevice):
+    """Converts timeout ioctls to regular writes that can be intercepted from a named pipe."""
+    timeout = 60
+
+    def get_support(self):
+        return WatchdogInfo(WDIOF['MAGICCLOSE'] | WDIOF['SETTIMEOUT'], 0, "Watchdog test harness")
+
+    def set_timeout(self, timeout):
+        buf = "Ctimeout={0}\n".format(timeout).encode('utf8')
+        while len(buf):
+            buf = buf[os.write(self._fd, buf):]
+        self.timeout = timeout
+
+    def get_timeout(self):
+        return self.timeout

postgres0.yml

@@ -66,6 +66,7 @@ postgresql:
   connect_address: 127.0.0.1:5432
   data_dir: data/postgresql0
 #  bin_dir:
+#  config_dir:
   pgpass: /tmp/pgpass0
   authentication:
     replication:
@@ -76,6 +77,12 @@ postgresql:
       password: zalando
   parameters:
     unix_socket_directories: '.'
+
+#watchdog:
+#  mode: automatic # Allowed values: off, automatic, required
+#  device: /dev/watchdog
+#  safety_margin: 5
+
 tags:
     nofailover: false
     noloadbalance: false

postgres1.yml

@@ -64,6 +64,7 @@ postgresql:
   connect_address: 127.0.0.1:5433
   data_dir: data/postgresql1
 #  bin_dir:
+#  config_dir:
   pgpass: /tmp/pgpass1
   authentication:
     replication:

postgres2.yml

@@ -61,6 +61,7 @@ postgresql:
   connect_address: 127.0.0.1:5434
   data_dir: data/postgresql2
 #  bin_dir:
+#  config_dir:
   pgpass: /tmp/pgpass2
   authentication:
     replication:

requirements.txt

@@ -5,9 +5,11 @@ PyYAML
 requests
 six >= 1.7
 kazoo==2.2.1
-python-etcd==0.4.3
+python-etcd>=0.4.3,<0.5
 python-consul==0.7.0
 click>=4.1
 prettytable>=0.7
 tzlocal
 python-dateutil
+psutil
+cdiff

tests/test_api.py

@@ -6,6 +6,7 @@ import unittest
 from mock import Mock, patch
 from patroni.api import RestApiHandler, RestApiServer
 from patroni.dcs import ClusterConfig, Member
+from patroni.ha import _MemberStatus
 from patroni.utils import tzutc
 from six import BytesIO as IO
 from six.moves import BaseHTTPServer
@@ -25,6 +26,8 @@ class MockPostgresql(object):
     sysid = 'dummysysid'
     scope = 'dummy'
     pending_restart = True
+    wal_name = 'wal'
+    lsn_name = 'lsn'
 
     @staticmethod
     def connection():
@@ -35,9 +38,14 @@ class MockPostgresql(object):
         return str(postmaster_start_time)
 
 
+class MockWatchdog(object):
+    is_healthy = False
+
+
 class MockHa(object):
 
     state_handler = MockPostgresql()
+    watchdog = MockWatchdog()
 
     @staticmethod
     def reinitialize():
@@ -57,14 +65,14 @@ class MockHa(object):
 
     @staticmethod
     def fetch_nodes_statuses(members):
-        return [[None, True, None, None, {}]]
+        return [_MemberStatus(None, True, None, None, {}, False)]
 
     @staticmethod
     def schedule_future_restart(data):
         return True
 
     @staticmethod
-    def is_lagging(xlog):
+    def is_lagging(wal):
         return False
 
     @staticmethod
@@ -101,6 +109,9 @@ class MockRequest(object):
     def makefile(self, *args, **kwargs):
         return IO(self.request)
 
+    def sendall(self, *args, **kwargs):
+        pass
+
 
 class MockRestApiServer(RestApiServer):
 

tests/test_async_executor.py

@@ -1,7 +1,7 @@
 import unittest
 
 from mock import Mock, patch
-from patroni.async_executor import AsyncExecutor
+from patroni.async_executor import AsyncExecutor, CriticalTask
 from threading import Thread
 
 
@@ -16,3 +16,11 @@ class TestAsyncExecutor(unittest.TestCase):
 
     def test_run(self):
         self.a.run(Mock(side_effect=Exception()))
+
+
+class TestCriticalTask(unittest.TestCase):
+
+    def test_completed_task(self):
+        ct = CriticalTask()
+        ct.complete(1)
+        self.assertFalse(ct.cancel())

tests/test_aws.py

@@ -1,5 +1,4 @@
 import boto.ec2
-import requests
 import sys
 import unittest
 
@@ -11,82 +10,60 @@ from requests.exceptions import RequestException
 
 class MockEc2Connection(object):
 
-    def __init__(self, error=False):
-        self.error = error
-
-    def get_all_volumes(self, filters):
-        if self.error:
-            raise Exception("get_all_volumes")
+    @staticmethod
+    def get_all_volumes(*args, **kwargs):
         oid = namedtuple('Volume', 'id')
         return [oid(id='a'), oid(id='b')]
 
-    def create_tags(self, objects, tags):
-        if self.error or len(objects) == 0:
-            raise Exception("create_tags")
+    @staticmethod
+    def create_tags(objects, *args, **kwargs):
+        if len(objects) == 0:
+            raise boto.exception.BotoServerError(503, 'Service Unavailable', 'Request limit exceeded')
         return True
 
 
 class MockResponse(object):
+    ok = True
 
     def __init__(self, content):
         self.content = content
-        self.ok = True
 
     def json(self):
         return self.content
 
 
+def requests_get(url, **kwargs):
+    if url.split('/')[-1] == 'document':
+        result = {"instanceId": "012345", "region": "eu-west-1"}
+    else:
+        result = 'foo'
+    return MockResponse(result)
+
+
+@patch('boto.ec2.connect_to_region', Mock(return_value=MockEc2Connection()))
 class TestAWSConnection(unittest.TestCase):
 
-    def boto_ec2_connect_to_region(self, region):
-        return MockEc2Connection(self.error)
-
-    def requests_get(self, url, **kwargs):
-        if self.error:
-            raise RequestException("foo")
-        result = namedtuple('Request', 'ok content')
-        result.ok = True
-        if url.split('/')[-1] == 'document' and not self.json_error:
-            result = {"instanceId": "012345", "region": "eu-west-1"}
-        else:
-            result = 'foo'
-        return MockResponse(result)
-
+    @patch('requests.get', requests_get)
     def setUp(self):
-        self.error = False
-        self.json_error = False
-        requests.get = self.requests_get
-        boto.ec2.connect_to_region = self.boto_ec2_connect_to_region
         self.conn = AWSConnection('test')
 
-    def test_aws_available(self):
-        self.assertTrue(self.conn.aws_available())
-
     def test_on_role_change(self):
-        self.assertTrue(self.conn._tag_ebs('master'))
-        self.assertTrue(self.conn._tag_ec2('master'))
         self.assertTrue(self.conn.on_role_change('master'))
+        with patch.object(MockEc2Connection, 'get_all_volumes', Mock(return_value=[])):
+            self.conn._retry.max_tries = 1
+            self.assertFalse(self.conn.on_role_change('master'))
 
+    @patch('requests.get', Mock(side_effect=RequestException('foo')))
     def test_non_aws(self):
-        self.error = True
         conn = AWSConnection('test')
-        self.assertFalse(conn.aws_available())
-        self.assertFalse(conn._tag_ebs('master'))
-        self.assertFalse(conn._tag_ec2('master'))
+        self.assertFalse(conn.on_role_change("master"))
 
+    @patch('requests.get', Mock(return_value=MockResponse('foo')))
     def test_aws_bizare_response(self):
-        self.json_error = True
         conn = AWSConnection('test')
         self.assertFalse(conn.aws_available())
 
-    def test_aws_tag_ebs_error(self):
-        self.error = True
-        self.assertFalse(self.conn._tag_ebs("master"))
-
-    def test_aws_tag_ec2_error(self):
-        self.error = True
-        self.assertFalse(self.conn._tag_ec2("master"))
-
+    @patch('requests.get', requests_get)
     @patch('sys.exit', Mock())
     def test_main(self):
         self.assertIsNone(_main())

tests/test_config.py

@@ -39,6 +39,7 @@ class TestConfig(unittest.TestCase):
             'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432',
             'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432',
             'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0',
+            'PATRONI_POSTGRESQL_CONFIG_DIR': 'data/postgres0',
             'PATRONI_POSTGRESQL_PGPASS': '/tmp/pgpass0',
             'PATRONI_ETCD_HOST': '127.0.0.1:2379',
             'PATRONI_ETCD_URL': 'https://127.0.0.1:2379',

tests/test_ctl.py

@@ -7,7 +7,8 @@ import unittest
 from click.testing import CliRunner
 from mock import patch, Mock
 from patroni.ctl import ctl, members, store_config, load_config, output_members, request_patroni, get_dcs, parse_dcs, \
-    get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException
+    get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException, apply_config_changes, \
+    format_config_for_editing, show_diff, invoke_editor
 from patroni.dcs.etcd import Client
 from psycopg2 import OperationalError
 from test_etcd import etcd_read, requests_get, socket_getaddrinfo, MockResponse
@@ -438,3 +439,70 @@ class TestCtl(unittest.TestCase):
             with patch('requests.patch', Mock(side_effect=Exception)):
                 result = self.runner.invoke(ctl, ['resume', 'dummy'])
                 assert 'Can not find accessible cluster member' in result.output
+
+    def test_apply_config_changes(self):
+        config = {"postgresql": {"parameters": {"work_mem": "4MB"}, "use_pg_rewind": True}, "ttl": 30}
+
+        before_editing = format_config_for_editing(config)
+
+        # Spaces are allowed and stripped, numbers and booleans are interpreted
+        after_editing, changed_config = apply_config_changes(before_editing, config,
+                                                             ["postgresql.parameters.work_mem = 5MB",
+                                                              "ttl=15", "postgresql.use_pg_rewind=off", 'a.b=c'])
+        self.assertEquals(changed_config, {"a": {"b": "c"}, "postgresql": {"parameters": {"work_mem": "5MB"},
+                                                                           "use_pg_rewind": False}, "ttl": 15})
+
+        # postgresql.parameters namespace is flattened
+        after_editing, changed_config = apply_config_changes(before_editing, config,
+                                                             ["postgresql.parameters.work_mem.sub = x"])
+        self.assertEquals(changed_config, {"postgresql": {"parameters": {"work_mem": "4MB", "work_mem.sub": "x"},
+                                                          "use_pg_rewind": True}, "ttl": 30})
+
+        # Setting to null deletes
+        after_editing, changed_config = apply_config_changes(before_editing, config,
+                                                             ["postgresql.parameters.work_mem=null"])
+        self.assertEquals(changed_config, {"postgresql": {"use_pg_rewind": True}, "ttl": 30})
+        after_editing, changed_config = apply_config_changes(before_editing, config,
+                                                             ["postgresql.use_pg_rewind=null",
+                                                              "postgresql.parameters.work_mem=null"])
+        self.assertEquals(changed_config, {"ttl": 30})
+
+        self.assertRaises(PatroniCtlException, apply_config_changes, before_editing, config, ['a'])
+
+    @patch('sys.stdout.isatty', return_value=False)
+    @patch('cdiff.markup_to_pager')
+    def test_show_diff(self, mock_markup_to_pager, mock_isatty):
+        show_diff("foo:\n  bar: 1\n", "foo:\n  bar: 2\n")
+        mock_markup_to_pager.assert_not_called()
+
+        mock_isatty.return_value = True
+        show_diff("foo:\n  bar: 1\n", "foo:\n  bar: 2\n")
+        mock_markup_to_pager.assert_called_once()
+
+        # Test that unicode handling doesn't fail with an exception
+        show_diff(b"foo:\n  bar: \xc3\xb6\xc3\xb6\n".decode('utf-8'),
+                  b"foo:\n  bar: \xc3\xbc\xc3\xbc\n".decode('utf-8'))
+
+    def test_invoke_editor(self):
+        for e in ('', 'false'):
+            os.environ['EDITOR'] = e
+            self.assertRaises(PatroniCtlException, invoke_editor, 'foo: bar\n', 'test')
+
+    @patch('patroni.ctl.get_dcs')
+    def test_show_config(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        self.runner.invoke(ctl, ['show-config', 'dummy'])
+
+    @patch('patroni.ctl.get_dcs')
+    def test_edit_config(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        os.environ['EDITOR'] = 'true'
+        self.runner.invoke(ctl, ['edit-config', 'dummy'])
+        self.runner.invoke(ctl, ['edit-config', 'dummy', '-s', 'foo=bar'])
+        self.runner.invoke(ctl, ['edit-config', 'dummy', '--replace', 'postgres0.yml'])
+        self.runner.invoke(ctl, ['edit-config', 'dummy', '--apply', '-'], input='foo: bar')
+        self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar')
+        mock_get_dcs.return_value.set_config_value = Mock(return_value=True)
+        self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar')

tests/test_etcd.py

@@ -59,7 +59,7 @@ def etcd_watch(self, key, index=None, timeout=None, recursive=None):
         raise etcd.EtcdWatchTimedOut
     elif timeout == 5.0:
         return etcd.EtcdResult('delete', {})
-    elif timeout == 10.0:
+    elif 5 < timeout <= 10.0:
         raise etcd.EtcdException
     elif timeout == 20.0:
         raise etcd.EtcdEventIndexCleared
@@ -302,6 +302,7 @@ class TestEtcd(unittest.TestCase):
     def test_delete_cluster(self):
         self.assertFalse(self.etcd.delete_cluster())
 
+    @patch('time.sleep', Mock(side_effect=SleepException))
     @patch.object(etcd.Client, 'watch', etcd_watch)
     def test_watch(self):
         self.etcd.watch(None, 0)
@@ -310,7 +311,7 @@ class TestEtcd(unittest.TestCase):
         self.etcd.watch(20729, 4.5)
         with patch.object(AbstractDCS, 'watch', Mock()):
             self.assertTrue(self.etcd.watch(20729, 19.5))
-            self.etcd.watch(20729, 9.5)
+            self.assertRaises(SleepException, self.etcd.watch, 20729, 9.5)
 
     def test_other_exceptions(self):
         self.etcd.retry = Mock(side_effect=AttributeError('foo'))

tests/test_ha.py

@@ -7,9 +7,10 @@ from mock import Mock, MagicMock, PropertyMock, patch
 from patroni.config import Config
 from patroni.dcs import Cluster, ClusterConfig, Failover, Leader, Member, get_dcs, SyncState
 from patroni.dcs.etcd import Client
-from patroni.exceptions import DCSError, PostgresException
+from patroni.exceptions import DCSError, PostgresConnectionException, PatroniException
 from patroni.ha import Ha, _MemberStatus
 from patroni.postgresql import Postgresql
+from patroni.watchdog import Watchdog
 from patroni.utils import tzutc
 from test_etcd import socket_getaddrinfo, etcd_read, etcd_write, requests_get
 from test_postgresql import psycopg2_connect
@@ -54,12 +55,12 @@ def get_cluster_initialized_with_only_leader(failover=None):
     return get_cluster(True, l, [l], failover, None)
 
 
-def get_node_status(reachable=True, in_recovery=True, xlog_location=10, nofailover=False):
+def get_node_status(reachable=True, in_recovery=True, wal_position=10, nofailover=False, watchdog_failed=False):
     def fetch_node_status(e):
         tags = {}
         if nofailover:
             tags['nofailover'] = True
-        return _MemberStatus(e, reachable, in_recovery, xlog_location, tags)
+        return _MemberStatus(e, reachable, in_recovery, wal_position, tags, watchdog_failed)
     return fetch_node_status
 
 future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5)
@@ -84,6 +85,8 @@ postgresql:
   pg_rewind:
     username: postgres
     password: postgres
+watchdog:
+  mode: off
 zookeeper:
   exhibitor:
     hosts: [localhost]
@@ -101,6 +104,7 @@ zookeeper:
         self.nosync = False
         self.scheduled_restart = {'schedule': future_restart_time,
                                   'postmaster_start_time': str(postmaster_start_time)}
+        self.watchdog = Watchdog(self.config)
 
 
 def run_async(self, func, args=()):
@@ -109,13 +113,13 @@ def run_async(self, func, args=()):
 
 @patch.object(Postgresql, 'is_running', Mock(return_value=True))
 @patch.object(Postgresql, 'is_leader', Mock(return_value=True))
-@patch.object(Postgresql, 'xlog_position', Mock(return_value=10))
+@patch.object(Postgresql, 'wal_position', Mock(return_value=10))
 @patch.object(Postgresql, 'call_nowait', Mock(return_value=True))
 @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
 @patch.object(Postgresql, 'controldata', Mock(return_value={'Database system identifier': '1234567890'}))
 @patch.object(Postgresql, 'sync_replication_slots', Mock())
 @patch.object(Postgresql, 'write_pg_hba', Mock())
-@patch.object(Postgresql, 'write_pgpass', Mock())
+@patch.object(Postgresql, 'write_pgpass', Mock(return_value={}))
 @patch.object(Postgresql, 'write_recovery_conf', Mock())
 @patch.object(Postgresql, 'query', Mock())
 @patch.object(Postgresql, 'checkpoint', Mock())
@@ -126,10 +130,12 @@ def run_async(self, func, args=()):
 @patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=False))
 @patch('patroni.async_executor.AsyncExecutor.run_async', run_async)
 @patch('subprocess.call', Mock(return_value=0))
+@patch('time.sleep', Mock())
 class TestHa(unittest.TestCase):
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
     @patch('psycopg2.connect', psycopg2_connect)
+    @patch('patroni.dcs.dcs_modules', Mock(return_value=['foo', 'patroni.dcs.etcd']))
     @patch.object(etcd.Client, 'read', etcd_read)
     def setUp(self):
         with patch.object(Client, 'machines') as mock_machines:
@@ -154,11 +160,11 @@ class TestHa(unittest.TestCase):
             self.ha.is_synchronous_mode = false
 
     def test_update_lock(self):
-        self.p.last_operation = Mock(side_effect=PostgresException(''))
+        self.p.last_operation = Mock(side_effect=PostgresConnectionException(''))
         self.assertTrue(self.ha.update_lock(True))
 
     def test_touch_member(self):
-        self.p.xlog_position = Mock(side_effect=Exception)
+        self.p.wal_position = Mock(side_effect=Exception)
         self.ha.touch_member()
 
     def test_start_as_replica(self):
@@ -167,7 +173,6 @@ class TestHa(unittest.TestCase):
 
     def test_recover_replica_failed(self):
         self.p.controldata = lambda: {'Database cluster state': 'in production'}
-        self.p.is_healthy = false
         self.p.is_running = false
         self.p.follow = false
         self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
@@ -175,7 +180,6 @@ class TestHa(unittest.TestCase):
 
     def test_recover_master_failed(self):
         self.p.follow = false
-        self.p.is_healthy = false
         self.p.is_running = false
         self.p.name = 'leader'
         self.p.set_role('master')
@@ -183,6 +187,12 @@ class TestHa(unittest.TestCase):
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.assertEquals(self.ha.run_cycle(), 'starting as readonly because i had the session lock')
 
+    @patch.object(Postgresql, 'rewind_needed_and_possible', Mock(return_value=True))
+    def test_recover_with_rewind(self):
+        self.p.is_running = false
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.assertEquals(self.ha.run_cycle(), 'running pg_rewind from leader')
+
     @patch('sys.exit', return_value=1)
     @patch('patroni.ha.Ha.sysid_valid', MagicMock(return_value=True))
     def test_sysid_no_match(self, exit_mock):
@@ -229,6 +239,15 @@ class TestHa(unittest.TestCase):
         self.p.is_leader = false
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader because i had the session lock')
 
+    def test_promote_without_watchdog(self):
+        self.ha.cluster.is_unlocked = false
+        self.ha.has_lock = true
+        self.p.is_leader = true
+        with patch.object(Watchdog, 'activate', Mock(return_value=False)):
+            self.assertEquals(self.ha.run_cycle(), 'Demoting self because watchdog could not be activated')
+            self.p.is_leader = false
+            self.assertEquals(self.ha.run_cycle(), 'Not promoting self because watchdog could not be actived')
+
     def test_leader_with_lock(self):
         self.ha.cluster.is_unlocked = false
         self.ha.has_lock = true
@@ -236,7 +255,8 @@ class TestHa(unittest.TestCase):
 
     def test_demote_because_not_having_lock(self):
         self.ha.cluster.is_unlocked = false
-        self.assertEquals(self.ha.run_cycle(), 'demoting self because i do not have the lock and i was a leader')
+        with patch.object(Watchdog, 'is_running', PropertyMock(return_value=True)):
+            self.assertEquals(self.ha.run_cycle(), 'demoting self because i do not have the lock and i was a leader')
 
     def test_demote_because_update_lock_failed(self):
         self.ha.cluster.is_unlocked = false
@@ -260,10 +280,18 @@ class TestHa(unittest.TestCase):
         self.p.is_leader = false
         self.assertEquals(self.ha.run_cycle(), 'PAUSE: no action')
 
+    @patch.object(Postgresql, 'rewind_needed_and_possible', Mock(return_value=True))
+    def test_follow_triggers_rewind(self):
+        self.p.is_leader = false
+        self.p.trigger_check_diverged_lsn()
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.assertEquals(self.ha.run_cycle(), 'running pg_rewind from leader')
+
     def test_no_etcd_connection_master_demote(self):
         self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
         self.assertEquals(self.ha.run_cycle(), 'demoted self because DCS is not accessible and i was a leader')
 
+    @patch('time.sleep', Mock())
     def test_bootstrap_from_another_member(self):
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap from replica \'other\'')
@@ -284,14 +312,31 @@ class TestHa(unittest.TestCase):
     def test_bootstrap_initialized_new_cluster(self):
         self.ha.cluster = get_cluster_not_initialized_without_leader()
         self.e.initialize = true
-        self.assertEquals(self.ha.bootstrap(), 'initialized a new cluster')
+        self.assertEquals(self.ha.bootstrap(), 'trying to bootstrap a new cluster')
+        self.p.is_leader = false
+        self.assertEquals(self.ha.run_cycle(), 'waiting for end of recovery after bootstrap')
+        self.p.is_leader = true
+        self.assertEquals(self.ha.run_cycle(), 'running post_bootstrap')
+        self.assertEquals(self.ha.run_cycle(), 'initialized a new cluster')
 
     def test_bootstrap_release_initialize_key_on_failure(self):
         self.ha.cluster = get_cluster_not_initialized_without_leader()
         self.e.initialize = true
-        self.p.bootstrap = Mock(side_effect=PostgresException("Could not bootstrap master PostgreSQL"))
-        self.assertRaises(PostgresException, self.ha.bootstrap)
+        self.ha.bootstrap()
+        self.p.is_running = false
+        self.assertRaises(PatroniException, self.ha.post_bootstrap)
 
+    def test_bootstrap_release_initialize_key_on_watchdog_failure(self):
+        self.ha.cluster = get_cluster_not_initialized_without_leader()
+        self.e.initialize = true
+        self.ha.bootstrap()
+        self.p.is_running = true
+        self.p.is_leader = true
+        with patch.object(Watchdog, 'activate', Mock(return_value=False)):
+            self.assertEquals(self.ha.post_bootstrap(), 'running post_bootstrap')
+            self.assertRaises(PatroniException, self.ha.post_bootstrap)
+
+    @patch('psycopg2.connect', psycopg2_connect)
     def test_reinitialize(self):
         self.assertIsNotNone(self.ha.reinitialize())
 
@@ -303,6 +348,7 @@ class TestHa(unittest.TestCase):
         self.ha.state_handler.name = self.ha.cluster.leader.name
         self.assertIsNotNone(self.ha.reinitialize())
 
+    @patch('time.sleep', Mock())
     def test_restart(self):
         self.assertEquals(self.ha.restart({}), (True, 'restarted successfully'))
         self.p.restart = Mock(return_value=None)
@@ -315,11 +361,12 @@ class TestHa(unittest.TestCase):
         with patch.object(self.ha, "restart_matches", return_value=False):
             self.assertEquals(self.ha.restart({'foo': 'bar'}), (False, "restart conditions are not satisfied"))
 
+    @patch('os.kill', Mock())
     def test_restart_in_progress(self):
         with patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=True)):
             self.ha.restart({}, run_async=True)
             self.assertTrue(self.ha.restart_scheduled())
-            self.assertEquals(self.ha.run_cycle(), 'not healthy enough for leader race')
+            self.assertEquals(self.ha.run_cycle(), 'restart in progress')
 
             self.ha.cluster = get_cluster_initialized_with_leader()
             self.assertEquals(self.ha.run_cycle(), 'restart in progress')
@@ -328,10 +375,13 @@ class TestHa(unittest.TestCase):
             self.assertEquals(self.ha.run_cycle(), 'updated leader lock during restart')
 
             self.ha.update_lock = false
-            self.assertEquals(self.ha.run_cycle(), 'failed to update leader lock during restart')
+            self.p.set_role('master')
+            with patch('patroni.async_executor.CriticalTask.cancel', Mock(return_value=False)):
+                with patch('patroni.postgresql.Postgresql.stop') as stop_mock:
+                    self.assertEquals(self.ha.run_cycle(), 'lost leader lock during restart')
+                    stop_mock.assert_called()
 
     @patch('requests.get', requests_get)
-    @patch('time.sleep', Mock())
     def test_manual_failover_from_leader(self):
         self.ha.fetch_node_status = get_node_status()
         self.ha.has_lock = true
@@ -344,9 +394,13 @@ class TestHa(unittest.TestCase):
         f = Failover(0, self.p.name, '', None)
         self.ha.cluster = get_cluster_initialized_with_leader(f)
         self.assertEquals(self.ha.run_cycle(), 'manual failover: demoting myself')
+        self.p.rewind_needed_and_possible = true
+        self.assertEquals(self.ha.run_cycle(), 'manual failover: demoting myself')
         self.ha.fetch_node_status = get_node_status(nofailover=True)
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
-        self.ha.fetch_node_status = get_node_status(xlog_location=1)
+        self.ha.fetch_node_status = get_node_status(watchdog_failed=True)
+        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
+        self.ha.fetch_node_status = get_node_status(wal_position=1)
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
         # manual failover from the previous leader to us won't happen if we hold the nofailover flag
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, None))
@@ -384,7 +438,6 @@ class TestHa(unittest.TestCase):
         self.assertEquals('PAUSE: no action.  i am the leader with the lock', self.ha.run_cycle())
 
     @patch('requests.get', requests_get)
-    @patch('time.sleep', Mock())
     def test_manual_failover_process_no_leader(self):
         self.p.is_leader = false
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', self.p.name, None))
@@ -409,7 +462,6 @@ class TestHa(unittest.TestCase):
         self.ha.patroni.nofailover = True
         self.assertEquals(self.ha.run_cycle(), 'following a different leader because I am not allowed to promote')
 
-    @patch('time.sleep', Mock())
     def test_manual_failover_process_no_leader_in_pause(self):
         self.ha.is_paused = true
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
@@ -428,6 +480,8 @@ class TestHa(unittest.TestCase):
         self.ha.patroni.nofailover = False
         self.ha.fetch_node_status = get_node_status()
         self.assertTrue(self.ha.is_healthiest_node())
+        with patch.object(Watchdog, 'is_healthy', PropertyMock(return_value=False)):
+            self.assertFalse(self.ha.is_healthiest_node())
         with patch('patroni.postgresql.Postgresql.is_starting', return_value=True):
             self.assertFalse(self.ha.is_healthiest_node())
         self.ha.is_paused = true
@@ -440,9 +494,9 @@ class TestHa(unittest.TestCase):
         self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.fetch_node_status = get_node_status(in_recovery=False)  # accessible, not in_recovery
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
-        self.ha.fetch_node_status = get_node_status(xlog_location=11)  # accessible, in_recovery, xlog location ahead
+        self.ha.fetch_node_status = get_node_status(wal_position=11)  # accessible, in_recovery, wal position ahead
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
-        with patch('patroni.postgresql.Postgresql.xlog_position', return_value=1):
+        with patch('patroni.postgresql.Postgresql.wal_position', return_value=1):
             self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.patroni.nofailover = True
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
@@ -569,7 +623,6 @@ class TestHa(unittest.TestCase):
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am a secondary and i am following a leader')
         check_calls([(update_lock, False), (demote, False)])
 
-    @patch('time.sleep', Mock())
     def test_manual_failover_while_starting(self):
         self.ha.has_lock = true
         self.p.check_for_startup = true
@@ -589,15 +642,13 @@ class TestHa(unittest.TestCase):
         self.assertEquals(self.ha.run_cycle(), 'stopped PostgreSQL to fail over after a crash')
         demote.assert_called_once()
 
-    @patch('time.sleep', Mock())
     @patch('patroni.postgresql.Postgresql.follow')
     def test_demote_immediate(self, follow):
         self.ha.has_lock = true
         self.e.get_cluster = Mock(return_value=get_cluster_initialized_without_leader())
         self.ha.demote('immediate')
-        follow.assert_called_once_with(None, None, True, None, True)
+        follow.assert_called_once_with(None)
 
-    @patch('time.sleep', Mock())
     def test_process_sync_replication(self):
         self.ha.has_lock = true
         mock_set_sync = self.p.set_synchronous_standby = Mock()
@@ -663,6 +714,13 @@ class TestHa(unittest.TestCase):
         self.ha.run_cycle()
         self.assertEquals(self.ha.dcs.write_sync_state.call_count, 1)
 
+        # Test sync set to '*' when synchronous_mode_strict is enabled
+        mock_set_sync.reset_mock()
+        self.ha.is_synchronous_mode_strict = true
+        self.p.pick_synchronous_standby = Mock(return_value=(None, False))
+        self.ha.run_cycle()
+        mock_set_sync.assert_called_once_with('*')
+
     def test_sync_replication_become_master(self):
         self.ha.is_synchronous_mode = true
 
@@ -717,8 +775,7 @@ class TestHa(unittest.TestCase):
         mock_promote.assert_called_once()
         mock_write_sync.assert_called_once_with('other', None, index=0)
 
-    @patch('time.sleep')
-    def test_disable_sync_when_restarting(self, mock_sleep):
+    def test_disable_sync_when_restarting(self):
         self.ha.is_synchronous_mode = true
 
         self.p.name = 'other'
@@ -731,10 +788,10 @@ class TestHa(unittest.TestCase):
             get_cluster_initialized_with_leader(sync=('leader', syncstandby))
             for syncstandby in ['other', None]])
 
-        self.ha.restart({})
-
-        mock_restart.assert_called_once()
-        mock_sleep.assert_called()
+        with patch('time.sleep') as mock_sleep:
+            self.ha.restart({})
+            mock_restart.assert_called_once()
+            mock_sleep.assert_called()
 
         # Restart is still called when DCS connection fails
         mock_restart.reset_mock()
@@ -771,3 +828,19 @@ class TestHa(unittest.TestCase):
 
     def test_wakup(self):
         self.ha.wakeup()
+
+    def test_shutdown(self):
+        self.p.is_running = false
+        self.ha.shutdown()
+
+    @patch('time.sleep', Mock())
+    def test_leader_with_empty_directory(self):
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.ha.has_lock = true
+        self.p.data_directory_empty = true
+        self.assertEquals(self.ha.run_cycle(), 'released leader key voluntarily as data dir empty and currently leader')
+
+        # as has_lock is mocked out, we need to fake the leader key release
+        self.ha.has_lock = false
+        # will not say bootstrap from leader as replica can't self elect
+        self.assertEquals(self.ha.run_cycle(), "trying to bootstrap from replica 'other'")

tests/test_patroni.py

@@ -105,6 +105,7 @@ class TestPatroni(unittest.TestCase):
     @patch('patroni.config.Config.reload_local_configuration', Mock(return_value=True))
     @patch.object(Postgresql, 'state', PropertyMock(return_value='running'))
     def test_run(self):
+        self.p.postgresql.set_role('replica')
         self.p.sighup_handler()
         self.p.ha.dcs.watch = Mock(side_effect=SleepException)
         self.p.api.start = Mock()

tests/test_postgresql.py

@@ -1,3 +1,4 @@
+import errno
 import mock  # for the mock.call method, importing it without a namespace breaks python3
 import os
 import psycopg2
@@ -6,8 +7,9 @@ import subprocess
 import unittest
 
 from mock import Mock, MagicMock, PropertyMock, patch, mock_open
+from patroni.async_executor import CriticalTask
 from patroni.dcs import Cluster, Leader, Member, SyncState
-from patroni.exceptions import PostgresException, PostgresConnectionException
+from patroni.exceptions import PostgresConnectionException
 from patroni.postgresql import Postgresql, STATE_REJECT, STATE_NO_RESPONSE
 from patroni.utils import RetryFailedError
 from six.moves import builtins
@@ -19,17 +21,20 @@ class MockCursor(object):
     def __init__(self, connection):
         self.connection = connection
         self.closed = False
+        self.rowcount = 0
         self.results = []
 
     def execute(self, sql, *params):
-        if sql.startswith('blabla') or sql == 'CHECKPOINT':
+        if sql.startswith('blabla'):
+            raise psycopg2.ProgrammingError()
+        elif sql == 'CHECKPOINT':
             raise psycopg2.OperationalError()
         elif sql.startswith('RetryFailedError'):
             raise RetryFailedError('retry')
         elif sql.startswith('SELECT slot_name'):
             self.results = [('blabla',), ('foobar',)]
         elif sql.startswith('SELECT CASE WHEN pg_is_in_recovery()'):
-            self.results = [(0,)]
+            self.results = [(2,)]
         elif sql == 'SELECT pg_is_in_recovery()':
             self.results = [(False, )]
         elif sql.startswith('WITH replication_info AS ('):
@@ -41,7 +46,15 @@ class MockCursor(object):
                             ('search_path', 'public', None, 'string', 'user'),
                             ('port', '5433', None, 'integer', 'postmaster'),
                             ('listen_addresses', '*', None, 'string', 'postmaster'),
-                            ('autovacuum', 'on', None, 'bool', 'sighup')]
+                            ('autovacuum', 'on', None, 'bool', 'sighup'),
+                            ('unix_socket_directories', '/tmp', None, 'string', 'postmaster')]
+        elif sql.startswith('IDENTIFY_SYSTEM'):
+            self.results = [('1', 2, '0/402EEC0', '')]
+        elif sql.startswith('TIMELINE_HISTORY '):
+            self.results = [('', b'x\t0/40159C0\tno recovery target specified\n\n' +
+                                 b'1\t0/40159C0\tno recovery target specified\n\n' +
+                                 b'2\t0/402DD98\tno recovery target specified\n\n' +
+                                 b'3\t0/403DD98\tno recovery target specified\n')]
         else:
             self.results = [(None, None, None, None, None, None, None, None, None, None)]
 
@@ -64,7 +77,7 @@ class MockCursor(object):
 
 class MockConnect(object):
 
-    server_version = '99999'
+    server_version = 99999
     autocommit = False
     closed = 0
 
@@ -137,21 +150,10 @@ Data page checksum version:           0
 """
 
 
-def postmaster_opts_string(*args, **kwargs):
-    return '/usr/local/pgsql/bin/postgres "-D" "data/postgresql0" "--listen_addresses=127.0.0.1" \
-"--port=5432" "--hot_standby=on" "--wal_keep_segments=8" "--wal_level=hot_standby" \
-"--archive_command=mkdir -p ../wal_archive && cp %p ../wal_archive/%f" "--wal_log_hints=on" \
-"--max_wal_senders=5" "--archive_timeout=1800s" "--archive_mode=on" "--max_replication_slots=5"\n'
-
-
 def psycopg2_connect(*args, **kwargs):
     return MockConnect()
 
 
-def fake_listdir(path):
-    return ["a", "b", "c"] if path.endswith('pg_xlog/archive_status') else []
-
-
 @patch('subprocess.call', Mock(return_value=0))
 @patch('psycopg2.connect', psycopg2_connect)
 class TestPostgresql(unittest.TestCase):
@@ -159,30 +161,30 @@ class TestPostgresql(unittest.TestCase):
                    'search_path': 'public', 'hot_standby': 'on', 'max_wal_senders': 5,
                    'wal_keep_segments': 8, 'wal_log_hints': 'on', 'max_locks_per_transaction': 64,
                    'max_worker_processes': 8, 'max_connections': 100, 'max_prepared_transactions': 0,
-                   'track_commit_timestamp': 'off'}
+                   'track_commit_timestamp': 'off', 'unix_socket_directories': '/tmp'}
 
     @patch('subprocess.call', Mock(return_value=0))
     @patch('psycopg2.connect', psycopg2_connect)
     @patch('os.rename', Mock())
-    @patch.object(Postgresql, 'get_major_version', Mock(return_value=9.6))
+    @patch.object(Postgresql, 'get_major_version', Mock(return_value=90600))
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def setUp(self):
         self.data_dir = 'data/test0'
+        self.config_dir = self.data_dir
         if not os.path.exists(self.data_dir):
             os.makedirs(self.data_dir)
-        self.p = Postgresql({'name': 'test0', 'scope': 'batman', 'data_dir': self.data_dir, 'retry_timeout': 10,
-                             'listen': '127.0.0.1, *:5432', 'connect_address': '127.0.0.2:5432',
+        self.p = Postgresql({'name': 'test0', 'scope': 'batman', 'data_dir': self.data_dir,
+                             'config_dir': self.config_dir, 'retry_timeout': 10,
+                             'listen': '127.0.0.2, 127.0.0.3:5432', 'connect_address': '127.0.0.2:5432',
                              'authentication': {'superuser': {'username': 'test', 'password': 'test'},
                                                 'replication': {'username': 'replicator', 'password': 'rep-pass'}},
                              'remove_data_directory_on_rewind_failure': True,
                              'use_pg_rewind': True, 'pg_ctl_timeout': 'bla',
                              'parameters': self._PARAMETERS,
                              'recovery_conf': {'foo': 'bar'},
-                             'callbacks': {'on_start': 'true', 'on_stop': 'true',
-                                           'on_restart': 'true', 'on_role_change': 'true',
-                                           'on_reload': 'true'
-                                           },
-                             'restore': 'true'})
+                             'pg_hba': ['host all all 0.0.0.0/0 md5'],
+                             'callbacks': {'on_start': 'true', 'on_stop': 'true', 'on_reload': 'true',
+                                           'on_restart': 'true', 'on_role_change': 'true'}})
         self.p._callback_executor = Mock()
         self.leadermem = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres'})
         self.leader = Leader(-1, 28, self.leadermem)
@@ -213,13 +215,13 @@ class TestPostgresql(unittest.TestCase):
         mock_is_running.return_value = True
         mock_wait_for_port_open.return_value = True
         mock_wait_for_startup.return_value = False
-        mock_popen.stdout.readline.return_value = '123'
+        mock_popen.return_value.stdout.readline.return_value = '123'
         self.assertTrue(self.p.start())
         mock_is_running.return_value = False
         open(os.path.join(self.data_dir, 'postmaster.pid'), 'w').close()
         pg_conf = os.path.join(self.data_dir, 'postgresql.conf')
         open(pg_conf, 'w').close()
-        self.assertFalse(self.p.start())
+        self.assertFalse(self.p.start(task=CriticalTask()))
         with open(pg_conf) as f:
             lines = f.readlines()
             self.assertTrue("f.oo = 'bar'\n" in lines)
@@ -230,6 +232,9 @@ class TestPostgresql(unittest.TestCase):
 
         mock_wait_for_port_open.return_value = False
         self.assertFalse(self.p.start())
+        task = CriticalTask()
+        task.cancel()
+        self.assertFalse(self.p.start(task=task))
 
     @patch.object(Postgresql, 'pg_isready')
     @patch.object(Postgresql, 'read_pid_file')
@@ -263,13 +268,29 @@ class TestPostgresql(unittest.TestCase):
         mock_pg_isready.return_value = 'garbage'
         self.assertTrue(self.p.wait_for_port_open(42, 100., 1))
 
+    @patch('time.sleep', Mock())
     @patch.object(Postgresql, 'is_running')
-    def test_stop(self, mock_is_running):
+    @patch.object(Postgresql, 'get_pid')
+    def test_stop(self, mock_get_pid, mock_is_running):
+        mock_callback = Mock()
+        mock_is_running.return_value = False
+        self.assertTrue(self.p.stop(on_safepoint=mock_callback))
+        mock_callback.assert_called()
         mock_is_running.return_value = True
-        self.assertTrue(self.p.stop())
-        with patch('subprocess.call', Mock(return_value=1)):
-            mock_is_running.return_value = False
+        mock_get_pid.return_value = 0
+        mock_callback.reset_mock()
+        self.assertTrue(self.p.stop(on_safepoint=mock_callback))
+        mock_callback.assert_called()
+        mock_get_pid.return_value = -1
+        self.assertFalse(self.p.stop())
+        mock_get_pid.return_value = 123
+        with patch('os.kill', Mock(side_effect=[OSError(errno.ESRCH, ''), OSError, None])):
             self.assertTrue(self.p.stop())
+            self.assertFalse(self.p.stop())
+            self.assertTrue(self.p.stop())
+        with patch.object(Postgresql, '_signal_postmaster_stop', Mock(return_value=(123, None))):
+            with patch.object(Postgresql, 'is_pid_running', Mock(side_effect=[True, False, False])):
+                self.assertTrue(self.p.stop())
 
     def test_restart(self):
         self.p.start = Mock(return_value=False)
@@ -292,39 +313,84 @@ class TestPostgresql(unittest.TestCase):
     @patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
     def test_pg_rewind(self, mock_call):
         r = {'user': '', 'host': '', 'port': '', 'database': '', 'password': ''}
-        self.assertTrue(self.p.rewind(r))
+        self.assertTrue(self.p.pg_rewind(r))
         subprocess.call = mock_call
-        self.assertFalse(self.p.rewind(r))
+        self.assertFalse(self.p.pg_rewind(r))
 
-    @patch('os.unlink', Mock(return_value=True))
-    @patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string))
-    @patch.object(Postgresql, 'remove_data_directory', Mock(return_value=True))
-    @patch.object(Postgresql, 'single_user_mode', Mock(return_value=1))
-    @patch.object(Postgresql, 'write_pgpass', Mock(return_value={}))
-    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
+    def test_check_recovery_conf(self):
+        self.p.write_recovery_conf({'primary_conninfo': 'foo'})
+        self.assertFalse(self.p.check_recovery_conf(None))
+        self.p.write_recovery_conf({})
+        self.assertTrue(self.p.check_recovery_conf(None))
+
+    @patch.object(Postgresql, 'start', Mock())
     @patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
-    @patch.object(Postgresql, 'rewind', return_value=False)
-    def test_follow(self, mock_pg_rewind):
-        with patch.object(Postgresql, 'check_recovery_conf', Mock(return_value=True)):
-            self.assertTrue(self.p.follow(None, None))  # nothing to do, recovery.conf has good primary_conninfo
+    def test__get_local_timeline_lsn(self):
+        self.p.trigger_check_diverged_lsn()
+        with patch.object(Postgresql, 'controldata', Mock(return_value={'Database cluster state': 'shut down'})):
+            self.p.rewind_needed_and_possible(self.leader)
+        with patch.object(Postgresql, 'controldata',
+                          Mock(return_value={'Database cluster state': 'shut down in recovery'})):
+            self.p.rewind_needed_and_possible(self.leader)
+        with patch.object(Postgresql, 'is_running', Mock(return_value=True)):
+            with patch.object(MockCursor, 'fetchone', Mock(side_effect=[(False, ), Exception])):
+                self.p.rewind_needed_and_possible(self.leader)
 
-        self.p.follow(self.me, self.me)  # follow is called when the node is holding leader lock
+    @patch.object(Postgresql, 'start', Mock())
+    @patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
+    @patch.object(Postgresql, '_get_local_timeline_lsn', Mock(return_value=(2, '0/40159C1')))
+    @patch.object(Postgresql, 'check_leader_is_not_in_recovery')
+    def test__check_timeline_and_lsn(self, mock_check_leader_is_not_in_recovery):
+        mock_check_leader_is_not_in_recovery.return_value = False
+        self.p.trigger_check_diverged_lsn()
+        self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
+        mock_check_leader_is_not_in_recovery.return_value = True
+        self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
+        self.p.trigger_check_diverged_lsn()
+        with patch('psycopg2.connect', Mock(side_effect=Exception)):
+            self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
+        with patch.object(MockCursor, 'fetchone',
+                          Mock(side_effect=[('', 2, '0/0'), ('', b'2\tG/40159C0\tno recovery target specified\n\n')])):
+            self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
+        self.p.trigger_check_diverged_lsn()
+        with patch.object(MockCursor, 'fetchone',
+                          Mock(side_effect=[('', 2, '0/0'), ('', b'3\t040159C0\tno recovery target specified\n')])):
+            self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
+        self.p.trigger_check_diverged_lsn()
+        with patch.object(MockCursor, 'fetchone', Mock(return_value=('', 1, '0/0'))):
+            with patch.object(Postgresql, '_get_local_timeline_lsn', Mock(return_value=(1, '0/0'))):
+                self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
+            self.p.trigger_check_diverged_lsn()
+            self.assertTrue(self.p.rewind_needed_and_possible(self.leader))
 
-        with patch.object(Postgresql, 'restart', Mock(return_value=False)):
-            self.p.set_role('replica')
-            self.p.follow(None, None)  # restart without rewind
+    @patch.object(MockCursor, 'fetchone', Mock(side_effect=[(True,), Exception]))
+    def test_check_leader_is_not_in_recovery(self):
+        self.p.check_leader_is_not_in_recovery()
+        self.p.check_leader_is_not_in_recovery()
 
-        with patch.object(Postgresql, 'stop', Mock(return_value=False)):
-            self.p.follow(self.leader, self.leader, need_rewind=True)  # failed to stop postgres
+    @patch.object(Postgresql, 'checkpoint', side_effect=['', '1'])
+    @patch.object(Postgresql, 'stop', Mock(return_value=False))
+    @patch.object(Postgresql, 'start', Mock())
+    def test_rewind(self, mock_checkpoint):
+        self.p.rewind(self.leader)
+        with patch.object(Postgresql, 'pg_rewind', Mock(return_value=False)):
+            mock_checkpoint.side_effect = ['1', '', '', '']
+            self.p.rewind(self.leader)
+            self.p.rewind(self.leader)
+            with patch.object(Postgresql, 'check_leader_is_not_in_recovery', Mock(return_value=False)):
+                self.p.rewind(self.leader)
+            self.p.config['remove_data_directory_on_rewind_failure'] = False
+            self.p.trigger_check_diverged_lsn()
+            self.p.rewind(self.leader)
+        with patch.object(Postgresql, 'is_running', Mock(return_value=True)):
+            self.p.rewind(self.leader)
+            self.p.is_leader = Mock(return_value=False)
+            self.p.rewind(self.leader)
 
-        self.p.follow(self.leader, self.leader)  # "leader" is not accessible or is_in_recovery
-
-        with patch.object(Postgresql, 'checkpoint', Mock(return_value=None)):
-            self.p.follow(self.leader, self.leader)
-            mock_pg_rewind.return_value = True
-            self.p.follow(self.leader, self.leader, need_rewind=True)
-
-        self.p.follow(None, None)  # check_recovery_conf...
+    @patch.object(Postgresql, 'is_running', Mock(return_value=False))
+    @patch.object(Postgresql, 'start', Mock())
+    def test_follow(self):
+        self.p.follow(None)
 
     @patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string))
     def test_can_rewind(self):
@@ -378,7 +444,7 @@ class TestPostgresql(unittest.TestCase):
             assert "test-3" in errorlog_mock.call_args[0][1]
             assert "test.3" in errorlog_mock.call_args[0][1]
 
-    @patch.object(MockConnect, 'closed', 2)
+    @patch.object(MockCursor, 'execute', Mock(side_effect=psycopg2.OperationalError))
     def test__query(self):
         self.assertRaises(PostgresConnectionException, self.p._query, 'blabla')
         self.p._state = 'restarting'
@@ -387,7 +453,7 @@ class TestPostgresql(unittest.TestCase):
     def test_query(self):
         self.p.query('select 1')
         self.assertRaises(PostgresConnectionException, self.p.query, 'RetryFailedError')
-        self.assertRaises(psycopg2.OperationalError, self.p.query, 'blabla')
+        self.assertRaises(psycopg2.ProgrammingError, self.p.query, 'blabla')
 
     @patch.object(Postgresql, 'pg_isready', Mock(return_value=STATE_REJECT))
     def test_is_leader(self):
@@ -406,12 +472,12 @@ class TestPostgresql(unittest.TestCase):
         self.assertFalse(self.p.is_healthy())
 
     def test_promote(self):
-        self.p._role = 'replica'
+        self.p.set_role('replica')
         self.assertTrue(self.p.promote())
         self.assertTrue(self.p.promote())
 
     def test_last_operation(self):
-        self.assertEquals(self.p.last_operation(), '0')
+        self.assertEquals(self.p.last_operation(), '2')
         Thread(target=self.p.last_operation).start()
 
     @patch('os.path.isfile', Mock(return_value=True))
@@ -425,6 +491,9 @@ class TestPostgresql(unittest.TestCase):
 
     @patch('shlex.split', Mock(side_effect=OSError))
     def test_call_nowait(self):
+        self.p.set_role('replica')
+        self.assertIsNone(self.p.call_nowait('on_start'))
+        self.p.bootstrapping = True
         self.assertIsNone(self.p.call_nowait('on_start'))
 
     def test_non_existing_callback(self):
@@ -446,20 +515,74 @@ class TestPostgresql(unittest.TestCase):
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def test_bootstrap(self):
         with patch('subprocess.call', Mock(return_value=1)):
-            self.assertRaises(PostgresException, self.p.bootstrap, {})
+            self.assertFalse(self.p.bootstrap({}))
 
-        with patch.object(Postgresql, 'run_bootstrap_post_init', Mock(return_value=False)):
-            self.assertRaises(PostgresException, self.p.bootstrap, {})
+        config = {'users': {'replicator': {'password': 'rep-pass', 'options': ['replication']}}}
 
-        self.p.bootstrap({'users': {'replicator': {'password': 'rep-pass', 'options': ['replication']}},
-                          'pg_hba': ['host replication replicator 127.0.0.1/32 md5',
-                                     'hostssl all all 0.0.0.0/0 md5',
-                                     'host all all 0.0.0.0/0 md5'],
-                          'post_init': '/bin/false'})
+        self.p.bootstrap(config)
+        with open(os.path.join(self.config_dir, 'pg_hba.conf')) as f:
+            lines = f.readlines()
+            self.assertTrue('host all all 0.0.0.0/0 md5\n' in lines)
+
+        self.p.config.pop('pg_hba')
+        config.update({'post_init': '/bin/false',
+                       'pg_hba': ['host replication replicator 127.0.0.1/32 md5',
+                                  'hostssl all all 0.0.0.0/0 md5',
+                                  'host all all 0.0.0.0/0 md5']})
+        self.p.bootstrap(config)
         with open(os.path.join(self.data_dir, 'pg_hba.conf')) as f:
             lines = f.readlines()
-            assert 'host replication replicator 127.0.0.1/32 md5\n' in lines
-            assert 'host all all 0.0.0.0/0 md5\n' in lines
+            self.assertTrue('host replication replicator 127.0.0.1/32 md5\n' in lines)
+
+    def test_custom_bootstrap(self):
+        config = {'method': 'foo', 'foo': {'command': 'bar'}}
+        with patch('subprocess.call', Mock(return_value=1)):
+            self.assertFalse(self.p.bootstrap(config))
+        with patch('subprocess.call', Mock(side_effect=Exception)):
+            self.assertFalse(self.p.bootstrap(config))
+        with patch('subprocess.call', Mock(return_value=0)),\
+                patch('subprocess.Popen', Mock(side_effect=Exception("42"))),\
+                patch('os.path.isfile', Mock(return_value=True)),\
+                patch('os.unlink', Mock()),\
+                patch.object(Postgresql, 'save_configuration_files', Mock()),\
+                patch.object(Postgresql, 'restore_configuration_files', Mock()),\
+                patch.object(Postgresql, 'write_recovery_conf', Mock()):
+            with self.assertRaises(Exception) as e:
+                self.p.bootstrap(config)
+            self.assertEqual(str(e.exception), '42')
+
+            config['foo']['recovery_conf'] = {'foo': 'bar'}
+
+            with self.assertRaises(Exception) as e:
+                self.p.bootstrap(config)
+            self.assertEqual(str(e.exception), '42')
+
+    @patch('time.sleep', Mock())
+    @patch('os.unlink', Mock())
+    @patch.object(Postgresql, 'run_bootstrap_post_init', Mock(return_value=True))
+    @patch.object(Postgresql, '_custom_bootstrap', Mock(return_value=True))
+    @patch.object(Postgresql, 'start', Mock(return_value=True))
+    def test_post_bootstrap(self):
+        config = {'method': 'foo', 'foo': {'command': 'bar'}}
+        self.p.bootstrap(config)
+
+        task = CriticalTask()
+        with patch.object(Postgresql, 'create_or_update_role', Mock(side_effect=Exception)):
+            self.p.post_bootstrap({}, task)
+            self.assertFalse(task.result)
+
+        self.p.config.pop('pg_hba')
+        self.p.post_bootstrap({}, task)
+        self.assertTrue(task.result)
+
+        self.p.bootstrap(config)
+        self.p.set_state('stopped')
+        self.p.reload_config({'authentication': {'superuser': {'username': 'p', 'password': 'p'},
+                                                 'replication': {'username': 'r', 'password': 'r'}},
+                              'listen': '*', 'retry_timeout': 10, 'parameters': {'hba_file': 'foo'}})
+        with patch.object(Postgresql, 'restart', Mock()) as mock_restart:
+            self.p.post_bootstrap({}, task)
+            mock_restart.assert_called_once()
 
     def test_run_bootstrap_post_init(self):
         with patch('subprocess.call', Mock(return_value=1)):
@@ -471,11 +594,16 @@ class TestPostgresql(unittest.TestCase):
         with patch('subprocess.call', Mock(return_value=0)) as mock_method:
             self.p._superuser.pop('username')
             self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+            mock_method.assert_called()
+            args, kwargs = mock_method.call_args
+            self.assertTrue('PGPASSFILE' in kwargs['env'])
+            self.assertEquals(args[0], ['/bin/false', 'postgres://127.0.0.2:5432/postgres'])
 
-        mock_method.assert_called()
-        args, kwargs = mock_method.call_args
-        assert 'PGPASSFILE' in kwargs['env'].keys()
-        self.assertEquals(args[0], ['/bin/false', 'postgres://localhost:5432/postgres'])
+            mock_method.reset_mock()
+            self.p._local_address.pop('host')
+            self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+            mock_method.assert_called()
+            self.assertEquals(mock_method.call_args[0][0], ['/bin/false', 'postgres://:5432/postgres'])
 
     @patch('patroni.postgresql.Postgresql.create_replica', Mock(return_value=0))
     def test_clone(self):
@@ -507,65 +635,6 @@ class TestPostgresql(unittest.TestCase):
         with patch('subprocess.check_output', Mock(side_effect=subprocess.CalledProcessError(1, ''))):
             self.assertEquals(self.p.controldata(), {})
 
-    def test_read_postmaster_opts(self):
-        m = mock_open(read_data=postmaster_opts_string())
-        with patch.object(builtins, 'open', m):
-            data = self.p.read_postmaster_opts()
-            self.assertEquals(data['wal_level'], 'hot_standby')
-            self.assertEquals(int(data['max_replication_slots']), 5)
-            self.assertEqual(data.get('D'), None)
-
-            m.side_effect = IOError
-            data = self.p.read_postmaster_opts()
-            self.assertEqual(data, dict())
-
-    @patch('subprocess.Popen')
-    @patch.object(builtins, 'open', MagicMock(return_value=42))
-    def test_single_user_mode(self, subprocess_popen_mock):
-        subprocess_popen_mock.return_value.wait.return_value = 0
-        self.assertEquals(self.p.single_user_mode(options=dict(archive_mode='on', archive_command='false')), 0)
-        subprocess_popen_mock.assert_called_once_with(['postgres', '--single', '-D', self.data_dir,
-                                                      '-c', 'archive_command=false', '-c', 'archive_mode=on',
-                                                       'postgres'], stdin=subprocess.PIPE,
-                                                      stdout=42,
-                                                      stderr=subprocess.STDOUT)
-        subprocess_popen_mock.reset_mock()
-        self.assertEquals(self.p.single_user_mode(command="CHECKPOINT"), 0)
-        subprocess_popen_mock.assert_called_once_with(['postgres', '--single', '-D', self.data_dir,
-                                                      'postgres'], stdin=subprocess.PIPE,
-                                                      stdout=42,
-                                                      stderr=subprocess.STDOUT)
-        subprocess_popen_mock.return_value = None
-        self.assertEquals(self.p.single_user_mode(), 1)
-
-    @patch('os.listdir', MagicMock(side_effect=fake_listdir))
-    @patch('os.unlink', return_value=True)
-    @patch('os.remove', return_value=True)
-    @patch('os.path.islink', return_value=False)
-    @patch('os.path.isfile', return_value=True)
-    def test_cleanup_archive_status(self, mock_file, mock_link, mock_remove, mock_unlink):
-        ap = os.path.join(self.data_dir, 'pg_xlog', 'archive_status/')
-        self.p.cleanup_archive_status()
-        mock_remove.assert_has_calls([mock.call(ap + 'a'), mock.call(ap + 'b'), mock.call(ap + 'c')])
-        mock_unlink.assert_not_called()
-
-        mock_remove.reset_mock()
-
-        mock_file.return_value = False
-        mock_link.return_value = True
-        self.p.cleanup_archive_status()
-        mock_unlink.assert_has_calls([mock.call(ap + 'a'), mock.call(ap + 'b'), mock.call(ap + 'c')])
-        mock_remove.assert_not_called()
-
-        mock_unlink.reset_mock()
-        mock_remove.reset_mock()
-
-        mock_file.side_effect = OSError
-        mock_link.side_effect = OSError
-        self.p.cleanup_archive_status()
-        mock_unlink.assert_not_called()
-        mock_remove.assert_not_called()
-
     @patch('patroni.postgresql.Postgresql._version_file_exists', Mock(return_value=True))
     @patch('subprocess.check_output', MagicMock(return_value=0, side_effect=pg_controldata_string))
     def test_sysid(self):
@@ -600,21 +669,27 @@ class TestPostgresql(unittest.TestCase):
     def test_reload_config(self):
         parameters = self._PARAMETERS.copy()
         parameters.pop('f.oo')
-        self.p.reload_config({'retry_timeout': 10, 'listen': '*', 'parameters': parameters})
+        config = {'pg_hba': [''], 'use_unix_socket': True, 'authentication': {},
+                  'retry_timeout': 10, 'listen': '*', 'parameters': parameters}
+        self.p.reload_config(config)
         parameters['b.ar'] = 'bar'
-        self.p.reload_config({'retry_timeout': 10, 'listen': '*', 'parameters': parameters})
+        self.p.reload_config(config)
         parameters['autovacuum'] = 'on'
-        self.p.reload_config({'retry_timeout': 10, 'listen': '*', 'parameters': parameters})
+        self.p.reload_config(config)
         parameters['autovacuum'] = 'off'
         parameters.pop('search_path')
-        self.p.reload_config({'retry_timeout': 10, 'listen': '*:5433', 'parameters': parameters})
+        config['listen'] = '*:5433'
+        self.p.reload_config(config)
+        parameters['unix_socket_directories'] = '.'
+        self.p.reload_config(config)
+        self.p.resolve_connection_addresses()
 
     @patch.object(Postgresql, '_version_file_exists', Mock(return_value=True))
     def test_get_major_version(self):
         with patch.object(builtins, 'open', mock_open(read_data='9.4')):
-            self.assertEquals(self.p.get_major_version(), 9.4)
+            self.assertEquals(self.p.get_major_version(), 90400)
         with patch.object(builtins, 'open', Mock(side_effect=Exception)):
-            self.assertEquals(self.p.get_major_version(), 0.0)
+            self.assertEquals(self.p.get_major_version(), 0)
 
     def test_postmaster_start_time(self):
         with patch.object(MockCursor, "fetchone", Mock(return_value=('foo', True, '', '', '', '', False))):
@@ -767,5 +842,44 @@ class TestPostgresql(unittest.TestCase):
     def test_get_server_parameters(self):
         config = {'synchronous_mode': True, 'parameters': {'wal_level': 'hot_standby'}, 'listen': '0'}
         self.p.get_server_parameters(config)
+        config['synchronous_mode_strict'] = True
+        self.p.get_server_parameters(config)
         self.p.set_synchronous_standby('foo')
         self.p.get_server_parameters(config)
+
+    @patch.object(Postgresql, 'read_pid_file', Mock(return_value={'pid': 'z'}))
+    def test_get_pid(self):
+        self.p.get_pid()
+
+    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
+    @patch.object(Postgresql, '_signal_postmaster_stop', Mock(return_value=(123, None)))
+    @patch.object(Postgresql, 'get_pid', Mock(return_value=123))
+    @patch('time.sleep', Mock())
+    @patch.object(Postgresql, 'is_pid_running')
+    def test__wait_for_connection_close(self, mock_is_pid_running):
+        mock_is_pid_running.side_effect = [True, False, False]
+        mock_callback = Mock()
+        self.p.stop(on_safepoint=mock_callback)
+
+        mock_is_pid_running.side_effect = [True, False, False]
+        with patch.object(MockCursor, "execute", Mock(side_effect=psycopg2.Error)):
+            self.p.stop(on_safepoint=mock_callback)
+
+    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
+    @patch.object(Postgresql, '_signal_postmaster_stop', Mock(return_value=(123, None)))
+    @patch.object(Postgresql, 'get_pid', Mock(return_value=123))
+    @patch.object(Postgresql, 'is_pid_running', Mock(return_value=False))
+    @patch('psutil.Process')
+    def test__wait_for_user_backends_to_close(self, mock_psutil):
+        child = Mock()
+        child.cmdline.return_value = ['foo']
+        mock_psutil.return_value.children.return_value = [child]
+        mock_callback = Mock()
+        self.p.stop(on_safepoint=mock_callback)
+
+    @patch('os.kill', Mock(side_effect=[OSError(errno.ESRCH, ''), OSError]))
+    @patch('time.sleep', Mock())
+    @patch.object(Postgresql, 'is_pid_running', Mock(side_effect=[True, False]))
+    def test_terminate_starting_postmaster(self):
+        self.p.terminate_starting_postmaster(123)
+        self.p.terminate_starting_postmaster(123)

tests/test_wale_restore.py

@@ -2,41 +2,67 @@ import psycopg2
 import subprocess
 import unittest
 
-from mock import Mock, MagicMock, patch, mock_open
+from mock import Mock, PropertyMock, patch, mock_open
+from patroni.scripts import wale_restore
 from patroni.scripts.wale_restore import WALERestore, main as _main, get_major_version
 from six.moves import builtins
+from test_postgresql import MockConnect, psycopg2_connect
 
 
-wale_output = b'name last_modified expanded_size_bytes wal_segment_backup_start ' +\
-              b'wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop\n' +\
-              b'base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 ' +\
-              b'00000001000000000000007F 00000040 00000001000000000000007F 00000240\n'
+wale_output_header = (
+    b'name\tlast_modified\t'
+    b'expanded_size_bytes\t'
+    b'wal_segment_backup_start\twal_segment_offset_backup_start\t'
+    b'wal_segment_backup_stop\twal_segment_offset_backup_stop\n'
+)
+
+wale_output_values = (
+    b'base_00000001000000000000007F_00000040\t2015-05-18T10:13:25.000Z\t'
+    b'167772160\t'
+    b'00000001000000000000007F\t00000040\t'
+    b'00000001000000000000007F\t00000240\n'
+)
+
+wale_output = wale_output_header + wale_output_values
+
+wale_restore.RETRY_SLEEP_INTERVAL = 0.001  # Speed up retries
+WALE_TEST_RETRIES = 2
 
 
 @patch('os.access', Mock(return_value=True))
 @patch('os.makedirs', Mock(return_value=True))
 @patch('os.path.exists', Mock(return_value=True))
 @patch('os.path.isdir', Mock(return_value=True))
-@patch('psycopg2.extensions.cursor', Mock(autospec=True))
-@patch('psycopg2.extensions.connection', Mock(autospec=True))
-@patch('psycopg2.connect', MagicMock(autospec=True))
+@patch('psycopg2.connect', psycopg2_connect)
 @patch('subprocess.check_output', Mock(return_value=wale_output))
 class TestWALERestore(unittest.TestCase):
 
     def setUp(self):
-        self.wale_restore = WALERestore("batman", "/data", "host=batman port=5432 user=batman",
-                                        "/etc", 100, 100, 1, 0, 1)
+        self.wale_restore = WALERestore('batman', '/data', 'host=batman port=5432 user=batman',
+                                        '/etc', 100, 100, 1, 0, WALE_TEST_RETRIES)
 
     def test_should_use_s3_to_create_replica(self):
         self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
+        with patch.object(MockConnect, 'server_version', PropertyMock(return_value=100000)):
+            self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
+
+        with patch('subprocess.check_output', Mock(return_value=wale_output.replace(b'167772160', b'1'))):
+            self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
 
         with patch('psycopg2.connect', Mock(side_effect=psycopg2.Error("foo"))):
             save_no_master = self.wale_restore.no_master
             save_master_connection = self.wale_restore.master_connection
 
             self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-            self.wale_restore.no_master = 1
-            self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())  # this would do 2 retries 1 sec each
+
+            with patch('time.sleep', Mock(return_value=None)) as mock_sleep:
+                self.wale_restore.no_master = 1
+                assert self.wale_restore.should_use_s3_to_create_replica()
+                # verify retries
+                mock_sleep.assert_has_calls(
+                    [((wale_restore.RETRY_SLEEP_INTERVAL,),)] * WALE_TEST_RETRIES
+                )
+
             self.wale_restore.master_connection = ''
             self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
 
@@ -45,10 +71,9 @@ class TestWALERestore(unittest.TestCase):
 
         with patch('subprocess.check_output', Mock(side_effect=subprocess.CalledProcessError(1, "cmd", "foo"))):
             self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output', Mock(return_value=wale_output.split(b'\n')[0])):
+        with patch('subprocess.check_output', Mock(return_value=wale_output_header)):
             self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output',
-                   Mock(return_value=wale_output.replace(b' wal_segment_offset_backup_stop', b''))):
+        with patch('subprocess.check_output', Mock(return_value=wale_output + wale_output_values)):
             self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
         with patch('subprocess.check_output',
                    Mock(return_value=wale_output.replace(b'expanded_size_bytes', b'expanded_size_foo'))):
@@ -70,17 +95,22 @@ class TestWALERestore(unittest.TestCase):
         with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(return_value=True)):
             with patch.object(self.wale_restore, 'create_replica_with_s3', Mock(return_value=0)):
                 self.assertEqual(self.wale_restore.run(), 0)
-            with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(return_value=None)):
-                self.assertEqual(self.wale_restore.run(), 1)
-            with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(side_effect=Exception)):
-                self.assertEqual(self.wale_restore.run(), 2)
+        with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(return_value=False)):
+            self.assertEqual(self.wale_restore.run(), 2)
+        with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(return_value=None)):
+            self.assertEqual(self.wale_restore.run(), 1)
+        with patch.object(self.wale_restore, 'should_use_s3_to_create_replica', Mock(side_effect=Exception)):
+            self.assertEqual(self.wale_restore.run(), 2)
 
     @patch('sys.exit', Mock())
     def test_main(self):
         with patch.object(WALERestore, 'run', Mock(return_value=0)):
             self.assertEqual(_main(), 0)
-        with patch.object(WALERestore, 'run', Mock(return_value=1)):
+
+        with patch.object(WALERestore, 'run', Mock(return_value=1)), \
+                patch('time.sleep', Mock(return_value=None)) as mock_sleep:
             self.assertEqual(_main(), 1)
+            assert mock_sleep.call_count == WALE_TEST_RETRIES
 
     @patch('os.path.isfile', Mock(return_value=True))
     def test_get_major_version(self):

tests/test_watchdog.py

@@ -0,0 +1,207 @@
+import ctypes
+import patroni.watchdog.linux as linuxwd
+import sys
+import unittest
+
+from mock import patch, Mock, PropertyMock
+from patroni.watchdog import Watchdog, WatchdogError
+from patroni.watchdog.base import NullWatchdog
+from patroni.watchdog.linux import LinuxWatchdogDevice
+
+
+class MockDevice(object):
+    def __init__(self, fd, filename, flag):
+        self.fd = fd
+        self.filename = filename
+        self.flag = flag
+        self.timeout = 60
+        self.open = True
+        self.writes = []
+
+
+mock_devices = [None]
+
+
+def mock_open(filename, flag):
+    fd = len(mock_devices)
+    mock_devices.append(MockDevice(fd, filename, flag))
+    return fd
+
+
+def mock_ioctl(fd, op, arg=None, mutate_flag=False):
+    assert 0 < fd < len(mock_devices)
+    dev = mock_devices[fd]
+    sys.stderr.write("Ioctl %d %d %r\n" % (fd, op, arg))
+    if op == linuxwd.WDIOC_GETSUPPORT:
+        sys.stderr.write("Get support\n")
+        assert(mutate_flag is True)
+        arg.options = sum(map(linuxwd.WDIOF.get, ['SETTIMEOUT', 'KEEPALIVEPING']))
+        arg.identity = (ctypes.c_ubyte*32)(*map(ord, 'Mock Watchdog'))
+    elif op == linuxwd.WDIOC_GETTIMEOUT:
+        arg.value = dev.timeout
+    elif op == linuxwd.WDIOC_SETTIMEOUT:
+        sys.stderr.write("Set timeout called with %s\n" % arg.value)
+        assert 0 < arg.value < 65535
+        dev.timeout = arg.value - 1
+    else:
+        raise Exception("Unknown op %d", op)
+    return 0
+
+
+def mock_write(fd, string):
+    assert 0 < fd < len(mock_devices)
+    assert len(string) == 1
+    assert mock_devices[fd].open
+    mock_devices[fd].writes.append(string)
+
+
+def mock_close(fd):
+    assert 0 < fd < len(mock_devices)
+    assert mock_devices[fd].open
+    mock_devices[fd].open = False
+
+
+@patch('os.open', mock_open)
+@patch('os.write', mock_write)
+@patch('os.close', mock_close)
+@patch('fcntl.ioctl', mock_ioctl)
+class TestWatchdog(unittest.TestCase):
+    def setUp(self):
+        mock_devices[:] = [None]
+
+    @patch('platform.system', Mock(return_value='Linux'))
+    @patch.object(LinuxWatchdogDevice, 'can_be_disabled', PropertyMock(return_value=True))
+    def test_unsafe_timeout_disable_watchdog_and_exit(self):
+        watchdog = Watchdog({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'required', 'safety_margin': -1}})
+        self.assertEquals(watchdog.activate(), False)
+        self.assertEquals(watchdog.is_running, False)
+
+    @patch('platform.system', Mock(return_value='Linux'))
+    @patch.object(LinuxWatchdogDevice, 'get_timeout', Mock(return_value=16))
+    def test_timeout_does_not_ensure_safe_termination(self):
+        Watchdog({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'auto', 'safety_margin': -1}}).activate()
+        self.assertEquals(len(mock_devices), 2)
+
+    @patch('platform.system', Mock(return_value='Linux'))
+    @patch.object(Watchdog, 'is_running', PropertyMock(return_value=False))
+    def test_watchdog_not_activated(self):
+        self.assertFalse(Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'required'}}).activate())
+
+    @patch('platform.system', Mock(return_value='Linux'))
+    @patch.object(LinuxWatchdogDevice, 'is_running', PropertyMock(return_value=False))
+    def test_watchdog_activate(self):
+        with patch.object(LinuxWatchdogDevice, 'open', Mock(side_effect=WatchdogError(''))):
+            self.assertTrue(Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'auto'}}).activate())
+        self.assertFalse(Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'required'}}).activate())
+
+    @patch('platform.system', Mock(return_value='Linux'))
+    def test_basic_operation(self):
+        watchdog = Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'required'}})
+        watchdog.activate()
+
+        self.assertEquals(len(mock_devices), 2)
+        device = mock_devices[-1]
+        self.assertTrue(device.open)
+
+        self.assertEquals(device.timeout, 24)
+
+        watchdog.keepalive()
+        self.assertEquals(len(device.writes), 1)
+
+        watchdog.disable()
+        self.assertFalse(device.open)
+        self.assertEquals(device.writes[-1], b'V')
+
+    def test_invalid_timings(self):
+        watchdog = Watchdog({'ttl': 30, 'loop_wait': 20, 'watchdog': {'mode': 'automatic', 'safety_margin': -1}})
+        watchdog.activate()
+        self.assertEquals(len(mock_devices), 1)
+        self.assertFalse(watchdog.is_running)
+
+    def test_parse_mode(self):
+        with patch('patroni.watchdog.base.logger.warning', new_callable=Mock()) as warning_mock:
+            watchdog = Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'bad'}})
+            self.assertEquals(watchdog.config.mode, 'off')
+            warning_mock.assert_called_once()
+
+    @patch('platform.system', Mock(return_value='Unknown'))
+    def test_unsupported_platform(self):
+        self.assertRaises(SystemExit, Watchdog, {'ttl': 30, 'loop_wait': 10,
+                                                 'watchdog': {'mode': 'required', 'driver': 'bad'}})
+
+    def test_exceptions(self):
+        wd = Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'bad'}})
+        wd.impl.close = wd.impl.keepalive = Mock(side_effect=WatchdogError(''))
+        self.assertIsNone(wd.disable())
+        self.assertIsNone(wd.keepalive())
+
+    @patch('platform.system', Mock(return_value='Linux'))
+    def test_config_reload(self):
+        watchdog = Watchdog({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'required'}})
+        self.assertTrue(watchdog.activate())
+        self.assertTrue(watchdog.is_running)
+
+        watchdog.reload_config({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'off'}})
+        self.assertFalse(watchdog.is_running)
+
+        watchdog.reload_config({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'required'}})
+        self.assertFalse(watchdog.is_running)
+        watchdog.keepalive()
+        self.assertTrue(watchdog.is_running)
+
+        watchdog.disable()
+        watchdog.reload_config({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'required', 'driver': 'unknown'}})
+        self.assertFalse(watchdog.is_healthy)
+
+        self.assertFalse(watchdog.activate())
+        watchdog.reload_config({'ttl': 30, 'loop_wait': 15, 'watchdog': {'mode': 'required'}})
+        self.assertFalse(watchdog.is_running)
+        watchdog.keepalive()
+        self.assertTrue(watchdog.is_running)
+
+        watchdog.reload_config({'ttl': 60, 'loop_wait': 15, 'watchdog': {'mode': 'required'}})
+        watchdog.keepalive()
+
+
+class TestNullWatchdog(unittest.TestCase):
+
+    def test_basics(self):
+        watchdog = NullWatchdog()
+        self.assertTrue(watchdog.can_be_disabled)
+        self.assertRaises(WatchdogError, watchdog.set_timeout, 1)
+        self.assertEquals(watchdog.describe(), 'NullWatchdog')
+        self.assertIsInstance(NullWatchdog.from_config({}), NullWatchdog)
+
+
+class TestLinuxWatchdogDevice(unittest.TestCase):
+
+    def setUp(self):
+        self.impl = LinuxWatchdogDevice.from_config({})
+
+    @patch('os.open', Mock(return_value=3))
+    @patch('os.write', Mock(side_effect=OSError))
+    @patch('fcntl.ioctl', Mock(return_value=0))
+    def test_basics(self):
+        self.impl.open()
+        try:
+            if self.impl.get_support().has_foo:
+                self.assertFail()
+        except Exception as e:
+            self.assertTrue(isinstance(e, AttributeError))
+        self.assertRaises(WatchdogError, self.impl.close)
+        self.assertRaises(WatchdogError, self.impl.keepalive)
+        self.assertRaises(WatchdogError, self.impl.set_timeout, -1)
+
+    @patch('os.open', Mock(return_value=3))
+    @patch('fcntl.ioctl', Mock(return_value=-1))
+    def test__ioctl(self):
+        self.assertRaises(WatchdogError, self.impl.get_support)
+        self.impl.open()
+        self.assertRaises(IOError, self.impl.get_support)
+
+    def test_is_healthy(self):
+        self.assertFalse(self.impl.is_healthy)
+
+    @patch('os.open', Mock(side_effect=OSError))
+    def test_open(self):
+        self.assertRaises(WatchdogError, self.impl.open)