A few iprovements in patronictl (#601)

* make switchover work with an old patroni
* exclude leader from candidates when interactively running failover

.travis.yml

@@ -1,17 +1,28 @@
-sudo: false
+sudo: true
 dist: trusty
 language: python
-python:
-- "3.4"  # 2.7 and 3.5 are preinstalled by default
 env:
   global:
-  - ETCDVERSION=3.0.17 ZKVERSION=3.4.9 CONSULVERSION=0.7.4
-  - PYVERSIONS="2.7 3.4 3.5"
-  matrix:
-  - TEST_SUITE="python setup.py"
-  - DCS="etcd" TEST_SUITE="behave"
-  - DCS="exhibitor" TEST_SUITE="behave"
-  - DCS="consul" TEST_SUITE="behave"
+  - ETCDVERSION=3.0.17 ZKVERSION=3.4.11 CONSULVERSION=0.7.4
+  - PYVERSIONS="2.7 3.5 3.6"
+  - EXCLUDE_BEHAVE="3.5"
+  - BOTO_CONFIG=/doesnotexist
+matrix:
+  include:
+  - python: "3.5"
+    env: TEST_SUITE="python setup.py"
+  - python: "3.6"
+    env: DCS="etcd" TEST_SUITE="behave"
+  - python: "3.6"
+    env: DCS="exhibitor" TEST_SUITE="behave"
+  - python: "3.6"
+    env: DCS="consul" TEST_SUITE="behave"
+  - python: "3.6"
+    env: DCS="kubernetes" TEST_SUITE="behave"
+branches:
+  only:
+  - master
+  - /^v\d+\.\d+(\.\d+)?$/
 cache:
   directories:
   - $HOME/mycache
@@ -19,7 +30,7 @@ before_cache:
 - |
   rm -fr $HOME/mycache/python*
   for pv in $PYVERSIONS; do
-      if [[ $TEST_SUITE != "behave" || $pv != "3.4" ]]; then
+      if [[ $TEST_SUITE != "behave" || $pv != $EXCLUDE_BEHAVE ]]; then
           fpv=$(basename $(readlink $HOME/virtualenv/python${pv}))
           mv $HOME/virtualenv/${fpv} $HOME/mycache/${fpv}
       fi
@@ -51,6 +62,26 @@ install:
             ln -s $EC etcd
         }
 
+        function get_kubernetes() {
+            wget -O localkube "https://storage.googleapis.com/minikube/k8sReleases/v1.7.0/localkube-linux-amd64"
+            chmod +x localkube
+            sudo nohup ./localkube --logtostderr=true --enable-dns=false > localkube.log 2>&1 &
+
+            echo "Waiting for localkube to start..."
+            if ! timeout 120 sh -c "while ! curl -ks http://127.0.0.1:8080/ >/dev/null; do sleep 1; done"; then
+                sudo cat localkube.log
+                echo "localkube did not start"
+                exit 1
+            fi
+            echo "Check certificate permissions"
+            sudo chmod 644 /var/lib/localkube/certs/*
+            sudo ls -altr /var/lib/localkube/certs/
+
+            echo "Set up .kube/config"
+            mkdir ~/.kube
+            echo -e "apiVersion: v1\nclusters:\n- cluster:\n    certificate-authority: /var/lib/localkube/certs/ca.crt\n    server: https://127.0.0.1:8443\n  name: local\ncontexts:\n- context:\n    cluster: local\n    user: myself\n  name: local\ncurrent-context: local\nkind: Config\npreferences: {}\nusers:\n- name: myself\n  user:\n    client-certificate: /var/lib/localkube/certs/apiserver.crt\n    client-key: /var/lib/localkube/certs/apiserver.key\n" > ~/.kube/config
+        }
+
         function get_exhibitor() {
             ZC=~/mycache/zookeeper-${ZKVERSION}
             if [[ ! -d $ZC ]]; then
@@ -65,7 +96,6 @@ install:
                 echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \
                     | nc -l 8181 &> /dev/null
             done&
-            ZK_PID=$!
         }
 
         attempt_num=1
@@ -77,7 +107,7 @@ install:
     fi
 
     for pv in $PYVERSIONS; do
-        if [[ $TEST_SUITE != "behave" || $pv != "3.4" ]]; then
+        if [[ $TEST_SUITE != "behave" || $pv != $EXCLUDE_BEHAVE ]]; then
             fpv=$(basename $(readlink $HOME/virtualenv/python$pv))
             if [[ -d ~/mycache/${fpv} ]]; then
                 mv ~/virtualenv/${fpv} ~/virtualenv/${fpv}.bckp
@@ -93,13 +123,17 @@ install:
 script:
   - |
     for pv in $PYVERSIONS; do
+        if [[ $TEST_SUITE == "behave" && $pv == $EXCLUDE_BEHAVE ]]; then
+            continue
+        fi
+
         source ~/virtualenv/python${pv}/bin/activate
 
         if [[ $TEST_SUITE != "behave" ]]; then
             echo Running unit tests using python${pv}
             $TEST_SUITE test
             $TEST_SUITE flake8
-        elif [[ $pv != "3.4" ]]; then
+        elif [[ $pv != $EXCLUDE_BEHAVE ]]; then
             echo Running acceptance tests using python${pv}
             if ! PATH=.:/usr/lib/postgresql/9.6/bin:$PATH $TEST_SUITE; then
                 # output all log files when tests are failing
@@ -112,7 +146,8 @@ script:
     set +e
 after_success:
   # before_cache is executed earlier than after_success, so we need to restore one of virtualenv directories
-  - fpv=$(basename $(readlink $HOME/virtualenv/python3.5)) && mv $HOME/mycache/${fpv} $HOME/virtualenv/${fpv}
+  - fpv=$(basename $(readlink $HOME/virtualenv/python3.6)) && mv $HOME/mycache/${fpv} $HOME/virtualenv/${fpv}
   - coveralls
   - if [[ $TEST_SUITE != "behave" ]]; then python-codacy-coverage -r coverage.xml; fi
-  - if [[ $DCS == "exhibitor" ]]; then ~/mycache/zookeeper-${ZKVERSION}/bin/zkServer.sh stop; kill -9 $ZK_PID; fi
+  - if [[ $DCS == "exhibitor" ]]; then ~/mycache/zookeeper-${ZKVERSION}/bin/zkServer.sh stop; fi
+  - sudo kill $(jobs -p)

README.rst

@@ -1,27 +1,27 @@
 |Build Status| |Coverage Status|
 
 Patroni: A Template for PostgreSQL HA with ZooKeeper, etcd or Consul
-------------------------------------------------------------
+--------------------------------------------------------------------
 
 You can find a version of this documentation that is searchable and also easier to navigate at `patroni.readthedocs.io <https://patroni.readthedocs.io>`__.
 
 
 There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
 
-Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__ or `Consul <https://github.com/hashicorp/consul>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
+Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__, `Consul <https://github.com/hashicorp/consul>`__ or `Kubernetes <https://kubernetes.io>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
 
 We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely.
 
-**Note to Kubernetes users**: We're currently developing Patroni to be as useful as possible for teams running Kubernetes on top of Google Compute Engine; Patroni can be the HA solution for Postgres in such an environment. To this end, there is a `Helm chart <https://github.com/kubernetes/charts/tree/master/incubator/patroni>`__ that uses Patroni and  `Spilo  <https://github.com/zalando/spilo/>`__ to provision a five-node PostgreSQL HA cluster in a Kubernetes+GCE environment. (The Helm chart deploys Spilo Docker images, not just "bare" Patroni.)
+**Note to Kubernetes users**: Patroni can run natively on top of Kubernetes. Take a look at the `Kubernetes <https://github.com/zalando/patroni/blob/master/docs/kubernetes.rst>`__ chapter of the Patroni documentation.
 
 .. contents::
     :local:
     :depth: 1
     :backlinks: none
 
-==============
+=================
 How Patroni Works
-==============
+=================
 
 Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features.
 
@@ -33,30 +33,60 @@ For additional background info, see:
 * `PostgreSQL HA with Kubernetes and Patroni <https://www.youtube.com/watch?v=iruaCgeG7qs>`__, talk by Josh Berkus at KubeCon 2016 (video)
 * `Feb. 2016 Zalando Tech blog post <https://tech.zalando.de/blog/zalandos-patroni-a-template-for-high-availability-postgresql/>`__
 
-================
+==================
 Development Status
-================
+==================
 
 Patroni is in active development and accepts contributions. See our `Contributing <https://github.com/zalando/patroni/blob/master/docs/CONTRIBUTING.rst>`__ section below for more details.
 
 We report new releases information `here <https://github.com/zalando/patroni/releases>`__.
 
-===========================
+===================================
 Technical Requirements/Installation
-===========================
+===================================
 
-**For Mac**
+**Pre-requirements for Mac OS**
 
 To install requirements on a Mac, run the following:
 
 ::
 
     brew install postgresql etcd haproxy libyaml python
-    pip install psycopg2 pyyaml
 
-===================
+**General installation for pip**
+
+Patroni can be installed with pip:
+
+::
+
+    pip install patroni[dependencies]
+
+where dependencies can be either empty, or consist of one or more of the following:
+
+etcd
+    `python-etcd` module in order to use Etcd as DCS
+consul
+    `python-consul` module in order to use Consul as DCS
+zookeeper
+    `kazoo` module in order to use Zookeeper as DCS
+exhibitor
+    `kazoo` module in order to use Exhibitor as DCS (same dependencies as for Zookeeper)
+kubernetes
+    `kubernetes` module in order to use Kubernetes as DCS in Patroni
+aws
+    `boto` in order to use AWS callbacks
+
+For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is:
+
+::
+
+    pip install patroni[etcd,aws]
+
+Note that external tools to call in the replica creation or custom bootstap scripts (i.e. WAL-E) should be installed independently of Patroni.
+
+=======================
 Running and Configuring
-===================
+=======================
 
 To get started, do the following from different terminals:
 ::
@@ -80,9 +110,9 @@ run:
 
     > psql --host 127.0.0.1 --port 5000 postgres
 
-===============
+==================
 YAML Configuration
-===============
+==================
 
 Go `here <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
 
@@ -92,15 +122,15 @@ Environment Configuration
 
 Go `here <https://github.com/zalando/patroni/blob/master/docs/ENVIRONMENT.rst>`__ for comprehensive information about configuring(overriding) settings via environment variables.
 
-===============
+===================
 Replication Choices
-===============
+===================
 
 Patroni uses Postgres' streaming replication, which is asynchronous by default. Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the leader. This setting should be increased or decreased based on business requirements. It's also possible to use synchronous replication for better durability guarantees. See `replication modes documentation <https://github.com/zalando/patroni/blob/master/docs/replication_modes.rst>`__ for details.
 
-===============================
+======================================
 Applications Should Not Use Superusers
-===============================
+======================================
 
 When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
 

docs/ENVIRONMENT.rst

@@ -1,6 +1,5 @@
 .. _environment:
 
-==================================
 Environment Configuration Settings
 ==================================
 
@@ -25,10 +24,21 @@ Example: defining ``PATRONI_admin_PASSWORD=strongpasswd`` and ``PATRONI_admin_OP
 Consul
 ------
 -  **PATRONI\_CONSUL\_HOST**: the host:port for the Consul endpoint.
+-  **PATRONI\_CONSUL\_URL**: url for the Consul, in format: http(s)://host:port
+-  **PATRONI\_CONSUL\_PORT**: (optional) Consul port
+-  **PATRONI\_CONSUL\_SCHEME**: (optional) **http** or **https**, defaults to **http**
+-  **PATRONI\_CONSUL\_TOKEN**: (optional) ACL token
+-  **PATRONI\_CONSUL\_VERIFY**: (optional) whether to verify the SSL certificate for HTTPS requests
+-  **PATRONI\_CONSUL\_CACERT**: (optional) The ca certificate. If pressent it will enable validation.
+-  **PATRONI\_CONSUL\_CERT**: (optional) File with the client certificate
+-  **PATRONI\_CONSUL\_KEY**: (optional) File with the client key. Can be empty if the key is part of certificate.
+-  **PATRONI\_CONSUL\_DC**: (optional) Datacenter to communicate with. By default the datacenter of the host is used.
+-  **PATRONI\_CONSUL\_CHECKS**: (optional) list of Consul health checks used for the session. If not specified Consul will use "serfHealth" in additional to the TTL based check created by Patroni. Additional checks, in particular the "serfHealth", may cause the leader lock to expire faster than in `ttl` seconds when the leader instance becomes unavailable.
 
 Etcd
 ----
 -  **PATRONI\_ETCD\_HOST**: the host:port for the etcd endpoint.
+-  **PATRONI\_ETCD\_HOSTS**: list of etcd endpoints in format host1:port1,host2:port2,etc...
 -  **PATRONI\_ETCD\_URL**: url for the etcd, in format: http(s)://(username:password@)host:port
 -  **PATRONI\_ETCD\_PROXY**: proxy url for the etcd. If you are connecting to the etcd using proxy, use this parameter instead of **PATRONI\_ETCD\_URL**
 -  **PATRONI\_ETCD\_SRV**: Domain to search the SRV record(s) for cluster autodiscovery.
@@ -41,6 +51,18 @@ Exhibitor
 -  **PATRONI\_EXHIBITOR\_HOSTS**: initial list of Exhibitor (ZooKeeper) nodes in format: 'host1,host2,etc...'. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes.
 -  **PATRONI\_EXHIBITOR\_PORT**: Exhibitor port.
 
+.. _kubernetes_environment:
+
+Kubernetes
+----------
+-  **PATRONI\_KUBERNETES\_NAMESPACE**: (optional) Kubernetes namespace where the Patroni pod is running. Default value is `default`.
+-  **PATRONI\_KUBERNETES\_LABELS**: Labels in format ``{label1: value1, label2: value2}``. These labels will be used to find existing objects (Pods and either Endpoints or ConfigMaps) associated with the current cluster. Also Patroni will set them on every object (Endpoint or ConfigMap) it creates.
+-  **PATRONI\_KUBERNETES\_SCOPE\_LABEL**: (optional) name of the label containing cluster name. Default value is `cluster-name`.
+-  **PATRONI\_KUBERNETES\_ROLE\_LABEL**: (optional) name of the label containing Postgres role (`master` or `replica`). Patroni will set this label on the pod it is running in. Default value is `role`.
+-  **PATRONI\_KUBERNETES\_USE\_ENDPOINTS**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
+-  **PATRONI\_KUBERNETES\_POD\_IP**: (optional) IP address of the pod Patroni is running in. This value is required when `PATRONI_KUBERNETES_USE_ENDPOINTS` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
+-  **PATRONI\_KUBERNETES\_PORTS**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service wont work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``PATRONI_KUBERNETES_PORTS='{[{"name": "postgresql", "port": 5432}]}'`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `PATRONI_KUBERNETES_USE_ENDPOINTS` is set.
+
 PostgreSQL
 ----------
 -  **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.

docs/README.rst

@@ -1,8 +1,8 @@
 .. _readme:
 
-=================
-How Patroni Works
-=================
+============
+Introduction
+============
 
 Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features.
 
@@ -13,30 +13,65 @@ For additional background info, see:
 * `PostgreSQL HA with Kubernetes and Patroni <https://www.youtube.com/watch?v=iruaCgeG7qs>`__, talk by Josh Berkus at KubeCon 2016 (video)
 * `Feb. 2016 Zalando Tech blog post <https://tech.zalando.de/blog/zalandos-patroni-a-template-for-high-availability-postgresql/>`__
 
-==================
+
 Development Status
-==================
+------------------
 
 Patroni is in active development and accepts contributions. See our :ref:`Contributing <contributing>` section below for more details.
 
 We report new releases information :ref:`here <releases>`.
 
-===================================
-Technical Requirements/Installation
-===================================
 
-**For Mac**
+Technical Requirements/Installation
+-----------------------------------
+
+**Pre-requirements for Mac OS**
 
 To install requirements on a Mac, run the following:
 
 ::
 
     brew install postgresql etcd haproxy libyaml python
-    pip install psycopg2 pyyaml
 
-=======================
+**General installation for pip**
+
+Patroni can be installed with pip:
+
+::
+
+    pip install patroni[dependencies]
+
+where dependencies can be either empty, or consist of one or more of the following:
+
+etcd
+    `python-etcd` module in order to use Etcd as DCS
+consul
+    `python-consul` module in order to use Consul as DCS
+zookeeper
+    `kazoo` module in order to use Zookeeper as DCS
+exhibitor
+    `kazoo` module in order to use Exhibitor as DCS (same dependencies as for Zookeeper)
+kubernetes
+    `kubernetes` module in order to use Kubernetes as DCS in Patroni
+aws
+    `boto` in order to use AWS callbacks
+
+For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is:
+
+::
+
+    pip install patroni[etcd,aws]
+
+Note that external tools to call in the replica creation or custom bootstap scripts (i.e. WAL-E) should be installed
+independently of Patroni.
+
+
 Running and Configuring
-=======================
+-----------------------
+
+The following section assumes Patroni repository as being cloned from https://github.com/zalando/patroni. Namely, you
+will need example configuration files `postgres0.yml` and `postgres1.yml`. If you installed Patroni with pip, you can
+obtain those files from the git repository and replace `./patroni.py` below with `patroni` command.
 
 To get started, do the following from different terminals:
 ::
@@ -60,27 +95,27 @@ run:
 
     > psql --host 127.0.0.1 --port 5000 postgres
 
-==================
+
 YAML Configuration
-==================
+------------------
 
 Go :ref:`here <settings>` for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
 
-=========================
+
 Environment Configuration
-=========================
+-------------------------
 
 Go :ref:`here <environment>` for comprehensive information about configuring(overriding) settings via environment variables.
 
-===================
+
 Replication Choices
-===================
+-------------------
 
 Patroni uses Postgres' streaming replication, which is asynchronous by default. Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the leader. This setting should be increased or decreased based on business requirements. It's also possible to use synchronous replication for better durability guarantees. See :ref:`replication modes documentation <replication_modes>` for details.
 
-======================================
+
 Applications Should Not Use Superusers
-======================================
+--------------------------------------
 
 When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
 

docs/SETTINGS.rst

@@ -43,14 +43,30 @@ Bootstrap configuration
             -  **- createdb**
 -  **post\_bootstrap** or **post\_init**: An additional script that will be executed after initializing the cluster. The script receives a connection string URL (with the cluster superuser as a user name). The PGPASSFILE variable is set to the location of pgpass file.
 
+.. _consul_settings:
+
 Consul
 ------
--  **host**: the host:port for the Consul endpoint.
+Most of the parameters are optional, but you have to specify one of the **host** or **url**
+
+-  **host**: the host:port for the Consul endpoint, in format: http(s)://host:port
+-  **url**: url for the Consul endpoint
+-  **port**: (optional) Consul port
+-  **scheme**: (optional) **http** or **https**, defaults to **http**
+-  **token**: (optional) ACL token
+-  **verify** (optional) whether to verify the SSL certificate for HTTPS requests
+-  **cacert**: (optional) The ca certificate. If pressent it will enable validation.
+-  **cert**: (optional) file with the client certificate
+-  **key**: (optional) file with the client key. Can be empty if the key is part of **cert**.
+-  **dc**: (optional) Datacenter to communicate with. By default the datacenter of the host is used.
+-  **checks**: (optional) list of Consul health checks used for the session. If not specified Consul will use "serfHealth" in additional to the TTL based check created by Patroni. Additional checks, in particular the "serfHealth", may cause the leader lock to expire faster than in `ttl` seconds when the leader instance becomes unavailable
 
 Etcd
 ----
-Most of the parameters are optional, but you have to specify one of the **host**, **url**, **proxy** or **srv**
+Most of the parameters are optional, but you have to specify one of the **host**, **hosts**, **url**, **proxy** or **srv**
+
 -  **host**: the host:port for the etcd endpoint.
+-  **hosts**: list of etcd endpoint in format host1:port1,host2:port2,etc... Could be a comma separated string or an actual yaml list.
 -  **url**: url for the etcd
 -  **proxy**: proxy url for the etcd. If you are connecting to the etcd using proxy, use this parameter instead of **url**
 -  **srv**: Domain to search the SRV record(s) for cluster autodiscovery.
@@ -67,6 +83,18 @@ Exhibitor
 -  **poll\_interval**: how often the list of ZooKeeper and Exhibitor nodes should be updated from Exhibitor
 -  **port**: Exhibitor port.
 
+.. _kubernetes_settings:
+
+Kubernetes
+----------
+-  **namespace**: (optional) Kubernetes namespace where Patroni pod is running. Default value is `default`.
+-  **labels**: Labels in format ``{label1: value1, label2: value2}``. These labels will be used to find existing objects (Pods and either Endpoints or ConfigMaps) associated with the current cluster. Also Patroni will set them on every object (Endpoint or ConfigMap) it creates.
+-  **scope\_label**: (optional) name of the label containing cluster name. Default value is `cluster-name`.
+-  **role\_label**: (optional) name of the label containing role (master or replica). Patroni will set this label on the pod it runs in. Default value is ``role``.
+-  **use\_endpoints**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
+-  **pod\_ip**: (optional) IP address of the pod Patroni is running in. This value is required when `use_endpoints` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
+-  **ports**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service wont work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``kubernetes.ports: {[{"name": "postgresql", "port": 5432}]}`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `kubernetes.use_endpoints` is set.
+
 .. _postgresql_settings:
 
 PostgreSQL

docs/conf.py

@@ -18,9 +18,11 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
 
+import sys
+sys.path.insert(0, os.path.abspath('..'))
+
+from patroni.version import __version__
 
 # -- General configuration ------------------------------------------------
 
@@ -51,7 +53,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'Patroni'
-copyright = '2016, Zalando SE'
+copyright = '2015 Compose, Zalando SE'
 author = 'Zalando SE'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -59,9 +61,9 @@ author = 'Zalando SE'
 # built documents.
 #
 # The short X.Y version.
-version = '1.2'
+version = __version__[:__version__.rfind('.')]
 # The full version, including alpha/beta/rc tags.
-release = '1.2.2'
+release = __version__
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/index.rst

@@ -6,11 +6,11 @@
 Introduction
 ============
 
-Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__ or `Consul <https://github.com/hashicorp/consul>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
+Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__, `Consul <https://github.com/hashicorp/consul>`__ or `Kubernetes <https://kubernetes.io>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
 
 We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely. There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
 
-**Note to Kubernetes users**: We're currently developing Patroni to be as useful as possible for teams running Kubernetes on top of Google Compute Engine; Patroni can be the HA solution for Postgres in such an environment. To this end, we've created a `Helm Chart <https://github.com/kubernetes/charts/tree/master/incubator/patroni>`__ that enables you to deploy a five-node Patroni cluster using a Kubernetes PetSet.
+**Note to Kubernetes users**: Patroni can run natively on top of Kubernetes. Take a look at the :ref:`Kubernetes <kubernetes>` chapter of the Patroni documentation.
 
 
 .. toctree::
@@ -24,6 +24,8 @@ We call Patroni a "template" because it is far from being a one-size-fits-all or
    replica_bootstrap
    replication_modes
    pause
+   kubernetes
+   watchdog
    releases
    CONTRIBUTING
 
@@ -33,5 +35,3 @@ Indices and tables
 * :ref:`genindex`
 * :ref:`modindex`
 * :ref:`search`
-
-

docs/kubernetes.rst

@@ -0,0 +1,53 @@
+.. _kubernetes:
+
+Using Patroni with Kubernetes
+=============================
+
+Patroni can use Kubernetes objects in order to store the state of the cluster and manage the leader key. That makes it
+capable of operating Postgres in Kubernetes environment without any consistency store, namely, one doesn't
+need to run an extra Etcd deployment. There are two different type of Kubernetes objects Patroni can use to store the
+leader and the configuration keys, they are configured with the `kubernetes.use_endpoints` or `PATRONI_KUBERNETES_USE_ENDPOINTS`
+environment variable.
+
+Use Endpoints
+-------------
+
+Despite the fact that this is the recommended mode, it is turned off by default for compatibility reasons. When it is on, Patroni stores
+the cluster configuration and the leader key in the `metadata: annotations` fields of the respective `Endpoints` it creates.
+Changing the leader is safer than when using `ConfigMaps`, since both the annotations, containing the leader information, and the actual addresses
+pointing to the running leader pod are updated simultaneously in one go.
+
+Use ConfigMaps
+--------------
+
+In this mode, Patroni will create ConfigMaps instead of Endpoints and store keys inside meta-data of those ConfigMaps.
+Changing the leader takes at least two updates, one to the leader ConfigMap and another to the respective Endpoint.
+
+There are two ways to direct the traffic to the Postgres master:
+
+- use the `callback script <https://github.com/zalando/patroni/blob/master/kubernetes/callback.py>`_ provided by Patroni
+- configure the Kubernetes Postgres service to use the label selector with the `role_label` (configured in patroni configuration).
+
+Note that in some cases, for instance, when running on OpenShift, there is no alternative to using ConfigMaps.
+
+Configuration
+-------------
+
+Patroni Kubernetes :ref:`settings <kubernetes_settings>` and :ref:`environment variables <kubernetes_environment>` are described in the general chapters of the documentation.
+
+Examples
+--------
+
+- The `kubernetes <https://github.com/zalando/patroni/tree/master/kubernetes>`__ folder of the Patroni repository contains
+  examples of the Docker image, the Kubernetes manifest and the callback script in order to test Patroni Kubernetes setup.
+  Note that in the current state it will not be able to use PersistentVolumes because of permission issues.
+
+- You can find the full-featured Docker image that can use Persistent Volumes in the
+  `Spilo Project <https://github.com/zalando/spilo>`_.
+
+- There is also a `Helm chart <https://github.com/unguiculus/charts/tree/feature/patroni/incubator/patroni>`_
+  to deploy the Spilo image configured with Patroni running using Kubernetes.
+
+- In order to run your database clusters at scale using Patroni and Spilo, take a look at the
+  `postgres-operator <https://github.com/zalando-incubator/postgres-operator>`_ project. It implements the operator pattern
+  to manage Spilo clusters.

docs/releases.rst

@@ -3,6 +3,238 @@
 Release notes
 =============
 
+Version 1.4.1
+-------------
+
+**Fixes in patronictl**
+
+- Don't show current leader in suggested list of members to failover to. (Alexander Kukushkin)
+
+  patronictl failover could still work when there is leader in the cluster and it should be excluded from the list of member where it is possible to failover to.
+
+- Make patronictl switchover compatible with the old Patroni api (Alexander)
+
+  In case if POST /switchover REST API call has failed with status code 501 it will do it once again, but to /failover endpoint.
+
+
+Version 1.4
+-----------
+
+This version adds support for using Kubernetes as a DCS, allowing to run Patroni as a cloud-native agent in Kubernetes without any additional deployments of Etcd, Zookeeper or Consul.
+
+**Upgrade notice**
+
+Installing Patroni via pip will no longer bring in dependencies for (such as libraries for Etcd, Zookeper, Consul or Kubernetes, or support for AWS). In order to enable them one need to list them in pip install command explicitely, for instance `pip install patroni[etcd,kubernetes]`.
+
+**Kubernetes support**
+
+Implement Kubernetes-based DCS. The endpoints meta-data is used in order to store the configuration and the leader key. The meta-data field inside the pods definition is used to store the member-related data.
+In addition to using Endpoints, Patroni supports ConfigMaps. You can find more information about this feature in the :ref:`Kubernetes chapter of the documentation <kubernetes>`
+
+**Stability improvements**
+
+- Factor out postmaster process into a separate object (Ants Aasma)
+
+  This object identifies a running postmaster process via pid and start time and simplifies detection (and resolution) of situations when the postmaster was restarted behind our back or when postgres directory disappeared from the file system.
+
+- Minimize the amount of SELECT's issued by Patroni on every loop of HA cylce (Alexander Kukushkin)
+
+  On every iteration of HA loop Patroni needs to know recovery status and absolute wal position. From now on Patroni will run only single SELECT to get this information instead of two on the replica and three on the master. 
+  
+- Remove leader key on shutdown only when we have the lock (Ants)
+
+  Unconditional removal was generating unnecessary and missleading exceptions.
+
+**Improvements in patronictl**
+
+- Add version command to patronictl (Ants)
+
+  It will show the version of installed Patroni and versions of running Patroni instances (if the cluster name is specified).
+  
+- Make optional specifying cluster_name argument for some of patronictl commands (Alexander, Ants)
+
+  It will work if patronictl is using usual Patroni configuration file with the ``scope`` defined.
+
+- Show information about scheduled switchover and maintenance mode (Alexander)
+
+  Before that it was possible to get this information only from Patroni logs or directly from DCS.
+  
+- Improve ``patronictl reinit`` (Alexander)
+
+  Sometimes ``patronictl reinit`` refused to proceed when Patroni was busy with other actions, namely trying to start postgres. `patronictl` didn't provide any commands to cancel such long running actions and the only (dangerous) workarond was removing a data directory manually. The new implementation of `reinit` forcefully cancells other long-running actions before proceeding with reinit.
+  
+- Implement ``--wait`` flag in ``patronictl pause`` and ``patronictl resume`` (Alexander)
+
+  It will make ``patronictl`` wait until the requested action is acknowledged by all nodes in the cluster.
+  Such behaviour is achieved by exposing the ``pause`` flag for every node in DCS and via the REST API.
+
+- Rename ``patronictl failover`` into ``patronictl switchover`` (Alexander)
+
+  The previous ``failover`` was actually only capable of doing a switchover; it refused to proceed in a cluster without the leader.
+
+- Alter the behavior of ``patronictl failover`` (Alexander)
+
+  It will work even if there is no leader, but in that case you will have to explicitely specify a node which should become the new leader.
+
+**Expose information about timeline and history**
+
+- Expose current timeline in DCS and via API (Alexander)
+
+  Store information about the current timeline for each member of the cluster. This information is accessible via the API and is stored in the DCS
+
+- Store promotion history in the /history key in DCS (Alexander)
+
+  In addition, store the timeline history enriched with the timestamp of the corresponding promotion in the /history key in DCS and update it with each promote.
+
+**Add endpoints for getting synchronous and asynchronous replicas**
+
+- Add new /sync and /async endpoints (Alexander, Oleksii Kliukin)
+
+ Those endpoints (also accessible as /synchronous and /asynchronous) return 200 only for synchronous and asynchornous replicas correspondingly (exclusing those marked as `noloadbalance`).
+
+**Allow multiple hosts for Etcd**
+
+- Add a new `hosts` parameter to Etcd configuration (Alexander)
+
+  This parameter should contain the initial list of hosts that will be used to discover and populate the list of the running etcd cluster members. If for some reason during work this list of discovered hosts is exhausted (no available hosts from that list), Patroni will return to the initial list from the `hosts` parameter.
+
+
+Version 1.3.6
+-------------
+
+**Stability improvements**
+
+- Verify process start time when checking if postgres is running. (Ants Aasma)
+
+  After a crash that doesn't clean up postmaster.pid there could be a new process with the same pid, resulting in a false positive for is_running(), which will lead to all kinds of bad behavior.
+
+- Shutdown postgresql before bootstrap when we lost data directory (ainlolcat)
+  
+  When data directory on the master is forcefully removed, postgres process can still stay alive for some time and prevent the replica created in place of that former master from starting or replicating.
+  The fix makes Patroni cache the postmaster pid and its start time and let it terminate the old postmaster in case it is still running after the corresponding data directory has been removed.
+
+- Perform crash recovery in a single user mode if postgres master dies (Alexander Kukushkin)
+
+  It is unsafe to start immediately as a standby and not possible to run ``pg_rewind`` if postgres hasn't been shut down cleanly.
+  The single user crash recovery only kicks in if ``pg_rewind`` is enabled or there is no master at the moment.
+
+**Consul improvements**
+
+- Make it possible to provide datacenter configuration for Consul (Vilius Okockis, Alexander)
+
+  Before that Patroni was always communicating with datacenter of the host it runs on.
+
+- Always send a token in X-Consul-Token http header (Alexander)
+
+  If ``consul.token`` is defined in Patroni configuration, we will always send it in the 'X-Consul-Token' http header.
+  python-consul module tries to be "consistent" with Consul REST API, which doesn't accept token as a query parameter for `session API <https://www.consul.io/api/session.html>`__, but it still works with 'X-Consul-Token' header.
+
+- Adjust session TTL if supplied value is smaller than the minimum possible (Stas Fomin, Alexander)
+
+  It could happen that the TTL provided in the Patroni configuration is smaller than the minimum one supported by Consul. In that case, Consul agent fails to create a new session.
+  Without a session Patroni cannot create member and leader keys in the Consul KV store, resulting in an unhealthy cluster.
+
+**Other improvements**
+
+- Define custom log format via environment variable ``PATRONI_LOGFORMAT`` (Stas)
+
+  Allow disabling timestamps and other similar fields in Patroni logs if they are already added by the system logger (usually when Patroni runs as a service).
+
+Version 1.3.5
+-------------
+
+**Bugfix**
+
+- Set role to 'uninitialized' if data directory was removed (Alexander Kukushkin)
+
+  If the node was running as a master it was preventing from failover.
+
+**Stability improvement**
+
+- Try to run postmaster in a single-user mode if we tried and failed to start postgres (Alexander)
+
+  Usually such problem happens when node running as a master was terminated and timelines were diverged.
+  If ``recovery.conf`` has ``restore_command`` defined, there are really high chances that postgres will abort startup and leave controldata unchanged.
+  It makes impossible to use ``pg_rewind``, which requires a clean shutdown.
+
+**Consul improvements**
+
+- Make it possible to specify health checks when creating session (Alexander)
+
+  If not specified, Consul will use "serfHealth". From one side it allows fast detection of isolated master, but from another side it makes it impossible for Patroni to tolerate short network lags.
+
+**Bugfix**
+
+- Fix watchdog on Python 3 (Ants Aasma)
+
+  A misunderstanding of the ioctl() call interface. If mutable=False then fcntl.ioctl() actually returns the arg buffer back.
+  This accidentally worked on Python2 because int and str comparison did not return an error.
+  Error reporting is actually done by raising IOError on Python2 and OSError on Python3.
+
+Version 1.3.4
+-------------
+
+**Different Consul improvements**
+
+- Pass the consul token as a header (Andrew Colin Kissa)
+
+  Headers are now the prefered way to pass the token to the consul `API <https://www.consul.io/api/index.html#authentication>`__.
+  
+
+- Advanced configuration for Consul (Alexander Kukushkin)
+
+  possibility to specify ``scheme``, ``token``, client and ca certificates :ref:`details <consul_settings>`.
+
+- compatibility with python-consul-0.7.1 and above (Alexander)
+
+  new python-consul module has changed signature of some methods
+
+- "Could not take out TTL lock" message was never logged (Alexander)
+
+  Not a critical bug, but lack of proper logging complicates investigation in case of problems.
+
+
+**Quote synchronous_standby_names using quote_ident**
+
+- When writing ``synchronous_standby_names`` into the ``postgresql.conf`` its value must be quoted (Alexander)
+
+  If it is not quoted properly, PostgreSQL will effectively disable synchronous replication and continue to work.
+
+
+**Different bugfixes around pause state, mostly related to watchdog** (Alexander)
+
+- Do not send keepalives if watchdog is not active
+- Avoid activating watchdog in a pause mode
+- Set correct postgres state in pause mode
+- Do not try to run queries from API if postgres is stopped
+
+
+Version 1.3.3
+-------------
+
+**Bugfixes**
+
+- synchronous replication was disabled shortly after promotion even when synchronous_mode_strict was turned on (Alexander Kukushkin)
+- create empty ``pg_ident.conf`` file if it is missing after restoring from the backup (Alexander)
+- open access in ``pg_hba.conf`` to all databases, not only postgres (Franco Bellagamba)
+
+
+Version 1.3.2
+-------------
+
+**Bugfix**
+
+- patronictl edit-config didn't work with ZooKeeper (Alexander Kukushkin)
+
+
+Version 1.3.1
+-------------
+
+**Bugfix**
+
+- failover via API was broken due to change in ``_MemberStatus`` (Alexander Kukushkin)
+
+
 Version 1.3
 -----------
 

docs/replica_bootstrap.rst

@@ -95,4 +95,3 @@ running master or replicas. In that case, an empty string will be passed in a co
 restoring the formerly running cluster from the binary backup.
 
 If all replica creation methods fail, Patroni will try again all methods in order during the next event loop cycle.
-

docs/replication_modes.rst

@@ -65,4 +65,4 @@ On each HA loop iteration Patroni re-evaluates synchronous standby choice. If th
 
 .. [1] The data is still there, but recovering it requires a manual recovery effort by data recovery specialists. When Patroni is allowed to rewind with ``use_pg_rewind`` the forked timeline will be automatically erased to rejoin the failed master with the cluster.
 
-.. [2] Clients can change the behavior per transaction using PostgreSQL's ``synchronous_commit`` setting. Transactions with ``synchronous_commit`` values of ``off`` and ``local`` may be lost on fail over, but will not be blocked by replication delays.
+.. [2] Clients can change the behavior per transaction using PostgreSQL's ``synchronous_commit`` setting. Transactions with ``synchronous_commit`` values of ``off`` and ``local`` may be lost on fail over, but will not be blocked by replication delays.

docs/watchdog.rst

@@ -1,6 +1,5 @@
 .. _watchdog:
 
-================
 Watchdog support
 ================
 

extras/startup-scripts/patroni

@@ -1,144 +1,144 @@
-#!/bin/sh
-#
-### BEGIN INIT INFO
-# Provides:          patroni
-# Required-Start:    $remote_fs $syslog
-# Required-Stop:     $remote_fs $syslog
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description: Patroni init script
-# Description:       Runners to orchestrate a high-availability PostgreSQL
-### END INIT INFO
-
-### BEGIN USER CONFIGURATION
-
-CONF="/etc/patroni/postgres.yml"
-LOGFILE="/var/log/patroni.log"
-USER="postgres"
-GROUP="postgres"
-
-NAME=patroni
-PATRONI="/opt/patroni/$NAME.py"
-PIDFILE="/var/run/$NAME.pid"
-
-# Set this parameter, if you have several Postgres versions installed
-# POSTGRES_VERSION="9.4"
-POSTGRES_VERSION=""
-
-### END USER CONFIGURATION
-
-. /lib/lsb/init-functions
-
-# Loading this library for get_versions() function
-if test ! -e /usr/share/postgresql-common/init.d-functions; then
-  log_failure_msg "Probably postgresql-common does not installed."
-  exit 1
-else
-  . /usr/share/postgresql-common/init.d-functions
-fi
-
-# Is there Patroni executable?
-if test ! -e $PATRONI; then
-  log_failure_msg "Patroni executable $PATRONI does not exist."
-  exit 1
-fi
-
-# Is there Patroni configuration file?
-if test ! -e $CONF; then
-  log_failure_msg "Patroni configuration file $CONF does not exist."
-  exit 1
-fi
-
-# Create logfile if doesn't exist
-if test ! -e $LOGFILE; then
-  log_action_msg "Creating logfile for Patroni..."
-  touch $LOGFILE
-  chown $USER:$GROUP $LOGFILE
-fi
-
-prepare_pgpath() {
-  if [ "$POSTGRES_VERSION" != "" ]; then
-    if [ -x /usr/lib/postgresql/$POSTGRES_VERSION/bin/pg_ctl ]; then
-      PGPATH="/usr/lib/postgresql/$POSTGRES_VERSION/bin"
-    else
-      log_failure_msg "Postgres version incorrect, check POSTGRES_VERSION variable."
-      exit 0
-    fi
-  else
-    get_versions
-    if echo $versions | grep -q -e "\s"; then
-      log_warning_msg "You have several Postgres versions installed. Please, use POSTGRES_VERSION to define correct environment."
-    else
-      versions=`echo $versions | sed -e 's/^[ \t]*//'`
-      PGPATH="/usr/lib/postgresql/$versions/bin"
-    fi
-  fi
-}
-
-get_pid() {
-  if test -e $PIDFILE; then
-    PID=`cat $PIDFILE`
-    CHILDPID=`ps --ppid $PID -o %p --no-headers`
-  else
-    log_failure_msg "Could not find PID file. Patroni probably down."
-    exit 1
-  fi
-}
-
-
-case "$1" in
-  start)
-    prepare_pgpath
-    PGPATH=$PATH:$PGPATH
-    log_success_msg "Starting Patroni\n"
-    exec start-stop-daemon --start --quiet \
-        --background \
-        --pidfile $PIDFILE --make-pidfile \
-        --chuid $USER:$GROUP \
-        --chdir `eval echo ~$USER` \
-        --exec $PATRONI \
-        --startas /bin/sh -- \
-        -c "/usr/bin/env PATH=$PGPATH /usr/bin/python $PATRONI $CONF >> $LOGFILE 2>&1"
-    ;;
-
-  stop)
-    log_success_msg "Stopping Patroni"
-    get_pid
-    start-stop-daemon --stop --pid $CHILDPID
-    start-stop-daemon --stop --pidfile $PIDFILE --remove-pidfile --quiet
-    ;;
-
-  reload)
-    log_success_msg "Reloading Patroni configuration"
-    get_pid
-    kill -HUP $CHILDPID
-    ;;
-
-  status)
-    get_pid
-    if start-stop-daemon -T --pid $CHILDPID; then
-      log_success_msg "Patroni is running\n"
-      exit 0
-    else
-      log_warning_msg "Patroni in not running\n"
-    fi
-    ;;
-
-  restart)
-    $0 stop
-    $0 start
-    ;;
-
-  *)
-    echo "Usage: /etc/init.d/$NAME {start|stop|restart|reload|status}"
-    exit 1
-    ;;
-esac
-
-if [ $? -eq 0 ]; then
-        echo .
-        exit 0
-else
-        echo " failed"
-        exit 1
-fi
+#!/bin/sh
+#
+### BEGIN INIT INFO
+# Provides:          patroni
+# Required-Start:    $remote_fs $syslog
+# Required-Stop:     $remote_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Patroni init script
+# Description:       Runners to orchestrate a high-availability PostgreSQL
+### END INIT INFO
+
+### BEGIN USER CONFIGURATION
+
+CONF="/etc/patroni/postgres.yml"
+LOGFILE="/var/log/patroni.log"
+USER="postgres"
+GROUP="postgres"
+
+NAME=patroni
+PATRONI="/opt/patroni/$NAME.py"
+PIDFILE="/var/run/$NAME.pid"
+
+# Set this parameter, if you have several Postgres versions installed
+# POSTGRES_VERSION="9.4"
+POSTGRES_VERSION=""
+
+### END USER CONFIGURATION
+
+. /lib/lsb/init-functions
+
+# Loading this library for get_versions() function
+if test ! -e /usr/share/postgresql-common/init.d-functions; then
+  log_failure_msg "Probably postgresql-common does not installed."
+  exit 1
+else
+  . /usr/share/postgresql-common/init.d-functions
+fi
+
+# Is there Patroni executable?
+if test ! -e $PATRONI; then
+  log_failure_msg "Patroni executable $PATRONI does not exist."
+  exit 1
+fi
+
+# Is there Patroni configuration file?
+if test ! -e $CONF; then
+  log_failure_msg "Patroni configuration file $CONF does not exist."
+  exit 1
+fi
+
+# Create logfile if doesn't exist
+if test ! -e $LOGFILE; then
+  log_action_msg "Creating logfile for Patroni..."
+  touch $LOGFILE
+  chown $USER:$GROUP $LOGFILE
+fi
+
+prepare_pgpath() {
+  if [ "$POSTGRES_VERSION" != "" ]; then
+    if [ -x /usr/lib/postgresql/$POSTGRES_VERSION/bin/pg_ctl ]; then
+      PGPATH="/usr/lib/postgresql/$POSTGRES_VERSION/bin"
+    else
+      log_failure_msg "Postgres version incorrect, check POSTGRES_VERSION variable."
+      exit 0
+    fi
+  else
+    get_versions
+    if echo $versions | grep -q -e "\s"; then
+      log_warning_msg "You have several Postgres versions installed. Please, use POSTGRES_VERSION to define correct environment."
+    else
+      versions=`echo $versions | sed -e 's/^[ \t]*//'`
+      PGPATH="/usr/lib/postgresql/$versions/bin"
+    fi
+  fi
+}
+
+get_pid() {
+  if test -e $PIDFILE; then
+    PID=`cat $PIDFILE`
+    CHILDPID=`ps --ppid $PID -o %p --no-headers`
+  else
+    log_failure_msg "Could not find PID file. Patroni probably down."
+    exit 1
+  fi
+}
+
+
+case "$1" in
+  start)
+    prepare_pgpath
+    PGPATH=$PATH:$PGPATH
+    log_success_msg "Starting Patroni\n"
+    exec start-stop-daemon --start --quiet \
+        --background \
+        --pidfile $PIDFILE --make-pidfile \
+        --chuid $USER:$GROUP \
+        --chdir `eval echo ~$USER` \
+        --exec $PATRONI \
+        --startas /bin/sh -- \
+        -c "/usr/bin/env PATH=$PGPATH /usr/bin/python $PATRONI $CONF >> $LOGFILE 2>&1"
+    ;;
+
+  stop)
+    log_success_msg "Stopping Patroni"
+    get_pid
+    start-stop-daemon --stop --pid $CHILDPID
+    start-stop-daemon --stop --pidfile $PIDFILE --remove-pidfile --quiet
+    ;;
+
+  reload)
+    log_success_msg "Reloading Patroni configuration"
+    get_pid
+    kill -HUP $CHILDPID
+    ;;
+
+  status)
+    get_pid
+    if start-stop-daemon -T --pid $CHILDPID; then
+      log_success_msg "Patroni is running\n"
+      exit 0
+    else
+      log_warning_msg "Patroni in not running\n"
+    fi
+    ;;
+
+  restart)
+    $0 stop
+    $0 start
+    ;;
+
+  *)
+    echo "Usage: /etc/init.d/$NAME {start|stop|restart|reload|status}"
+    exit 1
+    ;;
+esac
+
+if [ $? -eq 0 ]; then
+        echo .
+        exit 0
+else
+        echo " failed"
+        exit 1
+fi

features/basic_replication.feature

@@ -19,11 +19,21 @@ Feature: basic replication
     And I run patronictl.py restart batman postgres1 --force
     Then I receive a response returncode 0
     And "sync" key in DCS has sync_standby=postgres2 after 10 seconds
+    And I sleep for 2 seconds
+    When I issue a GET request to http://127.0.0.1:8010/sync
+    Then I receive a response code 200
+    When I issue a GET request to http://127.0.0.1:8009/async
+    Then I receive a response code 200
 
   Scenario: check the basic failover in synchronous mode
-    When I kill postgres0
-    Then postgres2 role is the primary after 22 seconds
-    When I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_mode": null, "master_start_timeout": 0}
+    Given I run patronictl.py pause batman
+    Then I receive a response returncode 0
+    When I sleep for 2 seconds
+    And I shut down postgres0
+    And I run patronictl.py resume batman 
+    Then I receive a response returncode 0
+    And postgres2 role is the primary after 24 seconds
+    When I issue a PATCH request to http://127.0.0.1:8010/config with {"synchronous_mode": null, "master_start_timeout": 0}
     Then I receive a response code 200
     When I add the table bar to postgres2
     Then table bar is present on postgres1 after 20 seconds

features/environment.py

@@ -7,6 +7,7 @@ import kazoo.exceptions
 import os
 import psutil
 import psycopg2
+import json
 import shutil
 import signal
 import six
@@ -98,6 +99,7 @@ class PatroniController(AbstractController):
         else:
             self.watchdog = None
 
+        self._scope = (custom_config or {}).get('scope', 'batman')
         self._config = self._make_patroni_test_config(name, custom_config)
         self._closables = []
 
@@ -125,6 +127,9 @@ class PatroniController(AbstractController):
     def _start(self):
         if self.watchdog:
             self.watchdog.start()
+        if isinstance(self._context.dcs_ctl, KubernetesController):
+            self._context.dcs_ctl.create_pod(self._name[8:], self._scope)
+            os.environ['PATRONI_KUBERNETES_POD_IP'] = '10.0.0.' + self._name[-1]
         return subprocess.Popen(['coverage', 'run', '--source=patroni', '-p', 'patroni.py', self._config],
                                 stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory)
 
@@ -132,6 +137,8 @@ class PatroniController(AbstractController):
         if postgres:
             return subprocess.call(['pg_ctl', '-D', self._data_dir, 'stop', '-mi', '-w'])
         super(PatroniController, self).stop(kill, timeout)
+        if isinstance(self._context.dcs_ctl, KubernetesController):
+            self._context.dcs_ctl.delete_pod(self._name[8:])
         if self.watchdog:
             self.watchdog.stop()
 
@@ -147,7 +154,7 @@ class PatroniController(AbstractController):
 
         with open(patroni_config_name) as f:
             config = yaml.safe_load(f)
-            config.pop('etcd')
+            config.pop('etcd', None)
 
         host = config['postgresql']['listen'].split(':')[0]
 
@@ -228,7 +235,7 @@ class PatroniController(AbstractController):
             if not os.path.exists(pidfile):
                 return None
             return int(open(pidfile).readline().strip())
-        except:
+        except Exception:
             return None
 
     def database_is_running(self):
@@ -335,10 +342,6 @@ class AbstractDcsController(AbstractController):
     def query(self, key, scope='batman'):
         """ query for a value of a given key """
 
-    @abc.abstractmethod
-    def set(self, key, value):
-        """ set a value to a given key """
-
     @abc.abstractmethod
     def cleanup_service_tree(self):
         """ clean all contents stored in the tree used for the tests """
@@ -388,9 +391,6 @@ class ConsulController(AbstractDcsController):
         _, value = self._client.kv.get(self.path(key, scope))
         return value and value['Value'].decode('utf-8')
 
-    def set(self, key, value):
-        self._client.kv.put(self.path(key), value)
-
     def cleanup_service_tree(self):
         self._client.kv.delete(self.path(scope=''), recurse=True)
 
@@ -417,9 +417,6 @@ class EtcdController(AbstractDcsController):
         except etcd.EtcdKeyNotFound:
             return None
 
-    def set(self, key, value):
-        self._client.set(self.path(key), value)
-
     def cleanup_service_tree(self):
         try:
             self._client.delete(self.path(scope=''), recursive=True)
@@ -436,6 +433,76 @@ class EtcdController(AbstractDcsController):
             return False
 
 
+class KubernetesController(AbstractDcsController):
+
+    def __init__(self, context):
+        super(KubernetesController, self).__init__(context)
+        self._namespace = 'default'
+        self._labels = {"application": "patroni"}
+        self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
+        os.environ['PATRONI_KUBERNETES_LABELS'] = json.dumps(self._labels)
+        os.environ['PATRONI_KUBERNETES_USE_ENDPOINTS'] = 'true'
+
+        from kubernetes import client as k8s_client, config as k8s_config
+        k8s_config.load_kube_config(context='local')
+        self._client = k8s_client
+        self._api = self._client.CoreV1Api()
+
+    def _start(self):
+        pass
+
+    def create_pod(self, name, scope):
+        labels = self._labels.copy()
+        labels['cluster-name'] = scope
+        metadata = self._client.V1ObjectMeta(namespace=self._namespace, name=name, labels=labels)
+        spec = self._client.V1PodSpec(containers=[self._client.V1Container(name=name, image='empty')])
+        body = self._client.V1Pod(metadata=metadata, spec=spec)
+        self._api.create_namespaced_pod(self._namespace, body)
+
+    def delete_pod(self, name):
+        try:
+            self._api.delete_namespaced_pod(name, self._namespace, self._client.V1DeleteOptions())
+        except:
+            pass
+        while True:
+            try:
+                self._api.read_namespaced_pod(name, self._namespace)
+            except:
+                break
+
+    def query(self, key, scope='batman'):
+        if key.startswith('members/'):
+            pod = self._api.read_namespaced_pod(key[8:], self._namespace)
+            return (pod.metadata.annotations or {}).get('status', '')
+        else:
+            try:
+                e = self._api.read_namespaced_endpoints(scope + ('' if key == 'leader' else '-' + key), self._namespace)
+                if key == 'leader':
+                    return e.metadata.annotations[key]
+                else:
+                    return json.dumps(e.metadata.annotations)
+            except:
+                return None
+
+    def cleanup_service_tree(self):
+        try:
+            self._api.delete_collection_namespaced_pod(self._namespace, label_selector=self._label_selector)
+        except:
+            pass
+        try:
+            self._api.delete_collection_namespaced_endpoints(self._namespace, label_selector=self._label_selector)
+        except:
+            pass
+
+        while True:
+            result = self._api.list_namespaced_pod(self._namespace, label_selector=self._label_selector)
+            if len(result.items) < 1:
+                break
+
+    def _is_running(self):
+        return True
+
+
 class ZooKeeperController(AbstractDcsController):
 
     """ handles all zookeeper related tasks, used for the tests setup and cleanup """
@@ -455,9 +522,6 @@ class ZooKeeperController(AbstractDcsController):
         except kazoo.exceptions.NoNodeError:
             return None
 
-    def set(self, key, value):
-        self._client.set(self.path(key), value.encode('utf-8'))
-
     def cleanup_service_tree(self):
         try:
             self._client.delete(self.path(scope=''), recursive=True)
@@ -713,6 +777,7 @@ class WatchdogMonitor(object):
 
 # actions to execute on start/stop of the tests and before running invidual features
 def before_all(context):
+    os.environ.update({'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password'})
     context.ci = 'TRAVIS_BUILD_NUMBER' in os.environ or 'BUILD_NUMBER' in os.environ
     context.timeout_multiplier = 2 if context.ci else 1
     context.pctl = PatroniPoolController(context)

features/patroni_api.feature

@@ -13,17 +13,17 @@ Scenario: check API requests on a stand-alone server
 	When I run patronictl.py reinit batman postgres0 --force
 	Then I receive a response returncode 0
 	And I receive a response output "Failed: reinitialize for member postgres0, status code=503, (I am the leader, can not reinitialize)"
-	When I run patronictl.py failover batman --master postgres0 --force
+	When I run patronictl.py switchover batman --master postgres0 --force
 	Then I receive a response returncode 1
-	And I receive a response output "Error: No candidates found to failover to"
-	When I issue a POST request to http://127.0.0.1:8008/failover with {"leader": "postgres0"}
-	Then I receive a response code 500
-	And I receive a response text failover is not possible: cluster does not have members except leader
+	And I receive a response output "Error: No candidates found to switchover to"
+	When I issue a POST request to http://127.0.0.1:8008/switchover with {"leader": "postgres0"}
+	Then I receive a response code 412
+	And I receive a response text switchover is not possible: cluster does not have members except leader
 	When I issue an empty POST request to http://127.0.0.1:8008/failover
 	Then I receive a response code 400
 	When I issue a POST request to http://127.0.0.1:8008/failover with {"foo": "bar"}
 	Then I receive a response code 400
-	And I receive a response text "No values given for required parameters leader and candidate"
+	And I receive a response text "Failover could be performed only to a specific candidate"
 
 Scenario: check local configuration reload
 	Given I issue an empty POST request to http://127.0.0.1:8008/reload
@@ -64,21 +64,21 @@ Scenario: check API requests for the primary-replica pair in the pause mode
 	When I sleep for 10 seconds
 	Then postgres1 role is the secondary after 15 seconds
 
-Scenario: check the failover via the API in the pause mode
-	Given I issue a POST request to http://127.0.0.1:8008/failover with {"leader": "postgres0", "candidate": "postgres1"}
+Scenario: check the switchover via the API in the pause mode
+	Given I issue a POST request to http://127.0.0.1:8008/switchover with {"leader": "postgres0", "candidate": "postgres1"}
 	Then I receive a response code 200
 	And postgres1 is a leader after 5 seconds
 	And postgres1 role is the primary after 10 seconds
 	And postgres0 role is the secondary after 10 seconds
 	And replication works from postgres1 to postgres0 after 20 seconds
 
-Scenario: check the scheduled failover
-	Given I issue a scheduled failover from postgres1 to postgres0 in 3 seconds
+Scenario: check the scheduled switchover
+	Given I issue a scheduled switchover from postgres1 to postgres0 in 3 seconds
 	Then I receive a response returncode 1
-	And I receive a response output "Can't schedule failover in the paused state"
+	And I receive a response output "Can't schedule switchover in the paused state"
 	When I run patronictl.py resume batman
 	Then I receive a response returncode 0
-	Given I issue a scheduled failover from postgres1 to postgres0 in 3 seconds
+	Given I issue a scheduled switchover from postgres1 to postgres0 in 3 seconds
 	Then I receive a response returncode 0
 	And postgres0 is a leader after 20 seconds
 	And postgres0 role is the primary after 10 seconds

features/steps/patroni_api.py

@@ -1,3 +1,4 @@
+import base64
 import json
 import os
 import parse
@@ -78,11 +79,13 @@ def do_post_empty(context, url):
 @step('I issue a {request_method:w} request to {url:url} with {data}')
 def do_request(context, request_method, url, data):
     data = data and json.loads(data) or {}
+    headers = {'Authorization': 'Basic ' + base64.b64encode('username:password'.encode('utf-8')).decode('utf-8'),
+               'Content-Type': 'application/json'}
     try:
         if request_method == 'PATCH':
-            r = requests.patch(url, json=data)
+            r = requests.patch(url, headers=headers, json=data)
         else:
-            r = requests.post(url, json=data)
+            r = requests.post(url, headers=headers, json=data)
     except requests.exceptions.RequestException:
         context.status_code = None
         context.response = None
@@ -122,10 +125,10 @@ def check_response(context, component, data):
         assert str(context.response[component]) == str(data), "{0} does not contain {1}".format(component, data)
 
 
-@step('I issue a scheduled failover from {from_host:w} to {to_host:w} in {in_seconds:d} seconds')
-def scheduled_failover(context, from_host, to_host, in_seconds):
+@step('I issue a scheduled switchover from {from_host:w} to {to_host:w} in {in_seconds:d} seconds')
+def scheduled_switchover(context, from_host, to_host, in_seconds):
     context.execute_steps(u"""
-        Given I run patronictl.py failover batman --master {0} --candidate {1} --scheduled "{2}" --force
+        Given I run patronictl.py switchover batman --master {0} --candidate {1} --scheduled "{2}" --force
     """.format(from_host, to_host, datetime.now(tzutc) + timedelta(seconds=int(in_seconds))))
 
 

features/steps/watchdog.py

@@ -31,7 +31,7 @@ def watchdog_was_closed(context, name):
     assert context.pctl.get_watchdog(name).was_closed
 
 
-@step('I wait for next {name:w} watchdog ping')
+@step('I reset {name:w} watchdog state')
 def watchdog_reset_pinged(context, name):
     context.pctl.get_watchdog(name).reset()
 

features/watchdog.feature

@@ -1,19 +1,31 @@
 Feature: watchdog
   Verify that watchdog gets pinged and triggered under appropriate circumstances.
 
-  Scenario: watchdog is opened, pinged and closed
+  Scenario: watchdog is opened and pinged
     Given I start postgres0 with watchdog
     Then postgres0 is a leader after 10 seconds
     And postgres0 role is the primary after 10 seconds
     And postgres0 watchdog has been pinged after 10 seconds
-    When I shut down postgres0
+
+  Scenario: watchdog is disabled during pause
+    Given I run patronictl.py pause batman
+    Then I receive a response returncode 0
+    When I sleep for 2 seconds
     Then postgres0 watchdog has been closed
 
-  #TODO: test watchdog is disabled during pause
-  #TODO: test watchdog is disabled properly when shutting down
+  Scenario: watchdog is opened and pinged after resume
+    Given I reset postgres0 watchdog state
+    And I run patronictl.py resume batman
+    Then I receive a response returncode 0
+    And postgres0 watchdog has been pinged after 10 seconds
+
+  Scenario: watchdog is disabled when shutting down
+    Given I shut down postgres0
+    Then postgres0 watchdog has been closed
 
   Scenario: watchdog is triggered if patroni stops responding
-    Given I start postgres0 with watchdog
+    Given I reset postgres0 watchdog state
+    And I start postgres0 with watchdog
     Then postgres0 role is the primary after 10 seconds
     When postgres0 hangs for 30 seconds
     Then postgres0 watchdog is triggered after 30 seconds

kubernetes/Dockerfile

@@ -0,0 +1,32 @@
+FROM postgres:9.6
+MAINTAINER Alexander Kukushkin <[email protected]>
+
+RUN export DEBIAN_FRONTEND=noninteractive \
+    && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
+    && apt-get update -y \
+    && apt-get upgrade -y \
+    && apt-get install -y git curl jq python-psycopg2 python-yaml python-requests python-six python-pysocks \
+        python-dateutil python-pip python-prettytable python-wheel python-psutil python locales \
+
+    ## Make sure we have a en_US.UTF-8 locale available
+    && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
+
+    && pip install setuptools pip --upgrade \
+    && pip install 'git+https://github.com/zalando/patroni.git@feature/k8s#egg=patroni[kubernetes]' \
+
+    && mkdir -p /home/postgres \
+    && chown postgres:postgres /home/postgres \
+
+    # Clean up
+    && apt-get remove -y git python-pip python-setuptools \
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/* /root/.cache
+
+ADD entrypoint.sh callback.py /
+
+EXPOSE 5432 8008
+ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
+USER postgres
+WORKDIR /home/postgres
+CMD ["/bin/bash", "/entrypoint.sh"]

kubernetes/callback.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+
+import logging
+import os
+import socket
+import sys
+import time
+
+from kubernetes import client as k8s_client, config as k8s_config
+from urllib3.exceptions import HTTPError
+from six.moves.http_client import HTTPException
+
+logger = logging.getLogger(__name__)
+
+
+class CoreV1Api(k8s_client.CoreV1Api):
+
+    def retry(func):
+        def wrapped(*args, **kwargs):
+            count = 0
+            while True:
+                try:
+                    return func(*args, **kwargs)
+                except (HTTPException, HTTPError, socket.error, socket.timeout):
+                    if count >= 10:
+                        raise
+                    logger.info('Throttling API requests...')
+                    time.sleep(2 ** count * 0.5)
+                    count += 1
+        return wrapped
+
+    @retry
+    def patch_namespaced_endpoints(self, *args, **kwargs):
+        return super(CoreV1Api, self).patch_namespaced_endpoints(*args, **kwargs)
+
+
+def patch_master_endpoint(api, namespace, cluster):
+    addresses = [k8s_client.V1EndpointAddress(ip=os.environ['POD_IP'])]
+    ports = [k8s_client.V1EndpointPort(port=5432)]
+    subsets = [k8s_client.V1EndpointSubset(addresses=addresses, ports=ports)]
+    body = k8s_client.V1Endpoints(subsets=subsets)
+    return api.patch_namespaced_endpoints(cluster, namespace, body)
+
+
+def main():
+    logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
+    if len(sys.argv) != 4 or sys.argv[1] not in ('on_start', 'on_stop', 'on_role_change'):
+        sys.exit('Usage: %s <action> <role> <cluster_name>', sys.argv[0])
+
+    action, role, cluster = sys.argv[1:4]
+
+    k8s_config.load_incluster_config()
+    k8s_api = CoreV1Api()
+
+    namespace = os.environ['KUBERNETES_NAMESPACE']
+
+    if role == 'master' and action in ('on_start', 'on_role_change'):
+        patch_master_endpoint(k8s_api, namespace, cluster)
+
+
+if __name__ == '__main__':
+    main()

kubernetes/entrypoint.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+cat > /home/postgres/patroni.yml <<__EOF__
+bootstrap:
+  dcs:
+    postgresql:
+      use_pg_rewind: true
+  initdb:
+  - auth-host: md5
+  - auth-local: trust
+  - encoding: UTF8
+  - locale: en_US.UTF-8
+  - data-checksums
+  pg_hba:
+  - host all all 0.0.0.0/0 md5
+  - host replication ${PATRONI_REPLICATION_USERNAME} ${POD_IP}/16    md5
+restapi:
+  connect_address: '${POD_IP}:8008'
+postgresql:
+  connect_address: '${POD_IP}:5432'
+  authentication:
+    superuser:
+      password: '${PATRONI_SUPERUSER_PASSWORD}'
+    replication:
+      password: '${PATRONI_REPLICATION_PASSWORD}'
+  callbacks:
+    on_start: /callback.py
+    on_stop: /callback.py
+    on_role_change: /callback.py
+__EOF__
+
+unset PATRONI_SUPERUSER_PASSWORD PATRONI_REPLICATION_PASSWORD
+export KUBERNETES_NAMESPACE=$PATRONI_KUBERNETES_NAMESPACE
+export POD_NAME=$PATRONI_NAME
+
+exec /usr/bin/python /usr/local/bin/patroni /home/postgres/patroni.yml

kubernetes/patroni_k8s.yaml

@@ -0,0 +1,122 @@
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  name: &cluster_name patronidemo
+  labels:
+    application: patroni
+    cluster-name: *cluster_name
+spec:
+  replicas: 3
+  serviceName: *cluster_name
+  template:
+    metadata:
+      labels:
+        application: patroni
+        cluster-name: *cluster_name
+    spec:
+      containers:
+      - name: *cluster_name
+        image: patroni  # docker build -t patroni .
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 8008
+          protocol: TCP
+        - containerPort: 5432
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /home/postgres/pgdata
+          name: pgdata
+        env:
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        - name: PATRONI_KUBERNETES_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: PATRONI_KUBERNETES_LABELS
+          value: '{application: patroni, cluster-name: patronidemo}'
+        - name: PATRONI_SUPERUSER_USERNAME
+          value: postgres
+        - name: PATRONI_SUPERUSER_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: *cluster_name
+              key: superuser-password
+        - name: PATRONI_REPLICATION_USERNAME
+          value: standby
+        - name: PATRONI_REPLICATION_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: *cluster_name
+              key: replication-password
+        - name: PATRONI_SCOPE
+          value: *cluster_name
+        - name: PATRONI_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: PATRONI_POSTGRESQL_DATA_DIR
+          value: /home/postgres/pgdata/pgroot/data
+        - name: PATRONI_POSTGRESQL_PGPASS
+          value: /tmp/pgpass
+        - name: PATRONI_POSTGRESQL_LISTEN
+          value: '0.0.0.0:5432'
+        - name: PATRONI_RESTAPI_LISTEN
+          value: '0.0.0.0:8008'
+      terminationGracePeriodSeconds: 0
+      volumes:
+      - name: pgdata
+        emptyDir: {}
+#  volumeClaimTemplates:
+#  - metadata:
+#      labels:
+#        application: spilo
+#        spilo-cluster: *cluster_name
+#      annotations:
+#        volume.alpha.kubernetes.io/storage-class: anything
+#      name: pgdata
+#    spec:
+#      accessModes:
+#      - ReadWriteOnce
+#      resources:
+#        requests:
+#          storage: 5Gi
+
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: &cluster_name patronidemo
+  labels:
+    application: patroni
+    cluster-name: *cluster_name
+subsets: []
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: &cluster_name patronidemo
+  labels:
+    application: patroni
+    cluster-name: *cluster_name
+spec:
+  type: ClusterIP
+  ports:
+  - port: 5432
+    targetPort: 5432
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: &cluster_name patronidemo
+  labels:
+    application: patroni
+    cluster-name: *cluster_name
+type: Opaque
+data:
+  superuser-password: emFsYW5kbw==
+  replication-password: cmVwLXBhc3M=

patroni/__init__.py

@@ -39,7 +39,7 @@ class Patroni(object):
         while True:
             try:
                 cluster = self.dcs.get_cluster()
-                if cluster and cluster.config:
+                if cluster and cluster.config and cluster.config.data:
                     if self.config.set_dynamic_configuration(cluster.config):
                         self.dcs.reload_config(self.config)
                         self.watchdog.reload_config(self.config)
@@ -113,8 +113,8 @@ class Patroni(object):
 
             logger.info(self.ha.run_cycle())
 
-            cluster = self.dcs.cluster
-            if cluster and cluster.config and self.config.set_dynamic_configuration(cluster.config):
+            if self.dcs.cluster and self.dcs.cluster.config and self.dcs.cluster.config.data \
+                    and self.config.set_dynamic_configuration(self.dcs.cluster.config):
                 self.reload_config()
 
             if self.postgresql.role != 'uninitialized':
@@ -134,7 +134,8 @@ class Patroni(object):
 
 
 def patroni_main():
-    logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
+    logformat = os.environ.get('PATRONI_LOGFORMAT', '%(asctime)s %(levelname)s: %(message)s')
+    logging.basicConfig(format=logformat, level=logging.INFO)
     logging.getLogger('requests').setLevel(logging.WARNING)
 
     patroni = Patroni()

patroni/api.py

@@ -7,8 +7,8 @@ import time
 import dateutil.parser
 import datetime
 
-from patroni.exceptions import PostgresConnectionException
-from patroni.utils import deep_compare, patch_config, Retry, RetryFailedError, is_valid_pg_version, parse_int, tzutc
+from patroni.postgresql import PostgresConnectionException, PostgresException, Postgresql
+from patroni.utils import deep_compare, parse_bool, patch_config, Retry, RetryFailedError, parse_int, tzutc
 from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 from six.moves.socketserver import ThreadingMixIn
 from threading import Thread
@@ -24,9 +24,9 @@ def check_auth(func):
     def do_PUT_foo():
         pass
     """
-    def wrapper(handler):
+    def wrapper(handler, *args, **kwargs):
         if handler.check_auth_header():
-            return func(handler)
+            return func(handler, *args, **kwargs)
     return wrapper
 
 
@@ -70,6 +70,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
             response['scheduled_restart']['schedule'] = (response['scheduled_restart']['schedule']).isoformat()
         if not patroni.ha.watchdog.is_healthy:
             response['watchdog_failed'] = True
+        if patroni.ha.is_paused():
+            response['pause'] = True
         self._write_json_response(status_code, response)
 
     def do_GET(self, write_status_code_only=False):
@@ -80,6 +82,14 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         patroni = self.server.patroni
         cluster = patroni.dcs.cluster
+
+        def is_synchronous():
+            return (cluster.is_synchronous_mode() and cluster.sync
+                    and cluster.sync.sync_standby == patroni.postgresql.name)
+
+        def is_balanceable_replica():
+            return response.get('role') == 'replica' and not patroni.noloadbalance
+
         if cluster:  # dcs available
             if cluster.leader and cluster.leader.name == patroni.postgresql.name:  # is_leader
                 status_code = 200 if 'master' in path else 503
@@ -87,6 +97,10 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 status_code = 503
             elif response['role'] == 'master':  # running as master but without leader lock!!!!
                 status_code = 503
+            elif path in ('/sync', '/synchronous'):
+                status_code = 200 if is_balanceable_replica() and is_synchronous() else 503
+            elif path in ('/async', '/asynchronous'):
+                status_code = 200 if is_balanceable_replica() and not is_synchronous() else 503
             elif response['role'] in path:  # response['role'] != 'master'
                 status_code = 503 if patroni.noloadbalance else 200
             else:
@@ -221,9 +235,11 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     data = "PostgreSQL role should be either master or replica"
                     break
             elif k == 'postgres_version':
-                if not is_valid_pg_version(request[k]):
+                try:
+                    Postgresql.postgres_version_to_int(request[k])
+                except PostgresException as e:
                     status_code = 400
-                    data = "PostgreSQL version should be in the first.major.minor format"
+                    data = e.value
                     break
             elif k == 'timeout':
                 request[k] = parse_int(request[k], 's')
@@ -264,7 +280,14 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
     @check_auth
     def do_POST_reinitialize(self):
-        data = self.server.patroni.ha.reinitialize()
+        request = self._read_json_content(body_is_optional=True)
+
+        if request:
+            logger.debug('received reinitialize request: %s', request)
+
+        force = isinstance(request, dict) and parse_bool(request.get('force')) or False
+
+        data = self.server.patroni.ha.reinitialize(force)
         if data is None:
             status_code = 200
             data = 'reinitialize started'
@@ -289,26 +312,32 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 logger.debug('Exception occured during polling failover result: %s', e)
         return 503, 'Failover status unknown'
 
-    def is_failover_possible(self, cluster, leader, candidate):
+    def is_failover_possible(self, cluster, leader, candidate, action):
         if leader and (not cluster.leader or cluster.leader.name != leader):
             return 'leader name does not match'
         if candidate:
+            if action == 'switchover' and cluster.is_synchronous_mode() and cluster.sync.sync_standby != candidate:
+                return 'candidate name does not match with sync_standby'
             members = [m for m in cluster.members if m.name == candidate]
             if not members:
                 return 'candidate does not exists'
+        elif cluster.is_synchronous_mode():
+            members = [m for m in cluster.members if m.name == cluster.sync.sync_standby]
+            if not members:
+                return action + ' is not possible: can not find sync_standby'
         else:
             members = [m for m in cluster.members if m.name != cluster.leader.name and m.api_url]
             if not members:
-                return 'failover is not possible: cluster does not have members except leader'
+                return action + ' is not possible: cluster does not have members except leader'
         for st in self.server.patroni.ha.fetch_nodes_statuses(members):
             if st.failover_limitation() is None:
                 return None
-        return 'failover is not possible: no good candidates have been found'
+        return action + ' is not possible: no good candidates have been found'
 
     @check_auth
-    def do_POST_failover(self):
+    def do_POST_failover(self, action='failover'):
         request = self._read_json_content()
-        status_code = 500
+        (status_code, data) = (400, '')
         if not request:
             return
 
@@ -317,39 +346,46 @@ class RestApiHandler(BaseHTTPRequestHandler):
         scheduled_at = request.get('scheduled_at')
         cluster = self.server.patroni.dcs.get_cluster()
 
-        if scheduled_at and cluster.is_paused():
-            self._write_response(status_code, "Can't schedule failover in the paused state")
+        logger.info("received %s request with leader=%s candidate=%s scheduled_at=%s",
+                    action, leader, candidate, scheduled_at)
 
-        logger.info("received failover request with leader=%s candidate=%s scheduled_at=%s",
-                    leader, candidate, scheduled_at)
+        if action == 'failover' and not candidate:
+            data = 'Failover could be performed only to a specific candidate'
+        elif action == 'switchover' and not leader:
+            data = 'Switchover could be performed only from a specific leader'
 
-        data = ''
-        if leader or candidate:
-            if scheduled_at:
-                (_, data, scheduled_at) = self.parse_schedule(scheduled_at, "failover")
-                if _:
-                    status_code = _
-                elif self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
-                    self.server.patroni.ha.wakeup()
-                    data = 'Failover scheduled'
+        if not data and scheduled_at:
+            if not leader:
+                data = 'Scheduled {0} is possible only from a specific leader'.format(action)
+            if not data and cluster.is_paused():
+                data = "Can't schedule {0} in the paused state".format(action)
+            if not data:
+                (status_code, data, scheduled_at) = self.parse_schedule(scheduled_at, action)
+
+        if not data and cluster.is_paused() and not candidate:
+            data = action.title() + ' is possible only to a specific candidate in a paused state'
+
+        if not data and not scheduled_at:
+            data = self.is_failover_possible(cluster, leader, candidate, action)
+            if data:
+                status_code = 412
+
+        if not data:
+            if self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
+                self.server.patroni.ha.wakeup()
+                if scheduled_at:
+                    data = action.title() + ' scheduled'
                     status_code = 202
                 else:
-                    data = 'failed to write failover key into DCS'
-                    status_code = 503
+                    status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, candidate)
             else:
-                data = self.is_failover_possible(cluster, leader, candidate)
-                if not data:
-                    if self.server.patroni.dcs.manual_failover(leader, candidate):
-                        self.server.patroni.ha.wakeup()
-                        status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, candidate)
-                    else:
-                        data = 'failed to write failover key into DCS'
-                        status_code = 503
-        else:
-            status_code = 400
-            data = 'No values given for required parameters leader and candidate'
+                data = 'failed to write {0} key into DCS'.format(action)
+                status_code = 503
         self._write_response(status_code, data)
 
+    def do_POST_switchover(self):
+        self.do_POST_failover(action='switchover')
+
     def parse_request(self):
         """Override parse_request method to enrich basic functionality of `BaseHTTPRequestHandler` class
 
@@ -376,40 +412,46 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
     def get_postgresql_status(self, retry=False):
         try:
-            row = self.query("""WITH replication_info AS (
-                                    SELECT usename, application_name, client_addr, state, sync_state, sync_priority
-                                      FROM pg_stat_replication
-                                )
-                                SELECT to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
-                                       pg_is_in_recovery(),
-                                       CASE WHEN pg_is_in_recovery()
-                                            THEN 0
-                                            ELSE pg_{0}_{1}_diff(pg_current_{0}_{1}(), '0/0')::bigint
-                                       END,
-                                       pg_{0}_{1}_diff(COALESCE(pg_last_{0}_receive_{1}(),
-                                                                      pg_last_{0}_replay_{1}()), '0/0')::bigint,
-                                       pg_{0}_{1}_diff(pg_last_{0}_replay_{1}(), '0/0')::bigint,
-                                       to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
-                                       pg_is_in_recovery() AND pg_is_{0}_replay_paused(),
-                                       (SELECT array_to_json(array_agg(row_to_json(ri)))
-                                          FROM replication_info ri)""".format(self.server.patroni.postgresql.wal_name,
-                                                                              self.server.patroni.postgresql.lsn_name),
-                             retry=retry)[0]
+            if self.server.patroni.postgresql.state not in ('running', 'restarting', 'starting'):
+                raise RetryFailedError('')
+            stmt = ("WITH replication_info AS ("
+                    "SELECT usename, application_name, client_addr, state, sync_state, sync_priority"
+                    " FROM pg_stat_replication) SELECT"
+                    " to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),"
+                    " CASE WHEN pg_is_in_recovery() THEN 0"
+                    " ELSE ('x' || SUBSTR(pg_{0}file_name(pg_current_{0}_{1}()), 1, 8))::bit(32)::int END,"
+                    " CASE WHEN pg_is_in_recovery() THEN 0"
+                    " ELSE pg_{0}_{1}_diff(pg_current_{0}_{1}(), '0/0')::bigint END,"
+                    " pg_{0}_{1}_diff(COALESCE(pg_last_{0}_receive_{1}(), pg_last_{0}_replay_{1}()), '0/0')::bigint,"
+                    " pg_{0}_{1}_diff(pg_last_{0}_replay_{1}(), '0/0')::bigint,"
+                    " to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),"
+                    " pg_is_in_recovery() AND pg_is_{0}_replay_paused(),"
+                    " (SELECT array_to_json(array_agg(row_to_json(ri))) FROM replication_info ri)")
+
+            row = self.query(stmt.format(self.server.patroni.postgresql.wal_name,
+                                         self.server.patroni.postgresql.lsn_name), retry=retry)[0]
 
             result = {
                 'state': self.server.patroni.postgresql.state,
                 'postmaster_start_time': row[0],
-                'role': 'replica' if row[1] else 'master',
+                'role': 'replica' if row[1] == 0 else 'master',
                 'server_version': self.server.patroni.postgresql.server_version,
                 'xlog': ({
                     'received_location': row[3],
                     'replayed_location': row[4],
                     'replayed_timestamp': row[5],
-                    'paused': row[6]} if row[1] else {
+                    'paused': row[6]} if row[1] == 0 else {
                     'location': row[2]
                 })
             }
 
+            if row[1] > 0:
+                result['timeline'] = row[1]
+            else:
+                cluster = self.server.patroni.dcs.cluster
+                leader_timeline = None if not cluster or cluster.is_unlocked() else cluster.leader.timeline
+                result['timeline'] = self.server.patroni.postgresql.replica_cached_timeline(leader_timeline)
+
             if row[7]:
                 result['replication'] = row[7]
 
@@ -473,7 +515,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
     def __initialize(self, config):
         self.__ssl_options = self.__get_ssl_options(config)
         self.__listen = config['listen']
-        host, port = config['listen'].split(':')
+        host, port = config['listen'].rsplit(':', 1)
         HTTPServer.__init__(self, (host, int(port)), RestApiHandler)
         Thread.__init__(self, target=self.serve_forever)
         self._set_fd_cloexec(self.socket)

patroni/async_executor.py

@@ -1,5 +1,5 @@
 import logging
-from threading import Lock, RLock, Thread
+from threading import Event, Lock, RLock, Thread
 
 logger = logging.getLogger(__name__)
 
@@ -52,22 +52,27 @@ class CriticalTask(object):
 
 class AsyncExecutor(object):
 
-    def __init__(self, ha_wakeup):
+    def __init__(self, state_handler, ha_wakeup):
+        self.state_handler = state_handler
         self._ha_wakeup = ha_wakeup
         self._thread_lock = RLock()
         self._scheduled_action = None
         self._scheduled_action_lock = RLock()
+        self._is_cancelled = False
+        self._finish_event = Event()
         self.critical_task = CriticalTask()
 
     @property
     def busy(self):
         return self.scheduled_action is not None
 
-    def schedule(self, action, immediately=False):
+    def schedule(self, action):
         with self._scheduled_action_lock:
             if self._scheduled_action is not None:
                 return self._scheduled_action
             self._scheduled_action = action
+            self._is_cancelled = False
+            self._finish_event.set()
         return None
 
     @property
@@ -82,14 +87,21 @@ class AsyncExecutor(object):
     def run(self, func, args=()):
         wakeup = False
         try:
+            with self:
+                if self._is_cancelled:
+                    return
+                self._finish_event.clear()
+
+            self.state_handler.reset_is_cancelled()
             # if the func returned something (not None) - wake up main HA loop
             wakeup = func(*args) if args else func()
             return wakeup
-        except:
+        except Exception:
             logger.exception('Exception during execution of long running task %s', self.scheduled_action)
         finally:
             with self:
                 self.reset_scheduled_action()
+                self._finish_event.set()
                 with self.critical_task:
                     self.critical_task.reset()
             if wakeup is not None:
@@ -98,6 +110,20 @@ class AsyncExecutor(object):
     def run_async(self, func, args=()):
         Thread(target=self.run, args=(func, args)).start()
 
+    def cancel(self):
+        with self:
+            with self._scheduled_action_lock:
+                if self._scheduled_action is None:
+                    return
+                logger.warning('Cancelling long running task %s', self._scheduled_action)
+            self._is_cancelled = True
+
+        self.state_handler.cancel()
+        self._finish_event.wait()
+
+        with self:
+            self.reset_scheduled_action()
+
     def __enter__(self):
         self._thread_lock.acquire()
 

patroni/config.py

@@ -239,16 +239,25 @@ class Config(object):
 
         for param in list(os.environ.keys()):
             if param.startswith(Config.PATRONI_ENV_PREFIX):
-                name, suffix = (param[8:].rsplit('_', 1) + [''])[:2]
+                name, suffix = (param[8:].split('_', 1) + [''])[:2]
                 if name and suffix:
                     # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..)
-                    if suffix in ('HOST', 'HOSTS', 'PORT', 'SRV', 'URL', 'PROXY', 'CACERT', 'CERT', 'KEY') \
-                            and '_' not in name:
+                    if suffix in ('HOST', 'HOSTS', 'PORT', 'SRV', 'URL', 'PROXY', 'CACERT', 'CERT',
+                                  'KEY', 'VERIFY', 'TOKEN', 'CHECKS', 'DC', 'NAMESPACE', 'CONTEXT',
+                                  'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL', 'POD_IP', 'PORTS', 'LABELS'):
                         value = os.environ.pop(param)
                         if suffix == 'PORT':
                             value = value and parse_int(value)
-                        elif suffix == 'HOSTS':
+                        elif suffix in ('HOSTS', 'PORTS', 'CHECKS'):
                             value = value and _parse_list(value)
+                        elif suffix == 'LABELS':
+                            if not value.strip().startswith('{'):
+                                value = '{{{0}}}'.format(value)
+                            try:
+                                value = yaml.safe_load(value)
+                            except Exception:
+                                logger.exception('Exception when parsing dict %s', value)
+                                value = None
                         if value:
                             ret[name.lower()][suffix.lower()] = value
                     # PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>

patroni/ctl.py

@@ -30,7 +30,8 @@ from patroni.config import Config
 from patroni.dcs import get_dcs as _get_dcs
 from patroni.exceptions import PatroniException
 from patroni.postgresql import Postgresql
-from patroni.utils import is_valid_pg_version, patch_config
+from patroni.utils import patch_config, polling_loop
+from patroni.version import __version__
 from prettytable import PrettyTable
 from six.moves.urllib_parse import urlparse
 from six import text_type
@@ -96,10 +97,12 @@ def store_config(config, path):
         yaml.dump(config, fd)
 
 
-option_format = click.option('--format', '-f', 'fmt', help='Output format (pretty, json)', default='pretty')
+option_format = click.option('--format', '-f', 'fmt', help='Output format (pretty, json, yaml)', default='pretty')
 option_watchrefresh = click.option('-w', '--watch', type=float, help='Auto update the screen every X seconds')
 option_watch = click.option('-W', is_flag=True, help='Auto update the screen every 2 seconds')
 option_force = click.option('--force', is_flag=True, help='Do not ask for confirmation at any point')
+arg_cluster_name = click.argument('cluster_name', required=False,
+                                  default=lambda: click.get_current_context().obj.get('scope'))
 
 
 @click.group()
@@ -149,9 +152,12 @@ def print_output(columns, rows=None, alignment=None, fmt='pretty', header=True,
         click.echo(t)
         return
 
-    if fmt == 'json':
+    if fmt in ['json', 'yaml', 'yml']:
         elements = [dict(zip(columns, r)) for r in rows]
-        click.echo(json.dumps(elements))
+        if fmt == 'json':
+            click.echo(json.dumps(elements))
+        elif fmt in ('yaml', 'yml'):
+            click.echo(yaml.safe_dump(elements, encoding=None, default_flow_style=False, allow_unicode=True, width=200))
 
     if fmt == 'tsv':
         if columns is not None and header:
@@ -257,9 +263,9 @@ def get_members(cluster, cluster_name, member_names, role, force, action):
         member_names = [click.prompt('Which member do you want to {0} [{1}]?'.format(action,
                         ', '.join(candidates.keys())), type=str, default='')]
 
-    for mn in member_names:
-        if mn not in candidates:
-            raise PatroniCtlException('{0} is not a member of cluster'.format(mn))
+    for member_name in member_names:
+        if member_name not in candidates:
+            raise PatroniCtlException('{0} is not a member of cluster'.format(member_name))
 
     if not force:
         confirm = click.confirm('Are you sure you want to {0} members {1}?'.format(action, ', '.join(member_names)))
@@ -273,7 +279,7 @@ def get_members(cluster, cluster_name, member_names, role, force, action):
 @click.option('--role', '-r', help='Give a dsn of any member with this role', type=click.Choice(['master', 'replica',
               'any']), default=None)
 @click.option('--member', '-m', help='Generate a dsn for this member', type=str)
-@click.argument('cluster_name')
+@arg_cluster_name
 @click.pass_obj
 def dsn(obj, cluster_name, role, member):
     if role is not None and member is not None:
@@ -291,7 +297,7 @@ def dsn(obj, cluster_name, role, member):
 
 
 @ctl.command('query', help='Query a Patroni PostgreSQL member')
-@click.argument('cluster_name')
+@arg_cluster_name
 @option_format
 @click.option('--format', 'fmt', help='Output format (pretty, json)', default='tsv')
 @click.option('--file', '-f', 'p_file', help='Execute the SQL commands from this file', type=click.File('rb'))
@@ -406,7 +412,7 @@ def remove(obj, cluster_name, fmt):
     if message != confirm:
         raise PatroniCtlException('You did not exactly type "{0}"'.format(message))
 
-    if cluster.leader:
+    if cluster.leader and cluster.leader.name:
         confirm = click.prompt('This cluster currently is healthy. Please specify the master name to continue')
         if confirm != cluster.leader.name:
             raise PatroniCtlException('You did not specify the current master of the cluster')
@@ -419,8 +425,10 @@ def check_response(response, member_name, action_name, silent_success=False):
         click.echo('Failed: {0} for member {1}, status code={2}, ({3})'.format(
             action_name, member_name, response.status_code, response.text
         ))
+        return False
     elif not silent_success:
         click.echo('Success: {0} for member {1}'.format(action_name, member_name))
+    return True
 
 
 def parse_scheduled(scheduled):
@@ -469,11 +477,12 @@ def restart(obj, cluster_name, member_names, force, role, p_any, scheduled, vers
         content['restart_pending'] = True
 
     if version:
-        if not is_valid_pg_version(version):
-            message = 'PostgreSQL version should be in the first.major.minor format'
-            raise PatroniCtlException(message)
-        else:
-            content['postgres_version'] = version
+        try:
+            Postgresql.postgres_version_to_int(version)
+        except PatroniException as e:
+            raise PatroniCtlException(e.value)
+
+        content['postgres_version'] = version
 
     if scheduled is None and not force:
         scheduled = click.prompt('When should the restart take place (e.g. 2015-10-01T14:30) ', type=str, default='now')
@@ -516,35 +525,33 @@ def reinit(obj, cluster_name, member_names, force):
     members = get_members(cluster, cluster_name, member_names, None, force, 'reinitialize')
 
     for member in members:
-        r = request_patroni(member, 'post', 'reinitialize', headers=auth_header(obj))
-        check_response(r, member.name, 'reinitialize')
+        body = {'force': force}
+        while True:
+            r = request_patroni(member, 'post', 'reinitialize', body, auth_header(obj))
+            if not check_response(r, member.name, 'reinitialize') and r.text.endswith(' already in progress') \
+                    and not force and click.confirm('Do you want to cancel it and reinitialize anyway?'):
+                body['force'] = True
+                continue
+            break
 
 
-@ctl.command('failover', help='Failover to a replica')
-@click.argument('cluster_name')
-@click.option('--master', help='The name of the current master', default=None)
-@click.option('--candidate', help='The name of the candidate', default=None)
-@click.option('--scheduled', help='Timestamp of a scheduled failover in unambiguous format (e.g. ISO 8601)',
-              default=None)
-@option_force
-@click.pass_obj
-def failover(obj, cluster_name, master, candidate, force, scheduled):
+def _do_failover_or_switchover(obj, action, cluster_name, master, candidate, force, scheduled=None):
     """
-        We want to trigger a failover for the specified cluster name.
+        We want to trigger a failover or switchover for the specified cluster name.
 
         We verify that the cluster name, master name and candidate name are correct.
-        If so, we trigger a failover and keep the client up to date.
+        If so, we trigger an action and keep the client up to date.
     """
 
     dcs = get_dcs(obj, cluster_name)
     cluster = dcs.get_cluster()
 
-    if cluster.leader is None and not cluster.is_paused():
+    if action == 'switchover' and cluster.leader is None:
         raise PatroniCtlException('This cluster has no master')
 
-    if master is None and (not cluster.is_paused() or cluster.leader):
-        if force:
-            master = cluster.leader.member.name
+    if master is None:
+        if force or action == 'failover':
+            master = cluster.leader and cluster.leader.name
         else:
             master = click.prompt('Master', type=str, default=cluster.leader.member.name)
 
@@ -556,28 +563,32 @@ def failover(obj, cluster_name, master, candidate, force, scheduled):
     candidate_names.sort()
 
     if not candidate_names:
-        raise PatroniCtlException('No candidates found to failover to')
+        raise PatroniCtlException('No candidates found to {0} to'.format(action))
 
     if candidate is None and not force:
         candidate = click.prompt('Candidate ' + str(candidate_names), type=str, default='')
 
+    if action == 'failover' and not candidate:
+        raise PatroniCtlException('Failover could be performed only to a specific candidate')
+
     if candidate == master:
-        raise PatroniCtlException('Failover target and source are the same.')
+        raise PatroniCtlException(action.title() + ' target and source are the same.')
 
     if candidate and candidate not in candidate_names:
         raise PatroniCtlException('Member {0} does not exist in cluster {1}'.format(candidate, cluster_name))
 
-    if scheduled is None and not force:
-        scheduled = click.prompt('When should the failover take place (e.g. 2015-10-01T14:30) ', type=str,
-                                 default='now')
-
-    scheduled_at = parse_scheduled(scheduled)
-
     scheduled_at_str = None
-    if scheduled_at:
-        if cluster.is_paused():
-            raise PatroniCtlException("Can't schedule failover in the paused state")
-        scheduled_at_str = scheduled_at.isoformat()
+
+    if action == 'switchover':
+        if scheduled is None and not force:
+            scheduled = click.prompt('When should the switchover take place (e.g. 2015-10-01T14:30) ',
+                                     type=str, default='now')
+
+        scheduled_at = parse_scheduled(scheduled)
+        if scheduled_at:
+            if cluster.is_paused():
+                raise PatroniCtlException("Can't schedule switchover in the paused state")
+            scheduled_at_str = scheduled_at.isoformat()
 
     failover_value = {'leader': master, 'candidate': candidate, 'scheduled_at': scheduled_at_str}
 
@@ -588,46 +599,75 @@ def failover(obj, cluster_name, master, candidate, force, scheduled):
     output_members(dcs.get_cluster(), cluster_name)
 
     if not force:
-        a = \
-            click.confirm('Are you sure you want to failover cluster {0}, demoting current master {1}?'.format(
-                cluster_name, master))
-        if not a:
-            raise PatroniCtlException('Aborting failover')
+        demote_msg = ', demoting current master ' + master if master else ''
+
+        if not click.confirm('Are you sure you want to {0} cluster {1}{2}?'.format(action, cluster_name, demote_msg)):
+            raise PatroniCtlException('Aborting ' + action)
 
     r = None
     try:
-        member = cluster.leader.member if cluster.leader else [m for m in cluster.members if m.name == candidate][0]
+        member = cluster.leader.member if cluster.leader else cluster.get_member(candidate, False)
+
+        r = request_patroni(member, 'post', action, failover_value, auth_header(obj))
+
+        # probably old patroni, which doesn't support switchover yet
+        if r.status_code == 501 and action == 'switchover' and 'Server does not support this operation' in r.text:
+            r = request_patroni(member, 'post', 'failover', failover_value, auth_header(obj))
 
-        r = request_patroni(member, 'post', 'failover', failover_value, auth_header(obj))
         if r.status_code in (200, 202):
             logging.debug(r)
             cluster = dcs.get_cluster()
             logging.debug(cluster)
             click.echo('{0} {1}'.format(timestamp(), r.text))
         else:
-            click.echo('Failover failed, details: {0}, {1}'.format(r.status_code, r.text))
+            click.echo('{0} failed, details: {1}, {2}'.format(action.title(), r.status_code, r.text))
             return
     except Exception:
         logging.exception(r)
         logging.warning('Failing over to DCS')
-        click.echo(timestamp() + ' Could not failover using Patroni api, falling back to DCS')
-        click.echo(timestamp() + ' Initializing failover from master {0}'.format(master))
+        click.echo('{0} Could not {1} using Patroni api, falling back to DCS'.format(timestamp(), action))
         dcs.manual_failover(master, candidate, scheduled_at=scheduled_at)
 
     output_members(cluster, cluster_name)
 
 
+@ctl.command('failover', help='Failover to a replica')
+@arg_cluster_name
+@click.option('--master', help='The name of the current master', default=None)
+@click.option('--candidate', help='The name of the candidate', default=None)
+@option_force
+@click.pass_obj
+def failover(obj, cluster_name, master, candidate, force):
+    action = 'switchover' if master else 'failover'
+    _do_failover_or_switchover(obj, action, cluster_name, master, candidate, force)
+
+
+@ctl.command('switchover', help='Switchover to a replica')
+@arg_cluster_name
+@click.option('--master', help='The name of the current master', default=None)
+@click.option('--candidate', help='The name of the candidate', default=None)
+@click.option('--scheduled', help='Timestamp of a scheduled switchover in unambiguous format (e.g. ISO 8601)',
+              default=None)
+@option_force
+@click.pass_obj
+def switchover(obj, cluster_name, master, candidate, force, scheduled):
+    _do_failover_or_switchover(obj, 'switchover', cluster_name, master, candidate, force, scheduled)
+
+
 def output_members(cluster, name, extended=False, fmt='pretty'):
     rows = []
     logging.debug(cluster)
     leader_name = None
     if cluster.leader:
-        leader_name = cluster.leader.member.name
+        leader_name = cluster.leader.name
 
     xlog_location_cluster = cluster.last_leader_operation or 0
 
     # Mainly for consistent pretty printing and watching we sort the output
     cluster.members.sort(key=lambda x: x.name)
+
+    extended = extended or any(m.data.get('scheduled_restart') for m in cluster.members)
+
     for m in cluster.members:
         logging.debug(m)
 
@@ -637,21 +677,15 @@ def output_members(cluster, name, extended=False, fmt='pretty'):
         elif m.name == cluster.sync.sync_standby:
             role = 'Sync standby'
 
-        host = m.conn_kwargs()['host']
-
-        xlog_location = m.data.get('xlog_location') or 0
+        xlog_location = m.data.get('xlog_location')
         lag = ''
-        if xlog_location_cluster >= xlog_location:
+        if xlog_location is None:
+            lag = 'unknown'
+        elif xlog_location_cluster >= xlog_location:
             lag = round((xlog_location_cluster - xlog_location)/1024/1024)
 
-        row = [
-            name,
-            m.name,
-            host,
-            role,
-            m.data.get('state', ''),
-            lag,
-        ]
+        row = [name, m.name, m.conn_kwargs()['host'], role, m.data.get('state', ''), lag]
+
         if extended:
             value = ''
             scheduled_restart = m.data.get('scheduled_restart')
@@ -664,14 +698,7 @@ def output_members(cluster, name, extended=False, fmt='pretty'):
 
         rows.append(row)
 
-    columns = [
-        'Cluster',
-        'Member',
-        'Host',
-        'Role',
-        'State',
-        'Lag in MB',
-    ]
+    columns = ['Cluster', 'Member', 'Host', 'Role', 'State', 'Lag in MB']
     alignment = {'Cluster': 'l', 'Member': 'l', 'Host': 'l', 'Lag in MB': 'r'}
 
     if extended:
@@ -680,23 +707,44 @@ def output_members(cluster, name, extended=False, fmt='pretty'):
 
     print_output(columns, rows, alignment, fmt)
 
+    service_info = []
+    if cluster.is_paused():
+        service_info.append('Maintenance mode: on')
+
+    if cluster.failover and cluster.failover.scheduled_at:
+        info = 'Failover scheduled at: ' + cluster.failover.scheduled_at.isoformat()
+        if cluster.failover.leader:
+            info += '\n                  from: ' + cluster.failover.leader
+        if cluster.failover.candidate:
+            info += '\n                    to: ' + cluster.failover.candidate
+        service_info.append(info)
+
+    if service_info:
+        click.echo(' ' + '\n '.join(service_info))
+
 
 @ctl.command('list', help='List the Patroni members for a given Patroni')
 @click.argument('cluster_names', nargs=-1)
 @click.option('--extended', '-e', help='Show some extra information', is_flag=True)
+@click.option('--timestamp', '-t', 'ts', help='Print timestamp', is_flag=True)
 @option_format
 @option_watch
 @option_watchrefresh
 @click.pass_obj
-def members(obj, cluster_names, fmt, watch, w, extended):
+def members(obj, cluster_names, fmt, watch, w, extended, ts):
     if not cluster_names:
-        logging.warning('Listing members: No cluster names were provided')
-        return
+        if 'scope' in obj:
+            cluster_names = [obj['scope']]
+        if not cluster_names:
+            return logging.warning('Listing members: No cluster names were provided')
 
     for cluster_name in cluster_names:
         dcs = get_dcs(obj, cluster_name)
 
         for _ in watching(w, watch):
+            if ts:
+                click.echo(timestamp(0))
+
             cluster = dcs.get_cluster()
             output_members(cluster, cluster_name, extended, fmt)
 
@@ -789,8 +837,27 @@ def flush(obj, cluster_name, member_names, force, role, target):
                 click.echo('No scheduled restart for member {0}'.format(member.name))
 
 
-def toggle_pause(config, cluster_name, paused):
-    cluster = get_dcs(config, cluster_name).get_cluster()
+def wait_until_pause_is_applied(dcs, paused, old_cluster):
+    click.echo("'{0}' request sent, waiting until it is recognized by all nodes".format(paused and 'pause' or 'resume'))
+    old = {m.name: m.index for m in old_cluster.members if m.api_url}
+    loop_wait = old_cluster.config.data.get('loop_wait', dcs.loop_wait)
+
+    for _ in polling_loop(loop_wait + 1):
+        cluster = dcs.get_cluster()
+        if all(m.data.get('pause', False) == paused for m in cluster.members if m.name in old):
+            break
+    else:
+        remaining = [m.name for m in cluster.members if m.data.get('pause', False) != paused
+                     and m.name in old and old[m.name] != m.index]
+        if remaining:
+            return click.echo("{0} members didn't recognized pause state after {1} seconds"
+                              .format(', '.join(remaining), loop_wait))
+    return click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
+
+
+def toggle_pause(config, cluster_name, paused, wait):
+    dcs = get_dcs(config, cluster_name)
+    cluster = dcs.get_cluster()
     if cluster.is_paused() == paused:
         raise PatroniCtlException('Cluster is {0} paused'.format(paused and 'already' or 'not'))
 
@@ -807,7 +874,10 @@ def toggle_pause(config, cluster_name, paused):
             continue
 
         if r.status_code == 200:
-            click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
+            if wait:
+                wait_until_pause_is_applied(dcs, paused, cluster)
+            else:
+                click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
         else:
             click.echo('Failed: {0} cluster management status code={1}, ({2})'.format(
                        paused and 'pause' or 'resume', r.status_code, r.text))
@@ -817,17 +887,19 @@ def toggle_pause(config, cluster_name, paused):
 
 
 @ctl.command('pause', help='Disable auto failover')
-@click.argument('cluster_name')
+@arg_cluster_name
 @click.pass_obj
-def pause(obj, cluster_name):
-    return toggle_pause(obj, cluster_name, True)
+@click.option('--wait', help='Wait until pause is applied on all nodes', is_flag=True)
+def pause(obj, cluster_name, wait):
+    return toggle_pause(obj, cluster_name, True, wait)
 
 
 @ctl.command('resume', help='Resume auto failover')
-@click.argument('cluster_name')
+@arg_cluster_name
+@click.option('--wait', help='Wait until pause is cleared on all nodes', is_flag=True)
 @click.pass_obj
-def resume(obj, cluster_name):
-    return toggle_pause(obj, cluster_name, False)
+def resume(obj, cluster_name, wait):
+    return toggle_pause(obj, cluster_name, False, wait)
 
 
 @contextmanager
@@ -969,7 +1041,7 @@ def invoke_editor(before_editing, cluster_name):
 
 
 @ctl.command('edit-config', help="Edit cluster configuration")
-@click.argument('cluster_name')
+@arg_cluster_name
 @click.option('--quiet', '-q', is_flag=True, help='Do not show changes')
 @click.option('--set', '-s', 'kvpairs', multiple=True,
               help='Set specific configuration value. Can be specified multiple times')
@@ -1029,3 +1101,36 @@ def show_config(obj, cluster_name):
     cluster = get_dcs(obj, cluster_name).get_cluster()
 
     click.echo(format_config_for_editing(cluster.config.data))
+
+
+@ctl.command('version', help='Output version of patronictl command or a running Patroni instance')
+@click.argument('cluster_name', required=False)
+@click.argument('member_names', nargs=-1)
+@click.pass_obj
+def version(obj, cluster_name, member_names):
+    click.echo("patronictl version {0}".format(__version__))
+
+    if not cluster_name:
+        return
+
+    click.echo("")
+    cluster = get_dcs(obj, cluster_name).get_cluster()
+    for m in cluster.members:
+        if m.api_url:
+            if not member_names or m.name in member_names:
+                try:
+                    response = request_patroni(m, 'get', 'patroni')
+                    data = response.json()
+                    version = data.get('patroni', {}).get('version')
+                    pg_version = data.get('server_version')
+                    pg_version_str = " PostgreSQL {0}".format(format_pg_version(pg_version)) if pg_version else ""
+                    click.echo("{0}: Patroni {1}{2}".format(m.name, version, pg_version_str))
+                except Exception as e:
+                    click.echo("{0}: failed to get version: {1}".format(m.name, e))
+
+
+def format_pg_version(version):
+    if version < 100000:
+        return "{0}.{1}.{2}".format(version // 10000, version // 100 % 100, version % 100)
+    else:
+        return "{0}.{1}".format(version // 10000, version % 100)

patroni/dcs/__init__.py

@@ -177,40 +177,48 @@ class Leader(namedtuple('Leader', 'index,session,member')):
     def conn_url(self):
         return self.member.conn_url
 
+    @property
+    def timeline(self):
+        return self.member.data.get('timeline')
+
 
 class Failover(namedtuple('Failover', 'index,leader,candidate,scheduled_at')):
 
     """
     >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader"}'))
     True
+    >>> 'Failover' in str(Failover.from_node(1, {"leader": "cluster_leader"}))
+    True
     >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader", "member": "cluster_candidate"}'))
     True
     >>> Failover.from_node(1, 'null') is None
-    True
+    False
     >>> n = '{"leader": "cluster_leader", "member": "cluster_candidate", "scheduled_at": "2016-01-14T10:09:57.1394Z"}'
     >>> 'tzinfo=' in str(Failover.from_node(1, n))
     True
     >>> Failover.from_node(1, None) is None
-    True
+    False
     >>> Failover.from_node(1, '{}') is None
-    True
+    False
     >>> 'abc' in Failover.from_node(1, 'abc:def')
     True
     """
     @staticmethod
     def from_node(index, value):
-        if not value:
-            return None
-
-        try:
-            data = json.loads(value)
-            if not data:
-                return None
-        except ValueError:
-            t = [a.strip() for a in value.split(':')]
-            leader = t[0]
-            candidate = t[1] if len(t) > 1 else None
-            return Failover(index, leader, candidate, None) if leader or candidate else None
+        if isinstance(value, dict):
+            data = value
+        elif value:
+            try:
+                data = json.loads(value)
+                if not isinstance(data, dict):
+                    data = {}
+            except ValueError:
+                t = [a.strip() for a in value.split(':')]
+                leader = t[0]
+                candidate = t[1] if len(t) > 1 else None
+                return Failover(index, leader, candidate, None) if leader or candidate else None
+        else:
+            data = {}
 
         if data.get('scheduled_at'):
             data['scheduled_at'] = dateutil.parser.parse(data['scheduled_at'])
@@ -258,8 +266,12 @@ class SyncState(namedtuple('SyncState', 'index,leader,sync_standby')):
         True
         >>> SyncState.from_node(1, '{"leader": "leader"}').leader == "leader"
         True
+        >>> SyncState.from_node(1, {"leader": "leader"}).leader == "leader"
+        True
         """
-        if value:
+        if isinstance(value, dict):
+            data = value
+        elif value:
             try:
                 data = json.loads(value)
                 if not isinstance(data, dict):
@@ -289,7 +301,26 @@ class SyncState(namedtuple('SyncState', 'index,leader,sync_standby')):
         return name is not None and name in (self.leader, self.sync_standby)
 
 
-class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operation,members,failover,sync')):
+class TimelineHistory(namedtuple('TimelineHistory', 'index,lines')):
+    """Object representing timeline history file"""
+
+    @staticmethod
+    def from_node(index, value):
+        """
+        >>> h = TimelineHistory.from_node(1, 2)
+        >>> h.lines
+        []
+        """
+        try:
+            lines = json.loads(value)
+        except (TypeError, ValueError):
+            lines = None
+        if not isinstance(lines, list):
+            lines = []
+        return TimelineHistory(index, lines)
+
+
+class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operation,members,failover,sync,history')):
 
     """Immutable object (namedtuple) which represents PostgreSQL cluster.
     Consists of the following fields:
@@ -301,6 +332,7 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operat
     :param members: list of Member object, all PostgreSQL cluster members including leader
     :param failover: reference to `Failover` object
     :param sync: reference to `SyncState` object, last observed synchronous replication state.
+    :param history: reference to `TimelineHistory` object
     """
 
     def is_unlocked(self):
@@ -320,6 +352,12 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operat
     def is_paused(self):
         return self.config and self.config.data.get('pause', False) or False
 
+    def is_synchronous_mode(self):
+        return bool(self.config and self.config.data.get('synchronous_mode'))
+
+    def is_synchronous_mode_strict(self):
+        return bool(self.config and self.config.data.get('synchronous_mode_strict'))
+
 
 @six.add_metaclass(abc.ABCMeta)
 class AbstractDCS(object):
@@ -328,6 +366,7 @@ class AbstractDCS(object):
     _CONFIG = 'config'
     _LEADER = 'leader'
     _FAILOVER = 'failover'
+    _HISTORY = 'history'
     _MEMBERS = 'members/'
     _OPTIME = 'optime'
     _LEADER_OPTIME = _OPTIME + '/' + _LEADER
@@ -339,8 +378,7 @@ class AbstractDCS(object):
             i.e.: `zookeeper` for zookeeper, `etcd` for etcd, etc...
         """
         self._name = config['name']
-        self._namespace = '/{0}'.format(config.get('namespace', '/service/').strip('/'))
-        self._base_path = '/'.join([self._namespace, config['scope']])
+        self._base_path = os.path.join('/', config.get('namespace', '/service/').strip('/'), config['scope'])
         self._set_loop_wait(config.get('loop_wait', 10))
 
         self._ctl = bool(config.get('patronictl', False))
@@ -376,6 +414,10 @@ class AbstractDCS(object):
     def failover_path(self):
         return self.client_path(self._FAILOVER)
 
+    @property
+    def history_path(self):
+        return self.client_path(self._HISTORY)
+
     @property
     def leader_optime_path(self):
         return self.client_path(self._LEADER_OPTIME)
@@ -418,7 +460,7 @@ class AbstractDCS(object):
         with self._cluster_thread_lock:
             try:
                 self._load_cluster()
-            except:
+            except Exception:
                 self._cluster = None
                 raise
             return self._cluster
@@ -443,7 +485,7 @@ class AbstractDCS(object):
             self._last_leader_operation = last_operation
 
     @abc.abstractmethod
-    def update_leader(self):
+    def _update_leader(self):
         """Update leader key (or session) ttl
 
         :returns: `!True` if leader key (or session) has been updated successfully.
@@ -452,6 +494,18 @@ class AbstractDCS(object):
         You have to use CAS (Compare And Swap) operation in order to update leader key,
         for example for etcd `prevValue` parameter must be used."""
 
+    def update_leader(self, last_operation):
+        """Update leader key (or session) ttl and optime/leader
+
+        :param last_operation: absolute xlog location in bytes
+        :returns: `!True` if leader key (or session) has been updated successfully.
+            If not, `!False` must be returned and current instance would be demoted."""
+
+        ret = self._update_leader()
+        if ret and last_operation:
+            self.write_leader_optime(last_operation)
+        return ret
+
     @abc.abstractmethod
     def attempt_to_acquire_leader(self, permanent=False):
         """Attempt to acquire leader lock
@@ -477,7 +531,6 @@ class AbstractDCS(object):
 
         if scheduled_at:
             failover_value['scheduled_at'] = scheduled_at.isoformat()
-
         return self.set_failover_value(json.dumps(failover_value, separators=(',', ':')), index)
 
     @abc.abstractmethod
@@ -490,7 +543,7 @@ class AbstractDCS(object):
         This method should create or update key with the name = '/members/' + `~self._name`
         and value = data in a given DCS.
 
-        :param data: json serialized information about instance (including connection strings)
+        :param data: information about instance (including connection strings)
         :param ttl: ttl for member key, optional parameter. If it is None `~self.member_ttl will be used`
         :param permanent: if set to `!True`, the member key will never expire.
          Used in patronictl for the external master.
@@ -527,8 +580,18 @@ class AbstractDCS(object):
     def delete_cluster(self):
         """Delete cluster from DCS"""
 
+    @staticmethod
+    def sync_state(leader, sync_standby):
+        """Build sync_state dict"""
+        return {'leader': leader, 'sync_standby': sync_standby}
+
     def write_sync_state(self, leader, sync_standby, index=None):
-        return self.set_sync_state_value(json.dumps({'leader': leader, 'sync_standby': sync_standby}), index=index)
+        sync_value = self.sync_state(leader, sync_standby)
+        return self.set_sync_state_value(json.dumps(sync_value, separators=(',', ':')), index)
+
+    @abc.abstractmethod
+    def set_history_value(self, value):
+        """"""
 
     @abc.abstractmethod
     def set_sync_state_value(self, value, index=None):

patroni/dcs/consul.py

@@ -1,16 +1,18 @@
 from __future__ import absolute_import
+import json
 import logging
 import os
 import socket
+import ssl
 import time
 import urllib3
 
 from consul import ConsulException, NotFound, base
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
 from patroni.exceptions import DCSError
-from patroni.utils import Retry, RetryFailedError
+from patroni.utils import deep_compare, parse_bool, Retry, RetryFailedError, split_host_port
 from urllib3.exceptions import HTTPError
-from six.moves.urllib.parse import urlencode
+from six.moves.urllib.parse import urlencode, urlparse
 from six.moves.http_client import HTTPException
 
 logger = logging.getLogger(__name__)
@@ -24,21 +26,39 @@ class ConsulInternalError(ConsulException):
     """An internal Consul server error occurred"""
 
 
+class InvalidSessionTTL(ConsulInternalError):
+    """Session TTL is too small or too big"""
+
+
 class HTTPClient(object):
 
-    def __init__(self, host='127.0.0.1', port=8500, scheme='http', verify=True, timeout=10):
-        self.host = host
-        self.port = port
-        self.scheme = scheme
-        self.verify = verify
-        self.set_read_timeout(timeout)
-        self.base_uri = '{0}://{1}:{2}'.format(self.scheme, self.host, self.port)
-        self.http = urllib3.PoolManager(num_pools=10)
+    def __init__(self, host='127.0.0.1', port=8500, token=None, scheme='http', verify=True, cert=None, ca_cert=None):
+        self.token = token
+        self._read_timeout = 10
+        self.base_uri = '{0}://{1}:{2}'.format(scheme, host, port)
+        kwargs = {}
+        if cert:
+            if isinstance(cert, tuple):
+                # Key and cert are separate
+                kwargs['cert_file'] = cert[0]
+                kwargs['key_file'] = cert[1]
+            else:
+                # combined certificate
+                kwargs['cert_file'] = cert
+        if ca_cert:
+            kwargs['ca_certs'] = ca_cert
+        if verify or ca_cert:
+            kwargs['cert_reqs'] = ssl.CERT_REQUIRED
+        self.http = urllib3.PoolManager(num_pools=10, **kwargs)
         self._ttl = None
 
     def set_read_timeout(self, timeout):
         self._read_timeout = timeout/3.0
 
+    @property
+    def ttl(self):
+        return self._ttl
+
     def set_ttl(self, ttl):
         ret = self._ttl != ttl
         self._ttl = ttl
@@ -48,7 +68,11 @@ class HTTPClient(object):
     def response(response):
         data = response.data.decode('utf-8')
         if response.status == 500:
-            raise ConsulInternalError('{0} {1}'.format(response.status, data))
+            msg = '{0} {1}'.format(response.status, data)
+            if data.startswith('Invalid Session TTL'):
+                raise InvalidSessionTTL(msg)
+            else:
+                raise ConsulInternalError(msg)
         return base.Response(response.status, response.headers, data)
 
     def uri(self, path, params=None):
@@ -72,15 +96,30 @@ class HTTPClient(object):
                 kwargs['timeout'] = (float(params['wait'][:-1]) if 'wait' in params else 300) + 1
             else:
                 kwargs['timeout'] = self._read_timeout
+            token = params.pop('token', self.token) if isinstance(params, dict) else self.token
+            if token:
+                kwargs['headers'] = {'X-Consul-Token': token}
             return callback(self.response(self.http.request(method.upper(), self.uri(path, params), **kwargs)))
         return wrapper
 
 
 class ConsulClient(base.Consul):
 
-    @staticmethod
-    def connect(host, port, scheme, verify=True):
-        return HTTPClient(host, port, scheme, verify)
+    def __init__(self, *args, **kwargs):
+        self._cert = kwargs.pop('cert', None)
+        self._ca_cert = kwargs.pop('ca_cert', None)
+        self._token = kwargs.get('token')
+        super(ConsulClient, self).__init__(*args, **kwargs)
+
+    def connect(self, *args, **kwargs):
+        kwargs.update(dict(zip(['host', 'port', 'scheme', 'verify'], args)))
+        if self._cert:
+            kwargs['cert'] = self._cert
+        if self._ca_cert:
+            kwargs['ca_cert'] = self._ca_cert
+        if self._token:
+            kwargs['token'] = self._token
+        return HTTPClient(**kwargs)
 
 
 def catch_consul_errors(func):
@@ -103,12 +142,37 @@ class Consul(AbstractDCS):
                             retry_exceptions=(ConsulInternalError, HTTPException,
                                               HTTPError, socket.error, socket.timeout))
 
-        self._my_member_data = None
-        host, port = config.get('host', '127.0.0.1:8500').split(':')
-        self._client = ConsulClient(host=host, port=port)
+        self._my_member_data = {}
+        kwargs = {}
+        if 'url' in config:
+            r = urlparse(config['url'])
+            config.update({'scheme': r.scheme, 'host': r.hostname, 'port': r.port or 8500})
+        elif 'host' in config:
+            host, port = split_host_port(config.get('host', '127.0.0.1:8500'), 8500)
+            config['host'] = host
+            if 'port' not in config:
+                config['port'] = int(port)
+
+        if config.get('cacert'):
+            config['ca_cert'] = config.pop('cacert')
+
+        if config.get('key') and config.get('cert'):
+            config['cert'] = (config['cert'], config['key'])
+
+        config_keys = ('host', 'port', 'token', 'scheme', 'cert', 'ca_cert', 'dc')
+        kwargs = {p: config.get(p) for p in config_keys if config.get(p)}
+
+        verify = config.get('verify')
+        if not isinstance(verify, bool):
+            verify = parse_bool(verify)
+        if isinstance(verify, bool):
+            kwargs['verify'] = verify
+
+        self._client = ConsulClient(**kwargs)
         self.set_retry_timeout(config['retry_timeout'])
         self.set_ttl(config.get('ttl') or 30)
         self._last_session_refresh = 0
+        self.__session_checks = config.get('checks')
         if not self._ctl:
             self.create_session()
 
@@ -132,6 +196,15 @@ class Consul(AbstractDCS):
         self._retry.deadline = retry_timeout
         self._client.http.set_read_timeout(retry_timeout)
 
+    def adjust_ttl(self):
+        try:
+            settings = self._client.agent.self()
+            min_ttl = (settings['Config']['SessionTTLMin'] or 10000000000)/1000000000.0
+            logger.warning('Changing Session TTL from %s to %s', self._client.http.ttl, min_ttl)
+            self._client.http.set_ttl(min_ttl)
+        except Exception:
+            logger.exception('adjust_ttl')
+
     def _do_refresh_session(self):
         """:returns: `!True` if it had to create new session"""
         if self._session and self._last_session_refresh + self._loop_wait > time.time():
@@ -144,8 +217,15 @@ class Consul(AbstractDCS):
                 self._session = None
         ret = not self._session
         if ret:
-            self._session = self._client.session.create(name=self._scope + '-' + self._name,
-                                                        lock_delay=0.001, behavior='delete')
+            try:
+                self._session = self._client.session.create(name=self._scope + '-' + self._name,
+                                                            checks=self.__session_checks,
+                                                            lock_delay=0.001, behavior='delete')
+            except InvalidSessionTTL:
+                logger.exception('session.create')
+                self.adjust_ttl()
+                raise
+
         self._last_session_refresh = time.time()
         return ret
 
@@ -184,6 +264,10 @@ class Consul(AbstractDCS):
             config = nodes.get(self._CONFIG)
             config = config and ClusterConfig.from_node(config['ModifyIndex'], config['Value'])
 
+            # get timeline history
+            history = nodes.get(self._HISTORY)
+            history = history and TimelineHistory.from_node(history['ModifyIndex'], history['Value'])
+
             # get last leader operation
             last_leader_operation = nodes.get(self._LEADER_OPTIME)
             last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation['Value'])
@@ -213,17 +297,17 @@ class Consul(AbstractDCS):
             sync = nodes.get(self._SYNC)
             sync = SyncState.from_node(sync and sync['ModifyIndex'], sync and sync['Value'])
 
-            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
         except NotFound:
-            self._cluster = Cluster(None, None, None, None, [], None, None)
-        except:
+            self._cluster = Cluster(None, None, None, None, [], None, None, None)
+        except Exception:
             logger.exception('get_cluster')
             raise ConsulError('Consul is not responding properly')
 
-    def touch_member(self, data, **kwargs):
+    def touch_member(self, data, ttl=None, permanent=False):
         cluster = self.cluster
         member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
-        create_member = self.refresh_session()
+        create_member = not permanent and self.refresh_session()
 
         if member and (create_member or member.session != self._session):
             try:
@@ -232,12 +316,12 @@ class Consul(AbstractDCS):
             except Exception:
                 return False
 
-        if not create_member and member and data == self._my_member_data:
+        if not create_member and member and deep_compare(data, self._my_member_data):
             return True
 
         try:
-            args = {} if kwargs.get('permanent', False) else {'acquire': self._session}
-            self._client.kv.put(self.member_path, data, **args)
+            args = {} if permanent else {'acquire': self._session}
+            self._client.kv.put(self.member_path, json.dumps(data, separators=(',', ':')), **args)
             self._my_member_data = data
             return True
         except Exception:
@@ -245,12 +329,14 @@ class Consul(AbstractDCS):
         return False
 
     @catch_consul_errors
+    def _do_attempt_to_acquire_leader(self, kwargs):
+        return self.retry(self._client.kv.put, self.leader_path, self._name, **kwargs)
+
     def attempt_to_acquire_leader(self, permanent=False):
         if not self._session and not permanent:
             self.refresh_session()
 
-        args = {} if permanent else {'acquire': self._session}
-        ret = self.retry(self._client.kv.put, self.leader_path, self._name, **args)
+        ret = self._do_attempt_to_acquire_leader({} if permanent else {'acquire': self._session})
         if not ret:
             logger.info('Could not take out TTL lock')
         return ret
@@ -271,7 +357,7 @@ class Consul(AbstractDCS):
         return self._client.kv.put(self.leader_optime_path, last_operation)
 
     @catch_consul_errors
-    def update_leader(self):
+    def _update_leader(self):
         if self._session:
             self.retry(self._client.session.renew, self._session)
             self._last_session_refresh = time.time()
@@ -290,6 +376,10 @@ class Consul(AbstractDCS):
     def delete_cluster(self):
         return self.retry(self._client.kv.delete, self.client_path(''), recurse=True)
 
+    @catch_consul_errors
+    def set_history_value(self, value):
+        return self._client.kv.put(self.history_path, value)
+
     @catch_consul_errors
     def delete_leader(self):
         cluster = self.cluster
@@ -298,11 +388,11 @@ class Consul(AbstractDCS):
 
     @catch_consul_errors
     def set_sync_state_value(self, value, index=None):
-        return self._client.kv.put(self.sync_path, value, cas=index)
+        return self.retry(self._client.kv.put, self.sync_path, value, cas=index)
 
     @catch_consul_errors
     def delete_sync_state(self, index=None):
-        return self._client.kv.delete(self.sync_path, cas=index)
+        return self.retry(self._client.kv.delete, self.sync_path, cas=index)
 
     def watch(self, leader_index, timeout):
         if self.__do_not_watch:

patroni/dcs/etcd.py

@@ -1,18 +1,20 @@
 from __future__ import absolute_import
 import etcd
+import json
 import logging
 import os
 import urllib3.util.connection
 import random
 import requests
+import six
 import socket
 import time
 
 from dns.exception import DNSException
 from dns import resolver
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
 from patroni.exceptions import DCSError
-from patroni.utils import Retry, RetryFailedError
+from patroni.utils import Retry, RetryFailedError, split_host_port
 from urllib3.exceptions import HTTPError, ReadTimeoutError
 from requests.exceptions import RequestException
 from six.moves.queue import Queue
@@ -23,6 +25,10 @@ from threading import Thread
 logger = logging.getLogger(__name__)
 
 
+def uri(protocol, host, port, endpoint=''):
+    return '{0}://{1}:{2}{3}'.format(protocol, host, port, endpoint)
+
+
 class EtcdError(DCSError):
     pass
 
@@ -227,7 +233,7 @@ class Client(etcd.Client):
             protocol = 'https' if '-ssl' in r else 'http'
             endpoint = '/members' if '-server' in r else ''
             for host, port in self.get_srv_record('_etcd{0}._tcp.{1}'.format(r, srv)):
-                url = '{0}://{1}:{2}{3}'.format(protocol, host, port, endpoint)
+                url = uri(protocol, host, port, endpoint)
                 if endpoint:
                     try:
                         response = requests.get(url, timeout=self.read_timeout, verify=False)
@@ -254,10 +260,10 @@ class Client(etcd.Client):
                 host, port = sa[:2]
                 if af == socket.AF_INET6:
                     host = '[{0}]'.format(host)
-                ret.append('{0}://{1}:{2}'.format(self.protocol, host, port))
+                ret.append(uri(self.protocol, host, port))
             if ret:
                 return list(set(ret))
-        return ['{0}://{1}:{2}'.format(self.protocol, host, port)]
+        return [uri(self.protocol, host, port)]
 
     def _load_machines_cache(self):
         """This method should fill up `_machines_cache` from scratch.
@@ -267,17 +273,20 @@ class Client(etcd.Client):
 
         self._update_machines_cache = True
 
-        if 'srv' not in self._config and 'host' not in self._config:
-            raise Exception('Neither srv nor host url are defined in etcd section of config')
+        if 'srv' not in self._config and 'host' not in self._config and 'hosts' not in self._config:
+            raise Exception('Neither srv, hosts, host nor url are defined in etcd section of config')
 
         if self._use_proxies:
-            self._machines_cache = ['{0}://{1}:{2}'.format(self.protocol, self._config['host'], self._config['port'])]
+            self._machines_cache = [uri(self.protocol, self._config['host'], self._config['port'])]
         else:
             self._machines_cache = []
 
             if 'srv' in self._config:
                 self._machines_cache = self._get_machines_cache_from_srv(self._config['srv'])
 
+            if not self._machines_cache and 'hosts' in self._config:
+                self._machines_cache = list(self._config['hosts'])
+
             if not self._machines_cache and 'host' in self._config:
                 self._machines_cache = self._get_machines_cache_from_dns(self._config['host'], self._config['port'])
 
@@ -347,8 +356,20 @@ class Etcd(AbstractDCS):
             r = urlparse(config['url'])
             config.update({'protocol': r.scheme, 'host': r.hostname, 'port': r.port or 2379,
                            'username': r.username, 'password': r.password})
+        elif 'hosts' in config:
+            hosts = config.pop('hosts')
+            default_port = config.pop('port', 2379)
+            protocol = config.get('protocol', 'http')
+
+            if isinstance(hosts, six.string_types):
+                hosts = hosts.split(',')
+
+            config['hosts'] = []
+            for value in hosts:
+                if isinstance(value, six.string_types):
+                    config['hosts'].append(uri(protocol, *split_host_port(value, default_port)))
         elif 'host' in config:
-            host, port = (config['host'] + ':2379').split(':')[:2]
+            host, port = split_host_port(config['host'], 2379)
             config['host'] = host
             if 'port' not in config:
                 config['port'] = int(port)
@@ -434,6 +455,10 @@ class Etcd(AbstractDCS):
             config = nodes.get(self._CONFIG)
             config = config and ClusterConfig.from_node(config.modifiedIndex, config.value)
 
+            # get timeline history
+            history = nodes.get(self._HISTORY)
+            history = history and TimelineHistory.from_node(history.modifiedIndex, history.value)
+
             # get last leader operation
             last_leader_operation = nodes.get(self._LEADER_OPTIME)
             last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation.value)
@@ -458,15 +483,16 @@ class Etcd(AbstractDCS):
             sync = nodes.get(self._SYNC)
             sync = SyncState.from_node(sync and sync.modifiedIndex, sync and sync.value)
 
-            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
         except etcd.EtcdKeyNotFound:
-            self._cluster = Cluster(None, None, None, None, [], None, None)
+            self._cluster = Cluster(None, None, None, None, [], None, None, None)
         except Exception as e:
             self._handle_exception(e, 'get_cluster', raise_ex=EtcdError('Etcd is not responding properly'))
         self._has_failed = False
 
     @catch_etcd_errors
     def touch_member(self, data, ttl=None, permanent=False):
+        data = json.dumps(data, separators=(',', ':'))
         return self.retry(self._client.set, self.member_path, data, None if permanent else ttl or self._ttl)
 
     @catch_etcd_errors
@@ -499,7 +525,7 @@ class Etcd(AbstractDCS):
         return self._client.set(self.leader_optime_path, last_operation)
 
     @catch_etcd_errors
-    def update_leader(self):
+    def _update_leader(self):
         return self.retry(self._client.test_and_set, self.leader_path, self._name, self._name, self._ttl)
 
     @catch_etcd_errors
@@ -518,9 +544,13 @@ class Etcd(AbstractDCS):
     def delete_cluster(self):
         return self.retry(self._client.delete, self.client_path(''), recursive=True)
 
+    @catch_etcd_errors
+    def set_history_value(self, value):
+        return self._client.write(self.history_path, value)
+
     @catch_etcd_errors
     def set_sync_state_value(self, value, index=None):
-        return self._client.write(self.sync_path, value, prevIndex=index or 0)
+        return self.retry(self._client.write, self.sync_path, value, prevIndex=index or 0)
 
     @catch_etcd_errors
     def delete_sync_state(self, index=None):

patroni/dcs/kubernetes.py

@@ -0,0 +1,405 @@
+from __future__ import absolute_import
+import datetime
+import functools
+import json
+import logging
+import socket
+import sys
+import time
+
+from kubernetes import client as k8s_client, config as k8s_config, watch as k8s_watch
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
+from patroni.exceptions import DCSError
+from patroni.utils import deep_compare, tzutc, Retry, RetryFailedError
+from urllib3.exceptions import HTTPError
+from six.moves.http_client import HTTPException
+
+logger = logging.getLogger(__name__)
+
+
+class KubernetesError(DCSError):
+    pass
+
+
+class KubernetesRetriableException(k8s_client.rest.ApiException):
+
+    def __init__(self, orig):
+        super(KubernetesRetriableException, self).__init__(orig.status, orig.reason)
+        self.body = orig.body
+        self.headers = orig.headers
+
+
+class CoreV1Api(object):
+
+    def __init__(self, use_endpoints=False):
+        self._api = k8s_client.CoreV1Api()
+        self._request_timeout = None
+        self._use_endpoints = use_endpoints
+
+    def set_timeout(self, timeout):
+        self._request_timeout = (1, timeout / 3.0)
+
+    def __getattr__(self, func):
+        if func.endswith('_kind'):
+            func = func[:-4] + ('endpoints' if self._use_endpoints else 'config_map')
+
+        def wrapper(*args, **kwargs):
+            if '_request_timeout' not in kwargs:
+                kwargs['_request_timeout'] = self._request_timeout
+            try:
+                return getattr(self._api, func)(*args, **kwargs)
+            except k8s_client.rest.ApiException as e:
+                if e.status in (502, 503, 504):  # XXX
+                    raise KubernetesRetriableException(e)
+                raise
+        return wrapper
+
+
+class Kubernetes(AbstractDCS):
+
+    def __init__(self, config):
+        self._labels = config['labels']
+        self._labels[config.get('scope_label', 'cluster-name')] = config['scope']
+        self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
+        self._namespace = config.get('namespace') or 'default'
+        self._role_label = config.get('role_label', 'role')
+        config['namespace'] = ''
+        super(Kubernetes, self).__init__(config)
+        self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
+                            retry_exceptions=(KubernetesRetriableException, HTTPException,
+                                              HTTPError, socket.error, socket.timeout))
+        self._ttl = None
+        try:
+            k8s_config.load_incluster_config()
+        except k8s_config.ConfigException:
+            k8s_config.load_kube_config(context=config.get('context', 'local'))
+
+        self.__subsets = None
+        use_endpoints = config.get('use_endpoints') and (config.get('patronictl') or 'pod_ip' in config)
+        if use_endpoints:
+            addresses = [k8s_client.V1EndpointAddress(ip=config['pod_ip'])]
+            ports = []
+            for p in config.get('ports', [{}]):
+                port = {'port': int(p.get('port', '5432'))}
+                port.update({n: p[n] for n in ('name', 'protocol') if p.get(n)})
+                ports.append(k8s_client.V1EndpointPort(**port))
+            self.__subsets = [k8s_client.V1EndpointSubset(addresses=addresses, ports=ports)]
+        self._api = CoreV1Api(use_endpoints)
+        self.set_retry_timeout(config['retry_timeout'])
+        self.set_ttl(config.get('ttl') or 30)
+        self._leader_observed_record = {}
+        self._leader_observed_time = None
+        self._leader_resource_version = None
+        self._leader_observed_subsets = []
+        self.__do_not_watch = False
+
+    def retry(self, *args, **kwargs):
+        return self._retry.copy()(*args, **kwargs)
+
+    def catch_kubernetes_errors(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            except (RetryFailedError, k8s_client.rest.ApiException,
+                    HTTPException, HTTPError, socket.error, socket.timeout):
+                return False
+        return wrapper
+
+    def client_path(self, path):
+        return super(Kubernetes, self).client_path(path)[1:].replace('/', '-')
+
+    @property
+    def leader_path(self):
+        return self._base_path[1:] if self.__subsets else super(Kubernetes, self).leader_path
+
+    def set_ttl(self, ttl):
+        ttl = int(ttl)
+        self.__do_not_watch = self._ttl != ttl
+        self._ttl = ttl
+
+    def set_retry_timeout(self, retry_timeout):
+        self._retry.deadline = retry_timeout
+        self._api.set_timeout(retry_timeout)
+
+    @staticmethod
+    def member(pod):
+        annotations = pod.metadata.annotations or {}
+        member = Member.from_node(pod.metadata.resource_version, pod.metadata.name, None, annotations.get('status', ''))
+        member.data['pod_labels'] = pod.metadata.labels
+        return member
+
+    def _load_cluster(self):
+        try:
+            # get list of members
+            response = self.retry(self._api.list_namespaced_pod, self._namespace, label_selector=self._label_selector)
+            members = [self.member(pod) for pod in response.items]
+
+            response = self.retry(self._api.list_namespaced_kind, self._namespace, label_selector=self._label_selector)
+            nodes = {item.metadata.name: item for item in response.items}
+
+            config = nodes.get(self.config_path)
+            metadata = config and config.metadata
+            annotations = metadata and metadata.annotations or {}
+
+            # get initialize flag
+            initialize = annotations.get(self._INITIALIZE)
+
+            # get global dynamic configuration
+            config = ClusterConfig.from_node(metadata and metadata.resource_version,
+                                             annotations.get(self._CONFIG) or '{}')
+
+            # get timeline history
+            history = TimelineHistory.from_node(metadata and metadata.resource_version,
+                                                annotations.get(self._HISTORY) or '[]')
+
+            leader = nodes.get(self.leader_path)
+            metadata = leader and leader.metadata
+            self._leader_resource_version = metadata.resource_version if metadata else None
+            self._leader_observed_subsets = leader.subsets if self.__subsets and leader else []
+            annotations = metadata and metadata.annotations or {}
+
+            # get last leader operation
+            last_leader_operation = annotations.get(self._OPTIME)
+            last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation)
+
+            # get leader
+            leader_record = {n: annotations.get(n) for n in (self._LEADER, 'acquireTime',
+                             'ttl', 'renewTime', 'transitions') if n in annotations}
+            if (leader_record or self._leader_observed_record) and leader_record != self._leader_observed_record:
+                self._leader_observed_record = leader_record
+                self._leader_observed_time = time.time()
+
+            leader = leader_record.get(self._LEADER)
+            try:
+                ttl = int(leader_record.get('ttl')) or self._ttl
+            except (TypeError, ValueError):
+                ttl = self._ttl
+
+            if not metadata or not self._leader_observed_time or self._leader_observed_time + ttl < time.time():
+                leader = None
+
+            if metadata:
+                member = Member(-1, leader, None, {})
+                member = ([m for m in members if m.name == leader] or [member])[0]
+                leader = Leader(response.metadata.resource_version, None, member)
+
+            # failover key
+            failover = nodes.get(self.failover_path)
+            metadata = failover and failover.metadata
+            failover = Failover.from_node(metadata and metadata.resource_version, metadata and metadata.annotations)
+
+            # get synchronization state
+            sync = nodes.get(self.sync_path)
+            metadata = sync and sync.metadata
+            sync = SyncState.from_node(metadata and metadata.resource_version,  metadata and metadata.annotations)
+
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
+        except Exception:
+            logger.exception('get_cluster')
+            raise KubernetesError('Kubernetes API is not responding properly')
+
+    @staticmethod
+    def compare_ports(p1, p2):
+        return p1.name == p2.name and p1.port == p2.port and (p1.protocol or 'TCP') == (p2.protocol or 'TCP')
+
+    @staticmethod
+    def subsets_changed(last_observed_subsets, subsets):
+        """
+        >>> Kubernetes.subsets_changed([], [])
+        False
+        >>> Kubernetes.subsets_changed([], [k8s_client.V1EndpointSubset()])
+        True
+        >>> s1 = [k8s_client.V1EndpointSubset(addresses=[k8s_client.V1EndpointAddress(ip='1.2.3.4')])]
+        >>> s2 = [k8s_client.V1EndpointSubset(addresses=[k8s_client.V1EndpointAddress(ip='1.2.3.5')])]
+        >>> Kubernetes.subsets_changed(s1, s2)
+        True
+        >>> a = [k8s_client.V1EndpointAddress(ip='1.2.3.4')]
+        >>> s1 = [k8s_client.V1EndpointSubset(addresses=a, ports=[k8s_client.V1EndpointPort(protocol='TCP', port=1)])]
+        >>> s2 = [k8s_client.V1EndpointSubset(addresses=a, ports=[k8s_client.V1EndpointPort(port=5432)])]
+        >>> Kubernetes.subsets_changed(s1, s2)
+        True
+        >>> p1 = k8s_client.V1EndpointPort(name='port1', port=1)
+        >>> p2 = k8s_client.V1EndpointPort(name='port2', port=2)
+        >>> p3 = k8s_client.V1EndpointPort(name='port3', port=3)
+        >>> s1 = [k8s_client.V1EndpointSubset(addresses=a, ports=[p1, p2])]
+        >>> s2 = [k8s_client.V1EndpointSubset(addresses=a, ports=[p2, p3])]
+        >>> Kubernetes.subsets_changed(s1, s2)
+        True
+        >>> s2 = [k8s_client.V1EndpointSubset(addresses=a, ports=[p2, p1])]
+        >>> Kubernetes.subsets_changed(s1, s2)
+        False
+        """
+        if len(last_observed_subsets) != len(subsets):
+            return True
+        if subsets == []:
+            return False
+        if len(last_observed_subsets[0].addresses or []) != 1 or \
+                last_observed_subsets[0].addresses[0].ip != subsets[0].addresses[0].ip or \
+                len(last_observed_subsets[0].ports) != len(subsets[0].ports):
+            return True
+        if len(subsets[0].ports) == 1:
+            return not Kubernetes.compare_ports(last_observed_subsets[0].ports[0], subsets[0].ports[0])
+        observed_ports = {p.name: p for p in last_observed_subsets[0].ports}
+        for p in subsets[0].ports:
+            if p.name not in observed_ports or not Kubernetes.compare_ports(p, observed_ports.pop(p.name)):
+                return True
+        return False
+
+    @catch_kubernetes_errors
+    def patch_or_create(self, name, annotations, resource_version=None, patch=False, retry=True, subsets=None):
+        metadata = {'namespace': self._namespace, 'name': name, 'labels': self._labels, 'annotations': annotations}
+        if patch or resource_version:
+            if resource_version is not None:
+                metadata['resource_version'] = resource_version
+            func = functools.partial(self._api.patch_namespaced_kind, name)
+        else:
+            func = functools.partial(self._api.create_namespaced_kind)
+            # skip annotations with null values
+            metadata['annotations'] = {k: v for k, v in metadata['annotations'].items() if v is not None}
+
+        metadata = k8s_client.V1ObjectMeta(**metadata)
+        if subsets is not None and self.__subsets:
+            endpoints = {'metadata': metadata}
+            if self.subsets_changed(self._leader_observed_subsets, subsets):
+                endpoints['subsets'] = subsets
+            body = k8s_client.V1Endpoints(**endpoints)
+        else:
+            body = k8s_client.V1ConfigMap(metadata=metadata)
+        return self.retry(func, self._namespace, body) if retry else func(self._namespace, body)
+
+    def _write_leader_optime(self, last_operation):
+        """Unused"""
+
+    def _update_leader(self):
+        """Unused"""
+
+    def update_leader(self, last_operation):
+        now = datetime.datetime.now(tzutc).isoformat()
+        annotations = {self._LEADER: self._name, 'ttl': str(self._ttl), 'renewTime': now,
+                       'acquireTime': self._leader_observed_record.get('acquireTime') or now,
+                       'transitions': self._leader_observed_record.get('transitions') or '0'}
+        if last_operation:
+            annotations[self._OPTIME] = last_operation
+
+        ret = self.patch_or_create(self.leader_path, annotations, self._leader_resource_version, subsets=self.__subsets)
+        if ret:
+            self._leader_resource_version = ret.metadata.resource_version
+        return ret
+
+    def attempt_to_acquire_leader(self, permanent=False):
+        now = datetime.datetime.now(tzutc).isoformat()
+        annotations = {self._LEADER: self._name, 'ttl': str(sys.maxsize if permanent else self._ttl),
+                       'renewTime': now, 'acquireTime': now, 'transitions': '0'}
+        if self._leader_observed_record:
+            try:
+                transitions = int(self._leader_observed_record.get('transitions'))
+            except (TypeError, ValueError):
+                transitions = 0
+
+            if self._leader_observed_record.get(self._LEADER) != self._name:
+                transitions += 1
+            else:
+                annotations['acquireTime'] = self._leader_observed_record.get('acquireTime') or now
+            annotations['transitions'] = str(transitions)
+        ret = self.patch_or_create(self.leader_path, annotations, self._leader_resource_version, subsets=self.__subsets)
+        if ret:
+            self._leader_resource_version = ret.metadata.resource_version
+        else:
+            logger.info('Could not take out TTL lock')
+        return ret
+
+    def take_leader(self):
+        return self.attempt_to_acquire_leader()
+
+    def set_failover_value(self, value, index=None):
+        """Unused"""
+
+    def manual_failover(self, leader, candidate, scheduled_at=None, index=None):
+        annotations = {'leader': leader or None, 'member': candidate or None, 'scheduled_at': scheduled_at}
+        patch = bool(self.cluster and isinstance(self.cluster.failover, Failover) and self.cluster.failover.index)
+        return self.patch_or_create(self.failover_path, annotations, index, bool(index or patch), False)
+
+    def set_config_value(self, value, index=None):
+        patch = bool(index or self.cluster and self.cluster.config and self.cluster.config.index)
+        return self.patch_or_create(self.config_path, {self._CONFIG: value}, index, patch, False)
+
+    @catch_kubernetes_errors
+    def touch_member(self, data, ttl=None, permanent=False):
+        cluster = self.cluster
+        if cluster and cluster.leader and cluster.leader.name == self._name:
+            role = 'master'
+        elif data['state'] == 'running' and data['role'] != 'master':
+            role = data['role']
+        else:
+            role = None
+
+        member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
+        pod_labels = member and member.data.pop('pod_labels', None)
+        ret = pod_labels is not None and pod_labels.get(self._role_label) == role and deep_compare(data, member.data)
+
+        if not ret:
+            metadata = {'namespace': self._namespace, 'name': self._name, 'labels': {self._role_label: role},
+                        'annotations': {'status': json.dumps(data, separators=(',', ':'))}}
+            body = k8s_client.V1Pod(metadata=k8s_client.V1ObjectMeta(**metadata))
+            ret = self._api.patch_namespaced_pod(self._name, self._namespace, body)
+        return ret
+
+    def initialize(self, create_new=True, sysid=""):
+        cluster = self.cluster
+        resource_version = cluster.config.index if cluster and cluster.config and cluster.config.index else None
+        return self.patch_or_create(self.config_path, {self._INITIALIZE: sysid}, resource_version)
+
+    def delete_leader(self):
+        if self.cluster and isinstance(self.cluster.leader, Leader) and self.cluster.leader.name == self._name:
+            self.patch_or_create(self.leader_path, {self._LEADER: None}, self._leader_resource_version, True, False, [])
+            self.reset_cluster()
+
+    def cancel_initialization(self):
+        self.patch_or_create(self.config_path, {self._INITIALIZE: None}, self.cluster.config.index, True)
+
+    @catch_kubernetes_errors
+    def delete_cluster(self):
+        self.retry(self._api.delete_collection_namespaced_kind, self._namespace, label_selector=self._label_selector)
+
+    @catch_kubernetes_errors
+    def set_history_value(self, value):
+        patch = bool(self.cluster and self.cluster.config and self.cluster.config.index)
+        return self.patch_or_create(self.config_path, {self._HISTORY: value}, None, patch, False)
+
+    def set_sync_state_value(self, value, index=None):
+        """Unused"""
+
+    def write_sync_state(self, leader, sync_standby, index=None):
+        return self.patch_or_create(self.sync_path, self.sync_state(leader, sync_standby), index, False)
+
+    def delete_sync_state(self, index=None):
+        return self.write_sync_state(None, None, index)
+
+    def watch(self, leader_index, timeout):
+        if self.__do_not_watch:
+            self.__do_not_watch = False
+            return True
+
+        if leader_index:
+            end_time = time.time() + timeout
+            w = k8s_watch.Watch()
+            while timeout >= 1:
+                try:
+                    for event in w.stream(self._api.list_namespaced_kind, self._namespace,
+                                          resource_version=leader_index, timeout_seconds=int(timeout + 0.5),
+                                          field_selector='metadata.name=' + self.leader_path,
+                                          _request_timeout=(1, timeout + 1)):
+                        return event['raw_object'].get('metadata', {}).get('resourceVersion') != leader_index
+                    return False
+                except KeyboardInterrupt:
+                    raise
+                except Exception:
+                    logging.exception('watch')
+
+                timeout = end_time - time.time()
+
+        try:
+            return super(Kubernetes, self).watch(None, timeout)
+        finally:
+            self.event.clear()

patroni/dcs/zookeeper.py

@@ -1,11 +1,13 @@
+import json
 import logging
 import time
 
 from kazoo.client import KazooClient, KazooState, KazooRetry
 from kazoo.exceptions import NoNodeError, NodeExistsError
 from kazoo.handlers.threading import SequentialThreadingHandler
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
 from patroni.exceptions import DCSError
+from patroni.utils import deep_compare
 
 logger = logging.getLogger(__name__)
 
@@ -57,7 +59,7 @@ class ZooKeeper(AbstractDCS):
                                    max_delay=1, max_tries=-1, sleep_func=time.sleep))
         self._client.add_listener(self.session_listener)
 
-        self._my_member_data = None
+        self._my_member_data = {}
         self._fetch_cluster = True
 
         self._orig_kazoo_connect = self._client._connection._connect
@@ -159,6 +161,10 @@ class ZooKeeper(AbstractDCS):
         config = self.get_node(self.config_path, watch=self.cluster_watcher) if self._CONFIG in nodes else None
         config = config and ClusterConfig.from_node(config[1].version, config[0], config[1].mzxid)
 
+        # get timeline history
+        history = self.get_node(self.history_path, watch=self.cluster_watcher) if self._HISTORY in nodes else None
+        history = history and TimelineHistory.from_node(history[1].mzxid, history[0])
+
         # get last leader operation
         last_leader_operation = self._OPTIME in nodes and self._fetch_cluster and self.get_node(self.leader_optime_path)
         last_leader_operation = last_leader_operation and int(last_leader_operation[0]) or 0
@@ -191,7 +197,7 @@ class ZooKeeper(AbstractDCS):
         failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
         failover = failover and Failover.from_node(failover[1].version, failover[0])
 
-        self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
+        self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
 
     def _load_cluster(self):
         if self._fetch_cluster or self._cluster is None:
@@ -206,7 +212,7 @@ class ZooKeeper(AbstractDCS):
         try:
             self._client.retry(self._client.create, path, value.encode('utf-8'), **kwargs)
             return True
-        except:
+        except Exception:
             return False
 
     def attempt_to_acquire_leader(self, permanent=False):
@@ -215,16 +221,19 @@ class ZooKeeper(AbstractDCS):
             logger.info('Could not take out TTL lock')
         return ret
 
-    def set_failover_value(self, value, index=None):
+    def __set_failover_or_sync_state_value(self, key, value, index=None):
         try:
-            self._client.retry(self._client.set, self.failover_path, value.encode('utf-8'), version=index or -1)
+            self._client.retry(self._client.set, key, value.encode('utf-8'), version=index or -1)
             return True
         except NoNodeError:
-            return value == '' or (index is None and self._create(self.failover_path, value))
-        except:
+            return value == '' or (index is None and self._create(key, value))
+        except Exception:
             logging.exception('set_failover_value')
             return False
 
+    def set_failover_value(self, value, index=None):
+        return self.__set_failover_or_sync_state_value(self.failover_path, value, index)
+
     def set_config_value(self, value, index=None):
         try:
             self._client.retry(self._client.set, self.config_path, value.encode('utf-8'), version=index or -1)
@@ -242,22 +251,23 @@ class ZooKeeper(AbstractDCS):
     def touch_member(self, data, ttl=None, permanent=False):
         cluster = self.cluster
         member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
-        data = data.encode('utf-8')
+        encoded_data = json.dumps(data, separators=(',', ':')).encode('utf-8')
         if member and self._client.client_id is not None and member.session != self._client.client_id[0]:
             try:
                 self._client.delete_async(self.member_path).get(timeout=1)
             except NoNodeError:
                 pass
-            except:
+            except Exception:
                 return False
             member = None
 
         if member:
-            if data == self._my_member_data:
+            if deep_compare(data, self._my_member_data):
                 return True
         else:
             try:
-                self._client.create_async(self.member_path, data, makepath=True, ephemeral=not permanent).get(timeout=1)
+                self._client.create_async(self.member_path, encoded_data, makepath=True,
+                                          ephemeral=not permanent).get(timeout=1)
                 self._my_member_data = data
                 return True
             except Exception as e:
@@ -265,10 +275,10 @@ class ZooKeeper(AbstractDCS):
                     logger.exception('touch_member')
                     return False
         try:
-            self._client.set_async(self.member_path, data).get(timeout=1)
+            self._client.set_async(self.member_path, encoded_data).get(timeout=1)
             self._my_member_data = data
             return True
-        except:
+        except Exception:
             logger.exception('touch_member')
 
         return False
@@ -276,22 +286,25 @@ class ZooKeeper(AbstractDCS):
     def take_leader(self):
         return self.attempt_to_acquire_leader()
 
-    def _write_leader_optime(self, last_operation):
-        last_operation = last_operation.encode('utf-8')
+    def __write_leader_optime_or_history_value(self, key, value):
+        value = value.encode('utf-8')
         try:
-            self._client.set_async(self.leader_optime_path, last_operation).get(timeout=1)
+            self._client.set_async(key, value).get(timeout=1)
             return True
         except NoNodeError:
             try:
-                self._client.create_async(self.leader_optime_path, last_operation, makepath=True).get(timeout=1)
+                self._client.create_async(key, value, makepath=True).get(timeout=1)
                 return True
-            except:
-                logger.exception('Failed to create %s', self.leader_optime_path)
-        except:
-            logger.exception('Failed to update %s', self.leader_optime_path)
+            except Exception:
+                logger.exception('Failed to create %s', key)
+        except Exception:
+            logger.exception('Failed to update %s', key)
         return False
 
-    def update_leader(self):
+    def _write_leader_optime(self, last_operation):
+        return self.__write_leader_optime_or_history_value(self.leader_optime_path, last_operation)
+
+    def _update_leader(self):
         return True
 
     def delete_leader(self):
@@ -307,7 +320,7 @@ class ZooKeeper(AbstractDCS):
     def cancel_initialization(self):
         try:
             self._client.retry(self._cancel_initialization)
-        except:
+        except Exception:
             logger.exception("Unable to delete initialize key")
 
     def delete_cluster(self):
@@ -316,15 +329,11 @@ class ZooKeeper(AbstractDCS):
         except NoNodeError:
             return True
 
+    def set_history_value(self, value):
+        return self.__write_leader_optime_or_history_value(self.history_path, value)
+
     def set_sync_state_value(self, value, index=None):
-        try:
-            self._client.retry(self._client.set, self.sync_path, value.encode('utf-8'), version=index or -1)
-            return True
-        except NoNodeError:
-            return value == '' or (index is None and self._create(self.sync_path, value))
-        except:
-            logging.exception('set_sync_state_value')
-            return False
+        return self.__set_failover_or_sync_state_value(self.sync_path, value, index)
 
     def delete_sync_state(self, index=None):
         return self.set_sync_state_value("{}", index)

patroni/ha.py

@@ -57,10 +57,12 @@ class Ha(object):
         self.dcs = patroni.dcs
         self.cluster = None
         self.old_cluster = None
+        self._leader_timeline = None
         self.recovering = False
         self._post_bootstrap_task = None
+        self._crash_recovery_executed = False
         self._start_timeout = None
-        self._async_executor = AsyncExecutor(self.wakeup)
+        self._async_executor = AsyncExecutor(self.state_handler, self.wakeup)
         self.watchdog = patroni.watchdog
 
         # Each member publishes various pieces of information to the DCS using touch_member. This lock protects
@@ -81,18 +83,21 @@ class Ha(object):
             self.old_cluster = cluster
         self.cluster = cluster
 
+        self._leader_timeline = None if cluster.is_unlocked() else cluster.leader.timeline
+
     def acquire_lock(self):
         return self.dcs.attempt_to_acquire_leader()
 
     def update_lock(self, write_leader_optime=False):
-        ret = self.dcs.update_leader()
+        last_operation = None
+        if write_leader_optime:
+            try:
+                last_operation = self.state_handler.last_operation()
+            except Exception:
+                logger.exception('Exception when called state_handler.last_operation()')
+        ret = self.dcs.update_leader(last_operation)
         if ret:
             self.watchdog.keepalive()
-            if write_leader_optime:
-                try:
-                    self.dcs.write_leader_optime(self.state_handler.last_operation())
-                except:
-                    pass
         return ret
 
     def has_lock(self):
@@ -121,17 +126,26 @@ class Ha(object):
                 data['tags'] = tags
             if self.state_handler.pending_restart:
                 data['pending_restart'] = True
-            if not self._async_executor.busy and data['state'] in ['running', 'restarting', 'starting']:
+            if self._async_executor.scheduled_action in (None, 'promote') \
+                    and data['state'] in ['running', 'restarting', 'starting']:
                 try:
-                    data['xlog_location'] = self.state_handler.wal_position(retry=False)
-                except:
+                    timeline, wal_position = self.state_handler.timeline_wal_position()
+                    data['xlog_location'] = wal_position
+                    if not timeline:
+                        timeline = self.state_handler.replica_cached_timeline(self._leader_timeline)
+                    if timeline:
+                        data['timeline'] = timeline
+                except Exception:
                     pass
             if self.patroni.scheduled_restart:
                 scheduled_restart_data = self.patroni.scheduled_restart.copy()
                 scheduled_restart_data['schedule'] = scheduled_restart_data['schedule'].isoformat()
                 data['scheduled_restart'] = scheduled_restart_data
 
-            return self.dcs.touch_member(json.dumps(data, separators=(',', ':')))
+            if self.is_paused():
+                data['pause'] = True
+
+            return self.dcs.touch_member(data)
 
     def clone(self, clone_member=None, msg='(without leader)'):
         if self.state_handler.clone(clone_member):
@@ -175,6 +189,11 @@ class Ha(object):
             self._async_executor.run_async(self.state_handler.rewind, (self.cluster.leader,))
             return True
 
+    def _start_crash_recovery(self, msg):
+        self._async_executor.schedule(msg)
+        self._async_executor.run_async(self.state_handler.fix_cluster_state)
+        return msg
+
     def recover(self):
         # Postgres is not running and we will restart in standby mode. Watchdog is not needed until we promote.
         self.watchdog.disable()
@@ -184,13 +203,22 @@ class Ha(object):
             if timeout == 0:
                 # We are requested to prefer failing over to restarting master. But see first if there
                 # is anyone to fail over to.
-                if self.is_failover_possible(self.cluster.members):
+                members = self.cluster.members
+                if self.is_synchronous_mode():
+                    members = [m for m in members if self.cluster.sync.matches(m.name)]
+                if self.is_failover_possible(members):
                     logger.info("Master crashed. Failing over.")
                     self.demote('immediate')
                     return 'stopped PostgreSQL to fail over after a crash'
         else:
             timeout = None
 
+        data = self.state_handler.controldata()
+        if data.get('Database cluster state') == 'in production' and not self._crash_recovery_executed and \
+                (self.cluster.is_unlocked() or self.state_handler.can_rewind):
+            self._crash_recovery_executed = True
+            return self._start_crash_recovery('doing crash recovery in a single user mode')
+
         self.load_cluster_from_dcs()
 
         if self.has_lock():
@@ -204,6 +232,13 @@ class Ha(object):
             msg = "starting as a secondary"
             node_to_follow = self._get_node_to_follow(self.cluster)
 
+        # once we already tried to start postgres but failed, single user mode is a rescue in this case
+        if self.recovering and not self.state_handler.rewind_executed \
+                and not self._crash_recovery_executed and self.state_handler.can_rewind \
+                and data.get('Database cluster state') not in ('shut down', 'shut down in recovery'):
+            self.recovering = False
+            return self._start_crash_recovery('fixing cluster state in a single user mode')
+
         self.recovering = True
 
         self._async_executor.schedule('restarting after failure')
@@ -249,10 +284,10 @@ class Ha(object):
         return follow_reason
 
     def is_synchronous_mode(self):
-        return bool(self.cluster and self.cluster.config and self.cluster.config.data.get('synchronous_mode'))
+        return bool(self.cluster and self.cluster.is_synchronous_mode())
 
     def is_synchronous_mode_strict(self):
-        return bool(self.cluster and self.cluster.config and self.cluster.config.data.get('synchronous_mode_strict'))
+        return bool(self.cluster and self.cluster.is_synchronous_mode_strict())
 
     def process_sync_replication(self):
         """Process synchronous standby beahvior.
@@ -340,20 +375,41 @@ class Ha(object):
             with self._member_state_lock:
                 self._disable_sync -= 1
 
-    def enforce_master_role(self, message, promote_message):
-        if not self.watchdog.is_running:
-            if not self.watchdog.activate():
-                if self.state_handler.is_leader():
-                    self.demote('immediate')
-                    return 'Demoting self because watchdog could not be activated'
-                else:
-                    self.release_leader_key_voluntarily()
-                    return 'Not promoting self because watchdog could not be actived'
+    def update_cluster_history(self):
+        master_timeline = self.state_handler.get_master_timeline()
+        cluster_history = self.cluster.history and self.cluster.history.lines
+        if master_timeline == 1:
+            if cluster_history:
+                self.dcs.set_history_value('[]')
+        elif not cluster_history or cluster_history[-1][0] != master_timeline - 1 or len(cluster_history[-1]) != 4:
+            cluster_history = {l[0]: l for l in cluster_history or []}
+            history = self.state_handler.get_history(master_timeline)
+            if history:
+                for line in history:
+                    # enrich current history with promotion timestamps stored in DCS
+                    if len(line) == 3 and line[0] in cluster_history \
+                            and len(cluster_history[line[0]]) == 4 \
+                            and cluster_history[line[0]][1] == line[1]:
+                        line.append(cluster_history[line[0]][3])
+                self.dcs.set_history_value(json.dumps(history, separators=(',', ':')))
 
-        if self.state_handler.is_leader() or self.state_handler.role == 'master':
+    def enforce_master_role(self, message, promote_message):
+        if not self.is_paused() and not self.watchdog.is_running and not self.watchdog.activate():
+            if self.state_handler.is_leader():
+                self.demote('immediate')
+                return 'Demoting self because watchdog could not be activated'
+            else:
+                self.release_leader_key_voluntarily()
+                return 'Not promoting self because watchdog could not be activated'
+
+        if self.state_handler.is_leader():
             # Inform the state handler about its master role.
             # It may be unaware of it if postgres is promoted manually.
             self.state_handler.set_role('master')
+            self.process_sync_replication()
+            self.update_cluster_history()
+            return message
+        elif self.state_handler.role == 'master':
             self.process_sync_replication()
             return message
         else:
@@ -364,8 +420,10 @@ class Ha(object):
                     # Somebody else updated sync state, it may be due to us losing the lock. To be safe, postpone
                     # promotion until next cycle. TODO: trigger immediate retry of run_cycle
                     return 'Postponing promotion because synchronous replication state was updated by somebody else'
-                self.state_handler.set_synchronous_standby(None)
-            self.state_handler.promote()
+                self.state_handler.set_synchronous_standby('*' if self.is_synchronous_mode_strict() else None)
+            if self.state_handler.role != 'master':
+                self._async_executor.schedule('promote')
+                self._async_executor.run_async(self.state_handler.promote, args=(self.dcs.loop_wait,))
             return promote_message
 
     @staticmethod
@@ -401,7 +459,7 @@ class Ha(object):
     def _is_healthiest_node(self, members, check_replication_lag=True):
         """This method tries to determine whether I am healthy enough to became a new leader candidate or not."""
 
-        my_wal_position = self.state_handler.wal_position()
+        _, my_wal_position = self.state_handler.timeline_wal_position()
         if check_replication_lag and self.is_lagging(my_wal_position):
             return False  # Too far behind last reported wal position on master
 
@@ -549,8 +607,8 @@ class Ha(object):
         self.state_handler.set_role('demoted')
 
         if mode_control['release']:
-                self.release_leader_key_voluntarily()
-                time.sleep(2)  # Give a time to somebody to take the leader lock
+            self.release_leader_key_voluntarily()
+            time.sleep(2)  # Give a time to somebody to take the leader lock
         if mode_control['offline']:
             node_to_follow, leader = None, None
         else:
@@ -564,6 +622,8 @@ class Ha(object):
             self._async_executor.schedule('starting after demotion')
             self._async_executor.run_async(self.state_handler.follow, (node_to_follow,))
         else:
+            if self.is_synchronous_mode():
+                self.state_handler.set_synchronous_standby(None)
             if self.state_handler.rewind_needed_and_possible(leader):
                 return False  # do not start postgres, but run pg_rewind on the next iteration
             self.state_handler.follow(node_to_follow)
@@ -622,8 +682,16 @@ class Ha(object):
                 if not failover.candidate and self.is_paused():
                     logger.warning('Failover is possible only to a specific candidate in a paused state')
                 else:
-                    members = [m for m in self.cluster.members
-                               if not failover.candidate or m.name == failover.candidate]
+                    if self.is_synchronous_mode():
+                        if failover.candidate and not self.cluster.sync.matches(failover.candidate):
+                            logger.warning('Failover candidate=%s does not match with sync_standby=%s',
+                                           failover.candidate, self.cluster.sync.sync_standby)
+                            members = []
+                        else:
+                            members = [m for m in self.cluster.members if self.cluster.sync.matches(m.name)]
+                    else:
+                        members = [m for m in self.cluster.members
+                                   if not failover.candidate or m.name == failover.candidate]
                     if self.is_failover_possible(members):  # check that there are healthy members
                         self._async_executor.schedule('manual failover: demote')
                         self._async_executor.run_async(self.demote, ('graceful',))
@@ -821,7 +889,7 @@ class Ha(object):
         member_role = 'leader' if clone_member == self.cluster.leader else 'replica'
         return self.clone(clone_member, "from {0} '{1}'".format(member_role, clone_member.name))
 
-    def reinitialize(self):
+    def reinitialize(self, force=False):
         with self._async_executor:
             self.load_cluster_from_dcs()
 
@@ -831,7 +899,11 @@ class Ha(object):
             if self.cluster.leader.name == self.state_handler.name:
                 return 'I am the leader, can not reinitialize'
 
-            action = self._async_executor.schedule('reinitialize', immediately=True)
+        if force:
+            self._async_executor.cancel()
+
+        with self._async_executor:
+            action = self._async_executor.schedule('reinitialize')
             if action is not None:
                 return '{0} already in progress'.format(action)
 
@@ -849,7 +921,7 @@ class Ha(object):
                     # background thread and has not even written a pid file yet.
                     with self._async_executor.critical_task as task:
                         if not task.cancel():
-                            self.state_handler.terminate_starting_postmaster(pid=task.result)
+                            self.state_handler.terminate_starting_postmaster(postmaster=task.result)
                 self.demote('immediate-nolock')
                 return 'lost leader lock during ' + self._async_executor.scheduled_action
 
@@ -874,6 +946,7 @@ class Ha(object):
                 self.dcs.reset_cluster()
                 return 'removed leader key after trying and failing to start postgres'
             return 'failed to start postgres'
+        self._crash_recovery_executed = False
         return None
 
     def cancel_initialization(self):
@@ -915,6 +988,8 @@ class Ha(object):
         # Check if we are in startup, when paused defer to main loop for manual failovers.
         if not self.state_handler.check_for_startup() or self.is_paused():
             self.set_start_timeout(None)
+            if self.is_paused():
+                self.state_handler.set_state(self.state_handler.is_running() and 'running' or 'stopped')
             return None
 
         # state_handler.state == 'starting' here
@@ -954,6 +1029,7 @@ class Ha(object):
     def _run_cycle(self):
         dcs_failed = False
         try:
+            self.state_handler.reset_cluster_info_state()
             self.load_cluster_from_dcs()
 
             if self.is_paused():
@@ -990,7 +1066,9 @@ class Ha(object):
 
             # is data directory empty?
             if self.state_handler.data_directory_empty():
-                # In case datadir went away while we were master. TODO: check for this and try to stop postgresql.
+                self.state_handler.set_role('uninitialized')
+                self.state_handler.stop('immediate')
+                # In case datadir went away while we were master.
                 self.watchdog.disable()
 
                 # is this instance the leader?
@@ -1068,7 +1146,8 @@ class Ha(object):
             disable_wd = self.watchdog.disable if self.watchdog.is_running else None
             self.while_not_sync_standby(lambda: self.state_handler.stop(checkpoint=False, on_safepoint=disable_wd))
             if not self.state_handler.is_running():
-                self.dcs.delete_leader()
+                if self.has_lock():
+                    self.dcs.delete_leader()
             else:
                 # XXX: what about when Patroni is started as the wrong user that has access to the watchdog device
                 # but cannot shut down PostgreSQL. Root would be the obvious example. Would be nice to not kill the

patroni/postgresql.py

@@ -1,12 +1,9 @@
 import logging
-import errno
 import os
 import psycopg2
-import psutil
 import re
 import shlex
 import shutil
-import signal
 import socket
 import subprocess
 import tempfile
@@ -14,10 +11,10 @@ import time
 
 from collections import defaultdict
 from contextlib import contextmanager
-from patroni import call_self
 from patroni.callback_executor import CallbackExecutor
-from patroni.exceptions import PostgresConnectionException
-from patroni.utils import compare_values, parse_bool, parse_int, Retry, RetryFailedError, polling_loop, null_context
+from patroni.exceptions import PostgresConnectionException, PostgresException
+from patroni.utils import compare_values, parse_bool, parse_int, Retry, RetryFailedError, polling_loop, split_host_port
+from patroni.postmaster import PostmasterProcess
 from six import string_types
 from six.moves.urllib.parse import quote_plus
 from threading import current_thread, Lock
@@ -35,13 +32,21 @@ STATE_REJECT = 'rejecting connections'
 STATE_NO_RESPONSE = 'not responding'
 STATE_UNKNOWN = 'unknown'
 
-STOP_SIGNALS = {
-    'smart': signal.SIGTERM,
-    'fast': signal.SIGINT,
-    'immediate': signal.SIGQUIT,
-}
 STOP_POLLING_INTERVAL = 1
 REWIND_STATUS = type('Enum', (), {'INITIAL': 0, 'CHECK': 1, 'NEED': 2, 'NOT_NEED': 3, 'SUCCESS': 4, 'FAILED': 5})
+sync_standby_name_re = re.compile('^[A-Za-z_][A-Za-z_0-9\$]*$')
+
+cluster_info_query = ("SELECT CASE WHEN pg_is_in_recovery() THEN 0 "
+                      "ELSE ('x' || SUBSTR(pg_{0}file_name(pg_current_{0}_{1}()), 1, 8))::bit(32)::int END, "
+                      "CASE WHEN pg_is_in_recovery() THEN GREATEST("
+                      " pg_{0}_{1}_diff(COALESCE(pg_last_{0}_receive_{1}(), '0/0'), '0/0')::bigint,"
+                      " pg_{0}_{1}_diff(pg_last_{0}_replay_{1}(), '0/0')::bigint)"
+                      "ELSE pg_{0}_{1}_diff(pg_current_{0}_{1}(), '0/0')::bigint END")
+
+
+def quote_ident(value):
+    """Very simplified version of quote_ident"""
+    return value if sync_standby_name_re.match(value) else '"' + value + '"'
 
 
 def slot_name_from_member_name(member_name):
@@ -60,6 +65,11 @@ def slot_name_from_member_name(member_name):
     return slot_name[0:63]
 
 
+@contextmanager
+def null_context():
+    yield
+
+
 class Postgresql(object):
 
     # List of parameters which must be always passed to postmaster as command line options
@@ -83,12 +93,12 @@ class Postgresql(object):
         'wal_level': ('hot_standby', lambda v: v.lower() in ('hot_standby', 'replica', 'logical'), 90100),
         'hot_standby': ('on', lambda _: False, 90100),
         'max_connections': (100, lambda v: int(v) >= 100, 90100),
-        'max_wal_senders': (5, lambda v: int(v) >= 5, 90100),
+        'max_wal_senders': (10, lambda v: int(v) >= 10, 90100),
         'wal_keep_segments': (8, lambda v: int(v) >= 8, 90100),
         'max_prepared_transactions': (0, lambda v: int(v) >= 0, 90100),
         'max_locks_per_transaction': (64, lambda v: int(v) >= 64, 90100),
         'track_commit_timestamp': ('off', lambda v: parse_bool(v) is not None, 90500),
-        'max_replication_slots': (5, lambda v: int(v) >= 5, 90400),
+        'max_replication_slots': (10, lambda v: int(v) >= 10, 90400),
         'max_worker_processes': (8, lambda v: int(v) >= 8, 90400),
         'wal_log_hints': ('on', lambda _: False, 90400)
     }
@@ -134,6 +144,10 @@ class Postgresql(object):
         self._trigger_file = config.get('recovery_conf', {}).get('trigger_file') or 'promote'
         self._trigger_file = os.path.abspath(os.path.join(self._data_dir, self._trigger_file))
 
+        self._is_cancelled = False
+        self._cancellable = None
+        self._cancellable_lock = Lock()
+
         self._connection_lock = Lock()
         self._connection = None
         self._cursor_holder = None
@@ -153,6 +167,12 @@ class Postgresql(object):
 
         self._state_entry_timestamp = None
 
+        self._cluster_info_state = {}
+        self._cached_replica_timeline = None
+
+        # Last known running process
+        self._postmaster_proc = None
+
         if self.is_running():
             self.set_state('running')
             self.set_role('master' if self.is_leader() else 'replica')
@@ -209,8 +229,8 @@ class Postgresql(object):
 
     def get_server_parameters(self, config):
         parameters = config['parameters'].copy()
-        listen_addresses, port = (config['listen'] + ':5432').split(':')[:2]
-        parameters.update({'cluster_name': self.scope, 'listen_addresses': listen_addresses, 'port': port})
+        listen_addresses, port = split_host_port(config['listen'], 5432)
+        parameters.update({'cluster_name': self.scope, 'listen_addresses': listen_addresses, 'port': str(port)})
         if config.get('synchronous_mode', False):
             if self._synchronous_standby_names is None:
                 if config.get('synchronous_mode_strict', False):
@@ -530,7 +550,7 @@ class Postgresql(object):
         params = ['--scope=' + self.scope, '--datadir=' + self._data_dir]
         try:
             logger.info('Running custom bootstrap script: %s', config['command'])
-            if subprocess.call(shlex.split(config['command']) + params) != 0:
+            if self.cancellable_subprocess_call(shlex.split(config['command']) + params) != 0:
                 self.set_state('custom bootstrap failed')
                 return False
         except Exception:
@@ -576,7 +596,7 @@ class Postgresql(object):
             env = self.write_pgpass(r) if 'password' in r else None
 
             try:
-                ret = subprocess.call(shlex.split(cmd) + [connstring], env=env)
+                ret = self.cancellable_subprocess_call(shlex.split(cmd) + [connstring], env=env)
             except OSError:
                 logger.error('post_init script %s failed', cmd)
                 return False
@@ -640,6 +660,9 @@ class Postgresql(object):
         # go through them in priority order
         ret = 1
         for replica_method in replica_methods:
+            with self._cancellable_lock:
+                if self._is_cancelled:
+                    break
             # if the method is basebackup, then use the built-in
             if replica_method == "basebackup":
                 ret = self.basebackup(connstring, env)
@@ -669,7 +692,7 @@ class Postgresql(object):
                 params = ["--{0}={1}".format(arg, val) for arg, val in method_config.items()]
                 try:
                     # call script with the full set of parameters
-                    ret = subprocess.call(shlex.split(cmd) + params, env=env)
+                    ret = self.cancellable_subprocess_call(shlex.split(cmd) + params, env=env)
                     # if we succeeded, stop
                     if ret == 0:
                         logger.info('replica has been created using %s', replica_method)
@@ -684,21 +707,40 @@ class Postgresql(object):
         self.set_state('stopped')
         return ret
 
+    def reset_cluster_info_state(self):
+        self._cluster_info_state = {}
+
+    def _cluster_info_state_get(self, name):
+        if not self._cluster_info_state:
+            stmt = cluster_info_query.format(self.wal_name, self.lsn_name)
+            try:
+                result = self._is_leader_retry(self._query, stmt).fetchone()
+                self._cluster_info_state = dict(zip(['timeline', 'wal_position'], result))
+            except RetryFailedError as e:  # SELECT failed two times
+                self._cluster_info_state = {'error': str(e)}
+                if not self.is_starting() and self.pg_isready() == STATE_REJECT:
+                    self.set_state('starting')
+
+        if 'error' in self._cluster_info_state:
+            raise PostgresConnectionException(self._cluster_info_state['error'])
+
+        return self._cluster_info_state.get(name)
+
     def is_leader(self):
-        try:
-            return not self._is_leader_retry(self._query, 'SELECT pg_is_in_recovery()').fetchone()[0]
-        except RetryFailedError as e:  # SELECT pg_is_in_recovery() failed two times
-            if not self.is_starting() and self.pg_isready() == STATE_REJECT:
-                self.set_state('starting')
-            raise PostgresConnectionException(str(e))
+        return bool(self._cluster_info_state_get('timeline'))
 
     def is_running(self):
-        if not (self._version_file_exists() and os.path.isfile(self._postmaster_pid)):
-            # XXX: This is dangerous in case somebody deletes the data directory while PostgreSQL is still running.
-            return False
-        return self.is_pid_running(self.get_pid())
+        """Returns PostmasterProcess if one is running on the data directory or None. If most recently seen process
+        is running udpates the cached process based on pid file."""
+        if self._postmaster_proc:
+            if self._postmaster_proc.is_running():
+                return self._postmaster_proc
+            self._postmaster_proc = None
 
-    def read_pid_file(self):
+        self._postmaster_proc = PostmasterProcess.from_pidfile(self._read_pid_file())
+        return self._postmaster_proc
+
+    def _read_pid_file(self):
         """Reads and parses postmaster.pid from the data directory
 
         :returns dictionary of values if successful, empty dictionary otherwise
@@ -710,27 +752,6 @@ class Postgresql(object):
         except IOError:
             return {}
 
-    def get_pid(self):
-        """Fetches pid value from postmaster.pid using read_pid_file
-
-        :returns pid if successful, 0 if pid file is not present"""
-        # TODO: figure out what to do on permission errors
-        pid = self.read_pid_file().get('pid', 0)
-        try:
-            return int(pid)
-        except ValueError:
-            logger.warning("Garbage pid in postmaster.pid: {0!r}".format(pid))
-            return 0
-
-    @staticmethod
-    def is_pid_running(pid):
-        try:
-            if pid < 0:
-                pid = -pid
-            return pid > 0 and pid != os.getpid() and pid != os.getppid() and (os.kill(pid, 0) or True)
-        except Exception:
-            return False
-
     @property
     def cb_called(self):
         return self.__cb_called
@@ -775,30 +796,24 @@ class Postgresql(object):
     def is_starting(self):
         return self.state == 'starting'
 
-    def wait_for_port_open(self, pid, initiated, timeout):
+    def wait_for_port_open(self, postmaster, timeout):
         """Waits until PostgreSQL opens ports."""
         for _ in polling_loop(timeout):
-            pid_file = self.read_pid_file()
-            if len(pid_file) > 5:
-                try:
-                    pmpid = int(pid_file['pid'])
-                    pmstart = int(pid_file['start_time'])
+            with self._cancellable_lock:
+                if self._is_cancelled:
+                    return False
 
-                    if pmstart >= initiated - 2 and pmpid == pid:
-                        isready = self.pg_isready()
-                        if isready != STATE_NO_RESPONSE:
-                            if isready not in [STATE_REJECT, STATE_RUNNING]:
-                                logger.warning("Can't determine PostgreSQL startup status, assuming running")
-                            return True
-                except ValueError:
-                    # Garbage in the pid file
-                    pass
-
-            if not self.is_pid_running(pid):
+            if not postmaster.is_running():
                 logger.error('postmaster is not running')
                 self.set_state('start failed')
                 return False
 
+            isready = self.pg_isready()
+            if isready != STATE_NO_RESPONSE:
+                if isready not in [STATE_REJECT, STATE_RUNNING]:
+                    logger.warning("Can't determine PostgreSQL startup status, assuming running")
+                return True
+
         logger.warning("Timed out waiting for PostgreSQL to start")
         return False
 
@@ -834,33 +849,22 @@ class Postgresql(object):
         options = ['--{0}={1}'.format(p, self._server_parameters[p]) for p in self.CMDLINE_OPTIONS
                    if p in self._server_parameters and p != 'wal_keep_segments']
 
-        # Unfortunately `pg_ctl start` does not return postmaster pid to us. Without this information
-        # it is hard to know the current state of postgres startup, so we had to reimplement pg_ctl start
-        # in python. It will start postgres, wait for port to be open and wait until postgres will start
-        # accepting connections.
-        # Important!!! We can't just start postgres using subprocess.Popen, because in this case it
-        # will be our child for the rest of our live and we will have to take care of it (`waitpid`).
-        # So we will use the same approach as pg_ctl uses: start a new process, which will start postgres.
-        # This process will write postmaster pid to stdout and exit immediately. Now it's responsibility
-        # of init process to take care about postmaster.
-        # In order to make everything portable we can't use fork&exec approach here, so  we will call
-        # ourselves and pass list of arguments which must be used to start postgres.
+        with self._cancellable_lock:
+            if self._is_cancelled:
+                return False
+
         with task or null_context():
             if task and task.is_cancelled:
                 logger.info("PostgreSQL start cancelled.")
                 return False
 
-            start_initiated = time.time()
-            proc = call_self(['pg_ctl_start', self._pgcommand('postgres'), '-D', self._data_dir,
-                             '--config-file={}'.format(self._postgresql_conf)] + options, close_fds=True,
-                             preexec_fn=os.setsid, stdout=subprocess.PIPE,
-                             env={p: os.environ[p] for p in ('PATH', 'LC_ALL', 'LANG') if p in os.environ})
-            pid = int(proc.stdout.readline().strip())
-            proc.wait()
-            logger.info('postmaster pid=%s', pid)
+            self._postmaster_proc = PostmasterProcess.start(self._pgcommand('postgres'),
+                                                            self._data_dir,
+                                                            self._postgresql_conf,
+                                                            options)
 
             if task:
-                task.complete(pid)
+                task.complete(self._postmaster_proc)
 
         start_timeout = timeout
         if not start_timeout:
@@ -870,7 +874,7 @@ class Postgresql(object):
                 start_timeout = 60
 
         # We want postmaster to open ports before we continue
-        if not self.wait_for_port_open(pid, start_initiated, start_timeout):
+        if not self._postmaster_proc or not self.wait_for_port_open(self._postmaster_proc, start_timeout):
             return False
 
         ret = self.wait_for_startup(start_timeout)
@@ -898,7 +902,7 @@ class Postgresql(object):
             logging.exception('Exception during CHECKPOINT')
             return 'not accessible or not healty'
 
-    def stop(self, mode='fast', block_callbacks=False, checkpoint=True, on_safepoint=None):
+    def stop(self, mode='fast', block_callbacks=False, checkpoint=None, on_safepoint=None):
         """Stop PostgreSQL
 
         Supports a callback when a safepoint is reached. A safepoint is when no user backend can return a successful
@@ -907,6 +911,9 @@ class Postgresql(object):
 
         :param on_safepoint: This callback is called when no user backends are running.
         """
+        if checkpoint is None:
+            checkpoint = False if mode == 'immediate' else True
+
         success, pg_signaled = self._do_stop(mode, block_callbacks, checkpoint, on_safepoint)
         if success:
             # block_callbacks is used during restart to avoid
@@ -921,7 +928,8 @@ class Postgresql(object):
         return success
 
     def _do_stop(self, mode, block_callbacks, checkpoint, on_safepoint):
-        if not self.is_running():
+        postmaster = self.is_running()
+        if not postmaster:
             if on_safepoint:
                 on_safepoint()
             return True, False
@@ -933,85 +941,39 @@ class Postgresql(object):
             self.set_state('stopping')
 
         # Send signal to postmaster to stop
-        pid, result = self._signal_postmaster_stop(mode)
-        if result is not None:
-            if result and on_safepoint:
+        success = postmaster.signal_stop(mode)
+        if success is not None:
+            if success and on_safepoint:
                 on_safepoint()
-            return result, True
+            return success, True
 
         # We can skip safepoint detection if we don't have a callback
         if on_safepoint:
             # Wait for our connection to terminate so we can be sure that no new connections are being initiated
-            self._wait_for_connection_close(pid)
-            self._wait_for_user_backends_to_close(pid)
+            self._wait_for_connection_close(postmaster)
+            postmaster.wait_for_user_backends_to_close()
             on_safepoint()
 
-        self._wait_for_postmaster_stop(pid)
+        postmaster.wait()
 
         return True, True
 
-    def _wait_for_postmaster_stop(self, pid):
-        # This wait loop differs subtly from pg_ctl as we check for both the pid file going
-        # away and if the pid is running. This seems safer.
-        while pid == self.get_pid() and self.is_pid_running(pid):
-            time.sleep(STOP_POLLING_INTERVAL)
-
-    def _signal_postmaster_stop(self, mode):
-        pid = self.get_pid()
-        if pid == 0:
-            return None, True
-        elif pid < 0:
-            logger.warning("Cannot stop server; single-user server is running (PID: {0})".format(-pid))
-            return None, False
-        try:
-            os.kill(pid, STOP_SIGNALS[mode])
-        except OSError as e:
-            if e.errno == errno.ESRCH:
-                return None, True
-            else:
-                logger.warning("Could not send stop signal to PostgreSQL (error: {0})".format(e.errno))
-                return None, False
-        return pid, None
-
-    def terminate_starting_postmaster(self, pid):
+    @staticmethod
+    def terminate_starting_postmaster(postmaster):
         """Terminates a postmaster that has not yet opened ports or possibly even written a pid file. Blocks
         until the process goes away."""
-        try:
-            os.kill(pid, STOP_SIGNALS['immediate'])
-        except OSError as e:
-            if e.errno == errno.ESRCH:
-                return
-            logger.warning("Could not send stop signal to PostgreSQL (error: {0})".format(e.errno))
+        postmaster.signal_stop('immediate')
+        postmaster.wait()
 
-        while self.is_pid_running(pid):
-            time.sleep(STOP_POLLING_INTERVAL)
-
-    def _wait_for_connection_close(self, pid):
+    def _wait_for_connection_close(self, postmaster):
         try:
             with self.connection().cursor() as cur:
-                while pid == self.get_pid() and self.is_pid_running(pid):  # Need a timeout here?
+                while postmaster.is_running():  # Need a timeout here?
                     cur.execute("SELECT 1")
                     time.sleep(STOP_POLLING_INTERVAL)
         except psycopg2.Error:
             pass
 
-    @staticmethod
-    def _wait_for_user_backends_to_close(postmaster_pid):
-        # These regexps are cross checked against versions PostgreSQL 9.1 .. 9.6
-        aux_proc_re = re.compile("(?:postgres:)( .*:)? (?:""(?:startup|logger|checkpointer|writer|wal writer|"
-                                 "autovacuum launcher|autovacuum worker|stats collector|wal receiver|archiver|"
-                                 "wal sender) process|bgworker: )")
-
-        try:
-            postmaster = psutil.Process(postmaster_pid)
-            user_backends = [p for p in postmaster.children() if not aux_proc_re.match(p.cmdline()[0])]
-            logger.debug("Waiting for user backends {0} to close".format(
-                ",".join(p.cmdline()[0] for p in user_backends)))
-            psutil.wait_procs(user_backends)
-            logger.debug("Backends closed")
-        except psutil.NoSuchProcess:
-            return
-
     def reload(self):
         ret = self.pg_ctl('reload')
         if ret:
@@ -1066,6 +1028,9 @@ class Postgresql(object):
             logger.warning("wait_for_startup() called when not in starting state")
 
         while not self.check_startup_state_changed():
+            with self._cancellable_lock:
+                if self._is_cancelled:
+                    return None
             if timeout and self.time_in_state() > timeout:
                 return None
             time.sleep(1)
@@ -1137,7 +1102,7 @@ class Postgresql(object):
             with open(self._pg_hba_conf, 'w') as f:
                 f.write(self._CONFIG_WARNING_HEADER)
                 for address, t in addresses.items():
-                    f.write('{0}\t{1}\t{2}\t{3}\ttrust\n'.format(t, self._database,
+                    f.write('{0}\t{1}\t{2}\t{3}\ttrust\n'.format(t, 'all',
                                                                  self._superuser.get('username') or 'all', address))
         elif not self._server_parameters.get('hba_file') and self.config.get('pg_hba'):
             with open(self._pg_hba_conf, 'w') as f:
@@ -1186,10 +1151,9 @@ class Postgresql(object):
         dsn = " ".join("{0}={1}".format(k, v) for k, v in dsn_attrs if v is not None)
         logger.info('running pg_rewind from %s', dsn)
         try:
-            return subprocess.call([self._pgcommand('pg_rewind'),
-                                    '-D', self._data_dir,
-                                    '--source-server', dsn,
-                                    ], env=env) == 0
+            return self.cancellable_subprocess_call([self._pgcommand('pg_rewind'),
+                                                     '-D', self._data_dir,
+                                                     '--source-server', dsn], env=env) == 0
         except OSError:
             return False
 
@@ -1238,29 +1202,57 @@ class Postgresql(object):
         except Exception:
             return logger.exception('Exception when working with leader')
 
-    def _get_local_timeline_lsn(self):
+    def _get_local_timeline_lsn_from_replication_connection(self):
         timeline = lsn = None
+        try:
+            with self._get_replication_connection_cursor(**self._local_replication_address) as cur:
+                cur.execute('IDENTIFY_SYSTEM')
+                timeline, lsn = cur.fetchone()[1:3]
+        except Exception:
+            logger.exception('Can not fetch local timeline and lsn from replication connection')
+        return timeline, lsn
+
+    def _get_local_timeline_lsn_from_controldata(self):
+        timeline = lsn = None
+        data = self.controldata()
+        try:
+            if data.get('Database cluster state') == 'shut down in recovery':
+                lsn = data.get('Minimum recovery ending location')
+                timeline = int(data.get("Min recovery ending loc's timeline"))
+                if lsn == '0/0' or timeline == 0:  # it was a master when it crashed
+                    data['Database cluster state'] = 'shut down'
+            if data.get('Database cluster state') == 'shut down':
+                lsn = data.get('Latest checkpoint location')
+                timeline = int(data.get("Latest checkpoint's TimeLineID"))
+        except (TypeError, ValueError):
+            logger.exception('Failed to get local timeline and lsn from pg_controldata output')
+        return timeline, lsn
+
+    def _get_local_timeline_lsn(self):
         if self.is_running():  # if postgres is running - get timeline and lsn from replication connection
-            try:
-                with self._get_replication_connection_cursor(**self._local_replication_address) as cur:
-                    cur.execute('IDENTIFY_SYSTEM')
-                    timeline, lsn = cur.fetchone()[1:3]
-            except Exception:
-                logger.exception('Can not fetch local timeline and lsn from replication connection')
+            timeline, lsn = self._get_local_timeline_lsn_from_replication_connection()
         else:  # otherwise analyze pg_controldata output
-            data = self.controldata()
-            try:
-                if data.get('Database cluster state') == 'shut down':
-                    lsn = data.get('Latest checkpoint location')
-                    timeline = int(data.get("Latest checkpoint's TimeLineID"))
-                elif data.get('Database cluster state') == 'shut down in recovery':
-                    lsn = data.get('Minimum recovery ending location')
-                    timeline = int(data.get("Min recovery ending loc's timeline"))
-            except (TypeError, ValueError):
-                logger.exception('Failed to get local timeline and lsn from pg_controldata output')
+            timeline, lsn = self._get_local_timeline_lsn_from_controldata()
         logger.info('Local timeline=%s lsn=%s', timeline, lsn)
         return timeline, lsn
 
+    @staticmethod
+    def parse_lsn(lsn):
+        t = lsn.split('/')
+        return int(t[0], 16) * 0x100000000 + int(t[1], 16)
+
+    @staticmethod
+    def parse_history(data):
+        for line in data.split('\n'):
+            values = line.strip().split('\t')
+            if len(values) == 3:
+                try:
+                    values[0] = int(values[0])
+                    values[1] = Postgresql.parse_lsn(values[1])
+                    yield values
+                except (IndexError, ValueError):
+                    logger.exception('Exception when parsing timeline history line "%s"', values)
+
     def _check_timeline_and_lsn(self, leader):
         local_timeline, local_lsn = self._get_local_timeline_lsn()
         if local_timeline is None or local_lsn is None:
@@ -1287,28 +1279,44 @@ class Postgresql(object):
             return logger.exception('Exception when working with master via replication connection')
 
         if history is not None:
-            def parse_lsn(lsn):
-                t = lsn.split('/')
-                return int(t[0], 16) * 0x100000000 + int(t[1], 16)
-
-            for line in history.split('\n'):
-                line = line.strip().split('\t')
-                if len(line) == 3:
+            for parent_timeline, switchpoint, _ in self.parse_history(history):
+                if parent_timeline == local_timeline:
                     try:
-                        timeline = int(line[0])
-                        if timeline == local_timeline:
-                            try:
-                                need_rewind = parse_lsn(local_lsn) >= parse_lsn(line[1])
-                            except ValueError:
-                                logger.exception('Exception when parsing lsn')
-                            break
-                        elif timeline > local_timeline:
-                            break
-                    except ValueError:
-                        continue
+                        need_rewind = self.parse_lsn(local_lsn) >= switchpoint
+                    except (IndexError, ValueError):
+                        logger.exception('Exception when parsing lsn')
+                    break
+                elif parent_timeline > local_timeline:
+                    break
 
         self._rewind_state = need_rewind and REWIND_STATUS.NEED or REWIND_STATUS.NOT_NEED
 
+    def get_replica_timeline(self):
+        return self._get_local_timeline_lsn_from_replication_connection()[0]
+
+    def replica_cached_timeline(self, master_timeline):
+        if not self._cached_replica_timeline or not master_timeline or self._cached_replica_timeline != master_timeline:
+            self._cached_replica_timeline = self.get_replica_timeline()
+        return self._cached_replica_timeline
+
+    def get_master_timeline(self):
+        return self._cluster_info_state_get('timeline')
+
+    def get_history(self, timeline):
+        history_path = 'pg_{0}/{1:08X}.history'.format(self.wal_name, timeline)
+        try:
+            cursor = self._cursor()
+            cursor.execute('SELECT isdir, modification FROM pg_stat_file(%s)', (history_path,))
+            isdir, modification = cursor.fetchone()
+            if not isdir:
+                cursor.execute('SELECT pg_read_file(%s)', (history_path,))
+                history = list(self.parse_history(cursor.fetchone()[0]))
+                if history[-1][0] == timeline - 1:
+                    history[-1].append(modification.isoformat())
+                return history
+        except Exception:
+            logger.exception('Failed to read and parse %s', (history_path,))
+
     def rewind(self, leader):
         if self.is_running() and not self.stop(checkpoint=False):
             return logger.warning('Can not run pg_rewind because postgres is still running')
@@ -1396,20 +1404,31 @@ class Postgresql(object):
             for f in self._configuration_to_save:
                 config_file = os.path.join(self._config_dir, f)
                 backup_file = os.path.join(self._data_dir, f + '.backup')
-                if not os.path.isfile(config_file) and os.path.isfile(backup_file):
-                    shutil.copy(backup_file, config_file)
+                if not os.path.isfile(config_file):
+                    if os.path.isfile(backup_file):
+                        shutil.copy(backup_file, config_file)
+                    # Previously we didn't backup pg_ident.conf, if file is missing just create empty
+                    elif f == 'pg_ident.conf':
+                        open(config_file, 'w').close()
         except IOError:
             logger.exception('unable to restore configuration files from backup')
 
-    def promote(self):
+    def _wait_promote(self, wait_seconds):
+        for _ in polling_loop(wait_seconds - 1):
+            data = self.controldata()
+            if data.get('Database cluster state') == 'in production':
+                return True
+
+    def promote(self, wait_seconds):
         if self.role == 'master':
             return True
-        ret = self.pg_ctl('promote')
+        ret = self.pg_ctl('promote', '-W')
         if ret:
             self.set_role('master')
             logger.info("cleared rewind state after becoming the leader")
             self._rewind_state = REWIND_STATUS.INITIAL
             self.call_nowait(ACTION_ON_ROLE_CHANGE)
+            ret = self._wait_promote(wait_seconds)
         return ret
 
     def create_or_update_role(self, name, password, options):
@@ -1429,22 +1448,15 @@ BEGIN
 END;
 $$""".format(name, ' '.join(options)), name, password, password)
 
-    def wal_position(self, retry=True):
-        stmt = """SELECT CASE WHEN pg_is_in_recovery()
-                              THEN GREATEST(pg_{0}_{1}_diff(COALESCE(pg_last_{0}_receive_{1}(), '0/0'),
-                                                                  '0/0')::bigint,
-                                            pg_{0}_{1}_diff(pg_last_{0}_replay_{1}(), '0/0')::bigint)
-                              ELSE pg_{0}_{1}_diff(pg_current_{0}_{1}(), '0/0')::bigint
-                          END""".format(self.wal_name, self.lsn_name)
-
+    def timeline_wal_position(self):
         # This method could be called from different threads (simultaneously with some other `_query` calls).
         # If it is called not from main thread we will create a new cursor to execute statement.
         if current_thread().ident == self.__thread_ident:
-            return (self.query(stmt) if retry else self._query(stmt)).fetchone()[0]
+            return self._cluster_info_state_get('timeline'), self._cluster_info_state_get('wal_position')
 
         with self.connection().cursor() as cursor:
-            cursor.execute(stmt)
-            return cursor.fetchone()[0]
+            cursor.execute(cluster_info_query.format(self.wal_name, self.lsn_name))
+            return cursor.fetchone()[:2]
 
     def load_replication_slots(self):
         if self.use_slots and self._schedule_load_slots:
@@ -1507,7 +1519,7 @@ $$""".format(name, ' '.join(options)), name, password, password)
                 self._schedule_load_slots = True
 
     def last_operation(self):
-        return str(self.wal_position())
+        return str(self._cluster_info_state_get('wal_position'))
 
     def _post_restore(self):
         self.delete_trigger_file()
@@ -1603,11 +1615,6 @@ $$""".format(name, ' '.join(options)), name, password, password)
             self.move_data_directory()
 
     def basebackup(self, conn_url, env):
-        # save environ to restore it later
-        old_env = os.environ.copy()
-        os.environ.clear()
-        os.environ.update(env)
-
         # creates a replica data dir using pg_basebackup.
         # this is the default, built-in create_replica_method
         # tries twice, then returns failure (as 1)
@@ -1615,12 +1622,15 @@ $$""".format(name, ' '.join(options)), name, password, password)
         maxfailures = 2
         ret = 1
         for bbfailures in range(0, maxfailures):
+            with self._cancellable_lock:
+                if self._is_cancelled:
+                    break
             if not self.data_directory_empty():
                 self.remove_data_directory()
 
             try:
-                ret = subprocess.call([self._pgcommand('pg_basebackup'), '--pgdata=' + self._data_dir,
-                                       '-X', 'stream', '--dbname=' + conn_url])
+                ret = self.cancellable_subprocess_call([self._pgcommand('pg_basebackup'), '--pgdata=' + self._data_dir,
+                                                       '-X', 'stream', '--dbname=' + conn_url], env=env)
                 if ret == 0:
                     break
                 else:
@@ -1633,10 +1643,6 @@ $$""".format(name, ' '.join(options)), name, password, password)
                 logger.warning('Trying again in 5 seconds')
                 time.sleep(5)
 
-        # restore environ
-        os.environ.clear()
-        os.environ.update(old_env)
-
         return ret
 
     def pick_synchronous_standby(self, cluster):
@@ -1648,12 +1654,13 @@ $$""".format(name, ' '.join(options)), name, password, password)
         :returns tuple of candidate name or None, and bool showing if the member is the active synchronous standby.
         """
         current = cluster.sync.sync_standby
-        members = {m.name: m for m in cluster.members}
+        current = current.lower() if current else current
+        members = {m.name.lower(): m for m in cluster.members}
         candidates = []
         # Pick candidates based on who has flushed WAL farthest.
         # TODO: for synchronous_commit = remote_write we actually want to order on write_location
         for app_name, state, sync_state in self.query(
-                """SELECT application_name, state, sync_state
+                """SELECT LOWER(application_name), state, sync_state
                      FROM pg_stat_replication
                     ORDER BY flush_{0} DESC""".format(self.lsn_name)):
             member = members.get(app_name)
@@ -1673,18 +1680,21 @@ $$""".format(name, ' '.join(options)), name, password, password)
 
     def set_synchronous_standby(self, name):
         """Sets a node to be synchronous standby and if changed does a reload for PostgreSQL."""
+        if name and name != '*':
+            name = quote_ident(name)
         if name != self._synchronous_standby_names:
             if name is None:
                 self._server_parameters.pop('synchronous_standby_names', None)
             else:
                 self._server_parameters['synchronous_standby_names'] = name
             self._synchronous_standby_names = name
-            self._write_postgresql_conf()
-            self.reload()
+            if self.state == 'running':
+                self._write_postgresql_conf()
+                self.reload()
 
     @staticmethod
     def postgres_version_to_int(pg_version):
-        """ Convert the server_version to integer
+        """Convert the server_version to integer
 
         >>> Postgresql.postgres_version_to_int('9.5.3')
         90503
@@ -1692,29 +1702,34 @@ $$""".format(name, ' '.join(options)), name, password, password)
         90313
         >>> Postgresql.postgres_version_to_int('10.1')
         100001
-        >>> Postgresql.postgres_version_to_int('10')
+        >>> Postgresql.postgres_version_to_int('10')  # doctest: +IGNORE_EXCEPTION_DETAIL
         Traceback (most recent call last):
             ...
-        Exception: Invalid PostgreSQL format: X.Y or X.Y.Z is accepted: 10
-        >>> Postgresql.postgres_version_to_int('a.b.c')
+        PostgresException: 'Invalid PostgreSQL version format: X.Y or X.Y.Z is accepted: 10'
+        >>> Postgresql.postgres_version_to_int('9.6')  # doctest: +IGNORE_EXCEPTION_DETAIL
         Traceback (most recent call last):
             ...
-        Exception: Invalid PostgreSQL version: a.b.c
+        PostgresException: 'Invalid PostgreSQL version format: X.Y or X.Y.Z is accepted: 9.6'
+        >>> Postgresql.postgres_version_to_int('a.b.c')  # doctest: +IGNORE_EXCEPTION_DETAIL
+        Traceback (most recent call last):
+            ...
+        PostgresException: 'Invalid PostgreSQL version: a.b.c'
         """
-        components = pg_version.split('.')
 
-        result = []
-        if len(components) < 2 or len(components) > 3:
-            raise Exception("Invalid PostgreSQL format: X.Y or X.Y.Z is accepted: {0}".format(pg_version))
+        try:
+            components = list(map(int, pg_version.split('.')))
+        except ValueError:
+            raise PostgresException('Invalid PostgreSQL version: {0}'.format(pg_version))
+
+        if len(components) < 2 or len(components) == 2 and components[0] < 10 or len(components) > 3:
+            raise PostgresException('Invalid PostgreSQL version format: X.Y or X.Y.Z is accepted: {0}'
+                                    .format(pg_version))
+
         if len(components) == 2:
             # new style verion numbers, i.e. 10.1 becomes 100001
-            components.insert(1, '0')
-        try:
-            result = [c if int(c) > 10 else '0{0}'.format(c) for c in components]
-            result = int(''.join(result))
-        except ValueError:
-            raise Exception("Invalid PostgreSQL version: {0}".format(pg_version))
-        return result
+            components.insert(1, 0)
+
+        return int(''.join('{0:02d}'.format(c) for c in components))
 
     @staticmethod
     def postgres_major_version_to_int(pg_version):
@@ -1725,3 +1740,96 @@ $$""".format(name, ' '.join(options)), name, password, password)
         90600
         """
         return Postgresql.postgres_version_to_int(pg_version + '.0')
+
+    def read_postmaster_opts(self):
+        """returns the list of option names/values from postgres.opts, Empty dict if read failed or no file"""
+        result = {}
+        try:
+            with open(os.path.join(self._data_dir, 'postmaster.opts')) as f:
+                data = f.read()
+                for opt in data.split('" "'):
+                    if '=' in opt and opt.startswith('--'):
+                        name, val = opt.split('=', 1)
+                        result[name.strip('-')] = val.rstrip('"\n')
+        except IOError:
+            logger.exception('Error when reading postmaster.opts')
+        return result
+
+    def single_user_mode(self, command=None, options=None):
+        """run a given command in a single-user mode. If the command is empty - then just start and stop"""
+        cmd = [self._pgcommand('postgres'), '--single', '-D', self._data_dir]
+        for opt, val in sorted((options or {}).items()):
+            cmd.extend(['-c', '{0}={1}'.format(opt, val)])
+        # need a database name to connect
+        cmd.append(self._database)
+        return self.cancellable_subprocess_call(cmd, communicate_input=command)
+
+    def cleanup_archive_status(self):
+        status_dir = os.path.join(self._data_dir, 'pg_' + self.wal_name, 'archive_status')
+        try:
+            for f in os.listdir(status_dir):
+                path = os.path.join(status_dir, f)
+                try:
+                    if os.path.islink(path):
+                        os.unlink(path)
+                    elif os.path.isfile(path):
+                        os.remove(path)
+                except OSError:
+                    logger.exception('Unable to remove %s', path)
+        except OSError:
+            logger.exception('Unable to list %s', status_dir)
+
+    def fix_cluster_state(self):
+        self.cleanup_archive_status()
+
+        # Start in a single user mode and stop to produce a clean shutdown
+        opts = self.read_postmaster_opts()
+        opts.update({'archive_mode': 'on', 'archive_command': 'false'})
+        if os.path.isfile(self._recovery_conf) or os.path.islink(self._recovery_conf):
+            os.unlink(self._recovery_conf)
+        return self.single_user_mode(options=opts) == 0 or None
+
+    def cancellable_subprocess_call(self, *args, **kwargs):
+        communicate_input = kwargs.pop('communicate_input', None)
+        for s in ('stdin', 'stdout', 'stderr'):
+            kwargs.pop(s, None)
+
+        try:
+            with self._cancellable_lock:
+                if self._is_cancelled:
+                    raise PostgresException('cancelled')
+
+                self._is_cancelled = False
+                self._cancellable = subprocess.Popen(*args, **kwargs)
+
+            if communicate_input:
+                kwargs['stdin'] = subprocess.PIPE
+                if communicate_input[-1] != '\n':
+                    communicate_input += '\n'
+                self._cancellable.communicate(communicate_input + '\n')
+                self._cancellable.stdin.close()
+
+            return self._cancellable.wait()
+        finally:
+            with self._cancellable_lock:
+                self._cancellable = None
+
+    def reset_is_cancelled(self):
+        with self._cancellable_lock:
+            self._is_cancelled = False
+
+    def cancel(self):
+        with self._cancellable_lock:
+            self._is_cancelled = True
+            if self._cancellable is None or self._cancellable.returncode is not None:
+                return
+            self._cancellable.terminate()
+
+        for _ in polling_loop(10):
+            with self._cancellable_lock:
+                if self._cancellable is None or self._cancellable.returncode is not None:
+                    return
+
+        with self._cancellable_lock:
+            if self._cancellable is not None and self._cancellable.returncode is None:
+                self._cancellable.kill()

patroni/postmaster.py

@@ -0,0 +1,115 @@
+import logging
+import os
+import psutil
+import re
+import signal
+import subprocess
+
+from patroni import call_self
+
+logger = logging.getLogger(__name__)
+
+STOP_SIGNALS = {
+    'smart': signal.SIGTERM,
+    'fast': signal.SIGINT,
+    'immediate': signal.SIGQUIT,
+}
+
+
+class PostmasterProcess(psutil.Process):
+    def __init__(self, pid):
+        self.is_single_user = False
+        if pid < 0:
+            pid = -pid
+            self.is_single_user = True
+        super(PostmasterProcess, self).__init__(pid)
+
+    @classmethod
+    def from_pidfile(cls, pidfile):
+        try:
+            pid = int(pidfile.get('pid', 0))
+            if not pid:
+                return None
+        except ValueError:
+            return None
+
+        try:
+            proc = cls(pid)
+        except psutil.NoSuchProcess:
+            return None
+
+        try:
+            start_time = int(pidfile.get('start_time', 0))
+            if start_time and abs(proc.create_time() - start_time) > 3:
+                return None
+        except ValueError:
+            logger.warning("Garbage start time value in pid file: %r", pidfile.get('start_time'))
+
+        # Extra safety check. The process can't be ourselves, our parent or our direct child.
+        if proc.pid == os.getpid() or proc.pid == os.getppid() or proc.parent() == os.getpid():
+            return None
+
+        return proc
+
+    @classmethod
+    def from_pid(cls, pid):
+        try:
+            return cls(pid)
+        except psutil.NoSuchProcess:
+            return None
+
+    def signal_stop(self, mode):
+        """Signal postmaster process to stop
+
+        :returns None if signaled, True if process is already gone, False if error
+        """
+        if self.is_single_user:
+            logger.warning("Cannot stop server; single-user server is running (PID: {0})".format(self.pid))
+            return False
+        try:
+            self.send_signal(STOP_SIGNALS[mode])
+        except psutil.NoSuchProcess:
+            return True
+        except psutil.AccessDenied as e:
+            logger.warning("Could not send stop signal to PostgreSQL (error: {0})".format(e))
+            return False
+
+        return None
+
+    def wait_for_user_backends_to_close(self):
+        # These regexps are cross checked against versions PostgreSQL 9.1 .. 9.6
+        aux_proc_re = re.compile("(?:postgres:)( .*:)? (?:""(?:startup|logger|checkpointer|writer|wal writer|"
+                                 "autovacuum launcher|autovacuum worker|stats collector|wal receiver|archiver|"
+                                 "wal sender) process|bgworker: )")
+
+        user_backends = [p for p in self.children() if not aux_proc_re.match(p.cmdline()[0])]
+        logger.debug("Waiting for user backends {0} to close".format(
+            ",".join(p.cmdline()[0] for p in user_backends)))
+        psutil.wait_procs(user_backends)
+        logger.debug("Backends closed")
+
+    @classmethod
+    def start(cls, pgcommand, data_dir, conf, options):
+        # Unfortunately `pg_ctl start` does not return postmaster pid to us. Without this information
+        # it is hard to know the current state of postgres startup, so we had to reimplement pg_ctl start
+        # in python. It will start postgres, wait for port to be open and wait until postgres will start
+        # accepting connections.
+        # Important!!! We can't just start postgres using subprocess.Popen, because in this case it
+        # will be our child for the rest of our live and we will have to take care of it (`waitpid`).
+        # So we will use the same approach as pg_ctl uses: start a new process, which will start postgres.
+        # This process will write postmaster pid to stdout and exit immediately. Now it's responsibility
+        # of init process to take care about postmaster.
+        # In order to make everything portable we can't use fork&exec approach here, so  we will call
+        # ourselves and pass list of arguments which must be used to start postgres.
+        proc = call_self(['pg_ctl_start', pgcommand, '-D', data_dir,
+                          '--config-file={}'.format(conf)] + options, close_fds=True,
+                         preexec_fn=os.setsid, stdout=subprocess.PIPE,
+                         env={p: os.environ[p] for p in ('PATH', 'LC_ALL', 'LANG') if p in os.environ})
+        pid = int(proc.stdout.readline().strip())
+        proc.wait()
+        logger.info('postmaster pid=%s', pid)
+
+        # TODO: In an extremely unlikely case, the process could have exited and the pid reassigned. The start
+        # initiation time is not accurate enough to compare to create time as start time would also likely
+        # be relatively close. We need the subprocess extract pid+start_time in a race free manner.
+        return PostmasterProcess.from_pid(pid)

patroni/utils.py

@@ -1,7 +1,5 @@
-import contextlib
 import random
 import time
-import re
 
 from dateutil import tz
 from patroni.exceptions import PatroniException
@@ -96,17 +94,17 @@ def strtol(value, strict=True):
     True
     """
     value = str(value).strip()
-    l = len(value)
+    ln = len(value)
     i = 0
     # skip sign:
-    if i < l and value[i] in ('-', '+'):
+    if i < ln and value[i] in ('-', '+'):
         i += 1
 
     # we always expect to get digit in the beginning
-    if i < l and value[i].isdigit():
+    if i < ln and value[i].isdigit():
         if value[i] == '0':
             i += 1
-            if i < l and value[i] in ('x', 'X'):  # '0' followed by 'x': HEX
+            if i < ln and value[i] in ('x', 'X'):  # '0' followed by 'x': HEX
                 base = 16
                 i += 1
             else:  # just starts with '0': OCT
@@ -115,7 +113,7 @@ def strtol(value, strict=True):
             base = 10
 
         ret = None
-        while i <= l:
+        while i <= ln:
             try:  # try to find maximally long number
                 i += 1  # by giving to `int` longer and longer strings
                 ret = int(value[:i], base)
@@ -195,10 +193,6 @@ def _sleep(interval):
     time.sleep(interval)
 
 
-def is_valid_pg_version(version):
-    return re.match(r'[1-9][0-9]?(\.(0|([1-9][0-9]?))){2}$', version)
-
-
 class RetryFailedError(PatroniException):
 
     """Raised when retrying an operation ultimately failed, after retrying the maximum number of attempts."""
@@ -283,6 +277,7 @@ def polling_loop(timeout, interval=1):
         time.sleep(interval)
 
 
-@contextlib.contextmanager
-def null_context():
-    yield
+def split_host_port(value, default_port):
+    t = value.rsplit(':', 1)
+    t.append(default_port)
+    return t[0], int(t[1])

patroni/version.py

@@ -1 +1 @@
-__version__ = '1.3.2'
+__version__ = '1.4.1'

patroni/watchdog/base.py

@@ -133,6 +133,7 @@ class Watchdog(object):
 
         try:
             self.impl.open()
+            actual_timeout = self._set_timeout()
         except WatchdogError as e:
             logger.warning("Could not activate %s: %s", self.impl.describe(), e)
             self.impl = NullWatchdog()
@@ -141,8 +142,6 @@ class Watchdog(object):
             logger.warning("Watchdog implementation can't be disabled."
                            " Watchdog will trigger after Patroni loses leader key.")
 
-        actual_timeout = self._set_timeout()
-
         if not self.impl.is_running or actual_timeout > self.config.timeout:
             if self.config.mode == MODE_REQUIRED:
                 if self.impl.is_null:
@@ -202,7 +201,8 @@ class Watchdog(object):
     @synchronized
     def keepalive(self):
         try:
-            self.impl.keepalive()
+            if self.active:
+                self.impl.keepalive()
             # In case there are any pending configuration changes apply them now.
             if self.active and self.config != self.active_config:
                 if self.config.mode != MODE_OFF and self.active_config.mode == MODE_OFF:

patroni/watchdog/linux.py

@@ -155,21 +155,25 @@ class LinuxWatchdogDevice(WatchdogBase):
     def can_be_disabled(self):
         return self.get_support().has_MAGICCLOSE
 
-    def _ioctl(self, func, arg, mutate_arg=False):
+    def _ioctl(self, func, arg):
+        """Runs the specified ioctl on the underlying fd.
+
+        Raises WatchdogError if the device is closed.
+        Raises OSError or IOError (Python 2) when the ioctl fails."""
         if self._fd is None:
             raise WatchdogError("Watchdog device is closed")
-
-        result = fcntl.ioctl(self._fd, func, arg, mutate_arg)
-        if result < 0:
-            raise IOError(result)
+        fcntl.ioctl(self._fd, func, arg, True)
 
     def get_support(self):
         if self._support_cache is None:
             info = watchdog_info()
-            self._ioctl(WDIOC_GETSUPPORT, info, True)
+            try:
+                self._ioctl(WDIOC_GETSUPPORT, info)
+            except (WatchdogError, OSError, IOError) as e:
+                raise WatchdogError("Could not get information about watchdog device: {}".format(e))
             self._support_cache = WatchdogInfo(info.options,
                                                info.firmware_version,
-                                               str(bytearray(info.identity)).rstrip('\x00'))
+                                               bytearray(info.identity).decode(errors='ignore').rstrip('\x00'))
         return self._support_cache
 
     def describe(self):
@@ -180,7 +184,7 @@ class LinuxWatchdogDevice(WatchdogBase):
             try:
                 _, version, identity = self.get_support()
                 ver_str = " (firmware {0})".format(version) if version else ""
-            except WatchdogError:  # XXX: Can it really be raise when self._fd is not None?
+            except WatchdogError:
                 pass
 
         return identity + ver_str + dev_str
@@ -199,11 +203,17 @@ class LinuxWatchdogDevice(WatchdogBase):
         timeout = int(timeout)
         if not 0 < timeout < 0xFFFF:
             raise WatchdogError("Invalid timeout {0}. Supported values are between 1 and 65535".format(timeout))
-        self._ioctl(WDIOC_SETTIMEOUT, ctypes.c_int(timeout))
+        try:
+            self._ioctl(WDIOC_SETTIMEOUT, ctypes.c_int(timeout))
+        except (WatchdogError, OSError, IOError) as e:
+            raise WatchdogError("Could not set timeout on watchdog device: {}".format(e))
 
     def get_timeout(self):
         timeout = ctypes.c_int()
-        self._ioctl(WDIOC_GETTIMEOUT, timeout, True)
+        try:
+            self._ioctl(WDIOC_GETTIMEOUT, timeout)
+        except (WatchdogError, OSError, IOError) as e:
+            raise WatchdogError("Could not get timeout on watchdog device: {}".format(e))
         return timeout.value
 
 

requirements.txt

@@ -1,15 +1,16 @@
-urllib3>=1.9
+urllib3>=1.19.1,!=1.21
 boto
-psycopg2>=2.6.1
+psycopg2>=2.5.4
 PyYAML
 requests
 six >= 1.7
-kazoo==2.2.1
+kazoo>=1.3.1
 python-etcd>=0.4.3,<0.5
-python-consul==0.7.0
+python-consul>=0.7.0
 click>=4.1
 prettytable>=0.7
 tzlocal
 python-dateutil
 psutil
 cdiff
+kubernetes==3.0.0

setup.py

@@ -87,7 +87,7 @@ class PyTest(TestCommand):
     def run_tests(self):
         try:
             import pytest
-        except:
+        except Exception:
             raise RuntimeError('py.test is not installed, run: pip install pytest')
         params = {'args': self.test_args}
         if self.cov:
@@ -115,13 +115,23 @@ def read(fname):
 
 def setup_package():
     # Assemble additional setup commands
-    cmdclass = {}
-    cmdclass['test'] = PyTest
+    cmdclass = {'test': PyTest}
 
     # Some helper variables
     version = os.getenv('GO_PIPELINE_LABEL', VERSION)
 
-    install_reqs = get_install_requirements('requirements.txt')
+    install_requires = []
+    extras_require = {'aws': ['boto'], 'etcd': ['python-etcd'], 'consul': ['python-consul'],
+                      'exhibitor': ['kazoo'], 'zookeeper': ['kazoo'], 'kubernetes': ['kubernetes']}
+
+    for r in get_install_requirements('requirements.txt'):
+        extra = False
+        for e, v in extras_require.items():
+            if r.startswith(v[0]):
+                extras_require[e] = [r]
+                extra = True
+        if not extra:
+            install_requires.append(r)
 
     command_options = {'test': {'test_suite': ('setup.py', 'tests')}}
     if JUNIT_XML:
@@ -145,10 +155,10 @@ def setup_package():
         test_suite='tests',
         packages=find_packages(exclude=['tests', 'tests.*']),
         package_data={MAIN_PACKAGE: ["*.json"]},
-        install_requires=install_reqs,
-        setup_requires=['flake8'],
+        install_requires=install_requires,
+        extras_require=extras_require,
         cmdclass=cmdclass,
-        tests_require=['mock>=2.0.0', 'pytest-cov', 'pytest'],
+        tests_require=['flake8', 'mock>=2.0.0', 'pytest-cov', 'pytest'],
         command_options=command_options,
         entry_points={'console_scripts': CONSOLE_SCRIPTS},
     )

tests/test_api.py

@@ -3,7 +3,7 @@ import json
 import psycopg2
 import unittest
 
-from mock import Mock, patch
+from mock import Mock, PropertyMock, patch
 from patroni.api import RestApiHandler, RestApiServer
 from patroni.dcs import ClusterConfig, Member
 from patroni.ha import _MemberStatus
@@ -37,6 +37,10 @@ class MockPostgresql(object):
     def postmaster_start_time():
         return str(postmaster_start_time)
 
+    @staticmethod
+    def replica_cached_timeline(_):
+        return 2
+
 
 class MockWatchdog(object):
     is_healthy = False
@@ -48,7 +52,7 @@ class MockHa(object):
     watchdog = MockWatchdog()
 
     @staticmethod
-    def reinitialize():
+    def reinitialize(_):
         return 'reinitialize'
 
     @staticmethod
@@ -83,6 +87,10 @@ class MockHa(object):
     def wakeup():
         pass
 
+    @staticmethod
+    def is_paused():
+        return True
+
 
 class MockPatroni(object):
 
@@ -92,7 +100,7 @@ class MockPatroni(object):
     dcs = Mock()
     tags = {}
     version = '0.00'
-    noloadbalance = Mock(return_value=False)
+    noloadbalance = PropertyMock(return_value=False)
     scheduled_restart = {'schedule': future_restart_time,
                          'postmaster_start_time': postgresql.postmaster_start_time()}
 
@@ -140,6 +148,12 @@ class TestRestApiHandler(unittest.TestCase):
         with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'master'})):
             MockRestApiServer(RestApiHandler, 'GET /replica')
         MockRestApiServer(RestApiHandler, 'GET /master')
+        MockPatroni.dcs.cluster.sync.sync_standby = MockPostgresql.name
+        MockPatroni.dcs.cluster.is_synchronous_mode = Mock(return_value=True)
+        with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'replica'})):
+            MockRestApiServer(RestApiHandler, 'GET /synchronous')
+        with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'replica'})):
+            MockRestApiServer(RestApiHandler, 'GET /asynchronous')
         MockPatroni.dcs.cluster.leader.name = MockPostgresql.name
         MockRestApiServer(RestApiHandler, 'GET /replica')
         MockPatroni.dcs.cluster = None
@@ -148,10 +162,13 @@ class TestRestApiHandler(unittest.TestCase):
         with patch.object(MockHa, 'restart_scheduled', Mock(return_value=True)):
             MockRestApiServer(RestApiHandler, 'GET /master')
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /master'))
+        with patch.object(RestApiServer, 'query', Mock(return_value=[('', 1, '', '', '', '', False, '')])):
+            self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
 
     def test_do_OPTIONS(self):
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'OPTIONS / HTTP/1.0'))
 
+    @patch.object(MockPostgresql, 'state', PropertyMock(return_value='stopped'))
     def test_do_GET_patroni(self):
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
 
@@ -263,7 +280,7 @@ class TestRestApiHandler(unittest.TestCase):
     def test_do_POST_reinitialize(self, mock_dcs):
         cluster = mock_dcs.get_cluster.return_value
         cluster.is_paused.return_value = False
-        request = 'POST /reinitialize HTTP/1.0' + self._authorization
+        request = 'POST /reinitialize HTTP/1.0' + self._authorization + '\nContent-Length: 15\n\n{"force": true}'
         MockRestApiServer(RestApiHandler, request)
         with patch.object(MockHa, 'reinitialize', Mock(return_value=None)):
             MockRestApiServer(RestApiHandler, request)
@@ -277,11 +294,13 @@ class TestRestApiHandler(unittest.TestCase):
 
     @patch('time.sleep', Mock())
     @patch.object(MockPatroni, 'dcs')
-    def test_do_POST_failover(self, dcs):
+    def test_do_POST_switchover(self, dcs):
         dcs.loop_wait = 10
         cluster = dcs.get_cluster.return_value
+        cluster.is_synchronous_mode.return_value = False
+        cluster.is_paused.return_value = False
 
-        post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
+        post = 'POST /switchover HTTP/1.0' + self._authorization + '\nContent-Length: '
 
         MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}')
 
@@ -291,14 +310,22 @@ class TestRestApiHandler(unittest.TestCase):
         cluster.leader.name = 'postgresql1'
         MockRestApiServer(RestApiHandler, request)
 
-        MockRestApiServer(RestApiHandler, post + '25\n\n{"leader": "postgresql1"}')
+        request = post + '25\n\n{"leader": "postgresql1"}'
+
+        cluster.is_paused.return_value = True
+        MockRestApiServer(RestApiHandler, request)
+
+        cluster.is_paused.return_value = False
+        for cluster.is_synchronous_mode.return_value in (True, False):
+            MockRestApiServer(RestApiHandler, request)
 
         cluster.leader.name = 'postgresql2'
         request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
         MockRestApiServer(RestApiHandler, request)
 
         cluster.leader.name = 'postgresql1'
-        MockRestApiServer(RestApiHandler, request)
+        for cluster.is_synchronous_mode.return_value in (True, False):
+            MockRestApiServer(RestApiHandler, request)
 
         cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
                            Member(0, 'postgresql2', 30, {'api_url': 'http'})]
@@ -346,3 +373,9 @@ class TestRestApiHandler(unittest.TestCase):
 
         # Invalid date
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}'))
+
+    @patch.object(MockPatroni, 'dcs', Mock())
+    def test_do_POST_failover(self):
+        post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
+        MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}')
+        MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}')

tests/test_async_executor.py

@@ -8,7 +8,7 @@ from threading import Thread
 class TestAsyncExecutor(unittest.TestCase):
 
     def setUp(self):
-        self.a = AsyncExecutor(Mock())
+        self.a = AsyncExecutor(Mock(), Mock())
 
     @patch.object(Thread, 'start', Mock())
     def test_run_async(self):
@@ -17,6 +17,12 @@ class TestAsyncExecutor(unittest.TestCase):
     def test_run(self):
         self.a.run(Mock(side_effect=Exception()))
 
+    def test_cancel(self):
+        self.a.cancel()
+        self.a.schedule('foo')
+        self.a.cancel()
+        self.a.run(Mock())
+
 
 class TestCriticalTask(unittest.TestCase):
 

tests/test_config.py

@@ -49,6 +49,9 @@ class TestConfig(unittest.TestCase):
             'PATRONI_ETCD_CERT': '/cert',
             'PATRONI_ETCD_KEY': '/key',
             'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
+            'PATRONI_KUBERNETES_LABELS': 'a:b:c',
+            'PATRONI_KUBERNETES_SCOPE_LABEL': 'a',
+            'PATRONI_KUBERNETES_PORTS': '[{"name": "postgresql"}]',
             'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
             'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',
             'PATRONI_EXHIBITOR_PORT': '8181',

tests/test_consul.py

@@ -3,7 +3,8 @@ import unittest
 
 from consul import ConsulException, NotFound
 from mock import Mock, patch
-from patroni.dcs.consul import AbstractDCS, Cluster, Consul, ConsulInternalError, ConsulError, HTTPClient
+from patroni.dcs.consul import AbstractDCS, Cluster, Consul, ConsulInternalError, \
+                                ConsulError, HTTPClient, InvalidSessionTTL
 from test_etcd import SleepException
 
 
@@ -45,9 +46,12 @@ class TestHTTPClient(unittest.TestCase):
 
     def test_get(self):
         self.client.get(Mock(), '')
-        self.client.get(Mock(), '', {'wait': '1s', 'index': 1})
+        self.client.get(Mock(), '', {'wait': '1s', 'index': 1, 'token': 'foo'})
         self.client.http.request.return_value.status = 500
+        self.client.http.request.return_value.data = b'Foo'
         self.assertRaises(ConsulInternalError, self.client.get, Mock(), '')
+        self.client.http.request.return_value.data = b"Invalid Session TTL '3000000000', must be between [10s=24h0m0s]"
+        self.assertRaises(InvalidSessionTTL, self.client.get, Mock(), '')
 
     def test_unknown_method(self):
         try:
@@ -69,6 +73,10 @@ class TestConsul(unittest.TestCase):
     @patch.object(consul.Consul.KV, 'get', kv_get)
     @patch.object(consul.Consul.KV, 'delete', Mock())
     def setUp(self):
+        Consul({'ttl': 30, 'scope': 't', 'name': 'p', 'url': 'https://l:1', 'retry_timeout': 10,
+                'verify': 'on', 'key': 'foo', 'cert': 'bar', 'cacert': 'buz', 'token': 'asd', 'dc': 'dc1'})
+        Consul({'ttl': 30, 'scope': 't', 'name': 'p', 'url': 'https://l:1', 'retry_timeout': 10,
+                'verify': 'on', 'cert': 'bar', 'cacert': 'buz'})
         self.c = Consul({'ttl': 30, 'scope': 'test', 'name': 'postgresql1', 'host': 'localhost:1', 'retry_timeout': 10})
         self.c._base_path = '/service/good'
         self.c._load_cluster()
@@ -80,7 +88,9 @@ class TestConsul(unittest.TestCase):
         self.assertRaises(SleepException, self.c.create_session)
 
     @patch.object(consul.Consul.Session, 'renew', Mock(side_effect=NotFound))
-    @patch.object(consul.Consul.Session, 'create', Mock(side_effect=ConsulException))
+    @patch.object(consul.Consul.Session, 'create', Mock(side_effect=[InvalidSessionTTL, ConsulException]))
+    @patch.object(consul.Consul.Agent, 'self', Mock(return_value={'Config': {'SessionTTLMin': 0}}))
+    @patch.object(HTTPClient, 'set_ttl', Mock(side_effect=ValueError))
     def test_referesh_session(self):
         self.c._session = '1'
         self.assertFalse(self.c.refresh_session())
@@ -102,11 +112,11 @@ class TestConsul(unittest.TestCase):
     @patch.object(consul.Consul.KV, 'put', Mock(side_effect=[True, ConsulException]))
     def test_touch_member(self):
         self.c.refresh_session = Mock(return_value=True)
-        self.c.touch_member('balbla')
-        self.c.touch_member('balbla')
-        self.c.touch_member('balbla')
+        self.c.touch_member({'balbla': 'blabla'})
+        self.c.touch_member({'balbla': 'blabla'})
+        self.c.touch_member({'balbla': 'blabla'})
         self.c.refresh_session = Mock(return_value=False)
-        self.c.touch_member('balbla')
+        self.c.touch_member({'balbla': 'blabla'})
 
     @patch.object(consul.Consul.KV, 'put', Mock(return_value=False))
     def test_take_leader(self):
@@ -128,7 +138,7 @@ class TestConsul(unittest.TestCase):
 
     @patch.object(consul.Consul.Session, 'renew', Mock())
     def test_update_leader(self):
-        self.c.update_leader()
+        self.c.update_leader(None)
 
     @patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
     def test_delete_leader(self):
@@ -162,3 +172,7 @@ class TestConsul(unittest.TestCase):
     def test_sync_state(self):
         self.assertTrue(self.c.set_sync_state_value('{}'))
         self.assertTrue(self.c.delete_sync_state())
+
+    @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
+    def test_set_history_value(self):
+        self.assertTrue(self.c.set_history_value('{}'))

tests/test_ctl.py

@@ -5,15 +5,17 @@ import sys
 import unittest
 
 from click.testing import CliRunner
+from datetime import datetime, timedelta
 from mock import patch, Mock
-from patroni.ctl import ctl, members, store_config, load_config, output_members, request_patroni, get_dcs, parse_dcs, \
+from patroni.ctl import ctl, store_config, load_config, output_members, request_patroni, get_dcs, parse_dcs, \
     get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException, apply_config_changes, \
-    format_config_for_editing, show_diff, invoke_editor
-from patroni.dcs.etcd import Client
+    format_config_for_editing, show_diff, invoke_editor, format_pg_version
+from patroni.dcs.etcd import Client, Failover
+from patroni.utils import tzutc
 from psycopg2 import OperationalError
 from test_etcd import etcd_read, requests_get, socket_getaddrinfo, MockResponse
 from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \
-    get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader
+    get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader, get_cluster, Member
 from test_postgresql import MockConnect, psycopg2_connect
 
 CONFIG_FILE_PATH = './test-ctl.yaml'
@@ -31,7 +33,7 @@ def test_rw_config():
 
 
 @patch('patroni.ctl.load_config',
-       Mock(return_value={'postgresql': {'data_dir': '.', 'parameters': {}, 'retry_timeout': 5},
+       Mock(return_value={'scope': 'alpha', 'postgresql': {'data_dir': '.', 'parameters': {}, 'retry_timeout': 5},
                           'restapi': {'auth': 'u:p', 'listen': ''}, 'etcd': {'host': 'localhost:2379'}}))
 class TestCtl(unittest.TestCase):
 
@@ -64,76 +66,92 @@ class TestCtl(unittest.TestCase):
         self.assertRaises(PatroniCtlException, parse_dcs, 'invalid://test')
 
     def test_output_members(self):
-        cluster = get_cluster_initialized_with_leader()
+        scheduled_at = datetime.now(tzutc) + timedelta(seconds=600)
+        cluster = get_cluster_initialized_with_leader(Failover(1, 'foo', 'bar', scheduled_at))
         self.assertIsNone(output_members(cluster, name='abc', fmt='pretty'))
         self.assertIsNone(output_members(cluster, name='abc', fmt='json'))
+        self.assertIsNone(output_members(cluster, name='abc', fmt='yaml'))
         self.assertIsNone(output_members(cluster, name='abc', fmt='tsv'))
 
+    @patch('patroni.ctl.get_dcs')
+    @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
+    def test_switchover(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        mock_get_dcs.return_value.set_failover_value = Mock()
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+        assert 'leader' in result.output
+
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
+        assert result.exit_code == 0
+
+        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+            result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00'])
+            assert result.exit_code == 1
+
+        # Aborting switchover, as we anser NO to the confirmation
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\nN')
+        assert result.exit_code == 1
+
+        # Target and source are equal
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nleader\n\ny')
+        assert result.exit_code == 1
+
+        # Reality is not part of this cluster
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nReality\n\ny')
+        assert result.exit_code == 1
+
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force'])
+        assert 'Member' in result.output
+
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
+        assert result.exit_code == 0
+
+        # Invalid timestamp
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', 'invalid'])
+        assert result.exit_code != 0
+
+        # Invalid timestamp
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', '2115-02-30T12:00:00+01:00'])
+        assert result.exit_code != 0
+
+        # Specifying wrong leader
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='dummy')
+        assert result.exit_code == 1
+
+        with patch('patroni.ctl.request_patroni', Mock(side_effect=Exception)):
+            # Non-responding patroni
+            result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
+            assert 'falling back to DCS' in result.output
+
+        with patch('patroni.ctl.request_patroni') as mocked:
+            mocked.return_value.status_code = 500
+            result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+            assert 'Switchover failed' in result.output
+
+            mocked.return_value.status_code = 501
+            mocked.return_value.text = 'Server does not support this operation'
+            result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+            assert 'Switchover failed' in result.output
+
+        # No members available
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+        assert result.exit_code == 1
+
+        # No master available
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+        assert result.exit_code == 1
+
     @patch('patroni.ctl.get_dcs')
     @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
     def test_failover(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
         mock_get_dcs.return_value.set_failover_value = Mock()
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
-        assert 'leader' in result.output
-
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
-        assert result.exit_code == 0
-
-        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00'])
-            assert result.exit_code == 1
-
-        # Aborting failover,as we anser NO to the confirmation
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\nN')
-        assert result.exit_code == 1
-
-        # Target and source are equal
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nleader\n\ny')
-        assert result.exit_code == 1
-
-        # Reality is not part of this cluster
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nReality\n\ny')
-        assert result.exit_code == 1
-
-        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force'])
-        assert 'Member' in result.output
-
-        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
-        assert result.exit_code == 0
-
-        # Invalid timestamp
-        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', 'invalid'])
-        assert result.exit_code != 0
-
-        # Invalid timestamp
-        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2115-02-30T12:00:00+01:00'])
-        assert result.exit_code != 0
-
-        # Specifying wrong leader
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='dummy')
-        assert result.exit_code == 1
-
-        with patch('patroni.ctl.request_patroni', Mock(side_effect=Exception)):
-            # Non-responding patroni
-            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
-            assert 'falling back to DCS' in result.output
-
-        with patch('patroni.ctl.request_patroni') as mocked:
-            mocked.return_value.status_code = 500
-            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
-            assert 'Failover failed' in result.output
-
-        # No members available
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
-        assert result.exit_code == 1
-
-        # No master available
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
-        assert result.exit_code == 1
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='\n')
+        assert 'Failover could be performed only to a specific candidate' in result.output
 
     def test_get_dcs(self):
         self.assertRaises(PatroniCtlException, get_dcs, {'dummy': {}}, 'dummy')
@@ -219,7 +237,7 @@ class TestCtl(unittest.TestCase):
         assert result.exit_code == 1
 
         # successful reinit
-        result = self.runner.invoke(ctl, ['reinit', 'alpha', 'other'], input='y')
+        result = self.runner.invoke(ctl, ['reinit', 'alpha', 'other'], input='y\ny')
         assert result.exit_code == 0
 
         # Aborted restart
@@ -235,7 +253,7 @@ class TestCtl(unittest.TestCase):
 
         # Wrong pg version
         result = self.runner.invoke(ctl, ['restart', 'alpha', '--any', '--pg-version', '9.1'], input='y')
-        assert 'Error: PostgreSQL version' in result.output
+        assert 'Error: Invalid PostgreSQL version format' in result.output
         assert result.exit_code == 1
 
         result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force', '--timeout', '10min'])
@@ -261,7 +279,7 @@ class TestCtl(unittest.TestCase):
         with patch('requests.post', Mock(return_value=MockResponse(204))):
             # get restart with the non-200 return code
             # normal restart, the schedule is actually parsed, but not validated in patronictl
-            result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0.0',
+            result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0',
                                         '--scheduled', '2300-10-01T14:30'], input='y')
             assert result.exit_code == 0
 
@@ -342,9 +360,11 @@ class TestCtl(unittest.TestCase):
     @patch('patroni.ctl.get_dcs')
     def test_members(self, mock_get_dcs):
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
-        result = self.runner.invoke(members, ['alpha'])
+        result = self.runner.invoke(ctl, ['list'])
         assert '127.0.0.1' in result.output
         assert result.exit_code == 0
+        with patch('patroni.ctl.load_config', Mock(return_value={})):
+            self.runner.invoke(ctl, ['list'])
 
     def test_configure(self):
         result = self.runner.invoke(configure, ['--dcs', 'abc', '-c', 'dummy', '-n', 'bla'])
@@ -380,7 +400,7 @@ class TestCtl(unittest.TestCase):
         cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
         mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster)
 
-        result = self.runner.invoke(ctl, ['list', 'dummy', '--extended'])
+        result = self.runner.invoke(ctl, ['list', 'dummy', '--extended', '--timestamp'])
         assert '2100' in result.output
         assert 'Scheduled restart' in result.output
 
@@ -400,14 +420,11 @@ class TestCtl(unittest.TestCase):
             assert 'Failed: flush scheduled restart' in result.output
 
     @patch('patroni.ctl.get_dcs')
+    @patch('patroni.ctl.polling_loop', Mock(return_value=[1]))
     def test_pause_cluster(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
 
-        with patch('requests.patch', Mock(return_value=MockResponse(200))):
-            result = self.runner.invoke(ctl, ['pause', 'dummy'])
-            assert 'Success' in result.output
-
         with patch('requests.patch', Mock(return_value=MockResponse(500))):
             result = self.runner.invoke(ctl, ['pause', 'dummy'])
             assert 'Failed' in result.output
@@ -417,6 +434,17 @@ class TestCtl(unittest.TestCase):
             result = self.runner.invoke(ctl, ['pause', 'dummy'])
             assert 'Cluster is already paused' in result.output
 
+        with patch('requests.patch', Mock(return_value=MockResponse(200))):
+            result = self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
+            assert "'pause' request sent" in result.output
+            mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
+                                                                      get_cluster(None, None, [], None, None)])
+            self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
+            member = Member(1, 'other', 28, {})
+            mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
+                                                                      get_cluster(None, None, [member], None, None)])
+            self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
+
     @patch('patroni.ctl.get_dcs')
     def test_resume_cluster(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
@@ -506,3 +534,21 @@ class TestCtl(unittest.TestCase):
         self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar')
         mock_get_dcs.return_value.set_config_value = Mock(return_value=True)
         self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar')
+
+    @patch('patroni.ctl.get_dcs')
+    def test_version(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        with patch('patroni.ctl.request_patroni') as mocked:
+            result = self.runner.invoke(ctl, ['version'])
+            assert 'patronictl version' in result.output
+            mocked.return_value.json = lambda: {'patroni': {'version': '1.2.3'}, 'server_version': 100001}
+            result = self.runner.invoke(ctl, ['version', 'dummy'])
+            assert '1.2.3' in result.output
+        with patch('requests.get', Mock(side_effect=Exception)):
+            result = self.runner.invoke(ctl, ['version', 'dummy'])
+            assert 'failed to get version' in result.output
+
+    def test_format_pg_version(self):
+        self.assertEquals(format_pg_version(100001), '10.1')
+        self.assertEquals(format_pg_version(90605), '9.6.5')

tests/test_etcd.py

@@ -18,7 +18,6 @@ class MockResponse(object):
         self.status_code = status_code
         self.content = '{}'
         self.ok = True
-        self.text = ''
 
     def json(self):
         return json.loads(self.content)
@@ -27,6 +26,10 @@ class MockResponse(object):
     def data(self):
         return self.content.encode('utf-8')
 
+    @property
+    def text(self):
+        return self.content
+
     @property
     def status(self):
         return self.status_code
@@ -48,6 +51,12 @@ def requests_get(url, **kwargs):
         response.content = '[{}]' if url.startswith('http://error') else members
     elif url.startswith('http://exhibitor'):
         response.content = '{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}'
+    elif url.endswith(':8011/reinitialize'):
+        data = kwargs.get('data', '')
+        if ' false}' in data:
+            response.status_code = 503
+            response.ok = False
+            response.content = 'restarting after failure already in progress'
     else:
         response.status_code = 404
         response.ok = False
@@ -262,6 +271,8 @@ class TestEtcd(unittest.TestCase):
                                   {'url': 'https://test:2379', 'retry_timeout': 10})
                 self.assertRaises(SleepException, self.etcd.get_etcd_client,
                                   {'proxy': 'https://user:password@test:2379', 'retry_timeout': 10})
+                self.assertRaises(SleepException, self.etcd.get_etcd_client,
+                                  {'hosts': 'foo:4001,bar', 'retry_timeout': 10})
 
     def test_get_cluster(self):
         self.assertIsInstance(self.etcd.get_cluster(), Cluster)
@@ -288,7 +299,7 @@ class TestEtcd(unittest.TestCase):
         self.etcd.write_leader_optime('0')
 
     def test_update_leader(self):
-        self.assertTrue(self.etcd.update_leader())
+        self.assertTrue(self.etcd.update_leader(None))
 
     def test_initialize(self):
         self.assertFalse(self.etcd.initialize())
@@ -324,3 +335,6 @@ class TestEtcd(unittest.TestCase):
     def test_sync_state(self):
         self.assertFalse(self.etcd.write_sync_state('leader', None))
         self.assertFalse(self.etcd.delete_sync_state())
+
+    def test_set_history_value(self):
+        self.assertFalse(self.etcd.set_history_value('{}'))

tests/test_ha.py

@@ -5,7 +5,7 @@ import unittest
 
 from mock import Mock, MagicMock, PropertyMock, patch
 from patroni.config import Config
-from patroni.dcs import Cluster, ClusterConfig, Failover, Leader, Member, get_dcs, SyncState
+from patroni.dcs import Cluster, ClusterConfig, Failover, Leader, Member, get_dcs, SyncState, TimelineHistory
 from patroni.dcs.etcd import Client
 from patroni.exceptions import DCSError, PostgresConnectionException, PatroniException
 from patroni.ha import Ha, _MemberStatus
@@ -13,7 +13,7 @@ from patroni.postgresql import Postgresql
 from patroni.watchdog import Watchdog
 from patroni.utils import tzutc
 from test_etcd import socket_getaddrinfo, etcd_read, etcd_write, requests_get
-from test_postgresql import psycopg2_connect
+from test_postgresql import psycopg2_connect, MockPostmaster
 
 
 def true(*args, **kwargs):
@@ -25,7 +25,8 @@ def false(*args, **kwargs):
 
 
 def get_cluster(initialize, leader, members, failover, sync):
-    return Cluster(initialize, ClusterConfig(1, {1: 2}, 1), leader, 10, members, failover, sync)
+    history = TimelineHistory(1, [(1, 67197376, 'no recovery target specified', datetime.datetime.now().isoformat())])
+    return Cluster(initialize, ClusterConfig(1, {1: 2}, 1), leader, 10, members, failover, sync, history)
 
 
 def get_cluster_not_initialized_without_leader():
@@ -35,15 +36,16 @@ def get_cluster_not_initialized_without_leader():
 def get_cluster_initialized_without_leader(leader=False, failover=None, sync=None):
     m1 = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres',
                                   'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location': 4})
-    l = Leader(0, 0, m1) if leader else None
+    leader = Leader(0, 0, m1) if leader else None
     m2 = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
                                  'api_url': 'http://127.0.0.1:8011/patroni',
                                  'state': 'running',
+                                 'pause': True,
                                  'tags': {'clonefrom': True},
                                  'scheduled_restart': {'schedule': "2100-01-01 10:53:07.560445+00:00",
                                                        'postgres_version': '99.0.0'}})
     syncstate = SyncState(0 if sync else None, sync and sync[0], sync and sync[1])
-    return get_cluster(True, l, [m1, m2], failover, syncstate)
+    return get_cluster(True, leader, [m1, m2], failover, syncstate)
 
 
 def get_cluster_initialized_with_leader(failover=None, sync=None):
@@ -51,8 +53,8 @@ def get_cluster_initialized_with_leader(failover=None, sync=None):
 
 
 def get_cluster_initialized_with_only_leader(failover=None):
-    l = get_cluster_initialized_without_leader(leader=True, failover=failover).leader
-    return get_cluster(True, l, [l], failover, None)
+    leader = get_cluster_initialized_without_leader(leader=True, failover=failover).leader
+    return get_cluster(True, leader, [leader], failover, None)
 
 
 def get_node_status(reachable=True, in_recovery=True, wal_position=10, nofailover=False, watchdog_failed=False):
@@ -63,6 +65,7 @@ def get_node_status(reachable=True, in_recovery=True, wal_position=10, nofailove
         return _MemberStatus(e, reachable, in_recovery, wal_position, tags, watchdog_failed)
     return fetch_node_status
 
+
 future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5)
 postmaster_start_time = datetime.datetime.now(tzutc)
 
@@ -111,9 +114,10 @@ def run_async(self, func, args=()):
     return func(*args) if args else func()
 
 
-@patch.object(Postgresql, 'is_running', Mock(return_value=True))
+@patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
 @patch.object(Postgresql, 'is_leader', Mock(return_value=True))
-@patch.object(Postgresql, 'wal_position', Mock(return_value=10))
+@patch.object(Postgresql, 'timeline_wal_position', Mock(return_value=(1, 10)))
+@patch.object(Postgresql, '_cluster_info_state_get', Mock(return_value=3))
 @patch.object(Postgresql, 'call_nowait', Mock(return_value=True))
 @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
 @patch.object(Postgresql, 'controldata', Mock(return_value={'Database system identifier': '1234567890'}))
@@ -124,9 +128,11 @@ def run_async(self, func, args=()):
 @patch.object(Postgresql, 'query', Mock())
 @patch.object(Postgresql, 'checkpoint', Mock())
 @patch.object(Postgresql, 'call_nowait', Mock())
+@patch.object(Postgresql, 'cancellable_subprocess_call', Mock(return_value=0))
 @patch.object(etcd.Client, 'write', etcd_write)
 @patch.object(etcd.Client, 'read', etcd_read)
 @patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
+@patch('patroni.postgresql.polling_loop', Mock(return_value=range(1)))
 @patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=False))
 @patch('patroni.async_executor.AsyncExecutor.run_async', run_async)
 @patch('subprocess.call', Mock(return_value=0))
@@ -135,7 +141,7 @@ class TestHa(unittest.TestCase):
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
     @patch('psycopg2.connect', psycopg2_connect)
-    @patch('patroni.dcs.dcs_modules', Mock(return_value=['foo', 'patroni.dcs.etcd']))
+    @patch('patroni.dcs.dcs_modules', Mock(return_value=['patroni.dcs.foo', 'patroni.dcs.etcd']))
     @patch.object(etcd.Client, 'read', etcd_read)
     def setUp(self):
         with patch.object(Client, 'machines') as mock_machines:
@@ -157,14 +163,14 @@ class TestHa(unittest.TestCase):
             self.ha.old_cluster = self.e.get_cluster()
             self.ha.cluster = get_cluster_not_initialized_without_leader()
             self.ha.load_cluster_from_dcs = Mock()
-            self.ha.is_synchronous_mode = false
 
     def test_update_lock(self):
         self.p.last_operation = Mock(side_effect=PostgresConnectionException(''))
         self.assertTrue(self.ha.update_lock(True))
 
     def test_touch_member(self):
-        self.p.wal_position = Mock(side_effect=Exception)
+        self.p.timeline_wal_position = Mock(return_value=(0, 1))
+        self.p.replica_cached_timeline = Mock(side_effect=Exception)
         self.ha.touch_member()
 
     def test_start_as_replica(self):
@@ -172,27 +178,42 @@ class TestHa(unittest.TestCase):
         self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
 
     def test_recover_replica_failed(self):
-        self.p.controldata = lambda: {'Database cluster state': 'in production'}
+        self.p.controldata = lambda: {'Database cluster state': 'in recovery'}
         self.p.is_running = false
         self.p.follow = false
         self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
         self.assertEquals(self.ha.run_cycle(), 'failed to start postgres')
 
-    def test_recover_master_failed(self):
+    def test_recover_former_master(self):
         self.p.follow = false
         self.p.is_running = false
         self.p.name = 'leader'
         self.p.set_role('master')
-        self.p.controldata = lambda: {'Database cluster state': 'in production'}
+        self.p.controldata = lambda: {'Database cluster state': 'shut down'}
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.assertEquals(self.ha.run_cycle(), 'starting as readonly because i had the session lock')
 
+    @patch.object(Postgresql, 'fix_cluster_state', Mock())
+    def test_crash_recovery(self):
+        self.p.is_running = false
+        self.p.controldata = lambda: {'Database cluster state': 'in production'}
+        self.assertEquals(self.ha.run_cycle(), 'doing crash recovery in a single user mode')
+
     @patch.object(Postgresql, 'rewind_needed_and_possible', Mock(return_value=True))
     def test_recover_with_rewind(self):
         self.p.is_running = false
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.assertEquals(self.ha.run_cycle(), 'running pg_rewind from leader')
 
+    @patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
+    @patch.object(Postgresql, 'fix_cluster_state', Mock())
+    def test_single_user_after_recover_failed(self):
+        self.p.controldata = lambda: {'Database cluster state': 'in recovery'}
+        self.p.is_running = false
+        self.p.follow = false
+        self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
+        self.assertEquals(self.ha.run_cycle(), 'fixing cluster state in a single user mode')
+
     @patch('sys.exit', return_value=1)
     @patch('patroni.ha.Ha.sysid_valid', MagicMock(return_value=True))
     def test_sysid_no_match(self, exit_mock):
@@ -204,8 +225,10 @@ class TestHa(unittest.TestCase):
         self.p.is_leader = false
         self.p.is_healthy = true
         self.ha.has_lock = true
+        self.p.controldata = lambda: {'Database cluster state': 'in production'}
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader because i had the session lock')
 
+    @patch('psycopg2.connect', psycopg2_connect)
     def test_acquire_lock_as_master(self):
         self.assertEquals(self.ha.run_cycle(), 'acquired session lock as a leader')
 
@@ -214,6 +237,13 @@ class TestHa(unittest.TestCase):
         self.p.is_leader = false
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
 
+    def test_long_promote(self):
+        self.ha.cluster.is_unlocked = false
+        self.ha.has_lock = true
+        self.p.is_leader = false
+        self.p.set_role('master')
+        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
+
     def test_demote_after_failing_to_obtain_lock(self):
         self.ha.acquire_lock = false
         self.assertEquals(self.ha.run_cycle(), 'demoted self after trying and failing to obtain lock')
@@ -246,7 +276,7 @@ class TestHa(unittest.TestCase):
         with patch.object(Watchdog, 'activate', Mock(return_value=False)):
             self.assertEquals(self.ha.run_cycle(), 'Demoting self because watchdog could not be activated')
             self.p.is_leader = false
-            self.assertEquals(self.ha.run_cycle(), 'Not promoting self because watchdog could not be actived')
+            self.assertEquals(self.ha.run_cycle(), 'Not promoting self because watchdog could not be activated')
 
     def test_leader_with_lock(self):
         self.ha.cluster.is_unlocked = false
@@ -330,7 +360,7 @@ class TestHa(unittest.TestCase):
         self.ha.cluster = get_cluster_not_initialized_without_leader()
         self.e.initialize = true
         self.ha.bootstrap()
-        self.p.is_running = true
+        self.p.is_running.return_value = MockPostmaster()
         self.p.is_leader = true
         with patch.object(Watchdog, 'activate', Mock(return_value=False)):
             self.assertEquals(self.ha.post_bootstrap(), 'running post_bootstrap')
@@ -341,7 +371,7 @@ class TestHa(unittest.TestCase):
         self.assertIsNotNone(self.ha.reinitialize())
 
         self.ha.cluster = get_cluster_initialized_with_leader()
-        self.assertIsNone(self.ha.reinitialize())
+        self.assertIsNone(self.ha.reinitialize(True))
 
         self.assertIsNotNone(self.ha.reinitialize())
 
@@ -377,9 +407,9 @@ class TestHa(unittest.TestCase):
             self.ha.update_lock = false
             self.p.set_role('master')
             with patch('patroni.async_executor.CriticalTask.cancel', Mock(return_value=False)):
-                with patch('patroni.postgresql.Postgresql.stop') as stop_mock:
+                with patch('patroni.postgresql.Postgresql.terminate_starting_postmaster') as mock_terminate:
                     self.assertEquals(self.ha.run_cycle(), 'lost leader lock during restart')
-                    stop_mock.assert_called()
+                    mock_terminate.assert_called()
 
     @patch('requests.get', requests_get)
     def test_manual_failover_from_leader(self):
@@ -437,6 +467,19 @@ class TestHa(unittest.TestCase):
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, '', None))
         self.assertEquals('PAUSE: no action.  i am the leader with the lock', self.ha.run_cycle())
 
+    @patch('requests.get', requests_get)
+    def test_manual_failover_from_leader_in_synchronous_mode(self):
+        self.p.is_leader = true
+        self.ha.has_lock = true
+        self.ha.is_synchronous_mode = true
+        self.ha.is_failover_possible = false
+        self.ha.process_sync_replication = Mock()
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'a', None), (self.p.name, None))
+        self.assertEquals('no action.  i am the leader with the lock', self.ha.run_cycle())
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, 'a', None), (self.p.name, 'a'))
+        self.ha.is_failover_possible = true
+        self.assertEquals('manual failover: demoting myself', self.ha.run_cycle())
+
     @patch('requests.get', requests_get)
     def test_manual_failover_process_no_leader(self):
         self.p.is_leader = false
@@ -496,7 +539,7 @@ class TestHa(unittest.TestCase):
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.fetch_node_status = get_node_status(wal_position=11)  # accessible, in_recovery, wal position ahead
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
-        with patch('patroni.postgresql.Postgresql.wal_position', return_value=1):
+        with patch('patroni.postgresql.Postgresql.timeline_wal_position', return_value=(1, 1)):
             self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.patroni.nofailover = True
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
@@ -634,7 +677,8 @@ class TestHa(unittest.TestCase):
     @patch('patroni.ha.Ha.demote')
     def test_failover_immediately_on_zero_master_start_timeout(self, demote):
         self.p.is_running = false
-        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.ha.cluster = get_cluster_initialized_with_leader(sync=(self.p.name, 'other'))
+        self.ha.cluster.config.data['synchronous_mode'] = True
         self.ha.patroni.config.set_dynamic_configuration({'master_start_timeout': 0})
         self.ha.has_lock = true
         self.ha.update_lock = true
@@ -831,6 +875,7 @@ class TestHa(unittest.TestCase):
 
     def test_shutdown(self):
         self.p.is_running = false
+        self.ha.has_lock = true
         self.ha.shutdown()
 
     @patch('time.sleep', Mock())
@@ -839,8 +884,15 @@ class TestHa(unittest.TestCase):
         self.ha.has_lock = true
         self.p.data_directory_empty = true
         self.assertEquals(self.ha.run_cycle(), 'released leader key voluntarily as data dir empty and currently leader')
+        self.assertEquals(self.p.role, 'uninitialized')
 
         # as has_lock is mocked out, we need to fake the leader key release
         self.ha.has_lock = false
         # will not say bootstrap from leader as replica can't self elect
         self.assertEquals(self.ha.run_cycle(), "trying to bootstrap from replica 'other'")
+
+    def test_update_cluster_history(self):
+        self.p.get_master_timeline = Mock(return_value=1)
+        self.ha.has_lock = true
+        self.ha.cluster.is_unlocked = false
+        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')

tests/test_kubernetes.py

@@ -0,0 +1,106 @@
+import unittest
+
+from mock import Mock, patch
+from patroni.dcs.kubernetes import Kubernetes, KubernetesError, k8s_client, k8s_watch
+
+
+def mock_list_namespaced_config_map(self, *args, **kwargs):
+    metadata = {'resource_version': '1', 'labels': {'f': 'b'}, 'name': 'test-config',
+                'annotations': {'initialize': '123', 'config': '{}'}}
+    items = [k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata))]
+    metadata.update({'name': 'test-leader', 'annotations': {'optime': '1234', 'leader': 'p-0', 'ttl': '30s'}})
+    items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
+    metadata.update({'name': 'test-failover', 'annotations': {'leader': 'p-0'}})
+    items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
+    metadata.update({'name': 'test-sync', 'annotations': {'leader': 'p-0'}})
+    items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
+    metadata = k8s_client.V1ObjectMeta(resource_version='1')
+    return k8s_client.V1ConfigMapList(metadata=metadata, items=items)
+
+
+def mock_list_namespaced_pod(self, *args, **kwargs):
+    metadata = k8s_client.V1ObjectMeta(resource_version='1', name='p-0', annotations={'status': '{}'})
+    items = [k8s_client.V1Pod(metadata=metadata)]
+    return k8s_client.V1PodList(items=items)
+
+
+@patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', Mock())
+@patch.object(k8s_client.CoreV1Api, 'create_namespaced_config_map', Mock())
+class TestKubernetes(unittest.TestCase):
+
+    @patch('kubernetes.config.load_kube_config', Mock())
+    @patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map)
+    @patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', mock_list_namespaced_pod)
+    def setUp(self):
+        self.k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10, 'labels': {'f': 'b'}})
+        with patch('time.time', Mock(return_value=1)):
+            self.k.get_cluster()
+
+    @patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map)
+    @patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', mock_list_namespaced_pod)
+    def test_get_cluster(self):
+        self.k.get_cluster()
+        with patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', Mock(side_effect=Exception)):
+            self.assertRaises(KubernetesError, self.k.get_cluster)
+
+    @patch('kubernetes.config.load_kube_config', Mock())
+    @patch.object(k8s_client.CoreV1Api, 'create_namespaced_endpoints', Mock())
+    def test_update_leader(self):
+        k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10,
+                        'labels': {'f': 'b'}, 'use_endpoints': True, 'pod_ip': '10.0.0.0'})
+        self.assertIsNotNone(k.update_leader('123'))
+
+    def test_take_leader(self):
+        self.k.take_leader()
+        self.k._leader_observed_record['leader'] = 'test'
+        self.k.patch_or_create = Mock(return_value=False)
+        self.k.take_leader()
+
+    def test_manual_failover(self):
+        self.k.manual_failover('foo', 'bar')
+
+    def test_set_config_value(self):
+        self.k.set_config_value('{}')
+
+    @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_pod', Mock(return_value=True))
+    def test_touch_member(self):
+        self.k.touch_member({})
+        self.k._name = 'p-1'
+        self.k.touch_member({'state': 'running', 'role': 'replica'})
+        self.k.touch_member({'state': 'stopped', 'role': 'master'})
+
+    def test_initialize(self):
+        self.k.initialize()
+
+    def test_delete_leader(self):
+        self.k.delete_leader()
+
+    def test_cancel_initialization(self):
+        self.k.cancel_initialization()
+
+    @patch.object(k8s_client.CoreV1Api, 'delete_collection_namespaced_config_map',
+                  Mock(side_effect=k8s_client.rest.ApiException(500, '')))
+    def test_delete_cluster(self):
+        self.k.delete_cluster()
+
+    @patch('kubernetes.config.load_kube_config', Mock())
+    @patch.object(k8s_client.CoreV1Api, 'create_namespaced_endpoints',
+                  Mock(side_effect=[k8s_client.rest.ApiException(502, ''), k8s_client.rest.ApiException(500, '')]))
+    def test_delete_sync_state(self):
+        k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10,
+                        'labels': {'f': 'b'}, 'use_endpoints': True, 'pod_ip': '10.0.0.0'})
+        self.assertFalse(k.delete_sync_state())
+
+    def test_watch(self):
+        self.k.set_ttl(10)
+        self.k.watch(None, 0)
+        self.k.watch(None, 0)
+        with patch.object(k8s_watch.Watch, 'stream',
+                          Mock(side_effect=[Exception, [], KeyboardInterrupt,
+                                            [{'raw_object': {'metadata': {'resourceVersion': '2'}}}]])):
+            self.assertFalse(self.k.watch('1', 2))
+            self.assertRaises(KeyboardInterrupt, self.k.watch, '1', 2)
+            self.assertTrue(self.k.watch('1', 2))
+
+    def test_set_history_value(self):
+        self.k.set_history_value('{}')

tests/test_patroni.py

@@ -12,7 +12,7 @@ from patroni.exceptions import DCSError
 from patroni import Patroni, main as _main, patroni_main
 from six.moves import BaseHTTPServer
 from test_etcd import SleepException, etcd_read, etcd_write
-from test_postgresql import Postgresql, psycopg2_connect
+from test_postgresql import Postgresql, psycopg2_connect, MockPostmaster
 
 
 class MockFrozenImporter(object):
@@ -26,7 +26,7 @@ class MockFrozenImporter(object):
 @patch.object(Postgresql, 'write_pg_hba', Mock())
 @patch.object(Postgresql, '_write_postgresql_conf', Mock())
 @patch.object(Postgresql, 'write_recovery_conf', Mock())
-@patch.object(Postgresql, 'is_running', Mock(return_value=True))
+@patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
 @patch.object(Postgresql, 'call_nowait', Mock())
 @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
 @patch.object(AsyncExecutor, 'run', Mock())
@@ -104,6 +104,7 @@ class TestPatroni(unittest.TestCase):
     @patch('patroni.config.Config.save_cache', Mock())
     @patch('patroni.config.Config.reload_local_configuration', Mock(return_value=True))
     @patch.object(Postgresql, 'state', PropertyMock(return_value='running'))
+    @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
     def test_run(self):
         self.p.postgresql.set_role('replica')
         self.p.sighup_handler()

tests/test_postgresql.py

@@ -1,4 +1,4 @@
-import errno
+import datetime
 import mock  # for the mock.call method, importing it without a namespace breaks python3
 import os
 import psycopg2
@@ -9,8 +9,9 @@ import unittest
 from mock import Mock, MagicMock, PropertyMock, patch, mock_open
 from patroni.async_executor import CriticalTask
 from patroni.dcs import Cluster, Leader, Member, SyncState
-from patroni.exceptions import PostgresConnectionException
+from patroni.exceptions import PostgresConnectionException, PostgresException
 from patroni.postgresql import Postgresql, STATE_REJECT, STATE_NO_RESPONSE
+from patroni.postmaster import PostmasterProcess
 from patroni.utils import RetryFailedError
 from six.moves import builtins
 from threading import Thread
@@ -34,13 +35,13 @@ class MockCursor(object):
         elif sql.startswith('SELECT slot_name'):
             self.results = [('blabla',), ('foobar',)]
         elif sql.startswith('SELECT CASE WHEN pg_is_in_recovery()'):
-            self.results = [(2,)]
-        elif sql == 'SELECT pg_is_in_recovery()':
-            self.results = [(False, )]
+            self.results = [(1, 2)]
+        elif sql.startswith('SELECT pg_is_in_recovery()'):
+            self.results = [(False, 2)]
         elif sql.startswith('WITH replication_info AS ('):
             replication_info = '[{"application_name":"walreceiver","client_addr":"1.2.3.4",' +\
                                '"state":"streaming","sync_state":"async","sync_priority":0}]'
-            self.results = [('', True, '', '', '', '', False, replication_info)]
+            self.results = [('', 0, '', '', '', '', False, replication_info)]
         elif sql.startswith('SELECT name, setting'):
             self.results = [('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
                             ('search_path', 'public', None, 'string', 'user'),
@@ -50,6 +51,11 @@ class MockCursor(object):
                             ('unix_socket_directories', '/tmp', None, 'string', 'postmaster')]
         elif sql.startswith('IDENTIFY_SYSTEM'):
             self.results = [('1', 2, '0/402EEC0', '')]
+        elif sql.startswith('SELECT isdir, modification'):
+            self.results = [(False, datetime.datetime.now())]
+        elif sql.startswith('SELECT pg_read_file'):
+            self.results = [('1\t0/40159C0\tno recovery target specified\n\n' +
+                             '2\t1/40159C0\tno recovery target specified\n',)]
         elif sql.startswith('TIMELINE_HISTORY '):
             self.results = [('', b'x\t0/40159C0\tno recovery target specified\n\n' +
                                  b'1\t0/40159C0\tno recovery target specified\n\n' +
@@ -95,6 +101,15 @@ class MockConnect(object):
         pass
 
 
+class MockPostmaster(object):
+    def __init__(self, is_running=True, is_single_master=False):
+        self.is_running = Mock(return_value=is_running)
+        self.is_single_master = Mock(return_value=is_single_master)
+        self.wait_for_user_backends_to_close = Mock()
+        self.signal_stop = Mock(return_value=None)
+        self.wait = Mock()
+
+
 def pg_controldata_string(*args, **kwargs):
     return b"""
 pg_control version number:            942
@@ -174,7 +189,7 @@ class TestPostgresql(unittest.TestCase):
         if not os.path.exists(self.data_dir):
             os.makedirs(self.data_dir)
         self.p = Postgresql({'name': 'test0', 'scope': 'batman', 'data_dir': self.data_dir,
-                             'config_dir': self.config_dir, 'retry_timeout': 10,
+                             'config_dir': self.config_dir, 'retry_timeout': 10, 'pgpass': '/tmp/pgpass0',
                              'listen': '127.0.0.2, 127.0.0.3:5432', 'connect_address': '127.0.0.2:5432',
                              'authentication': {'superuser': {'username': 'test', 'password': 'test'},
                                                 'replication': {'username': 'replicator', 'password': 'rep-pass'}},
@@ -212,85 +227,84 @@ class TestPostgresql(unittest.TestCase):
     @patch.object(Postgresql, 'wait_for_port_open')
     @patch.object(Postgresql, 'is_running')
     def test_start(self, mock_is_running, mock_wait_for_port_open, mock_wait_for_startup, mock_popen):
-        mock_is_running.return_value = True
+        mock_is_running.return_value = MockPostmaster()
         mock_wait_for_port_open.return_value = True
         mock_wait_for_startup.return_value = False
         mock_popen.return_value.stdout.readline.return_value = '123'
         self.assertTrue(self.p.start())
-        mock_is_running.return_value = False
-        open(os.path.join(self.data_dir, 'postmaster.pid'), 'w').close()
-        pg_conf = os.path.join(self.data_dir, 'postgresql.conf')
-        open(pg_conf, 'w').close()
-        self.assertFalse(self.p.start(task=CriticalTask()))
-        with open(pg_conf) as f:
-            lines = f.readlines()
-            self.assertTrue("f.oo = 'bar'\n" in lines)
+        mock_is_running.return_value = None
 
-        mock_wait_for_startup.return_value = None
-        self.assertFalse(self.p.start(10))
-        self.assertIsNone(self.p.start())
+        mock_postmaster = MockPostmaster()
+        with patch.object(PostmasterProcess, 'start', return_value=mock_postmaster):
+            pg_conf = os.path.join(self.data_dir, 'postgresql.conf')
+            open(pg_conf, 'w').close()
+            self.assertFalse(self.p.start(task=CriticalTask()))
 
-        mock_wait_for_port_open.return_value = False
+            with open(pg_conf) as f:
+                lines = f.readlines()
+                self.assertTrue("f.oo = 'bar'\n" in lines)
+
+            mock_wait_for_startup.return_value = None
+            self.assertFalse(self.p.start(10))
+            self.assertIsNone(self.p.start())
+
+            mock_wait_for_port_open.return_value = False
+            self.assertFalse(self.p.start())
+            task = CriticalTask()
+            task.cancel()
+            self.assertFalse(self.p.start(task=task))
+
+        self.p.cancel()
         self.assertFalse(self.p.start())
-        task = CriticalTask()
-        task.cancel()
-        self.assertFalse(self.p.start(task=task))
 
     @patch.object(Postgresql, 'pg_isready')
-    @patch.object(Postgresql, 'read_pid_file')
-    @patch.object(Postgresql, 'is_pid_running')
     @patch('patroni.postgresql.polling_loop', Mock(return_value=range(1)))
-    def test_wait_for_port_open(self, mock_is_pid_running, mock_read_pid_file, mock_pg_isready):
-        mock_is_pid_running.return_value = False
+    def test_wait_for_port_open(self, mock_pg_isready):
         mock_pg_isready.return_value = STATE_NO_RESPONSE
+        mock_postmaster = MockPostmaster(is_running=False)
 
         # No pid file and postmaster death
-        mock_read_pid_file.return_value = {}
-        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
+        self.assertFalse(self.p.wait_for_port_open(mock_postmaster, 1))
 
-        mock_is_pid_running.return_value = True
+        mock_postmaster.is_running.return_value = True
 
         # timeout
-        mock_read_pid_file.return_value = {'pid', 1}
-        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
-
-        # Garbage pid
-        mock_read_pid_file.return_value = {'pid': 'garbage', 'start_time': '101', 'data_dir': '',
-                                           'socket_dir': '', 'port': '', 'listen_addr': ''}
-        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
-
-        # Not ready
-        mock_read_pid_file.return_value = {'pid': '42', 'start_time': '101', 'data_dir': '',
-                                           'socket_dir': '', 'port': '', 'listen_addr': ''}
-        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
+        self.assertFalse(self.p.wait_for_port_open(mock_postmaster, 1))
 
         # pg_isready failure
         mock_pg_isready.return_value = 'garbage'
-        self.assertTrue(self.p.wait_for_port_open(42, 100., 1))
+        self.assertTrue(self.p.wait_for_port_open(mock_postmaster, 1))
+
+        # cancelled
+        self.p.cancel()
+        self.assertFalse(self.p.wait_for_port_open(mock_postmaster, 1))
 
     @patch('time.sleep', Mock())
     @patch.object(Postgresql, 'is_running')
-    @patch.object(Postgresql, 'get_pid')
-    def test_stop(self, mock_get_pid, mock_is_running):
+    @patch.object(Postgresql, '_wait_for_connection_close', Mock())
+    def test_stop(self, mock_is_running):
+        # Postmaster is not running
         mock_callback = Mock()
-        mock_is_running.return_value = False
+        mock_is_running.return_value = None
         self.assertTrue(self.p.stop(on_safepoint=mock_callback))
         mock_callback.assert_called()
-        mock_is_running.return_value = True
-        mock_get_pid.return_value = 0
+
+        # Is running, stopped successfully
+        mock_is_running.return_value = mock_postmaster = MockPostmaster()
         mock_callback.reset_mock()
         self.assertTrue(self.p.stop(on_safepoint=mock_callback))
         mock_callback.assert_called()
-        mock_get_pid.return_value = -1
+        mock_postmaster.signal_stop.assert_called()
+
+        # Stop signal failed
+        mock_postmaster.signal_stop.return_value = False
         self.assertFalse(self.p.stop())
-        mock_get_pid.return_value = 123
-        with patch('os.kill', Mock(side_effect=[OSError(errno.ESRCH, ''), OSError, None])):
-            self.assertTrue(self.p.stop())
-            self.assertFalse(self.p.stop())
-            self.assertTrue(self.p.stop())
-        with patch.object(Postgresql, '_signal_postmaster_stop', Mock(return_value=(123, None))):
-            with patch.object(Postgresql, 'is_pid_running', Mock(side_effect=[True, False, False])):
-                self.assertTrue(self.p.stop())
+
+        # Stop signal failed to find process
+        mock_postmaster.signal_stop.return_value = True
+        mock_callback.reset_mock()
+        self.assertTrue(self.p.stop(on_safepoint=mock_callback))
+        mock_callback.assert_called()
 
     def test_restart(self):
         self.p.start = Mock(return_value=False)
@@ -309,12 +323,13 @@ class TestPostgresql(unittest.TestCase):
             self.assertIsNone(self.p.checkpoint())
         self.assertEquals(self.p.checkpoint(), 'not accessible or not healty')
 
-    @patch('subprocess.call', side_effect=OSError)
+    @patch.object(Postgresql, 'cancellable_subprocess_call')
     @patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
-    def test_pg_rewind(self, mock_call):
+    def test_pg_rewind(self, mock_cancellable_subprocess_call):
         r = {'user': '', 'host': '', 'port': '', 'database': '', 'password': ''}
+        mock_cancellable_subprocess_call.return_value = 0
         self.assertTrue(self.p.pg_rewind(r))
-        subprocess.call = mock_call
+        mock_cancellable_subprocess_call.side_effect = OSError
         self.assertFalse(self.p.pg_rewind(r))
 
     def test_check_recovery_conf(self):
@@ -327,10 +342,10 @@ class TestPostgresql(unittest.TestCase):
     @patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
     def test__get_local_timeline_lsn(self):
         self.p.trigger_check_diverged_lsn()
-        with patch.object(Postgresql, 'controldata', Mock(return_value={'Database cluster state': 'shut down'})):
-            self.p.rewind_needed_and_possible(self.leader)
         with patch.object(Postgresql, 'controldata',
-                          Mock(return_value={'Database cluster state': 'shut down in recovery'})):
+                          Mock(return_value={'Database cluster state': 'shut down in recovery',
+                                             'Minimum recovery ending location': '0/0',
+                                             "Min recovery ending loc's timeline": '0'})):
             self.p.rewind_needed_and_possible(self.leader)
         with patch.object(Postgresql, 'is_running', Mock(return_value=True)):
             with patch.object(MockCursor, 'fetchone', Mock(side_effect=[(False, ), Exception])):
@@ -338,7 +353,7 @@ class TestPostgresql(unittest.TestCase):
 
     @patch.object(Postgresql, 'start', Mock())
     @patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
-    @patch.object(Postgresql, '_get_local_timeline_lsn', Mock(return_value=(2, '0/40159C1')))
+    @patch.object(Postgresql, '_get_local_timeline_lsn', Mock(return_value=(2, '40159C1')))
     @patch.object(Postgresql, 'check_leader_is_not_in_recovery')
     def test__check_timeline_and_lsn(self, mock_check_leader_is_not_in_recovery):
         mock_check_leader_is_not_in_recovery.return_value = False
@@ -349,12 +364,8 @@ class TestPostgresql(unittest.TestCase):
         self.p.trigger_check_diverged_lsn()
         with patch('psycopg2.connect', Mock(side_effect=Exception)):
             self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
-        with patch.object(MockCursor, 'fetchone',
-                          Mock(side_effect=[('', 2, '0/0'), ('', b'2\tG/40159C0\tno recovery target specified\n\n')])):
-            self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
         self.p.trigger_check_diverged_lsn()
-        with patch.object(MockCursor, 'fetchone',
-                          Mock(side_effect=[('', 2, '0/0'), ('', b'3\t040159C0\tno recovery target specified\n')])):
+        with patch.object(MockCursor, 'fetchone', Mock(side_effect=[('', 2, '0/0'), ('', b'3\t0/40159C0\tn\n')])):
             self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
         self.p.trigger_check_diverged_lsn()
         with patch.object(MockCursor, 'fetchone', Mock(return_value=('', 1, '0/0'))):
@@ -368,6 +379,7 @@ class TestPostgresql(unittest.TestCase):
         self.p.check_leader_is_not_in_recovery()
         self.p.check_leader_is_not_in_recovery()
 
+    @patch.object(Postgresql, 'cancellable_subprocess_call', Mock(return_value=0))
     @patch.object(Postgresql, 'checkpoint', side_effect=['', '1'])
     @patch.object(Postgresql, 'stop', Mock(return_value=False))
     @patch.object(Postgresql, 'start', Mock())
@@ -404,31 +416,41 @@ class TestPostgresql(unittest.TestCase):
         self.assertFalse(self.p.can_rewind)
 
     @patch('time.sleep', Mock())
+    @patch.object(Postgresql, 'cancellable_subprocess_call')
     @patch.object(Postgresql, 'remove_data_directory', Mock(return_value=True))
-    def test_create_replica(self):
+    def test_create_replica(self, mock_cancellable_subprocess_call):
         self.p.delete_trigger_file = Mock(side_effect=OSError)
-        with patch('subprocess.call', Mock(side_effect=[1, 0])):
-            self.assertEquals(self.p.create_replica(self.leader), 0)
-        with patch('subprocess.call', Mock(side_effect=[Exception(), 0])):
-            self.assertEquals(self.p.create_replica(self.leader), 0)
 
         self.p.config['create_replica_method'] = ['wale', 'basebackup']
         self.p.config['wale'] = {'command': 'foo'}
-        with patch('subprocess.call', Mock(return_value=0)):
-            self.assertEquals(self.p.create_replica(self.leader), 0)
-            del self.p.config['wale']
-            self.assertEquals(self.p.create_replica(self.leader), 0)
+        mock_cancellable_subprocess_call.return_value = 0
+        self.assertEquals(self.p.create_replica(self.leader), 0)
+        del self.p.config['wale']
+        self.assertEquals(self.p.create_replica(self.leader), 0)
 
-        with patch('subprocess.call', Mock(side_effect=Exception("foo"))):
-            self.assertEquals(self.p.create_replica(self.leader), 1)
+        mock_cancellable_subprocess_call.return_value = 1
+        self.assertEquals(self.p.create_replica(self.leader), 1)
 
-        with patch('subprocess.call', Mock(return_value=1)):
-            self.assertEquals(self.p.create_replica(self.leader), 1)
+        mock_cancellable_subprocess_call.side_effect = Exception('foo')
+        self.assertEquals(self.p.create_replica(self.leader), 1)
+
+        mock_cancellable_subprocess_call.side_effect = [1, 0]
+        self.assertEquals(self.p.create_replica(self.leader), 0)
+
+        mock_cancellable_subprocess_call.side_effect = [Exception(), 0]
+        self.assertEquals(self.p.create_replica(self.leader), 0)
+
+        self.p.cancel()
+        self.assertEquals(self.p.create_replica(self.leader), 1)
+
+    def test_basebackup(self):
+        self.p.cancel()
+        self.p.basebackup(None, None)
 
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def test_sync_replication_slots(self):
         self.p.start()
-        cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None, None)
+        cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None, None, None)
         with mock.patch('patroni.postgresql.Postgresql._query', Mock(side_effect=psycopg2.OperationalError)):
             self.p.sync_replication_slots(cluster)
         self.p.sync_replication_slots(cluster)
@@ -458,6 +480,7 @@ class TestPostgresql(unittest.TestCase):
     @patch.object(Postgresql, 'pg_isready', Mock(return_value=STATE_REJECT))
     def test_is_leader(self):
         self.assertTrue(self.p.is_leader())
+        self.p.reset_cluster_info_state()
         with patch.object(Postgresql, '_query', Mock(side_effect=RetryFailedError(''))):
             self.assertRaises(PostgresConnectionException, self.p.is_leader)
 
@@ -473,21 +496,29 @@ class TestPostgresql(unittest.TestCase):
 
     def test_promote(self):
         self.p.set_role('replica')
-        self.assertTrue(self.p.promote())
-        self.assertTrue(self.p.promote())
+        self.assertIsNone(self.p.promote(0))
+        self.assertTrue(self.p.promote(0))
 
-    def test_last_operation(self):
-        self.assertEquals(self.p.last_operation(), '2')
-        Thread(target=self.p.last_operation).start()
+    def test_timeline_wal_position(self):
+        self.assertEquals(self.p.timeline_wal_position(), (1, 2))
+        Thread(target=self.p.timeline_wal_position).start()
 
-    @patch('os.path.isfile', Mock(return_value=True))
-    @patch('os.kill', Mock(side_effect=Exception))
-    @patch('os.getpid', Mock(return_value=2))
-    @patch('os.getppid', Mock(return_value=2))
-    @patch.object(builtins, 'open', mock_open(read_data='-1'))
-    @patch.object(Postgresql, '_version_file_exists', Mock(return_value=True))
-    def test_is_running(self):
-        self.assertFalse(self.p.is_running())
+    @patch.object(PostmasterProcess, 'from_pidfile')
+    def test_is_running(self, mock_frompidfile):
+        # Cached postmaster running
+        mock_postmaster = self.p._postmaster_proc = MockPostmaster()
+        self.assertEquals(self.p.is_running(), mock_postmaster)
+
+        # Cached postmaster not running, no postmaster running
+        mock_postmaster.is_running.return_value = False
+        mock_frompidfile.return_value = None
+        self.assertEquals(self.p.is_running(), None)
+        self.assertEquals(self.p._postmaster_proc, None)
+
+        # No cached postmaster, postmaster running
+        mock_frompidfile.return_value = mock_postmaster2 = MockPostmaster()
+        self.assertEquals(self.p.is_running(), mock_postmaster2)
+        self.assertEquals(self.p._postmaster_proc, mock_postmaster2)
 
     @patch('shlex.split', Mock(side_effect=OSError))
     def test_call_nowait(self):
@@ -499,7 +530,7 @@ class TestPostgresql(unittest.TestCase):
     def test_non_existing_callback(self):
         self.assertFalse(self.p.call_nowait('foobar'))
 
-    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
+    @patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
     def test_is_leader_exception(self):
         self.p.start()
         self.p.query = Mock(side_effect=psycopg2.OperationalError("not supported"))
@@ -534,14 +565,15 @@ class TestPostgresql(unittest.TestCase):
             lines = f.readlines()
             self.assertTrue('host replication replicator 127.0.0.1/32 md5\n' in lines)
 
-    def test_custom_bootstrap(self):
+    @patch.object(Postgresql, 'cancellable_subprocess_call')
+    def test_custom_bootstrap(self, mock_cancellable_subprocess_call):
         config = {'method': 'foo', 'foo': {'command': 'bar'}}
-        with patch('subprocess.call', Mock(return_value=1)):
-            self.assertFalse(self.p.bootstrap(config))
-        with patch('subprocess.call', Mock(side_effect=Exception)):
-            self.assertFalse(self.p.bootstrap(config))
-        with patch('subprocess.call', Mock(return_value=0)),\
-                patch('subprocess.Popen', Mock(side_effect=Exception("42"))),\
+
+        mock_cancellable_subprocess_call.return_value = 1
+        self.assertFalse(self.p.bootstrap(config))
+
+        mock_cancellable_subprocess_call.return_value = 0
+        with patch('subprocess.Popen', Mock(side_effect=Exception("42"))),\
                 patch('os.path.isfile', Mock(return_value=True)),\
                 patch('os.unlink', Mock()),\
                 patch.object(Postgresql, 'save_configuration_files', Mock()),\
@@ -557,6 +589,9 @@ class TestPostgresql(unittest.TestCase):
                 self.p.bootstrap(config)
             self.assertEqual(str(e.exception), '42')
 
+        mock_cancellable_subprocess_call.side_effect = Exception
+        self.assertFalse(self.p.bootstrap(config))
+
     @patch('time.sleep', Mock())
     @patch('os.unlink', Mock())
     @patch.object(Postgresql, 'run_bootstrap_post_init', Mock(return_value=True))
@@ -584,26 +619,27 @@ class TestPostgresql(unittest.TestCase):
             self.p.post_bootstrap({}, task)
             mock_restart.assert_called_once()
 
-    def test_run_bootstrap_post_init(self):
-        with patch('subprocess.call', Mock(return_value=1)):
-            self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+    @patch.object(Postgresql, 'cancellable_subprocess_call')
+    def test_run_bootstrap_post_init(self, mock_cancellable_subprocess_call):
+        mock_cancellable_subprocess_call.return_value = 1
+        self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
 
-        with patch('subprocess.call', Mock(side_effect=OSError)):
-            self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+        mock_cancellable_subprocess_call.return_value = 0
+        self.p._superuser.pop('username')
+        self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+        mock_cancellable_subprocess_call.assert_called()
+        args, kwargs = mock_cancellable_subprocess_call.call_args
+        self.assertTrue('PGPASSFILE' in kwargs['env'])
+        self.assertEquals(args[0], ['/bin/false', 'postgres://127.0.0.2:5432/postgres'])
 
-        with patch('subprocess.call', Mock(return_value=0)) as mock_method:
-            self.p._superuser.pop('username')
-            self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
-            mock_method.assert_called()
-            args, kwargs = mock_method.call_args
-            self.assertTrue('PGPASSFILE' in kwargs['env'])
-            self.assertEquals(args[0], ['/bin/false', 'postgres://127.0.0.2:5432/postgres'])
+        mock_cancellable_subprocess_call.reset_mock()
+        self.p._local_address.pop('host')
+        self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+        mock_cancellable_subprocess_call.assert_called()
+        self.assertEquals(mock_cancellable_subprocess_call.call_args[0][0], ['/bin/false', 'postgres://:5432/postgres'])
 
-            mock_method.reset_mock()
-            self.p._local_address.pop('host')
-            self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
-            mock_method.assert_called()
-            self.assertEquals(mock_method.call_args[0][0], ['/bin/false', 'postgres://:5432/postgres'])
+        mock_cancellable_subprocess_call.side_effect = OSError
+        self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
 
     @patch('patroni.postgresql.Postgresql.create_replica', Mock(return_value=0))
     def test_clone(self):
@@ -766,22 +802,24 @@ class TestPostgresql(unittest.TestCase):
                 self.assertFalse(self.p.wait_for_startup(timeout=2))
                 self.assertEquals(state['sleeps'], 3)
 
+        with patch.object(Postgresql, 'check_startup_state_changed', Mock(return_value=False)):
+            self.p.cancel()
+            self.p._state = 'starting'
+            self.assertIsNone(self.p.wait_for_startup())
+
     def test_read_pid_file(self):
         pidfile = os.path.join(self.data_dir, 'postmaster.pid')
         if os.path.exists(pidfile):
             os.remove(pidfile)
-        self.assertEquals(self.p.read_pid_file(), {})
-
-    @patch('os.kill')
-    def test_is_pid_running(self, mock_kill):
-        mock_kill.return_value = True
-        self.assertTrue(self.p.is_pid_running(-100))
-        self.assertFalse(self.p.is_pid_running(0))
-        self.assertFalse(self.p.is_pid_running(None))
+        self.assertEquals(self.p._read_pid_file(), {})
+        with open(pidfile, 'w') as fd:
+            fd.write("123\n/foo/bar\n123456789\n5432")
+        self.assertEquals(self.p._read_pid_file(), {"pid": "123", "data_dir": "/foo/bar",
+                                                    "start_time": "123456789", "port": "5432"})
 
     def test_pick_sync_standby(self):
         cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None,
-                          SyncState(0, self.me.name, self.leadermem.name))
+                          SyncState(0, self.me.name, self.leadermem.name), None)
 
         with patch.object(Postgresql, "query", return_value=[
                     (self.leadermem.name, 'streaming', 'sync'),
@@ -847,39 +885,72 @@ class TestPostgresql(unittest.TestCase):
         self.p.set_synchronous_standby('foo')
         self.p.get_server_parameters(config)
 
-    @patch.object(Postgresql, 'read_pid_file', Mock(return_value={'pid': 'z'}))
-    def test_get_pid(self):
-        self.p.get_pid()
-
-    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
-    @patch.object(Postgresql, '_signal_postmaster_stop', Mock(return_value=(123, None)))
-    @patch.object(Postgresql, 'get_pid', Mock(return_value=123))
     @patch('time.sleep', Mock())
-    @patch.object(Postgresql, 'is_pid_running')
-    def test__wait_for_connection_close(self, mock_is_pid_running):
-        mock_is_pid_running.side_effect = [True, False, False]
-        mock_callback = Mock()
-        self.p.stop(on_safepoint=mock_callback)
-
-        mock_is_pid_running.side_effect = [True, False, False]
-        with patch.object(MockCursor, "execute", Mock(side_effect=psycopg2.Error)):
+    def test__wait_for_connection_close(self):
+        mock_postmaster = MockPostmaster()
+        with patch.object(Postgresql, 'is_running', Mock(return_value=mock_postmaster)):
+            mock_postmaster.is_running.side_effect = [True, False, False]
+            mock_callback = Mock()
             self.p.stop(on_safepoint=mock_callback)
 
-    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
-    @patch.object(Postgresql, '_signal_postmaster_stop', Mock(return_value=(123, None)))
-    @patch.object(Postgresql, 'get_pid', Mock(return_value=123))
-    @patch.object(Postgresql, 'is_pid_running', Mock(return_value=False))
-    @patch('psutil.Process')
-    def test__wait_for_user_backends_to_close(self, mock_psutil):
-        child = Mock()
-        child.cmdline.return_value = ['foo']
-        mock_psutil.return_value.children.return_value = [child]
-        mock_callback = Mock()
-        self.p.stop(on_safepoint=mock_callback)
+            mock_postmaster.is_running.side_effect = [True, False, False]
+            with patch.object(MockCursor, "execute", Mock(side_effect=psycopg2.Error)):
+                self.p.stop(on_safepoint=mock_callback)
 
-    @patch('os.kill', Mock(side_effect=[OSError(errno.ESRCH, ''), OSError]))
-    @patch('time.sleep', Mock())
-    @patch.object(Postgresql, 'is_pid_running', Mock(side_effect=[True, False]))
     def test_terminate_starting_postmaster(self):
-        self.p.terminate_starting_postmaster(123)
-        self.p.terminate_starting_postmaster(123)
+        mock_postmaster = MockPostmaster()
+        self.p.terminate_starting_postmaster(mock_postmaster)
+        mock_postmaster.signal_stop.assert_called()
+        mock_postmaster.wait.assert_called()
+
+    def test_read_postmaster_opts(self):
+        m = mock_open(read_data='/usr/lib/postgres/9.6/bin/postgres "-D" "data/postgresql0" \
+"--listen_addresses=127.0.0.1" "--port=5432" "--hot_standby=on" "--wal_level=hot_standby" \
+"--wal_log_hints=on" "--max_wal_senders=5" "--max_replication_slots=5"\n')
+        with patch.object(builtins, 'open', m):
+            data = self.p.read_postmaster_opts()
+            self.assertEquals(data['wal_level'], 'hot_standby')
+            self.assertEquals(int(data['max_replication_slots']), 5)
+            self.assertEqual(data.get('D'), None)
+
+            m.side_effect = IOError
+            data = self.p.read_postmaster_opts()
+            self.assertEqual(data, dict())
+
+    @patch('subprocess.Popen')
+    def test_single_user_mode(self, subprocess_popen_mock):
+        subprocess_popen_mock.return_value.wait.return_value = 0
+        self.assertEquals(self.p.single_user_mode('CHECKPOINT', {'archive_mode': 'on'}), 0)
+
+    @patch('os.listdir', Mock(side_effect=[OSError, ['a', 'b']]))
+    @patch('os.unlink', Mock(side_effect=OSError))
+    @patch('os.remove', Mock())
+    @patch('os.path.islink', Mock(side_effect=[True, False]))
+    @patch('os.path.isfile', Mock(return_value=True))
+    def test_cleanup_archive_status(self):
+        self.p.cleanup_archive_status()
+        self.p.cleanup_archive_status()
+
+    @patch('os.unlink', Mock())
+    @patch('os.path.isfile', Mock(return_value=True))
+    @patch.object(Postgresql, 'single_user_mode', Mock(return_value=0))
+    def test_fix_cluster_state(self):
+        self.assertTrue(self.p.fix_cluster_state())
+
+    def test_replica_cached_timeline(self):
+        self.assertEquals(self.p.replica_cached_timeline(1), 2)
+
+    def test_get_master_timeline(self):
+        self.assertEquals(self.p.get_master_timeline(), 1)
+
+    def test_cancellable_subprocess_call(self):
+        self.p.cancel()
+        self.assertRaises(PostgresException, self.p.cancellable_subprocess_call)
+
+    @patch('patroni.postgresql.polling_loop', Mock(return_value=[0, 0]))
+    def test_cancel(self):
+        self.p._cancellable = Mock()
+        self.p._cancellable.returncode = None
+        self.p.cancel()
+        type(self.p._cancellable).returncode = PropertyMock(side_effect=[None, -15])
+        self.p.cancel()

tests/test_postmaster.py

@@ -0,0 +1,78 @@
+import unittest
+
+from mock import Mock, patch
+from patroni.postmaster import PostmasterProcess
+import psutil
+
+
+class TestPostmasterProcess(unittest.TestCase):
+    @patch('psutil.Process.__init__', Mock())
+    def test_init(self):
+        proc = PostmasterProcess(-123)
+        self.assertTrue(proc.is_single_user)
+
+    @patch('psutil.Process.create_time')
+    @patch('psutil.Process.__init__')
+    def test_from_pidfile(self, mock_init, mock_create_time):
+        mock_init.side_effect = psutil.NoSuchProcess(123)
+        self.assertEquals(PostmasterProcess.from_pidfile({}), None)
+        self.assertEquals(PostmasterProcess.from_pidfile({"pid": "foo"}), None)
+        self.assertEquals(PostmasterProcess.from_pidfile({"pid": "123"}), None)
+
+        mock_init.side_effect = None
+        with patch.object(psutil.Process, 'pid', 123), \
+                patch.object(psutil.Process, 'parent', return_value=124), \
+                patch('os.getpid', return_value=125) as mock_ospid, \
+                patch('os.getppid', return_value=126):
+
+            self.assertNotEquals(PostmasterProcess.from_pidfile({"pid": "123"}), None)
+
+            mock_create_time.return_value = 100000
+            self.assertEquals(PostmasterProcess.from_pidfile({"pid": "123", "start_time": "200000"}), None)
+            self.assertNotEquals(PostmasterProcess.from_pidfile({"pid": "123", "start_time": "foobar"}), None)
+
+            mock_ospid.return_value = 123
+            self.assertEquals(PostmasterProcess.from_pidfile({"pid": "123", "start_time": "100000"}), None)
+
+    @patch('psutil.Process.__init__')
+    def test_from_pid(self, mock_init):
+        mock_init.side_effect = psutil.NoSuchProcess(123)
+        self.assertEquals(PostmasterProcess.from_pid(123), None)
+        mock_init.side_effect = None
+        self.assertNotEquals(PostmasterProcess.from_pid(123), None)
+
+    @patch('psutil.Process.__init__', Mock())
+    @patch('psutil.Process.send_signal')
+    @patch('psutil.Process.pid', Mock(return_value=123))
+    def test_signal_stop(self, mock_send_signal):
+        proc = PostmasterProcess(-123)
+        self.assertEquals(proc.signal_stop('immediate'), False)
+
+        mock_send_signal.side_effect = [None, psutil.NoSuchProcess(123), psutil.AccessDenied()]
+        proc = PostmasterProcess(123)
+        self.assertEquals(proc.signal_stop('immediate'), None)
+        self.assertEquals(proc.signal_stop('immediate'), True)
+        self.assertEquals(proc.signal_stop('immediate'), False)
+
+    @patch('psutil.Process.__init__', Mock())
+    @patch('psutil.wait_procs')
+    def test_wait_for_user_backends_to_close(self, mock_wait):
+        c1 = Mock()
+        c1.cmdline = Mock(return_value=["postgres: startup process"])
+        c2 = Mock()
+        c2.cmdline = Mock(return_value=["postgres: postgres postgres [local] idle"])
+        with patch('psutil.Process.children', Mock(return_value=[c1, c2])):
+            proc = PostmasterProcess(123)
+            self.assertIsNone(proc.wait_for_user_backends_to_close())
+            mock_wait.assert_called_with([c2])
+
+    @patch('subprocess.Popen')
+    @patch.object(PostmasterProcess, 'from_pid')
+    def test_start(self, mock_frompid, mock_popen):
+        mock_frompid.return_value = "proc 123"
+        mock_popen.return_value.stdout.readline.return_value = '123'
+        self.assertEquals(
+            PostmasterProcess.start('/bin/true', '/tmp/', '/tmp/test.conf', ['--foo=bar', '--bar=baz']),
+            "proc 123"
+        )
+        mock_frompid.assert_called_with(123)

tests/test_watchdog.py

@@ -132,8 +132,9 @@ class TestWatchdog(unittest.TestCase):
     def test_exceptions(self):
         wd = Watchdog({'ttl': 30, 'loop_wait': 10, 'watchdog': {'mode': 'bad'}})
         wd.impl.close = wd.impl.keepalive = Mock(side_effect=WatchdogError(''))
-        self.assertIsNone(wd.disable())
+        self.assertTrue(wd.activate())
         self.assertIsNone(wd.keepalive())
+        self.assertIsNone(wd.disable())
 
     @patch('platform.system', Mock(return_value='Linux'))
     def test_config_reload(self):
@@ -193,15 +194,24 @@ class TestLinuxWatchdogDevice(unittest.TestCase):
         self.assertRaises(WatchdogError, self.impl.set_timeout, -1)
 
     @patch('os.open', Mock(return_value=3))
-    @patch('fcntl.ioctl', Mock(return_value=-1))
+    @patch('fcntl.ioctl', Mock(side_effect=OSError))
     def test__ioctl(self):
         self.assertRaises(WatchdogError, self.impl.get_support)
         self.impl.open()
-        self.assertRaises(IOError, self.impl.get_support)
+        self.assertRaises(WatchdogError, self.impl.get_support)
 
     def test_is_healthy(self):
         self.assertFalse(self.impl.is_healthy)
 
+    @patch('os.open', Mock(return_value=3))
+    @patch('fcntl.ioctl', Mock(side_effect=OSError))
+    def test_error_handling(self):
+        self.impl.open()
+        self.assertRaises(WatchdogError, self.impl.get_timeout)
+        self.assertRaises(WatchdogError, self.impl.set_timeout, 10)
+        # We still try to output a reasonable string even if getting info errors
+        self.assertEquals(self.impl.describe(), "Linux watchdog device")
+
     @patch('os.open', Mock(side_effect=OSError))
     def test_open(self):
         self.assertRaises(WatchdogError, self.impl.open)

tests/test_zookeeper.py

@@ -58,11 +58,11 @@ class MockKazooClient(Mock):
             raise TypeError("Invalid type for 'path' (string expected)")
         if not isinstance(value, (six.binary_type,)):
             raise TypeError("Invalid type for 'value' (must be a byte string)")
-        if value == b'Exception':
+        if b'Exception' in value:
             raise Exception
         if path.endswith('/initialize') or path == '/service/test/optime/leader':
             raise Exception
-        elif value == b'retry' or (value == b'exists' and self.exists):
+        elif b'retry' in value or (b'exists' in value and self.exists):
             raise NodeExistsError
 
     def create_async(self, path, value=b"", acl=None, ephemeral=False, sequence=False, makepath=False):
@@ -76,10 +76,10 @@ class MockKazooClient(Mock):
             raise TypeError("Invalid type for 'value' (must be a byte string)")
         if path == '/service/bla/optime/leader':
             raise Exception
-        if path == '/service/test/members/bar' and value == b'retry':
+        if path == '/service/test/members/bar' and b'retry' in value:
             return
         if path in ('/service/test/failover', '/service/test/config', '/service/test/sync'):
-            if value == b'Exception':
+            if b'Exception' in value:
                 raise Exception
             elif value == b'ok':
                 return
@@ -145,7 +145,7 @@ class TestZooKeeper(unittest.TestCase):
         self.assertRaises(ZooKeeperError, self.zk.get_cluster)
         cluster = self.zk.get_cluster()
         self.assertIsInstance(cluster.leader, Leader)
-        self.zk.touch_member('foo')
+        self.zk.touch_member({'foo': 'foo'})
 
     def test_delete_leader(self):
         self.assertTrue(self.zk.delete_leader())
@@ -169,17 +169,17 @@ class TestZooKeeper(unittest.TestCase):
     def test_touch_member(self):
         self.zk._name = 'buzz'
         self.zk.get_cluster()
-        self.zk.touch_member('new')
+        self.zk.touch_member({'new': 'new'})
         self.zk._name = 'bar'
-        self.zk.touch_member('new')
+        self.zk.touch_member({'new': 'new'})
         self.zk._name = 'na'
         self.zk._client.exists = 1
-        self.zk.touch_member('Exception')
+        self.zk.touch_member({'Exception': 'Exception'})
         self.zk._name = 'bar'
-        self.zk.touch_member('retry')
+        self.zk.touch_member({'retry': 'retry'})
         self.zk._fetch_cluster = True
         self.zk.get_cluster()
-        self.zk.touch_member('retry')
+        self.zk.touch_member({'retry': 'retry'})
 
     def test_take_leader(self):
         self.zk.take_leader()
@@ -187,7 +187,7 @@ class TestZooKeeper(unittest.TestCase):
             self.zk.take_leader()
 
     def test_update_leader(self):
-        self.assertTrue(self.zk.update_leader())
+        self.assertTrue(self.zk.update_leader(None))
 
     def test_write_leader_optime(self):
         self.zk.last_leader_operation = '0'
@@ -217,3 +217,6 @@ class TestZooKeeper(unittest.TestCase):
         self.zk.set_sync_state_value('ok')
         self.zk.set_sync_state_value('Exception')
         self.zk.delete_sync_state()
+
+    def test_set_history_value(self):
+        self.zk.set_history_value('{}')