Release 1.4.3 (#635)

Bump version and update release notes

.travis.yml

@@ -1,17 +1,28 @@
-sudo: false
+sudo: true
 dist: trusty
 language: python
-python:
-- "3.4"  # 2.7 and 3.5 are preinstalled by default
 env:
   global:
-  - ETCDVERSION=3.0.17 ZKVERSION=3.4.9 CONSULVERSION=0.7.4
-  - PYVERSIONS="2.7 3.4 3.5"
-  matrix:
-  - TEST_SUITE="python setup.py"
-  - DCS="etcd" TEST_SUITE="behave"
-  - DCS="exhibitor" TEST_SUITE="behave"
-  - DCS="consul" TEST_SUITE="behave"
+  - ETCDVERSION=3.0.17 ZKVERSION=3.4.11 CONSULVERSION=0.7.4
+  - PYVERSIONS="2.7 3.5 3.6"
+  - EXCLUDE_BEHAVE="3.5"
+  - BOTO_CONFIG=/doesnotexist
+matrix:
+  include:
+  - python: "3.5"
+    env: TEST_SUITE="python setup.py"
+  - python: "3.6"
+    env: DCS="etcd" TEST_SUITE="behave"
+  - python: "3.6"
+    env: DCS="exhibitor" TEST_SUITE="behave"
+  - python: "3.6"
+    env: DCS="consul" TEST_SUITE="behave"
+  - python: "3.6"
+    env: DCS="kubernetes" TEST_SUITE="behave"
+branches:
+  only:
+  - master
+  - /^v\d+\.\d+(\.\d+)?$/
 cache:
   directories:
   - $HOME/mycache
@@ -19,7 +30,7 @@ before_cache:
 - |
   rm -fr $HOME/mycache/python*
   for pv in $PYVERSIONS; do
-      if [[ $TEST_SUITE != "behave" || $pv != "3.4" ]]; then
+      if [[ $TEST_SUITE != "behave" || $pv != $EXCLUDE_BEHAVE ]]; then
           fpv=$(basename $(readlink $HOME/virtualenv/python${pv}))
           mv $HOME/virtualenv/${fpv} $HOME/mycache/${fpv}
       fi
@@ -51,6 +62,26 @@ install:
             ln -s $EC etcd
         }
 
+        function get_kubernetes() {
+            wget -O localkube "https://storage.googleapis.com/minikube/k8sReleases/v1.7.0/localkube-linux-amd64"
+            chmod +x localkube
+            sudo nohup ./localkube --logtostderr=true --enable-dns=false > localkube.log 2>&1 &
+
+            echo "Waiting for localkube to start..."
+            if ! timeout 120 sh -c "while ! curl -ks http://127.0.0.1:8080/ >/dev/null; do sleep 1; done"; then
+                sudo cat localkube.log
+                echo "localkube did not start"
+                exit 1
+            fi
+            echo "Check certificate permissions"
+            sudo chmod 644 /var/lib/localkube/certs/*
+            sudo ls -altr /var/lib/localkube/certs/
+
+            echo "Set up .kube/config"
+            mkdir ~/.kube
+            echo -e "apiVersion: v1\nclusters:\n- cluster:\n    certificate-authority: /var/lib/localkube/certs/ca.crt\n    server: https://127.0.0.1:8443\n  name: local\ncontexts:\n- context:\n    cluster: local\n    user: myself\n  name: local\ncurrent-context: local\nkind: Config\npreferences: {}\nusers:\n- name: myself\n  user:\n    client-certificate: /var/lib/localkube/certs/apiserver.crt\n    client-key: /var/lib/localkube/certs/apiserver.key\n" > ~/.kube/config
+        }
+
         function get_exhibitor() {
             ZC=~/mycache/zookeeper-${ZKVERSION}
             if [[ ! -d $ZC ]]; then
@@ -65,7 +96,6 @@ install:
                 echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \
                     | nc -l 8181 &> /dev/null
             done&
-            ZK_PID=$!
         }
 
         attempt_num=1
@@ -77,7 +107,7 @@ install:
     fi
 
     for pv in $PYVERSIONS; do
-        if [[ $TEST_SUITE != "behave" || $pv != "3.4" ]]; then
+        if [[ $TEST_SUITE != "behave" || $pv != $EXCLUDE_BEHAVE ]]; then
             fpv=$(basename $(readlink $HOME/virtualenv/python$pv))
             if [[ -d ~/mycache/${fpv} ]]; then
                 mv ~/virtualenv/${fpv} ~/virtualenv/${fpv}.bckp
@@ -93,13 +123,17 @@ install:
 script:
   - |
     for pv in $PYVERSIONS; do
+        if [[ $TEST_SUITE == "behave" && $pv == $EXCLUDE_BEHAVE ]]; then
+            continue
+        fi
+
         source ~/virtualenv/python${pv}/bin/activate
 
         if [[ $TEST_SUITE != "behave" ]]; then
             echo Running unit tests using python${pv}
             $TEST_SUITE test
             $TEST_SUITE flake8
-        elif [[ $pv != "3.4" ]]; then
+        elif [[ $pv != $EXCLUDE_BEHAVE ]]; then
             echo Running acceptance tests using python${pv}
             if ! PATH=.:/usr/lib/postgresql/9.6/bin:$PATH $TEST_SUITE; then
                 # output all log files when tests are failing
@@ -112,7 +146,8 @@ script:
     set +e
 after_success:
   # before_cache is executed earlier than after_success, so we need to restore one of virtualenv directories
-  - fpv=$(basename $(readlink $HOME/virtualenv/python3.5)) && mv $HOME/mycache/${fpv} $HOME/virtualenv/${fpv}
+  - fpv=$(basename $(readlink $HOME/virtualenv/python3.6)) && mv $HOME/mycache/${fpv} $HOME/virtualenv/${fpv}
   - coveralls
   - if [[ $TEST_SUITE != "behave" ]]; then python-codacy-coverage -r coverage.xml; fi
-  - if [[ $DCS == "exhibitor" ]]; then ~/mycache/zookeeper-${ZKVERSION}/bin/zkServer.sh stop; kill -9 $ZK_PID; fi
+  - if [[ $DCS == "exhibitor" ]]; then ~/mycache/zookeeper-${ZKVERSION}/bin/zkServer.sh stop; fi
+  - sudo kill $(jobs -p)

README.rst

@@ -1,27 +1,27 @@
 |Build Status| |Coverage Status|
 
 Patroni: A Template for PostgreSQL HA with ZooKeeper, etcd or Consul
-------------------------------------------------------------
+--------------------------------------------------------------------
 
 You can find a version of this documentation that is searchable and also easier to navigate at `patroni.readthedocs.io <https://patroni.readthedocs.io>`__.
 
 
 There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
 
-Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__ or `Consul <https://github.com/hashicorp/consul>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
+Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__, `Consul <https://github.com/hashicorp/consul>`__ or `Kubernetes <https://kubernetes.io>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
 
 We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely.
 
-**Note to Kubernetes users**: We're currently developing Patroni to be as useful as possible for teams running Kubernetes on top of Google Compute Engine; Patroni can be the HA solution for Postgres in such an environment. To this end, there is a `Helm chart <https://github.com/kubernetes/charts/tree/master/incubator/patroni>`__ that uses Patroni and  `Spilo  <https://github.com/zalando/spilo/>`__ to provision a five-node PostgreSQL HA cluster in a Kubernetes+GCE environment. (The Helm chart deploys Spilo Docker images, not just "bare" Patroni.)
+**Note to Kubernetes users**: Patroni can run natively on top of Kubernetes. Take a look at the `Kubernetes <https://github.com/zalando/patroni/blob/master/docs/kubernetes.rst>`__ chapter of the Patroni documentation.
 
 .. contents::
     :local:
     :depth: 1
     :backlinks: none
 
-==============
+=================
 How Patroni Works
-==============
+=================
 
 Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features.
 
@@ -33,30 +33,60 @@ For additional background info, see:
 * `PostgreSQL HA with Kubernetes and Patroni <https://www.youtube.com/watch?v=iruaCgeG7qs>`__, talk by Josh Berkus at KubeCon 2016 (video)
 * `Feb. 2016 Zalando Tech blog post <https://tech.zalando.de/blog/zalandos-patroni-a-template-for-high-availability-postgresql/>`__
 
-================
+==================
 Development Status
-================
+==================
 
 Patroni is in active development and accepts contributions. See our `Contributing <https://github.com/zalando/patroni/blob/master/docs/CONTRIBUTING.rst>`__ section below for more details.
 
 We report new releases information `here <https://github.com/zalando/patroni/releases>`__.
 
-===========================
+===================================
 Technical Requirements/Installation
-===========================
+===================================
 
-**For Mac**
+**Pre-requirements for Mac OS**
 
 To install requirements on a Mac, run the following:
 
 ::
 
     brew install postgresql etcd haproxy libyaml python
-    pip install psycopg2 pyyaml
 
-===================
+**General installation for pip**
+
+Patroni can be installed with pip:
+
+::
+
+    pip install patroni[dependencies]
+
+where dependencies can be either empty, or consist of one or more of the following:
+
+etcd
+    `python-etcd` module in order to use Etcd as DCS
+consul
+    `python-consul` module in order to use Consul as DCS
+zookeeper
+    `kazoo` module in order to use Zookeeper as DCS
+exhibitor
+    `kazoo` module in order to use Exhibitor as DCS (same dependencies as for Zookeeper)
+kubernetes
+    `kubernetes` module in order to use Kubernetes as DCS in Patroni
+aws
+    `boto` in order to use AWS callbacks
+
+For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is:
+
+::
+
+    pip install patroni[etcd,aws]
+
+Note that external tools to call in the replica creation or custom bootstap scripts (i.e. WAL-E) should be installed independently of Patroni.
+
+=======================
 Running and Configuring
-===================
+=======================
 
 To get started, do the following from different terminals:
 ::
@@ -80,9 +110,9 @@ run:
 
     > psql --host 127.0.0.1 --port 5000 postgres
 
-===============
+==================
 YAML Configuration
-===============
+==================
 
 Go `here <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
 
@@ -92,15 +122,15 @@ Environment Configuration
 
 Go `here <https://github.com/zalando/patroni/blob/master/docs/ENVIRONMENT.rst>`__ for comprehensive information about configuring(overriding) settings via environment variables.
 
-===============
+===================
 Replication Choices
-===============
+===================
 
 Patroni uses Postgres' streaming replication, which is asynchronous by default. Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the leader. This setting should be increased or decreased based on business requirements. It's also possible to use synchronous replication for better durability guarantees. See `replication modes documentation <https://github.com/zalando/patroni/blob/master/docs/replication_modes.rst>`__ for details.
 
-===============================
+======================================
 Applications Should Not Use Superusers
-===============================
+======================================
 
 When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
 

docs/ENVIRONMENT.rst

@@ -1,6 +1,5 @@
 .. _environment:
 
-==================================
 Environment Configuration Settings
 ==================================
 
@@ -12,6 +11,8 @@ Global/Universal
 -  **PATRONI\_NAME**: name of the node where the current instance of Patroni is running. Must be unique for the cluster.
 -  **PATRONI\_NAMESPACE**: path within the configuration store where Patroni will keep information about the cluster. Default value: "/service"
 -  **PATRONI\_SCOPE**: cluster name
+-  **PATRONI\_LOGLEVEL**: sets the general logging level (see `the docs for Python logging <https://docs.python.org/3.6/library/logging.html#levels>`_)
+-  **PATRONI\_REQUESTS_LOGLEVEL**: sets the logging level for all HTTP requests e.g. Kubernetes API calls (see `the docs for Python logging <https://docs.python.org/3.6/library/logging.html#levels>`_)
 
 Bootstrap configuration
 -----------------------
@@ -39,6 +40,7 @@ Consul
 Etcd
 ----
 -  **PATRONI\_ETCD\_HOST**: the host:port for the etcd endpoint.
+-  **PATRONI\_ETCD\_HOSTS**: list of etcd endpoints in format host1:port1,host2:port2,etc...
 -  **PATRONI\_ETCD\_URL**: url for the etcd, in format: http(s)://(username:password@)host:port
 -  **PATRONI\_ETCD\_PROXY**: proxy url for the etcd. If you are connecting to the etcd using proxy, use this parameter instead of **PATRONI\_ETCD\_URL**
 -  **PATRONI\_ETCD\_SRV**: Domain to search the SRV record(s) for cluster autodiscovery.
@@ -51,6 +53,18 @@ Exhibitor
 -  **PATRONI\_EXHIBITOR\_HOSTS**: initial list of Exhibitor (ZooKeeper) nodes in format: 'host1,host2,etc...'. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes.
 -  **PATRONI\_EXHIBITOR\_PORT**: Exhibitor port.
 
+.. _kubernetes_environment:
+
+Kubernetes
+----------
+-  **PATRONI\_KUBERNETES\_NAMESPACE**: (optional) Kubernetes namespace where the Patroni pod is running. Default value is `default`.
+-  **PATRONI\_KUBERNETES\_LABELS**: Labels in format ``{label1: value1, label2: value2}``. These labels will be used to find existing objects (Pods and either Endpoints or ConfigMaps) associated with the current cluster. Also Patroni will set them on every object (Endpoint or ConfigMap) it creates.
+-  **PATRONI\_KUBERNETES\_SCOPE\_LABEL**: (optional) name of the label containing cluster name. Default value is `cluster-name`.
+-  **PATRONI\_KUBERNETES\_ROLE\_LABEL**: (optional) name of the label containing Postgres role (`master` or `replica`). Patroni will set this label on the pod it is running in. Default value is `role`.
+-  **PATRONI\_KUBERNETES\_USE\_ENDPOINTS**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
+-  **PATRONI\_KUBERNETES\_POD\_IP**: (optional) IP address of the pod Patroni is running in. This value is required when `PATRONI_KUBERNETES_USE_ENDPOINTS` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
+-  **PATRONI\_KUBERNETES\_PORTS**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service wont work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``PATRONI_KUBERNETES_PORTS='{[{"name": "postgresql", "port": 5432}]}'`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `PATRONI_KUBERNETES_USE_ENDPOINTS` is set.
+
 PostgreSQL
 ----------
 -  **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.

docs/README.rst

@@ -1,8 +1,8 @@
 .. _readme:
 
-=================
-How Patroni Works
-=================
+============
+Introduction
+============
 
 Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features.
 
@@ -13,30 +13,65 @@ For additional background info, see:
 * `PostgreSQL HA with Kubernetes and Patroni <https://www.youtube.com/watch?v=iruaCgeG7qs>`__, talk by Josh Berkus at KubeCon 2016 (video)
 * `Feb. 2016 Zalando Tech blog post <https://tech.zalando.de/blog/zalandos-patroni-a-template-for-high-availability-postgresql/>`__
 
-==================
+
 Development Status
-==================
+------------------
 
 Patroni is in active development and accepts contributions. See our :ref:`Contributing <contributing>` section below for more details.
 
 We report new releases information :ref:`here <releases>`.
 
-===================================
-Technical Requirements/Installation
-===================================
 
-**For Mac**
+Technical Requirements/Installation
+-----------------------------------
+
+**Pre-requirements for Mac OS**
 
 To install requirements on a Mac, run the following:
 
 ::
 
     brew install postgresql etcd haproxy libyaml python
-    pip install psycopg2 pyyaml
 
-=======================
+**General installation for pip**
+
+Patroni can be installed with pip:
+
+::
+
+    pip install patroni[dependencies]
+
+where dependencies can be either empty, or consist of one or more of the following:
+
+etcd
+    `python-etcd` module in order to use Etcd as DCS
+consul
+    `python-consul` module in order to use Consul as DCS
+zookeeper
+    `kazoo` module in order to use Zookeeper as DCS
+exhibitor
+    `kazoo` module in order to use Exhibitor as DCS (same dependencies as for Zookeeper)
+kubernetes
+    `kubernetes` module in order to use Kubernetes as DCS in Patroni
+aws
+    `boto` in order to use AWS callbacks
+
+For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is:
+
+::
+
+    pip install patroni[etcd,aws]
+
+Note that external tools to call in the replica creation or custom bootstap scripts (i.e. WAL-E) should be installed
+independently of Patroni.
+
+
 Running and Configuring
-=======================
+-----------------------
+
+The following section assumes Patroni repository as being cloned from https://github.com/zalando/patroni. Namely, you
+will need example configuration files `postgres0.yml` and `postgres1.yml`. If you installed Patroni with pip, you can
+obtain those files from the git repository and replace `./patroni.py` below with `patroni` command.
 
 To get started, do the following from different terminals:
 ::
@@ -60,27 +95,27 @@ run:
 
     > psql --host 127.0.0.1 --port 5000 postgres
 
-==================
+
 YAML Configuration
-==================
+------------------
 
 Go :ref:`here <settings>` for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
 
-=========================
+
 Environment Configuration
-=========================
+-------------------------
 
 Go :ref:`here <environment>` for comprehensive information about configuring(overriding) settings via environment variables.
 
-===================
+
 Replication Choices
-===================
+-------------------
 
 Patroni uses Postgres' streaming replication, which is asynchronous by default. Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the leader. This setting should be increased or decreased based on business requirements. It's also possible to use synchronous replication for better durability guarantees. See :ref:`replication modes documentation <replication_modes>` for details.
 
-======================================
+
 Applications Should Not Use Superusers
-======================================
+--------------------------------------
 
 When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
 

docs/SETTINGS.rst

@@ -63,9 +63,10 @@ Most of the parameters are optional, but you have to specify one of the **host**
 
 Etcd
 ----
-Most of the parameters are optional, but you have to specify one of the **host**, **url**, **proxy** or **srv**
+Most of the parameters are optional, but you have to specify one of the **host**, **hosts**, **url**, **proxy** or **srv**
 
 -  **host**: the host:port for the etcd endpoint.
+-  **hosts**: list of etcd endpoint in format host1:port1,host2:port2,etc... Could be a comma separated string or an actual yaml list.
 -  **url**: url for the etcd
 -  **proxy**: proxy url for the etcd. If you are connecting to the etcd using proxy, use this parameter instead of **url**
 -  **srv**: Domain to search the SRV record(s) for cluster autodiscovery.
@@ -82,6 +83,18 @@ Exhibitor
 -  **poll\_interval**: how often the list of ZooKeeper and Exhibitor nodes should be updated from Exhibitor
 -  **port**: Exhibitor port.
 
+.. _kubernetes_settings:
+
+Kubernetes
+----------
+-  **namespace**: (optional) Kubernetes namespace where Patroni pod is running. Default value is `default`.
+-  **labels**: Labels in format ``{label1: value1, label2: value2}``. These labels will be used to find existing objects (Pods and either Endpoints or ConfigMaps) associated with the current cluster. Also Patroni will set them on every object (Endpoint or ConfigMap) it creates.
+-  **scope\_label**: (optional) name of the label containing cluster name. Default value is `cluster-name`.
+-  **role\_label**: (optional) name of the label containing role (master or replica). Patroni will set this label on the pod it runs in. Default value is ``role``.
+-  **use\_endpoints**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
+-  **pod\_ip**: (optional) IP address of the pod Patroni is running in. This value is required when `use_endpoints` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
+-  **ports**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service wont work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``kubernetes.ports: {[{"name": "postgresql", "port": 5432}]}`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `kubernetes.use_endpoints` is set.
+
 .. _postgresql_settings:
 
 PostgreSQL

docs/conf.py

@@ -18,9 +18,11 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
 
+import sys
+sys.path.insert(0, os.path.abspath('..'))
+
+from patroni.version import __version__
 
 # -- General configuration ------------------------------------------------
 
@@ -51,7 +53,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'Patroni'
-copyright = '2016, Zalando SE'
+copyright = '2015 Compose, Zalando SE'
 author = 'Zalando SE'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -59,9 +61,9 @@ author = 'Zalando SE'
 # built documents.
 #
 # The short X.Y version.
-version = '1.2'
+version = __version__[:__version__.rfind('.')]
 # The full version, including alpha/beta/rc tags.
-release = '1.2.2'
+release = __version__
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/index.rst

@@ -6,11 +6,11 @@
 Introduction
 ============
 
-Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__ or `Consul <https://github.com/hashicorp/consul>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
+Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__, `Consul <https://github.com/hashicorp/consul>`__ or `Kubernetes <https://kubernetes.io>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
 
 We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely. There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
 
-**Note to Kubernetes users**: We're currently developing Patroni to be as useful as possible for teams running Kubernetes on top of Google Compute Engine; Patroni can be the HA solution for Postgres in such an environment. To this end, we've created a `Helm Chart <https://github.com/kubernetes/charts/tree/master/incubator/patroni>`__ that enables you to deploy a five-node Patroni cluster using a Kubernetes PetSet.
+**Note to Kubernetes users**: Patroni can run natively on top of Kubernetes. Take a look at the :ref:`Kubernetes <kubernetes>` chapter of the Patroni documentation.
 
 
 .. toctree::
@@ -24,6 +24,8 @@ We call Patroni a "template" because it is far from being a one-size-fits-all or
    replica_bootstrap
    replication_modes
    pause
+   kubernetes
+   watchdog
    releases
    CONTRIBUTING
 
@@ -33,5 +35,3 @@ Indices and tables
 * :ref:`genindex`
 * :ref:`modindex`
 * :ref:`search`
-
-

docs/kubernetes.rst

@@ -0,0 +1,53 @@
+.. _kubernetes:
+
+Using Patroni with Kubernetes
+=============================
+
+Patroni can use Kubernetes objects in order to store the state of the cluster and manage the leader key. That makes it
+capable of operating Postgres in Kubernetes environment without any consistency store, namely, one doesn't
+need to run an extra Etcd deployment. There are two different type of Kubernetes objects Patroni can use to store the
+leader and the configuration keys, they are configured with the `kubernetes.use_endpoints` or `PATRONI_KUBERNETES_USE_ENDPOINTS`
+environment variable.
+
+Use Endpoints
+-------------
+
+Despite the fact that this is the recommended mode, it is turned off by default for compatibility reasons. When it is on, Patroni stores
+the cluster configuration and the leader key in the `metadata: annotations` fields of the respective `Endpoints` it creates.
+Changing the leader is safer than when using `ConfigMaps`, since both the annotations, containing the leader information, and the actual addresses
+pointing to the running leader pod are updated simultaneously in one go.
+
+Use ConfigMaps
+--------------
+
+In this mode, Patroni will create ConfigMaps instead of Endpoints and store keys inside meta-data of those ConfigMaps.
+Changing the leader takes at least two updates, one to the leader ConfigMap and another to the respective Endpoint.
+
+There are two ways to direct the traffic to the Postgres master:
+
+- use the `callback script <https://github.com/zalando/patroni/blob/master/kubernetes/callback.py>`_ provided by Patroni
+- configure the Kubernetes Postgres service to use the label selector with the `role_label` (configured in patroni configuration).
+
+Note that in some cases, for instance, when running on OpenShift, there is no alternative to using ConfigMaps.
+
+Configuration
+-------------
+
+Patroni Kubernetes :ref:`settings <kubernetes_settings>` and :ref:`environment variables <kubernetes_environment>` are described in the general chapters of the documentation.
+
+Examples
+--------
+
+- The `kubernetes <https://github.com/zalando/patroni/tree/master/kubernetes>`__ folder of the Patroni repository contains
+  examples of the Docker image, the Kubernetes manifest and the callback script in order to test Patroni Kubernetes setup.
+  Note that in the current state it will not be able to use PersistentVolumes because of permission issues.
+
+- You can find the full-featured Docker image that can use Persistent Volumes in the
+  `Spilo Project <https://github.com/zalando/spilo>`_.
+
+- There is also a `Helm chart <https://github.com/unguiculus/charts/tree/feature/patroni/incubator/patroni>`_
+  to deploy the Spilo image configured with Patroni running using Kubernetes.
+
+- In order to run your database clusters at scale using Patroni and Spilo, take a look at the
+  `postgres-operator <https://github.com/zalando-incubator/postgres-operator>`_ project. It implements the operator pattern
+  to manage Spilo clusters.

docs/releases.rst

@@ -3,6 +3,171 @@
 Release notes
 =============
 
+Version 1.4.3
+-------------
+
+**Improvements in logging**
+
+- Make log level configurable from environment variables (Andy Newton, Keyvan Hedayati)
+
+  `PATRONI_LOGLEVEL` - sets the general logging level
+  `PATRONI_REQUESTS_LOGLEVEL` - sets the logging level for all HTTP requests e.g. Kubernetes API calls
+  See `the docs for Python logging <https://docs.python.org/3.6/library/logging.html#levels>` to get the names of possible log levels
+
+**Stability improvements and bug fixes**
+
+- Don't rediscover etcd cluster topology when watch timed out (Alexander Kukushkin)
+
+  If we have only one host in etcd configuration and exactly this host is not accessible, Patroni was starting discovery of cluster topology and never succeeding. Instead it should just switch to the next available node.
+
+- Write content of bootstrap.pg_hba into a pg_hba.conf after custom bootstrap (Alexander)
+
+  Now it behaves similarly to the usual bootstrap with `initdb`
+
+- Single user mode was waiting for user input and never finish (Alexander)
+
+  Regression was introduced in https://github.com/zalando/patroni/pull/576
+
+
+Version 1.4.2
+-------------
+
+**Improvements in patronictl**
+
+- Rename scheduled failover to scheduled switchover (Alexander Kukushkin)
+
+  Failover and switchover functions were separated in version 1.4, but `patronictl list` was still reporting `Scheduled failover` instead of `Scheduled switchover`.
+
+- Show information about pending restarts (Alexander)
+
+  In order to apply some configuration changes sometimes it is necessary to restart postgres. Patroni was already giving a hint about that in the REST API and when writing node status into DCS, but there were no easy way to display it.
+
+- Make show-config to work with cluster_name from config file (Alexander)
+
+  It works similar to the `patronictl edit-config`
+
+**Stability improvements**
+
+- Avoid calling pg_controldata during bootstrap (Alexander)
+
+  During initdb or custom bootstrap there is a time window when pgdata is not empty but pg_controldata has not been written yet. In such case pg_controldata call was failing with error messages.
+
+- Handle exceptions raised from psutil (Alexander)
+
+  cmdline is read and parsed every time when `cmdline()` method is called. It could happen that the process being examined
+  has already disappeared, in that case `NoSuchProcess` is raised.
+
+**Kubernetes support improvements**
+
+- Don't swallow errors from k8s API (Alexander)
+
+  A call to Kubernetes API could fail for a different number of reasons. In some cases such call should be retried, in some other cases we should log the error message and the exception stack trace. The change here will help debug Kubernetes permission issues.
+
+- Update Kubernetes example Dockerfile to install Patroni from the master branch (Maciej Szulik)
+
+  Before that it was using `feature/k8s`, which became outdated.
+
+- Add proper RBAC to run patroni on k8s (Maciej)
+
+  Add the Service account that is assigned to the pods of the cluster, the role that holds only the necessary permissions, and the rolebinding that connects the Service account and the Role.
+
+
+Version 1.4.1
+-------------
+
+**Fixes in patronictl**
+
+- Don't show current leader in suggested list of members to failover to. (Alexander Kukushkin)
+
+  patronictl failover could still work when there is leader in the cluster and it should be excluded from the list of member where it is possible to failover to.
+
+- Make patronictl switchover compatible with the old Patroni api (Alexander)
+
+  In case if POST /switchover REST API call has failed with status code 501 it will do it once again, but to /failover endpoint.
+
+
+Version 1.4
+-----------
+
+This version adds support for using Kubernetes as a DCS, allowing to run Patroni as a cloud-native agent in Kubernetes without any additional deployments of Etcd, Zookeeper or Consul.
+
+**Upgrade notice**
+
+Installing Patroni via pip will no longer bring in dependencies for (such as libraries for Etcd, Zookeper, Consul or Kubernetes, or support for AWS). In order to enable them one need to list them in pip install command explicitely, for instance `pip install patroni[etcd,kubernetes]`.
+
+**Kubernetes support**
+
+Implement Kubernetes-based DCS. The endpoints meta-data is used in order to store the configuration and the leader key. The meta-data field inside the pods definition is used to store the member-related data.
+In addition to using Endpoints, Patroni supports ConfigMaps. You can find more information about this feature in the :ref:`Kubernetes chapter of the documentation <kubernetes>`
+
+**Stability improvements**
+
+- Factor out postmaster process into a separate object (Ants Aasma)
+
+  This object identifies a running postmaster process via pid and start time and simplifies detection (and resolution) of situations when the postmaster was restarted behind our back or when postgres directory disappeared from the file system.
+
+- Minimize the amount of SELECT's issued by Patroni on every loop of HA cylce (Alexander Kukushkin)
+
+  On every iteration of HA loop Patroni needs to know recovery status and absolute wal position. From now on Patroni will run only single SELECT to get this information instead of two on the replica and three on the master. 
+  
+- Remove leader key on shutdown only when we have the lock (Ants)
+
+  Unconditional removal was generating unnecessary and missleading exceptions.
+
+**Improvements in patronictl**
+
+- Add version command to patronictl (Ants)
+
+  It will show the version of installed Patroni and versions of running Patroni instances (if the cluster name is specified).
+  
+- Make optional specifying cluster_name argument for some of patronictl commands (Alexander, Ants)
+
+  It will work if patronictl is using usual Patroni configuration file with the ``scope`` defined.
+
+- Show information about scheduled switchover and maintenance mode (Alexander)
+
+  Before that it was possible to get this information only from Patroni logs or directly from DCS.
+  
+- Improve ``patronictl reinit`` (Alexander)
+
+  Sometimes ``patronictl reinit`` refused to proceed when Patroni was busy with other actions, namely trying to start postgres. `patronictl` didn't provide any commands to cancel such long running actions and the only (dangerous) workarond was removing a data directory manually. The new implementation of `reinit` forcefully cancells other long-running actions before proceeding with reinit.
+  
+- Implement ``--wait`` flag in ``patronictl pause`` and ``patronictl resume`` (Alexander)
+
+  It will make ``patronictl`` wait until the requested action is acknowledged by all nodes in the cluster.
+  Such behaviour is achieved by exposing the ``pause`` flag for every node in DCS and via the REST API.
+
+- Rename ``patronictl failover`` into ``patronictl switchover`` (Alexander)
+
+  The previous ``failover`` was actually only capable of doing a switchover; it refused to proceed in a cluster without the leader.
+
+- Alter the behavior of ``patronictl failover`` (Alexander)
+
+  It will work even if there is no leader, but in that case you will have to explicitely specify a node which should become the new leader.
+
+**Expose information about timeline and history**
+
+- Expose current timeline in DCS and via API (Alexander)
+
+  Store information about the current timeline for each member of the cluster. This information is accessible via the API and is stored in the DCS
+
+- Store promotion history in the /history key in DCS (Alexander)
+
+  In addition, store the timeline history enriched with the timestamp of the corresponding promotion in the /history key in DCS and update it with each promote.
+
+**Add endpoints for getting synchronous and asynchronous replicas**
+
+- Add new /sync and /async endpoints (Alexander, Oleksii Kliukin)
+
+ Those endpoints (also accessible as /synchronous and /asynchronous) return 200 only for synchronous and asynchornous replicas correspondingly (exclusing those marked as `noloadbalance`).
+
+**Allow multiple hosts for Etcd**
+
+- Add a new `hosts` parameter to Etcd configuration (Alexander)
+
+  This parameter should contain the initial list of hosts that will be used to discover and populate the list of the running etcd cluster members. If for some reason during work this list of discovered hosts is exhausted (no available hosts from that list), Patroni will return to the initial list from the `hosts` parameter.
+
+
 Version 1.3.6
 -------------
 
@@ -24,7 +189,7 @@ Version 1.3.6
 
 **Consul improvements**
 
-- Make it possible to provide datacenter configuration for Consul (DeathBorn, Alexander)
+- Make it possible to provide datacenter configuration for Consul (Vilius Okockis, Alexander)
 
   Before that Patroni was always communicating with datacenter of the host it runs on.
 

docs/replica_bootstrap.rst

@@ -95,4 +95,3 @@ running master or replicas. In that case, an empty string will be passed in a co
 restoring the formerly running cluster from the binary backup.
 
 If all replica creation methods fail, Patroni will try again all methods in order during the next event loop cycle.
-

docs/replication_modes.rst

@@ -65,4 +65,4 @@ On each HA loop iteration Patroni re-evaluates synchronous standby choice. If th
 
 .. [1] The data is still there, but recovering it requires a manual recovery effort by data recovery specialists. When Patroni is allowed to rewind with ``use_pg_rewind`` the forked timeline will be automatically erased to rejoin the failed master with the cluster.
 
-.. [2] Clients can change the behavior per transaction using PostgreSQL's ``synchronous_commit`` setting. Transactions with ``synchronous_commit`` values of ``off`` and ``local`` may be lost on fail over, but will not be blocked by replication delays.
+.. [2] Clients can change the behavior per transaction using PostgreSQL's ``synchronous_commit`` setting. Transactions with ``synchronous_commit`` values of ``off`` and ``local`` may be lost on fail over, but will not be blocked by replication delays.

docs/watchdog.rst

@@ -1,6 +1,5 @@
 .. _watchdog:
 
-================
 Watchdog support
 ================
 

extras/confd/templates/haproxy.tmpl

@@ -10,19 +10,23 @@ defaults
 	timeout server 30m
 	timeout check 5s
 
-frontend master_postgresql
+listen stats
+	mode http
+	bind *:7000
+	stats enable
+	stats uri /
+
+listen master
 	bind *:5000
-	default_backend backend_master
-
-frontend replicas_postgresql
-	bind *:5001
-	default_backend backend_replicas
-
-backend backend_master
 	option httpchk OPTIONS /master
+	http-check expect status 200
+	default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
 {{range gets "/members/*"}}	server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}}
 {{end}}
-backend backend_replicas
+listen replicas
+	bind *:5001
 	option httpchk OPTIONS /replica
+	http-check expect status 200
+	default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
 {{range gets "/members/*"}}	server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}}
 {{end}}

features/basic_replication.feature

@@ -19,11 +19,21 @@ Feature: basic replication
     And I run patronictl.py restart batman postgres1 --force
     Then I receive a response returncode 0
     And "sync" key in DCS has sync_standby=postgres2 after 10 seconds
+    And I sleep for 2 seconds
+    When I issue a GET request to http://127.0.0.1:8010/sync
+    Then I receive a response code 200
+    When I issue a GET request to http://127.0.0.1:8009/async
+    Then I receive a response code 200
 
   Scenario: check the basic failover in synchronous mode
-    When I kill postgres0
-    Then postgres2 role is the primary after 22 seconds
-    When I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_mode": null, "master_start_timeout": 0}
+    Given I run patronictl.py pause batman
+    Then I receive a response returncode 0
+    When I sleep for 2 seconds
+    And I shut down postgres0
+    And I run patronictl.py resume batman 
+    Then I receive a response returncode 0
+    And postgres2 role is the primary after 24 seconds
+    When I issue a PATCH request to http://127.0.0.1:8010/config with {"synchronous_mode": null, "master_start_timeout": 0}
     Then I receive a response code 200
     When I add the table bar to postgres2
     Then table bar is present on postgres1 after 20 seconds

features/environment.py

@@ -7,6 +7,7 @@ import kazoo.exceptions
 import os
 import psutil
 import psycopg2
+import json
 import shutil
 import signal
 import six
@@ -98,6 +99,7 @@ class PatroniController(AbstractController):
         else:
             self.watchdog = None
 
+        self._scope = (custom_config or {}).get('scope', 'batman')
         self._config = self._make_patroni_test_config(name, custom_config)
         self._closables = []
 
@@ -125,6 +127,9 @@ class PatroniController(AbstractController):
     def _start(self):
         if self.watchdog:
             self.watchdog.start()
+        if isinstance(self._context.dcs_ctl, KubernetesController):
+            self._context.dcs_ctl.create_pod(self._name[8:], self._scope)
+            os.environ['PATRONI_KUBERNETES_POD_IP'] = '10.0.0.' + self._name[-1]
         return subprocess.Popen(['coverage', 'run', '--source=patroni', '-p', 'patroni.py', self._config],
                                 stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory)
 
@@ -132,6 +137,8 @@ class PatroniController(AbstractController):
         if postgres:
             return subprocess.call(['pg_ctl', '-D', self._data_dir, 'stop', '-mi', '-w'])
         super(PatroniController, self).stop(kill, timeout)
+        if isinstance(self._context.dcs_ctl, KubernetesController):
+            self._context.dcs_ctl.delete_pod(self._name[8:])
         if self.watchdog:
             self.watchdog.stop()
 
@@ -147,7 +154,7 @@ class PatroniController(AbstractController):
 
         with open(patroni_config_name) as f:
             config = yaml.safe_load(f)
-            config.pop('etcd')
+            config.pop('etcd', None)
 
         host = config['postgresql']['listen'].split(':')[0]
 
@@ -335,10 +342,6 @@ class AbstractDcsController(AbstractController):
     def query(self, key, scope='batman'):
         """ query for a value of a given key """
 
-    @abc.abstractmethod
-    def set(self, key, value):
-        """ set a value to a given key """
-
     @abc.abstractmethod
     def cleanup_service_tree(self):
         """ clean all contents stored in the tree used for the tests """
@@ -388,9 +391,6 @@ class ConsulController(AbstractDcsController):
         _, value = self._client.kv.get(self.path(key, scope))
         return value and value['Value'].decode('utf-8')
 
-    def set(self, key, value):
-        self._client.kv.put(self.path(key), value)
-
     def cleanup_service_tree(self):
         self._client.kv.delete(self.path(scope=''), recurse=True)
 
@@ -417,9 +417,6 @@ class EtcdController(AbstractDcsController):
         except etcd.EtcdKeyNotFound:
             return None
 
-    def set(self, key, value):
-        self._client.set(self.path(key), value)
-
     def cleanup_service_tree(self):
         try:
             self._client.delete(self.path(scope=''), recursive=True)
@@ -436,6 +433,76 @@ class EtcdController(AbstractDcsController):
             return False
 
 
+class KubernetesController(AbstractDcsController):
+
+    def __init__(self, context):
+        super(KubernetesController, self).__init__(context)
+        self._namespace = 'default'
+        self._labels = {"application": "patroni"}
+        self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
+        os.environ['PATRONI_KUBERNETES_LABELS'] = json.dumps(self._labels)
+        os.environ['PATRONI_KUBERNETES_USE_ENDPOINTS'] = 'true'
+
+        from kubernetes import client as k8s_client, config as k8s_config
+        k8s_config.load_kube_config(context='local')
+        self._client = k8s_client
+        self._api = self._client.CoreV1Api()
+
+    def _start(self):
+        pass
+
+    def create_pod(self, name, scope):
+        labels = self._labels.copy()
+        labels['cluster-name'] = scope
+        metadata = self._client.V1ObjectMeta(namespace=self._namespace, name=name, labels=labels)
+        spec = self._client.V1PodSpec(containers=[self._client.V1Container(name=name, image='empty')])
+        body = self._client.V1Pod(metadata=metadata, spec=spec)
+        self._api.create_namespaced_pod(self._namespace, body)
+
+    def delete_pod(self, name):
+        try:
+            self._api.delete_namespaced_pod(name, self._namespace, self._client.V1DeleteOptions())
+        except:
+            pass
+        while True:
+            try:
+                self._api.read_namespaced_pod(name, self._namespace)
+            except:
+                break
+
+    def query(self, key, scope='batman'):
+        if key.startswith('members/'):
+            pod = self._api.read_namespaced_pod(key[8:], self._namespace)
+            return (pod.metadata.annotations or {}).get('status', '')
+        else:
+            try:
+                e = self._api.read_namespaced_endpoints(scope + ('' if key == 'leader' else '-' + key), self._namespace)
+                if key == 'leader':
+                    return e.metadata.annotations[key]
+                else:
+                    return json.dumps(e.metadata.annotations)
+            except:
+                return None
+
+    def cleanup_service_tree(self):
+        try:
+            self._api.delete_collection_namespaced_pod(self._namespace, label_selector=self._label_selector)
+        except:
+            pass
+        try:
+            self._api.delete_collection_namespaced_endpoints(self._namespace, label_selector=self._label_selector)
+        except:
+            pass
+
+        while True:
+            result = self._api.list_namespaced_pod(self._namespace, label_selector=self._label_selector)
+            if len(result.items) < 1:
+                break
+
+    def _is_running(self):
+        return True
+
+
 class ZooKeeperController(AbstractDcsController):
 
     """ handles all zookeeper related tasks, used for the tests setup and cleanup """
@@ -455,9 +522,6 @@ class ZooKeeperController(AbstractDcsController):
         except kazoo.exceptions.NoNodeError:
             return None
 
-    def set(self, key, value):
-        self._client.set(self.path(key), value.encode('utf-8'))
-
     def cleanup_service_tree(self):
         try:
             self._client.delete(self.path(scope=''), recursive=True)
@@ -713,6 +777,7 @@ class WatchdogMonitor(object):
 
 # actions to execute on start/stop of the tests and before running invidual features
 def before_all(context):
+    os.environ.update({'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password'})
     context.ci = 'TRAVIS_BUILD_NUMBER' in os.environ or 'BUILD_NUMBER' in os.environ
     context.timeout_multiplier = 2 if context.ci else 1
     context.pctl = PatroniPoolController(context)

features/patroni_api.feature

@@ -13,17 +13,17 @@ Scenario: check API requests on a stand-alone server
 	When I run patronictl.py reinit batman postgres0 --force
 	Then I receive a response returncode 0
 	And I receive a response output "Failed: reinitialize for member postgres0, status code=503, (I am the leader, can not reinitialize)"
-	When I run patronictl.py failover batman --master postgres0 --force
+	When I run patronictl.py switchover batman --master postgres0 --force
 	Then I receive a response returncode 1
-	And I receive a response output "Error: No candidates found to failover to"
-	When I issue a POST request to http://127.0.0.1:8008/failover with {"leader": "postgres0"}
-	Then I receive a response code 500
-	And I receive a response text failover is not possible: cluster does not have members except leader
+	And I receive a response output "Error: No candidates found to switchover to"
+	When I issue a POST request to http://127.0.0.1:8008/switchover with {"leader": "postgres0"}
+	Then I receive a response code 412
+	And I receive a response text switchover is not possible: cluster does not have members except leader
 	When I issue an empty POST request to http://127.0.0.1:8008/failover
 	Then I receive a response code 400
 	When I issue a POST request to http://127.0.0.1:8008/failover with {"foo": "bar"}
 	Then I receive a response code 400
-	And I receive a response text "No values given for required parameters leader and candidate"
+	And I receive a response text "Failover could be performed only to a specific candidate"
 
 Scenario: check local configuration reload
 	Given I issue an empty POST request to http://127.0.0.1:8008/reload
@@ -64,21 +64,21 @@ Scenario: check API requests for the primary-replica pair in the pause mode
 	When I sleep for 10 seconds
 	Then postgres1 role is the secondary after 15 seconds
 
-Scenario: check the failover via the API in the pause mode
-	Given I issue a POST request to http://127.0.0.1:8008/failover with {"leader": "postgres0", "candidate": "postgres1"}
+Scenario: check the switchover via the API in the pause mode
+	Given I issue a POST request to http://127.0.0.1:8008/switchover with {"leader": "postgres0", "candidate": "postgres1"}
 	Then I receive a response code 200
 	And postgres1 is a leader after 5 seconds
 	And postgres1 role is the primary after 10 seconds
 	And postgres0 role is the secondary after 10 seconds
 	And replication works from postgres1 to postgres0 after 20 seconds
 
-Scenario: check the scheduled failover
-	Given I issue a scheduled failover from postgres1 to postgres0 in 3 seconds
+Scenario: check the scheduled switchover
+	Given I issue a scheduled switchover from postgres1 to postgres0 in 3 seconds
 	Then I receive a response returncode 1
-	And I receive a response output "Can't schedule failover in the paused state"
+	And I receive a response output "Can't schedule switchover in the paused state"
 	When I run patronictl.py resume batman
 	Then I receive a response returncode 0
-	Given I issue a scheduled failover from postgres1 to postgres0 in 3 seconds
+	Given I issue a scheduled switchover from postgres1 to postgres0 in 3 seconds
 	Then I receive a response returncode 0
 	And postgres0 is a leader after 20 seconds
 	And postgres0 role is the primary after 10 seconds

features/steps/patroni_api.py

@@ -1,3 +1,4 @@
+import base64
 import json
 import os
 import parse
@@ -78,11 +79,13 @@ def do_post_empty(context, url):
 @step('I issue a {request_method:w} request to {url:url} with {data}')
 def do_request(context, request_method, url, data):
     data = data and json.loads(data) or {}
+    headers = {'Authorization': 'Basic ' + base64.b64encode('username:password'.encode('utf-8')).decode('utf-8'),
+               'Content-Type': 'application/json'}
     try:
         if request_method == 'PATCH':
-            r = requests.patch(url, json=data)
+            r = requests.patch(url, headers=headers, json=data)
         else:
-            r = requests.post(url, json=data)
+            r = requests.post(url, headers=headers, json=data)
     except requests.exceptions.RequestException:
         context.status_code = None
         context.response = None
@@ -122,10 +125,10 @@ def check_response(context, component, data):
         assert str(context.response[component]) == str(data), "{0} does not contain {1}".format(component, data)
 
 
-@step('I issue a scheduled failover from {from_host:w} to {to_host:w} in {in_seconds:d} seconds')
-def scheduled_failover(context, from_host, to_host, in_seconds):
+@step('I issue a scheduled switchover from {from_host:w} to {to_host:w} in {in_seconds:d} seconds')
+def scheduled_switchover(context, from_host, to_host, in_seconds):
     context.execute_steps(u"""
-        Given I run patronictl.py failover batman --master {0} --candidate {1} --scheduled "{2}" --force
+        Given I run patronictl.py switchover batman --master {0} --candidate {1} --scheduled "{2}" --force
     """.format(from_host, to_host, datetime.now(tzutc) + timedelta(seconds=int(in_seconds))))
 
 

kubernetes/Dockerfile

@@ -0,0 +1,32 @@
+FROM postgres:9.6
+MAINTAINER Alexander Kukushkin <[email protected]>
+
+RUN export DEBIAN_FRONTEND=noninteractive \
+    && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
+    && apt-get update -y \
+    && apt-get upgrade -y \
+    && apt-get install -y git curl jq python-psycopg2 python-yaml python-requests python-six python-pysocks \
+        python-dateutil python-pip python-prettytable python-wheel python-psutil python locales \
+
+    ## Make sure we have a en_US.UTF-8 locale available
+    && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
+
+    && pip install setuptools pip --upgrade \
+    && pip install 'git+https://github.com/zalando/patroni.git#egg=patroni[kubernetes]' \
+
+    && mkdir -p /home/postgres \
+    && chown postgres:postgres /home/postgres \
+
+    # Clean up
+    && apt-get remove -y git python-pip python-setuptools \
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/* /root/.cache
+
+ADD entrypoint.sh callback.py /
+
+EXPOSE 5432 8008
+ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
+USER postgres
+WORKDIR /home/postgres
+CMD ["/bin/bash", "/entrypoint.sh"]

kubernetes/callback.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+
+import logging
+import os
+import socket
+import sys
+import time
+
+from kubernetes import client as k8s_client, config as k8s_config
+from urllib3.exceptions import HTTPError
+from six.moves.http_client import HTTPException
+
+logger = logging.getLogger(__name__)
+
+
+class CoreV1Api(k8s_client.CoreV1Api):
+
+    def retry(func):
+        def wrapped(*args, **kwargs):
+            count = 0
+            while True:
+                try:
+                    return func(*args, **kwargs)
+                except (HTTPException, HTTPError, socket.error, socket.timeout):
+                    if count >= 10:
+                        raise
+                    logger.info('Throttling API requests...')
+                    time.sleep(2 ** count * 0.5)
+                    count += 1
+        return wrapped
+
+    @retry
+    def patch_namespaced_endpoints(self, *args, **kwargs):
+        return super(CoreV1Api, self).patch_namespaced_endpoints(*args, **kwargs)
+
+
+def patch_master_endpoint(api, namespace, cluster):
+    addresses = [k8s_client.V1EndpointAddress(ip=os.environ['POD_IP'])]
+    ports = [k8s_client.V1EndpointPort(port=5432)]
+    subsets = [k8s_client.V1EndpointSubset(addresses=addresses, ports=ports)]
+    body = k8s_client.V1Endpoints(subsets=subsets)
+    return api.patch_namespaced_endpoints(cluster, namespace, body)
+
+
+def main():
+    logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
+    if len(sys.argv) != 4 or sys.argv[1] not in ('on_start', 'on_stop', 'on_role_change'):
+        sys.exit('Usage: %s <action> <role> <cluster_name>', sys.argv[0])
+
+    action, role, cluster = sys.argv[1:4]
+
+    k8s_config.load_incluster_config()
+    k8s_api = CoreV1Api()
+
+    namespace = os.environ['KUBERNETES_NAMESPACE']
+
+    if role == 'master' and action in ('on_start', 'on_role_change'):
+        patch_master_endpoint(k8s_api, namespace, cluster)
+
+
+if __name__ == '__main__':
+    main()

kubernetes/entrypoint.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+cat > /home/postgres/patroni.yml <<__EOF__
+bootstrap:
+  dcs:
+    postgresql:
+      use_pg_rewind: true
+  initdb:
+  - auth-host: md5
+  - auth-local: trust
+  - encoding: UTF8
+  - locale: en_US.UTF-8
+  - data-checksums
+  pg_hba:
+  - host all all 0.0.0.0/0 md5
+  - host replication ${PATRONI_REPLICATION_USERNAME} ${POD_IP}/16    md5
+restapi:
+  connect_address: '${POD_IP}:8008'
+postgresql:
+  connect_address: '${POD_IP}:5432'
+  authentication:
+    superuser:
+      password: '${PATRONI_SUPERUSER_PASSWORD}'
+    replication:
+      password: '${PATRONI_REPLICATION_PASSWORD}'
+  callbacks:
+    on_start: /callback.py
+    on_stop: /callback.py
+    on_role_change: /callback.py
+__EOF__
+
+unset PATRONI_SUPERUSER_PASSWORD PATRONI_REPLICATION_PASSWORD
+export KUBERNETES_NAMESPACE=$PATRONI_KUBERNETES_NAMESPACE
+export POD_NAME=$PATRONI_NAME
+
+exec /usr/bin/python /usr/local/bin/patroni /home/postgres/patroni.yml

kubernetes/patroni_k8s.yaml

@@ -0,0 +1,182 @@
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  name: &cluster_name patronidemo
+  labels:
+    application: patroni
+    cluster-name: *cluster_name
+spec:
+  replicas: 3
+  serviceName: *cluster_name
+  template:
+    metadata:
+      labels:
+        application: patroni
+        cluster-name: *cluster_name
+    spec:
+      serviceAccountName: patronidemo
+      containers:
+      - name: *cluster_name
+        image: patroni  # docker build -t patroni .
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 8008
+          protocol: TCP
+        - containerPort: 5432
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /home/postgres/pgdata
+          name: pgdata
+        env:
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        - name: PATRONI_KUBERNETES_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: PATRONI_KUBERNETES_LABELS
+          value: '{application: patroni, cluster-name: patronidemo}'
+        - name: PATRONI_SUPERUSER_USERNAME
+          value: postgres
+        - name: PATRONI_SUPERUSER_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: *cluster_name
+              key: superuser-password
+        - name: PATRONI_REPLICATION_USERNAME
+          value: standby
+        - name: PATRONI_REPLICATION_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: *cluster_name
+              key: replication-password
+        - name: PATRONI_SCOPE
+          value: *cluster_name
+        - name: PATRONI_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: PATRONI_POSTGRESQL_DATA_DIR
+          value: /home/postgres/pgdata/pgroot/data
+        - name: PATRONI_POSTGRESQL_PGPASS
+          value: /tmp/pgpass
+        - name: PATRONI_POSTGRESQL_LISTEN
+          value: '0.0.0.0:5432'
+        - name: PATRONI_RESTAPI_LISTEN
+          value: '0.0.0.0:8008'
+      terminationGracePeriodSeconds: 0
+      volumes:
+      - name: pgdata
+        emptyDir: {}
+#  volumeClaimTemplates:
+#  - metadata:
+#      labels:
+#        application: spilo
+#        spilo-cluster: *cluster_name
+#      annotations:
+#        volume.alpha.kubernetes.io/storage-class: anything
+#      name: pgdata
+#    spec:
+#      accessModes:
+#      - ReadWriteOnce
+#      resources:
+#        requests:
+#          storage: 5Gi
+
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: &cluster_name patronidemo
+  labels:
+    application: patroni
+    cluster-name: *cluster_name
+subsets: []
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: &cluster_name patronidemo
+  labels:
+    application: patroni
+    cluster-name: *cluster_name
+spec:
+  type: ClusterIP
+  ports:
+  - port: 5432
+    targetPort: 5432
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: &cluster_name patronidemo
+  labels:
+    application: patroni
+    cluster-name: *cluster_name
+type: Opaque
+data:
+  superuser-password: emFsYW5kbw==
+  replication-password: cmVwLXBhc3M=
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: patronidemo
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: patronidemo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - patch
+  - update
+  # the following three privileges are necessary only when using endpoints
+  - create
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: patronidemo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: patronidemo
+subjects:
+- kind: ServiceAccount
+  name: patronidemo

patroni/__init__.py

@@ -39,7 +39,7 @@ class Patroni(object):
         while True:
             try:
                 cluster = self.dcs.get_cluster()
-                if cluster and cluster.config:
+                if cluster and cluster.config and cluster.config.data:
                     if self.config.set_dynamic_configuration(cluster.config):
                         self.dcs.reload_config(self.config)
                         self.watchdog.reload_config(self.config)
@@ -113,8 +113,8 @@ class Patroni(object):
 
             logger.info(self.ha.run_cycle())
 
-            cluster = self.dcs.cluster
-            if cluster and cluster.config and self.config.set_dynamic_configuration(cluster.config):
+            if self.dcs.cluster and self.dcs.cluster.config and self.dcs.cluster.config.data \
+                    and self.config.set_dynamic_configuration(self.dcs.cluster.config):
                 self.reload_config()
 
             if self.postgresql.role != 'uninitialized':
@@ -135,8 +135,10 @@ class Patroni(object):
 
 def patroni_main():
     logformat = os.environ.get('PATRONI_LOGFORMAT', '%(asctime)s %(levelname)s: %(message)s')
-    logging.basicConfig(format=logformat, level=logging.INFO)
-    logging.getLogger('requests').setLevel(logging.WARNING)
+    loglevel = os.environ.get('PATRONI_LOGLEVEL', 'INFO')
+    requests_loglevel = os.environ.get('PATRONI_REQUESTS_LOGLEVEL', 'WARNING')
+    logging.basicConfig(format=logformat, level=loglevel)
+    logging.getLogger('requests').setLevel(requests_loglevel)
 
     patroni = Patroni()
     try:

patroni/api.py

@@ -7,8 +7,8 @@ import time
 import dateutil.parser
 import datetime
 
-from patroni.exceptions import PostgresConnectionException
-from patroni.utils import deep_compare, patch_config, Retry, RetryFailedError, is_valid_pg_version, parse_int, tzutc
+from patroni.postgresql import PostgresConnectionException, PostgresException, Postgresql
+from patroni.utils import deep_compare, parse_bool, patch_config, Retry, RetryFailedError, parse_int, tzutc
 from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 from six.moves.socketserver import ThreadingMixIn
 from threading import Thread
@@ -24,9 +24,9 @@ def check_auth(func):
     def do_PUT_foo():
         pass
     """
-    def wrapper(handler):
+    def wrapper(handler, *args, **kwargs):
         if handler.check_auth_header():
-            return func(handler)
+            return func(handler, *args, **kwargs)
     return wrapper
 
 
@@ -70,6 +70,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
             response['scheduled_restart']['schedule'] = (response['scheduled_restart']['schedule']).isoformat()
         if not patroni.ha.watchdog.is_healthy:
             response['watchdog_failed'] = True
+        if patroni.ha.is_paused():
+            response['pause'] = True
         self._write_json_response(status_code, response)
 
     def do_GET(self, write_status_code_only=False):
@@ -80,6 +82,14 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         patroni = self.server.patroni
         cluster = patroni.dcs.cluster
+
+        def is_synchronous():
+            return (cluster.is_synchronous_mode() and cluster.sync
+                    and cluster.sync.sync_standby == patroni.postgresql.name)
+
+        def is_balanceable_replica():
+            return response.get('role') == 'replica' and not patroni.noloadbalance
+
         if cluster:  # dcs available
             if cluster.leader and cluster.leader.name == patroni.postgresql.name:  # is_leader
                 status_code = 200 if 'master' in path else 503
@@ -87,6 +97,10 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 status_code = 503
             elif response['role'] == 'master':  # running as master but without leader lock!!!!
                 status_code = 503
+            elif path in ('/sync', '/synchronous'):
+                status_code = 200 if is_balanceable_replica() and is_synchronous() else 503
+            elif path in ('/async', '/asynchronous'):
+                status_code = 200 if is_balanceable_replica() and not is_synchronous() else 503
             elif response['role'] in path:  # response['role'] != 'master'
                 status_code = 503 if patroni.noloadbalance else 200
             else:
@@ -221,9 +235,11 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     data = "PostgreSQL role should be either master or replica"
                     break
             elif k == 'postgres_version':
-                if not is_valid_pg_version(request[k]):
+                try:
+                    Postgresql.postgres_version_to_int(request[k])
+                except PostgresException as e:
                     status_code = 400
-                    data = "PostgreSQL version should be in the first.major.minor format"
+                    data = e.value
                     break
             elif k == 'timeout':
                 request[k] = parse_int(request[k], 's')
@@ -264,7 +280,14 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
     @check_auth
     def do_POST_reinitialize(self):
-        data = self.server.patroni.ha.reinitialize()
+        request = self._read_json_content(body_is_optional=True)
+
+        if request:
+            logger.debug('received reinitialize request: %s', request)
+
+        force = isinstance(request, dict) and parse_bool(request.get('force')) or False
+
+        data = self.server.patroni.ha.reinitialize(force)
         if data is None:
             status_code = 200
             data = 'reinitialize started'
@@ -289,11 +312,11 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 logger.debug('Exception occured during polling failover result: %s', e)
         return 503, 'Failover status unknown'
 
-    def is_failover_possible(self, cluster, leader, candidate):
+    def is_failover_possible(self, cluster, leader, candidate, action):
         if leader and (not cluster.leader or cluster.leader.name != leader):
             return 'leader name does not match'
         if candidate:
-            if cluster.is_synchronous_mode() and cluster.sync.sync_standby != candidate:
+            if action == 'switchover' and cluster.is_synchronous_mode() and cluster.sync.sync_standby != candidate:
                 return 'candidate name does not match with sync_standby'
             members = [m for m in cluster.members if m.name == candidate]
             if not members:
@@ -301,20 +324,20 @@ class RestApiHandler(BaseHTTPRequestHandler):
         elif cluster.is_synchronous_mode():
             members = [m for m in cluster.members if m.name == cluster.sync.sync_standby]
             if not members:
-                return 'failover is not possible: can not find sync_standby'
+                return action + ' is not possible: can not find sync_standby'
         else:
             members = [m for m in cluster.members if m.name != cluster.leader.name and m.api_url]
             if not members:
-                return 'failover is not possible: cluster does not have members except leader'
+                return action + ' is not possible: cluster does not have members except leader'
         for st in self.server.patroni.ha.fetch_nodes_statuses(members):
             if st.failover_limitation() is None:
                 return None
-        return 'failover is not possible: no good candidates have been found'
+        return action + ' is not possible: no good candidates have been found'
 
     @check_auth
-    def do_POST_failover(self):
+    def do_POST_failover(self, action='failover'):
         request = self._read_json_content()
-        status_code = 500
+        (status_code, data) = (400, '')
         if not request:
             return
 
@@ -323,39 +346,46 @@ class RestApiHandler(BaseHTTPRequestHandler):
         scheduled_at = request.get('scheduled_at')
         cluster = self.server.patroni.dcs.get_cluster()
 
-        if scheduled_at and cluster.is_paused():
-            self._write_response(status_code, "Can't schedule failover in the paused state")
+        logger.info("received %s request with leader=%s candidate=%s scheduled_at=%s",
+                    action, leader, candidate, scheduled_at)
 
-        logger.info("received failover request with leader=%s candidate=%s scheduled_at=%s",
-                    leader, candidate, scheduled_at)
+        if action == 'failover' and not candidate:
+            data = 'Failover could be performed only to a specific candidate'
+        elif action == 'switchover' and not leader:
+            data = 'Switchover could be performed only from a specific leader'
 
-        data = ''
-        if leader or candidate:
-            if scheduled_at:
-                (_, data, scheduled_at) = self.parse_schedule(scheduled_at, "failover")
-                if _:
-                    status_code = _
-                elif self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
-                    self.server.patroni.ha.wakeup()
-                    data = 'Failover scheduled'
+        if not data and scheduled_at:
+            if not leader:
+                data = 'Scheduled {0} is possible only from a specific leader'.format(action)
+            if not data and cluster.is_paused():
+                data = "Can't schedule {0} in the paused state".format(action)
+            if not data:
+                (status_code, data, scheduled_at) = self.parse_schedule(scheduled_at, action)
+
+        if not data and cluster.is_paused() and not candidate:
+            data = action.title() + ' is possible only to a specific candidate in a paused state'
+
+        if not data and not scheduled_at:
+            data = self.is_failover_possible(cluster, leader, candidate, action)
+            if data:
+                status_code = 412
+
+        if not data:
+            if self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
+                self.server.patroni.ha.wakeup()
+                if scheduled_at:
+                    data = action.title() + ' scheduled'
                     status_code = 202
                 else:
-                    data = 'failed to write failover key into DCS'
-                    status_code = 503
+                    status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, candidate)
             else:
-                data = self.is_failover_possible(cluster, leader, candidate)
-                if not data:
-                    if self.server.patroni.dcs.manual_failover(leader, candidate):
-                        self.server.patroni.ha.wakeup()
-                        status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, candidate)
-                    else:
-                        data = 'failed to write failover key into DCS'
-                        status_code = 503
-        else:
-            status_code = 400
-            data = 'No values given for required parameters leader and candidate'
+                data = 'failed to write {0} key into DCS'.format(action)
+                status_code = 503
         self._write_response(status_code, data)
 
+    def do_POST_switchover(self):
+        self.do_POST_failover(action='switchover')
+
     def parse_request(self):
         """Override parse_request method to enrich basic functionality of `BaseHTTPRequestHandler` class
 
@@ -384,40 +414,44 @@ class RestApiHandler(BaseHTTPRequestHandler):
         try:
             if self.server.patroni.postgresql.state not in ('running', 'restarting', 'starting'):
                 raise RetryFailedError('')
-            row = self.query("""WITH replication_info AS (
-                                    SELECT usename, application_name, client_addr, state, sync_state, sync_priority
-                                      FROM pg_stat_replication
-                                )
-                                SELECT to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
-                                       pg_is_in_recovery(),
-                                       CASE WHEN pg_is_in_recovery()
-                                            THEN 0
-                                            ELSE pg_{0}_{1}_diff(pg_current_{0}_{1}(), '0/0')::bigint
-                                       END,
-                                       pg_{0}_{1}_diff(COALESCE(pg_last_{0}_receive_{1}(),
-                                                                      pg_last_{0}_replay_{1}()), '0/0')::bigint,
-                                       pg_{0}_{1}_diff(pg_last_{0}_replay_{1}(), '0/0')::bigint,
-                                       to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
-                                       pg_is_in_recovery() AND pg_is_{0}_replay_paused(),
-                                       (SELECT array_to_json(array_agg(row_to_json(ri)))
-                                          FROM replication_info ri)""".format(self.server.patroni.postgresql.wal_name,
-                                                                              self.server.patroni.postgresql.lsn_name),
-                             retry=retry)[0]
+            stmt = ("WITH replication_info AS ("
+                    "SELECT usename, application_name, client_addr, state, sync_state, sync_priority"
+                    " FROM pg_stat_replication) SELECT"
+                    " to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),"
+                    " CASE WHEN pg_is_in_recovery() THEN 0"
+                    " ELSE ('x' || SUBSTR(pg_{0}file_name(pg_current_{0}_{1}()), 1, 8))::bit(32)::int END,"
+                    " CASE WHEN pg_is_in_recovery() THEN 0"
+                    " ELSE pg_{0}_{1}_diff(pg_current_{0}_{1}(), '0/0')::bigint END,"
+                    " pg_{0}_{1}_diff(COALESCE(pg_last_{0}_receive_{1}(), pg_last_{0}_replay_{1}()), '0/0')::bigint,"
+                    " pg_{0}_{1}_diff(pg_last_{0}_replay_{1}(), '0/0')::bigint,"
+                    " to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),"
+                    " pg_is_in_recovery() AND pg_is_{0}_replay_paused(),"
+                    " (SELECT array_to_json(array_agg(row_to_json(ri))) FROM replication_info ri)")
+
+            row = self.query(stmt.format(self.server.patroni.postgresql.wal_name,
+                                         self.server.patroni.postgresql.lsn_name), retry=retry)[0]
 
             result = {
                 'state': self.server.patroni.postgresql.state,
                 'postmaster_start_time': row[0],
-                'role': 'replica' if row[1] else 'master',
+                'role': 'replica' if row[1] == 0 else 'master',
                 'server_version': self.server.patroni.postgresql.server_version,
                 'xlog': ({
                     'received_location': row[3],
                     'replayed_location': row[4],
                     'replayed_timestamp': row[5],
-                    'paused': row[6]} if row[1] else {
+                    'paused': row[6]} if row[1] == 0 else {
                     'location': row[2]
                 })
             }
 
+            if row[1] > 0:
+                result['timeline'] = row[1]
+            else:
+                cluster = self.server.patroni.dcs.cluster
+                leader_timeline = None if not cluster or cluster.is_unlocked() else cluster.leader.timeline
+                result['timeline'] = self.server.patroni.postgresql.replica_cached_timeline(leader_timeline)
+
             if row[7]:
                 result['replication'] = row[7]
 
@@ -481,7 +515,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
     def __initialize(self, config):
         self.__ssl_options = self.__get_ssl_options(config)
         self.__listen = config['listen']
-        host, port = config['listen'].split(':')
+        host, port = config['listen'].rsplit(':', 1)
         HTTPServer.__init__(self, (host, int(port)), RestApiHandler)
         Thread.__init__(self, target=self.serve_forever)
         self._set_fd_cloexec(self.socket)

patroni/async_executor.py

@@ -1,5 +1,5 @@
 import logging
-from threading import Lock, RLock, Thread
+from threading import Event, Lock, RLock, Thread
 
 logger = logging.getLogger(__name__)
 
@@ -52,22 +52,27 @@ class CriticalTask(object):
 
 class AsyncExecutor(object):
 
-    def __init__(self, ha_wakeup):
+    def __init__(self, state_handler, ha_wakeup):
+        self.state_handler = state_handler
         self._ha_wakeup = ha_wakeup
         self._thread_lock = RLock()
         self._scheduled_action = None
         self._scheduled_action_lock = RLock()
+        self._is_cancelled = False
+        self._finish_event = Event()
         self.critical_task = CriticalTask()
 
     @property
     def busy(self):
         return self.scheduled_action is not None
 
-    def schedule(self, action, immediately=False):
+    def schedule(self, action):
         with self._scheduled_action_lock:
             if self._scheduled_action is not None:
                 return self._scheduled_action
             self._scheduled_action = action
+            self._is_cancelled = False
+            self._finish_event.set()
         return None
 
     @property
@@ -82,6 +87,12 @@ class AsyncExecutor(object):
     def run(self, func, args=()):
         wakeup = False
         try:
+            with self:
+                if self._is_cancelled:
+                    return
+                self._finish_event.clear()
+
+            self.state_handler.reset_is_cancelled()
             # if the func returned something (not None) - wake up main HA loop
             wakeup = func(*args) if args else func()
             return wakeup
@@ -90,6 +101,7 @@ class AsyncExecutor(object):
         finally:
             with self:
                 self.reset_scheduled_action()
+                self._finish_event.set()
                 with self.critical_task:
                     self.critical_task.reset()
             if wakeup is not None:
@@ -98,6 +110,20 @@ class AsyncExecutor(object):
     def run_async(self, func, args=()):
         Thread(target=self.run, args=(func, args)).start()
 
+    def cancel(self):
+        with self:
+            with self._scheduled_action_lock:
+                if self._scheduled_action is None:
+                    return
+                logger.warning('Cancelling long running task %s', self._scheduled_action)
+            self._is_cancelled = True
+
+        self.state_handler.cancel()
+        self._finish_event.wait()
+
+        with self:
+            self.reset_scheduled_action()
+
     def __enter__(self):
         self._thread_lock.acquire()
 

patroni/config.py

@@ -239,16 +239,25 @@ class Config(object):
 
         for param in list(os.environ.keys()):
             if param.startswith(Config.PATRONI_ENV_PREFIX):
-                name, suffix = (param[8:].rsplit('_', 1) + [''])[:2]
+                name, suffix = (param[8:].split('_', 1) + [''])[:2]
                 if name and suffix:
                     # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..)
-                    if suffix in ('HOST', 'HOSTS', 'PORT', 'SRV', 'URL', 'PROXY', 'CACERT', 'CERT', 'KEY',
-                                  'VERIFY', 'TOKEN', 'CHECKS', 'DC') and '_' not in name:
+                    if suffix in ('HOST', 'HOSTS', 'PORT', 'SRV', 'URL', 'PROXY', 'CACERT', 'CERT',
+                                  'KEY', 'VERIFY', 'TOKEN', 'CHECKS', 'DC', 'NAMESPACE', 'CONTEXT',
+                                  'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL', 'POD_IP', 'PORTS', 'LABELS'):
                         value = os.environ.pop(param)
                         if suffix == 'PORT':
                             value = value and parse_int(value)
-                        elif suffix in ('HOSTS', 'CHECKS'):
+                        elif suffix in ('HOSTS', 'PORTS', 'CHECKS'):
                             value = value and _parse_list(value)
+                        elif suffix == 'LABELS':
+                            if not value.strip().startswith('{'):
+                                value = '{{{0}}}'.format(value)
+                            try:
+                                value = yaml.safe_load(value)
+                            except Exception:
+                                logger.exception('Exception when parsing dict %s', value)
+                                value = None
                         if value:
                             ret[name.lower()][suffix.lower()] = value
                     # PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>

patroni/ctl.py

@@ -30,7 +30,8 @@ from patroni.config import Config
 from patroni.dcs import get_dcs as _get_dcs
 from patroni.exceptions import PatroniException
 from patroni.postgresql import Postgresql
-from patroni.utils import is_valid_pg_version, patch_config
+from patroni.utils import patch_config, polling_loop
+from patroni.version import __version__
 from prettytable import PrettyTable
 from six.moves.urllib_parse import urlparse
 from six import text_type
@@ -96,10 +97,12 @@ def store_config(config, path):
         yaml.dump(config, fd)
 
 
-option_format = click.option('--format', '-f', 'fmt', help='Output format (pretty, json)', default='pretty')
+option_format = click.option('--format', '-f', 'fmt', help='Output format (pretty, json, yaml)', default='pretty')
 option_watchrefresh = click.option('-w', '--watch', type=float, help='Auto update the screen every X seconds')
 option_watch = click.option('-W', is_flag=True, help='Auto update the screen every 2 seconds')
 option_force = click.option('--force', is_flag=True, help='Do not ask for confirmation at any point')
+arg_cluster_name = click.argument('cluster_name', required=False,
+                                  default=lambda: click.get_current_context().obj.get('scope'))
 
 
 @click.group()
@@ -149,9 +152,12 @@ def print_output(columns, rows=None, alignment=None, fmt='pretty', header=True,
         click.echo(t)
         return
 
-    if fmt == 'json':
+    if fmt in ['json', 'yaml', 'yml']:
         elements = [dict(zip(columns, r)) for r in rows]
-        click.echo(json.dumps(elements))
+        if fmt == 'json':
+            click.echo(json.dumps(elements))
+        elif fmt in ('yaml', 'yml'):
+            click.echo(yaml.safe_dump(elements, encoding=None, default_flow_style=False, allow_unicode=True, width=200))
 
     if fmt == 'tsv':
         if columns is not None and header:
@@ -257,9 +263,9 @@ def get_members(cluster, cluster_name, member_names, role, force, action):
         member_names = [click.prompt('Which member do you want to {0} [{1}]?'.format(action,
                         ', '.join(candidates.keys())), type=str, default='')]
 
-    for mn in member_names:
-        if mn not in candidates:
-            raise PatroniCtlException('{0} is not a member of cluster'.format(mn))
+    for member_name in member_names:
+        if member_name not in candidates:
+            raise PatroniCtlException('{0} is not a member of cluster'.format(member_name))
 
     if not force:
         confirm = click.confirm('Are you sure you want to {0} members {1}?'.format(action, ', '.join(member_names)))
@@ -273,7 +279,7 @@ def get_members(cluster, cluster_name, member_names, role, force, action):
 @click.option('--role', '-r', help='Give a dsn of any member with this role', type=click.Choice(['master', 'replica',
               'any']), default=None)
 @click.option('--member', '-m', help='Generate a dsn for this member', type=str)
-@click.argument('cluster_name')
+@arg_cluster_name
 @click.pass_obj
 def dsn(obj, cluster_name, role, member):
     if role is not None and member is not None:
@@ -291,7 +297,7 @@ def dsn(obj, cluster_name, role, member):
 
 
 @ctl.command('query', help='Query a Patroni PostgreSQL member')
-@click.argument('cluster_name')
+@arg_cluster_name
 @option_format
 @click.option('--format', 'fmt', help='Output format (pretty, json)', default='tsv')
 @click.option('--file', '-f', 'p_file', help='Execute the SQL commands from this file', type=click.File('rb'))
@@ -406,7 +412,7 @@ def remove(obj, cluster_name, fmt):
     if message != confirm:
         raise PatroniCtlException('You did not exactly type "{0}"'.format(message))
 
-    if cluster.leader:
+    if cluster.leader and cluster.leader.name:
         confirm = click.prompt('This cluster currently is healthy. Please specify the master name to continue')
         if confirm != cluster.leader.name:
             raise PatroniCtlException('You did not specify the current master of the cluster')
@@ -419,8 +425,10 @@ def check_response(response, member_name, action_name, silent_success=False):
         click.echo('Failed: {0} for member {1}, status code={2}, ({3})'.format(
             action_name, member_name, response.status_code, response.text
         ))
+        return False
     elif not silent_success:
         click.echo('Success: {0} for member {1}'.format(action_name, member_name))
+    return True
 
 
 def parse_scheduled(scheduled):
@@ -469,11 +477,12 @@ def restart(obj, cluster_name, member_names, force, role, p_any, scheduled, vers
         content['restart_pending'] = True
 
     if version:
-        if not is_valid_pg_version(version):
-            message = 'PostgreSQL version should be in the first.major.minor format'
-            raise PatroniCtlException(message)
-        else:
-            content['postgres_version'] = version
+        try:
+            Postgresql.postgres_version_to_int(version)
+        except PatroniException as e:
+            raise PatroniCtlException(e.value)
+
+        content['postgres_version'] = version
 
     if scheduled is None and not force:
         scheduled = click.prompt('When should the restart take place (e.g. 2015-10-01T14:30) ', type=str, default='now')
@@ -516,35 +525,33 @@ def reinit(obj, cluster_name, member_names, force):
     members = get_members(cluster, cluster_name, member_names, None, force, 'reinitialize')
 
     for member in members:
-        r = request_patroni(member, 'post', 'reinitialize', headers=auth_header(obj))
-        check_response(r, member.name, 'reinitialize')
+        body = {'force': force}
+        while True:
+            r = request_patroni(member, 'post', 'reinitialize', body, auth_header(obj))
+            if not check_response(r, member.name, 'reinitialize') and r.text.endswith(' already in progress') \
+                    and not force and click.confirm('Do you want to cancel it and reinitialize anyway?'):
+                body['force'] = True
+                continue
+            break
 
 
-@ctl.command('failover', help='Failover to a replica')
-@click.argument('cluster_name')
-@click.option('--master', help='The name of the current master', default=None)
-@click.option('--candidate', help='The name of the candidate', default=None)
-@click.option('--scheduled', help='Timestamp of a scheduled failover in unambiguous format (e.g. ISO 8601)',
-              default=None)
-@option_force
-@click.pass_obj
-def failover(obj, cluster_name, master, candidate, force, scheduled):
+def _do_failover_or_switchover(obj, action, cluster_name, master, candidate, force, scheduled=None):
     """
-        We want to trigger a failover for the specified cluster name.
+        We want to trigger a failover or switchover for the specified cluster name.
 
         We verify that the cluster name, master name and candidate name are correct.
-        If so, we trigger a failover and keep the client up to date.
+        If so, we trigger an action and keep the client up to date.
     """
 
     dcs = get_dcs(obj, cluster_name)
     cluster = dcs.get_cluster()
 
-    if cluster.leader is None and not cluster.is_paused():
+    if action == 'switchover' and cluster.leader is None:
         raise PatroniCtlException('This cluster has no master')
 
-    if master is None and (not cluster.is_paused() or cluster.leader):
-        if force:
-            master = cluster.leader.member.name
+    if master is None:
+        if force or action == 'failover':
+            master = cluster.leader and cluster.leader.name
         else:
             master = click.prompt('Master', type=str, default=cluster.leader.member.name)
 
@@ -556,28 +563,32 @@ def failover(obj, cluster_name, master, candidate, force, scheduled):
     candidate_names.sort()
 
     if not candidate_names:
-        raise PatroniCtlException('No candidates found to failover to')
+        raise PatroniCtlException('No candidates found to {0} to'.format(action))
 
     if candidate is None and not force:
         candidate = click.prompt('Candidate ' + str(candidate_names), type=str, default='')
 
+    if action == 'failover' and not candidate:
+        raise PatroniCtlException('Failover could be performed only to a specific candidate')
+
     if candidate == master:
-        raise PatroniCtlException('Failover target and source are the same.')
+        raise PatroniCtlException(action.title() + ' target and source are the same.')
 
     if candidate and candidate not in candidate_names:
         raise PatroniCtlException('Member {0} does not exist in cluster {1}'.format(candidate, cluster_name))
 
-    if scheduled is None and not force:
-        scheduled = click.prompt('When should the failover take place (e.g. 2015-10-01T14:30) ', type=str,
-                                 default='now')
-
-    scheduled_at = parse_scheduled(scheduled)
-
     scheduled_at_str = None
-    if scheduled_at:
-        if cluster.is_paused():
-            raise PatroniCtlException("Can't schedule failover in the paused state")
-        scheduled_at_str = scheduled_at.isoformat()
+
+    if action == 'switchover':
+        if scheduled is None and not force:
+            scheduled = click.prompt('When should the switchover take place (e.g. 2015-10-01T14:30) ',
+                                     type=str, default='now')
+
+        scheduled_at = parse_scheduled(scheduled)
+        if scheduled_at:
+            if cluster.is_paused():
+                raise PatroniCtlException("Can't schedule switchover in the paused state")
+            scheduled_at_str = scheduled_at.isoformat()
 
     failover_value = {'leader': master, 'candidate': candidate, 'scheduled_at': scheduled_at_str}
 
@@ -588,46 +599,76 @@ def failover(obj, cluster_name, master, candidate, force, scheduled):
     output_members(dcs.get_cluster(), cluster_name)
 
     if not force:
-        a = \
-            click.confirm('Are you sure you want to failover cluster {0}, demoting current master {1}?'.format(
-                cluster_name, master))
-        if not a:
-            raise PatroniCtlException('Aborting failover')
+        demote_msg = ', demoting current master ' + master if master else ''
+
+        if not click.confirm('Are you sure you want to {0} cluster {1}{2}?'.format(action, cluster_name, demote_msg)):
+            raise PatroniCtlException('Aborting ' + action)
 
     r = None
     try:
-        member = cluster.leader.member if cluster.leader else [m for m in cluster.members if m.name == candidate][0]
+        member = cluster.leader.member if cluster.leader else cluster.get_member(candidate, False)
+
+        r = request_patroni(member, 'post', action, failover_value, auth_header(obj))
+
+        # probably old patroni, which doesn't support switchover yet
+        if r.status_code == 501 and action == 'switchover' and 'Server does not support this operation' in r.text:
+            r = request_patroni(member, 'post', 'failover', failover_value, auth_header(obj))
 
-        r = request_patroni(member, 'post', 'failover', failover_value, auth_header(obj))
         if r.status_code in (200, 202):
             logging.debug(r)
             cluster = dcs.get_cluster()
             logging.debug(cluster)
             click.echo('{0} {1}'.format(timestamp(), r.text))
         else:
-            click.echo('Failover failed, details: {0}, {1}'.format(r.status_code, r.text))
+            click.echo('{0} failed, details: {1}, {2}'.format(action.title(), r.status_code, r.text))
             return
     except Exception:
         logging.exception(r)
         logging.warning('Failing over to DCS')
-        click.echo(timestamp() + ' Could not failover using Patroni api, falling back to DCS')
-        click.echo(timestamp() + ' Initializing failover from master {0}'.format(master))
+        click.echo('{0} Could not {1} using Patroni api, falling back to DCS'.format(timestamp(), action))
         dcs.manual_failover(master, candidate, scheduled_at=scheduled_at)
 
     output_members(cluster, cluster_name)
 
 
+@ctl.command('failover', help='Failover to a replica')
+@arg_cluster_name
+@click.option('--master', help='The name of the current master', default=None)
+@click.option('--candidate', help='The name of the candidate', default=None)
+@option_force
+@click.pass_obj
+def failover(obj, cluster_name, master, candidate, force):
+    action = 'switchover' if master else 'failover'
+    _do_failover_or_switchover(obj, action, cluster_name, master, candidate, force)
+
+
+@ctl.command('switchover', help='Switchover to a replica')
+@arg_cluster_name
+@click.option('--master', help='The name of the current master', default=None)
+@click.option('--candidate', help='The name of the candidate', default=None)
+@click.option('--scheduled', help='Timestamp of a scheduled switchover in unambiguous format (e.g. ISO 8601)',
+              default=None)
+@option_force
+@click.pass_obj
+def switchover(obj, cluster_name, master, candidate, force, scheduled):
+    _do_failover_or_switchover(obj, 'switchover', cluster_name, master, candidate, force, scheduled)
+
+
 def output_members(cluster, name, extended=False, fmt='pretty'):
     rows = []
     logging.debug(cluster)
     leader_name = None
     if cluster.leader:
-        leader_name = cluster.leader.member.name
+        leader_name = cluster.leader.name
 
     xlog_location_cluster = cluster.last_leader_operation or 0
 
     # Mainly for consistent pretty printing and watching we sort the output
     cluster.members.sort(key=lambda x: x.name)
+
+    has_scheduled_restarts = any(m.data.get('scheduled_restart') for m in cluster.members)
+    has_pending_restarts = any(m.data.get('pending_restart') for m in cluster.members)
+
     for m in cluster.members:
         logging.debug(m)
 
@@ -637,22 +678,19 @@ def output_members(cluster, name, extended=False, fmt='pretty'):
         elif m.name == cluster.sync.sync_standby:
             role = 'Sync standby'
 
-        host = m.conn_kwargs()['host']
-
-        xlog_location = m.data.get('xlog_location') or 0
+        xlog_location = m.data.get('xlog_location')
         lag = ''
-        if xlog_location_cluster >= xlog_location:
+        if xlog_location is None:
+            lag = 'unknown'
+        elif xlog_location_cluster >= xlog_location:
             lag = round((xlog_location_cluster - xlog_location)/1024/1024)
 
-        row = [
-            name,
-            m.name,
-            host,
-            role,
-            m.data.get('state', ''),
-            lag,
-        ]
-        if extended:
+        row = [name, m.name, m.conn_kwargs()['host'], role, m.data.get('state', ''), lag]
+
+        if extended or has_pending_restarts:
+            row.append('*' if m.data.get('pending_restart') else '')
+
+        if extended or has_scheduled_restarts:
             value = ''
             scheduled_restart = m.data.get('scheduled_restart')
             if scheduled_restart:
@@ -664,39 +702,55 @@ def output_members(cluster, name, extended=False, fmt='pretty'):
 
         rows.append(row)
 
-    columns = [
-        'Cluster',
-        'Member',
-        'Host',
-        'Role',
-        'State',
-        'Lag in MB',
-    ]
-    alignment = {'Cluster': 'l', 'Member': 'l', 'Host': 'l', 'Lag in MB': 'r'}
+    columns = ['Cluster', 'Member', 'Host', 'Role', 'State', 'Lag in MB']
+    alignment = {'Lag in MB': 'r'}
 
-    if extended:
+    if extended or has_pending_restarts:
+        columns.append('Pending restart')
+
+    if extended or has_scheduled_restarts:
         columns.append('Scheduled restart')
-        alignment['Scheduled restart'] = 'l'
 
     print_output(columns, rows, alignment, fmt)
 
+    service_info = []
+    if cluster.is_paused():
+        service_info.append('Maintenance mode: on')
+
+    if cluster.failover and cluster.failover.scheduled_at:
+        info = 'Switchover scheduled at: ' + cluster.failover.scheduled_at.isoformat()
+        if cluster.failover.leader:
+            info += '\n                    from: ' + cluster.failover.leader
+        if cluster.failover.candidate:
+            info += '\n                      to: ' + cluster.failover.candidate
+        service_info.append(info)
+
+    if service_info:
+        click.echo(' ' + '\n '.join(service_info))
+
 
 @ctl.command('list', help='List the Patroni members for a given Patroni')
 @click.argument('cluster_names', nargs=-1)
 @click.option('--extended', '-e', help='Show some extra information', is_flag=True)
+@click.option('--timestamp', '-t', 'ts', help='Print timestamp', is_flag=True)
 @option_format
 @option_watch
 @option_watchrefresh
 @click.pass_obj
-def members(obj, cluster_names, fmt, watch, w, extended):
+def members(obj, cluster_names, fmt, watch, w, extended, ts):
     if not cluster_names:
-        logging.warning('Listing members: No cluster names were provided')
-        return
+        if 'scope' in obj:
+            cluster_names = [obj['scope']]
+        if not cluster_names:
+            return logging.warning('Listing members: No cluster names were provided')
 
     for cluster_name in cluster_names:
         dcs = get_dcs(obj, cluster_name)
 
         for _ in watching(w, watch):
+            if ts:
+                click.echo(timestamp(0))
+
             cluster = dcs.get_cluster()
             output_members(cluster, cluster_name, extended, fmt)
 
@@ -789,8 +843,27 @@ def flush(obj, cluster_name, member_names, force, role, target):
                 click.echo('No scheduled restart for member {0}'.format(member.name))
 
 
-def toggle_pause(config, cluster_name, paused):
-    cluster = get_dcs(config, cluster_name).get_cluster()
+def wait_until_pause_is_applied(dcs, paused, old_cluster):
+    click.echo("'{0}' request sent, waiting until it is recognized by all nodes".format(paused and 'pause' or 'resume'))
+    old = {m.name: m.index for m in old_cluster.members if m.api_url}
+    loop_wait = old_cluster.config.data.get('loop_wait', dcs.loop_wait)
+
+    for _ in polling_loop(loop_wait + 1):
+        cluster = dcs.get_cluster()
+        if all(m.data.get('pause', False) == paused for m in cluster.members if m.name in old):
+            break
+    else:
+        remaining = [m.name for m in cluster.members if m.data.get('pause', False) != paused
+                     and m.name in old and old[m.name] != m.index]
+        if remaining:
+            return click.echo("{0} members didn't recognized pause state after {1} seconds"
+                              .format(', '.join(remaining), loop_wait))
+    return click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
+
+
+def toggle_pause(config, cluster_name, paused, wait):
+    dcs = get_dcs(config, cluster_name)
+    cluster = dcs.get_cluster()
     if cluster.is_paused() == paused:
         raise PatroniCtlException('Cluster is {0} paused'.format(paused and 'already' or 'not'))
 
@@ -807,7 +880,10 @@ def toggle_pause(config, cluster_name, paused):
             continue
 
         if r.status_code == 200:
-            click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
+            if wait:
+                wait_until_pause_is_applied(dcs, paused, cluster)
+            else:
+                click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
         else:
             click.echo('Failed: {0} cluster management status code={1}, ({2})'.format(
                        paused and 'pause' or 'resume', r.status_code, r.text))
@@ -817,17 +893,19 @@ def toggle_pause(config, cluster_name, paused):
 
 
 @ctl.command('pause', help='Disable auto failover')
-@click.argument('cluster_name')
+@arg_cluster_name
 @click.pass_obj
-def pause(obj, cluster_name):
-    return toggle_pause(obj, cluster_name, True)
+@click.option('--wait', help='Wait until pause is applied on all nodes', is_flag=True)
+def pause(obj, cluster_name, wait):
+    return toggle_pause(obj, cluster_name, True, wait)
 
 
 @ctl.command('resume', help='Resume auto failover')
-@click.argument('cluster_name')
+@arg_cluster_name
+@click.option('--wait', help='Wait until pause is cleared on all nodes', is_flag=True)
 @click.pass_obj
-def resume(obj, cluster_name):
-    return toggle_pause(obj, cluster_name, False)
+def resume(obj, cluster_name, wait):
+    return toggle_pause(obj, cluster_name, False, wait)
 
 
 @contextmanager
@@ -969,7 +1047,7 @@ def invoke_editor(before_editing, cluster_name):
 
 
 @ctl.command('edit-config', help="Edit cluster configuration")
-@click.argument('cluster_name')
+@arg_cluster_name
 @click.option('--quiet', '-q', is_flag=True, help='Do not show changes')
 @click.option('--set', '-s', 'kvpairs', multiple=True,
               help='Set specific configuration value. Can be specified multiple times')
@@ -1023,9 +1101,42 @@ def edit_config(obj, cluster_name, force, quiet, kvpairs, pgkvpairs, apply_filen
 
 
 @ctl.command('show-config', help="Show cluster configuration")
-@click.argument('cluster_name')
+@arg_cluster_name
 @click.pass_obj
 def show_config(obj, cluster_name):
     cluster = get_dcs(obj, cluster_name).get_cluster()
 
     click.echo(format_config_for_editing(cluster.config.data))
+
+
+@ctl.command('version', help='Output version of patronictl command or a running Patroni instance')
+@click.argument('cluster_name', required=False)
+@click.argument('member_names', nargs=-1)
+@click.pass_obj
+def version(obj, cluster_name, member_names):
+    click.echo("patronictl version {0}".format(__version__))
+
+    if not cluster_name:
+        return
+
+    click.echo("")
+    cluster = get_dcs(obj, cluster_name).get_cluster()
+    for m in cluster.members:
+        if m.api_url:
+            if not member_names or m.name in member_names:
+                try:
+                    response = request_patroni(m, 'get', 'patroni')
+                    data = response.json()
+                    version = data.get('patroni', {}).get('version')
+                    pg_version = data.get('server_version')
+                    pg_version_str = " PostgreSQL {0}".format(format_pg_version(pg_version)) if pg_version else ""
+                    click.echo("{0}: Patroni {1}{2}".format(m.name, version, pg_version_str))
+                except Exception as e:
+                    click.echo("{0}: failed to get version: {1}".format(m.name, e))
+
+
+def format_pg_version(version):
+    if version < 100000:
+        return "{0}.{1}.{2}".format(version // 10000, version // 100 % 100, version % 100)
+    else:
+        return "{0}.{1}".format(version // 10000, version % 100)

patroni/dcs/__init__.py

@@ -177,40 +177,48 @@ class Leader(namedtuple('Leader', 'index,session,member')):
     def conn_url(self):
         return self.member.conn_url
 
+    @property
+    def timeline(self):
+        return self.member.data.get('timeline')
+
 
 class Failover(namedtuple('Failover', 'index,leader,candidate,scheduled_at')):
 
     """
     >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader"}'))
     True
+    >>> 'Failover' in str(Failover.from_node(1, {"leader": "cluster_leader"}))
+    True
     >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader", "member": "cluster_candidate"}'))
     True
     >>> Failover.from_node(1, 'null') is None
-    True
+    False
     >>> n = '{"leader": "cluster_leader", "member": "cluster_candidate", "scheduled_at": "2016-01-14T10:09:57.1394Z"}'
     >>> 'tzinfo=' in str(Failover.from_node(1, n))
     True
     >>> Failover.from_node(1, None) is None
-    True
+    False
     >>> Failover.from_node(1, '{}') is None
-    True
+    False
     >>> 'abc' in Failover.from_node(1, 'abc:def')
     True
     """
     @staticmethod
     def from_node(index, value):
-        if not value:
-            return None
-
-        try:
-            data = json.loads(value)
-            if not data:
-                return None
-        except ValueError:
-            t = [a.strip() for a in value.split(':')]
-            leader = t[0]
-            candidate = t[1] if len(t) > 1 else None
-            return Failover(index, leader, candidate, None) if leader or candidate else None
+        if isinstance(value, dict):
+            data = value
+        elif value:
+            try:
+                data = json.loads(value)
+                if not isinstance(data, dict):
+                    data = {}
+            except ValueError:
+                t = [a.strip() for a in value.split(':')]
+                leader = t[0]
+                candidate = t[1] if len(t) > 1 else None
+                return Failover(index, leader, candidate, None) if leader or candidate else None
+        else:
+            data = {}
 
         if data.get('scheduled_at'):
             data['scheduled_at'] = dateutil.parser.parse(data['scheduled_at'])
@@ -258,8 +266,12 @@ class SyncState(namedtuple('SyncState', 'index,leader,sync_standby')):
         True
         >>> SyncState.from_node(1, '{"leader": "leader"}').leader == "leader"
         True
+        >>> SyncState.from_node(1, {"leader": "leader"}).leader == "leader"
+        True
         """
-        if value:
+        if isinstance(value, dict):
+            data = value
+        elif value:
             try:
                 data = json.loads(value)
                 if not isinstance(data, dict):
@@ -289,7 +301,26 @@ class SyncState(namedtuple('SyncState', 'index,leader,sync_standby')):
         return name is not None and name in (self.leader, self.sync_standby)
 
 
-class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operation,members,failover,sync')):
+class TimelineHistory(namedtuple('TimelineHistory', 'index,lines')):
+    """Object representing timeline history file"""
+
+    @staticmethod
+    def from_node(index, value):
+        """
+        >>> h = TimelineHistory.from_node(1, 2)
+        >>> h.lines
+        []
+        """
+        try:
+            lines = json.loads(value)
+        except (TypeError, ValueError):
+            lines = None
+        if not isinstance(lines, list):
+            lines = []
+        return TimelineHistory(index, lines)
+
+
+class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operation,members,failover,sync,history')):
 
     """Immutable object (namedtuple) which represents PostgreSQL cluster.
     Consists of the following fields:
@@ -301,6 +332,7 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operat
     :param members: list of Member object, all PostgreSQL cluster members including leader
     :param failover: reference to `Failover` object
     :param sync: reference to `SyncState` object, last observed synchronous replication state.
+    :param history: reference to `TimelineHistory` object
     """
 
     def is_unlocked(self):
@@ -334,6 +366,7 @@ class AbstractDCS(object):
     _CONFIG = 'config'
     _LEADER = 'leader'
     _FAILOVER = 'failover'
+    _HISTORY = 'history'
     _MEMBERS = 'members/'
     _OPTIME = 'optime'
     _LEADER_OPTIME = _OPTIME + '/' + _LEADER
@@ -345,8 +378,7 @@ class AbstractDCS(object):
             i.e.: `zookeeper` for zookeeper, `etcd` for etcd, etc...
         """
         self._name = config['name']
-        self._namespace = '/{0}'.format(config.get('namespace', '/service/').strip('/'))
-        self._base_path = '/'.join([self._namespace, config['scope']])
+        self._base_path = os.path.join('/', config.get('namespace', '/service/').strip('/'), config['scope'])
         self._set_loop_wait(config.get('loop_wait', 10))
 
         self._ctl = bool(config.get('patronictl', False))
@@ -382,6 +414,10 @@ class AbstractDCS(object):
     def failover_path(self):
         return self.client_path(self._FAILOVER)
 
+    @property
+    def history_path(self):
+        return self.client_path(self._HISTORY)
+
     @property
     def leader_optime_path(self):
         return self.client_path(self._LEADER_OPTIME)
@@ -449,7 +485,7 @@ class AbstractDCS(object):
             self._last_leader_operation = last_operation
 
     @abc.abstractmethod
-    def update_leader(self):
+    def _update_leader(self):
         """Update leader key (or session) ttl
 
         :returns: `!True` if leader key (or session) has been updated successfully.
@@ -458,6 +494,18 @@ class AbstractDCS(object):
         You have to use CAS (Compare And Swap) operation in order to update leader key,
         for example for etcd `prevValue` parameter must be used."""
 
+    def update_leader(self, last_operation):
+        """Update leader key (or session) ttl and optime/leader
+
+        :param last_operation: absolute xlog location in bytes
+        :returns: `!True` if leader key (or session) has been updated successfully.
+            If not, `!False` must be returned and current instance would be demoted."""
+
+        ret = self._update_leader()
+        if ret and last_operation:
+            self.write_leader_optime(last_operation)
+        return ret
+
     @abc.abstractmethod
     def attempt_to_acquire_leader(self, permanent=False):
         """Attempt to acquire leader lock
@@ -483,7 +531,6 @@ class AbstractDCS(object):
 
         if scheduled_at:
             failover_value['scheduled_at'] = scheduled_at.isoformat()
-
         return self.set_failover_value(json.dumps(failover_value, separators=(',', ':')), index)
 
     @abc.abstractmethod
@@ -496,7 +543,7 @@ class AbstractDCS(object):
         This method should create or update key with the name = '/members/' + `~self._name`
         and value = data in a given DCS.
 
-        :param data: json serialized information about instance (including connection strings)
+        :param data: information about instance (including connection strings)
         :param ttl: ttl for member key, optional parameter. If it is None `~self.member_ttl will be used`
         :param permanent: if set to `!True`, the member key will never expire.
          Used in patronictl for the external master.
@@ -533,8 +580,18 @@ class AbstractDCS(object):
     def delete_cluster(self):
         """Delete cluster from DCS"""
 
+    @staticmethod
+    def sync_state(leader, sync_standby):
+        """Build sync_state dict"""
+        return {'leader': leader, 'sync_standby': sync_standby}
+
     def write_sync_state(self, leader, sync_standby, index=None):
-        return self.set_sync_state_value(json.dumps({'leader': leader, 'sync_standby': sync_standby}), index=index)
+        sync_value = self.sync_state(leader, sync_standby)
+        return self.set_sync_state_value(json.dumps(sync_value, separators=(',', ':')), index)
+
+    @abc.abstractmethod
+    def set_history_value(self, value):
+        """"""
 
     @abc.abstractmethod
     def set_sync_state_value(self, value, index=None):

patroni/dcs/consul.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+import json
 import logging
 import os
 import socket
@@ -7,9 +8,9 @@ import time
 import urllib3
 
 from consul import ConsulException, NotFound, base
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
 from patroni.exceptions import DCSError
-from patroni.utils import parse_bool, Retry, RetryFailedError
+from patroni.utils import deep_compare, parse_bool, Retry, RetryFailedError, split_host_port
 from urllib3.exceptions import HTTPError
 from six.moves.urllib.parse import urlencode, urlparse
 from six.moves.http_client import HTTPException
@@ -141,13 +142,13 @@ class Consul(AbstractDCS):
                             retry_exceptions=(ConsulInternalError, HTTPException,
                                               HTTPError, socket.error, socket.timeout))
 
-        self._my_member_data = None
+        self._my_member_data = {}
         kwargs = {}
         if 'url' in config:
             r = urlparse(config['url'])
             config.update({'scheme': r.scheme, 'host': r.hostname, 'port': r.port or 8500})
         elif 'host' in config:
-            host, port = (config.get('host', '127.0.0.1:8500') + ':8500').split(':')[:2]
+            host, port = split_host_port(config.get('host', '127.0.0.1:8500'), 8500)
             config['host'] = host
             if 'port' not in config:
                 config['port'] = int(port)
@@ -263,6 +264,10 @@ class Consul(AbstractDCS):
             config = nodes.get(self._CONFIG)
             config = config and ClusterConfig.from_node(config['ModifyIndex'], config['Value'])
 
+            # get timeline history
+            history = nodes.get(self._HISTORY)
+            history = history and TimelineHistory.from_node(history['ModifyIndex'], history['Value'])
+
             # get last leader operation
             last_leader_operation = nodes.get(self._LEADER_OPTIME)
             last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation['Value'])
@@ -292,9 +297,9 @@ class Consul(AbstractDCS):
             sync = nodes.get(self._SYNC)
             sync = SyncState.from_node(sync and sync['ModifyIndex'], sync and sync['Value'])
 
-            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
         except NotFound:
-            self._cluster = Cluster(None, None, None, None, [], None, None)
+            self._cluster = Cluster(None, None, None, None, [], None, None, None)
         except Exception:
             logger.exception('get_cluster')
             raise ConsulError('Consul is not responding properly')
@@ -311,12 +316,12 @@ class Consul(AbstractDCS):
             except Exception:
                 return False
 
-        if not create_member and member and data == self._my_member_data:
+        if not create_member and member and deep_compare(data, self._my_member_data):
             return True
 
         try:
             args = {} if permanent else {'acquire': self._session}
-            self._client.kv.put(self.member_path, data, **args)
+            self._client.kv.put(self.member_path, json.dumps(data, separators=(',', ':')), **args)
             self._my_member_data = data
             return True
         except Exception:
@@ -352,7 +357,7 @@ class Consul(AbstractDCS):
         return self._client.kv.put(self.leader_optime_path, last_operation)
 
     @catch_consul_errors
-    def update_leader(self):
+    def _update_leader(self):
         if self._session:
             self.retry(self._client.session.renew, self._session)
             self._last_session_refresh = time.time()
@@ -371,6 +376,10 @@ class Consul(AbstractDCS):
     def delete_cluster(self):
         return self.retry(self._client.kv.delete, self.client_path(''), recurse=True)
 
+    @catch_consul_errors
+    def set_history_value(self, value):
+        return self._client.kv.put(self.history_path, value)
+
     @catch_consul_errors
     def delete_leader(self):
         cluster = self.cluster
@@ -379,11 +388,11 @@ class Consul(AbstractDCS):
 
     @catch_consul_errors
     def set_sync_state_value(self, value, index=None):
-        return self._client.kv.put(self.sync_path, value, cas=index)
+        return self.retry(self._client.kv.put, self.sync_path, value, cas=index)
 
     @catch_consul_errors
     def delete_sync_state(self, index=None):
-        return self._client.kv.delete(self.sync_path, cas=index)
+        return self.retry(self._client.kv.delete, self.sync_path, cas=index)
 
     def watch(self, leader_index, timeout):
         if self.__do_not_watch:

patroni/dcs/etcd.py

@@ -1,18 +1,20 @@
 from __future__ import absolute_import
 import etcd
+import json
 import logging
 import os
 import urllib3.util.connection
 import random
 import requests
+import six
 import socket
 import time
 
 from dns.exception import DNSException
 from dns import resolver
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
 from patroni.exceptions import DCSError
-from patroni.utils import Retry, RetryFailedError
+from patroni.utils import Retry, RetryFailedError, split_host_port
 from urllib3.exceptions import HTTPError, ReadTimeoutError
 from requests.exceptions import RequestException
 from six.moves.queue import Queue
@@ -23,6 +25,10 @@ from threading import Thread
 logger = logging.getLogger(__name__)
 
 
+def uri(protocol, host, port, endpoint=''):
+    return '{0}://{1}:{2}{3}'.format(protocol, host, port, endpoint)
+
+
 class EtcdError(DCSError):
     pass
 
@@ -204,8 +210,11 @@ class Client(etcd.Client):
                 self._machines_cache = self.machines
                 if self._base_uri in self._machines_cache:
                     self._machines_cache.remove(self._base_uri)
-        except etcd.EtcdConnectionFailed:
-            self._update_machines_cache = True
+        except etcd.EtcdConnectionFailed as e:
+            if isinstance(e, etcd.EtcdWatchTimedOut) and self._machines_cache:
+                self._base_uri = self._next_server()
+            else:
+                self._update_machines_cache = True
             if not response:
                 raise
         return self._handle_server_response(response)
@@ -227,7 +236,7 @@ class Client(etcd.Client):
             protocol = 'https' if '-ssl' in r else 'http'
             endpoint = '/members' if '-server' in r else ''
             for host, port in self.get_srv_record('_etcd{0}._tcp.{1}'.format(r, srv)):
-                url = '{0}://{1}:{2}{3}'.format(protocol, host, port, endpoint)
+                url = uri(protocol, host, port, endpoint)
                 if endpoint:
                     try:
                         response = requests.get(url, timeout=self.read_timeout, verify=False)
@@ -254,10 +263,10 @@ class Client(etcd.Client):
                 host, port = sa[:2]
                 if af == socket.AF_INET6:
                     host = '[{0}]'.format(host)
-                ret.append('{0}://{1}:{2}'.format(self.protocol, host, port))
+                ret.append(uri(self.protocol, host, port))
             if ret:
                 return list(set(ret))
-        return ['{0}://{1}:{2}'.format(self.protocol, host, port)]
+        return [uri(self.protocol, host, port)]
 
     def _load_machines_cache(self):
         """This method should fill up `_machines_cache` from scratch.
@@ -267,17 +276,20 @@ class Client(etcd.Client):
 
         self._update_machines_cache = True
 
-        if 'srv' not in self._config and 'host' not in self._config:
-            raise Exception('Neither srv nor host url are defined in etcd section of config')
+        if 'srv' not in self._config and 'host' not in self._config and 'hosts' not in self._config:
+            raise Exception('Neither srv, hosts, host nor url are defined in etcd section of config')
 
         if self._use_proxies:
-            self._machines_cache = ['{0}://{1}:{2}'.format(self.protocol, self._config['host'], self._config['port'])]
+            self._machines_cache = [uri(self.protocol, self._config['host'], self._config['port'])]
         else:
             self._machines_cache = []
 
             if 'srv' in self._config:
                 self._machines_cache = self._get_machines_cache_from_srv(self._config['srv'])
 
+            if not self._machines_cache and 'hosts' in self._config:
+                self._machines_cache = list(self._config['hosts'])
+
             if not self._machines_cache and 'host' in self._config:
                 self._machines_cache = self._get_machines_cache_from_dns(self._config['host'], self._config['port'])
 
@@ -347,8 +359,20 @@ class Etcd(AbstractDCS):
             r = urlparse(config['url'])
             config.update({'protocol': r.scheme, 'host': r.hostname, 'port': r.port or 2379,
                            'username': r.username, 'password': r.password})
+        elif 'hosts' in config:
+            hosts = config.pop('hosts')
+            default_port = config.pop('port', 2379)
+            protocol = config.get('protocol', 'http')
+
+            if isinstance(hosts, six.string_types):
+                hosts = hosts.split(',')
+
+            config['hosts'] = []
+            for value in hosts:
+                if isinstance(value, six.string_types):
+                    config['hosts'].append(uri(protocol, *split_host_port(value, default_port)))
         elif 'host' in config:
-            host, port = (config['host'] + ':2379').split(':')[:2]
+            host, port = split_host_port(config['host'], 2379)
             config['host'] = host
             if 'port' not in config:
                 config['port'] = int(port)
@@ -434,6 +458,10 @@ class Etcd(AbstractDCS):
             config = nodes.get(self._CONFIG)
             config = config and ClusterConfig.from_node(config.modifiedIndex, config.value)
 
+            # get timeline history
+            history = nodes.get(self._HISTORY)
+            history = history and TimelineHistory.from_node(history.modifiedIndex, history.value)
+
             # get last leader operation
             last_leader_operation = nodes.get(self._LEADER_OPTIME)
             last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation.value)
@@ -458,15 +486,16 @@ class Etcd(AbstractDCS):
             sync = nodes.get(self._SYNC)
             sync = SyncState.from_node(sync and sync.modifiedIndex, sync and sync.value)
 
-            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
         except etcd.EtcdKeyNotFound:
-            self._cluster = Cluster(None, None, None, None, [], None, None)
+            self._cluster = Cluster(None, None, None, None, [], None, None, None)
         except Exception as e:
             self._handle_exception(e, 'get_cluster', raise_ex=EtcdError('Etcd is not responding properly'))
         self._has_failed = False
 
     @catch_etcd_errors
     def touch_member(self, data, ttl=None, permanent=False):
+        data = json.dumps(data, separators=(',', ':'))
         return self.retry(self._client.set, self.member_path, data, None if permanent else ttl or self._ttl)
 
     @catch_etcd_errors
@@ -499,7 +528,7 @@ class Etcd(AbstractDCS):
         return self._client.set(self.leader_optime_path, last_operation)
 
     @catch_etcd_errors
-    def update_leader(self):
+    def _update_leader(self):
         return self.retry(self._client.test_and_set, self.leader_path, self._name, self._name, self._ttl)
 
     @catch_etcd_errors
@@ -518,9 +547,13 @@ class Etcd(AbstractDCS):
     def delete_cluster(self):
         return self.retry(self._client.delete, self.client_path(''), recursive=True)
 
+    @catch_etcd_errors
+    def set_history_value(self, value):
+        return self._client.write(self.history_path, value)
+
     @catch_etcd_errors
     def set_sync_state_value(self, value, index=None):
-        return self._client.write(self.sync_path, value, prevIndex=index or 0)
+        return self.retry(self._client.write, self.sync_path, value, prevIndex=index or 0)
 
     @catch_etcd_errors
     def delete_sync_state(self, index=None):

patroni/dcs/kubernetes.py

@@ -0,0 +1,409 @@
+from __future__ import absolute_import
+import datetime
+import functools
+import json
+import logging
+import socket
+import sys
+import time
+
+from kubernetes import client as k8s_client, config as k8s_config, watch as k8s_watch
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
+from patroni.exceptions import DCSError
+from patroni.utils import deep_compare, tzutc, Retry, RetryFailedError
+from urllib3.exceptions import HTTPError
+from six.moves.http_client import HTTPException
+
+logger = logging.getLogger(__name__)
+
+
+class KubernetesError(DCSError):
+    pass
+
+
+class KubernetesRetriableException(k8s_client.rest.ApiException):
+
+    def __init__(self, orig):
+        super(KubernetesRetriableException, self).__init__(orig.status, orig.reason)
+        self.body = orig.body
+        self.headers = orig.headers
+
+
+class CoreV1ApiProxy(object):
+
+    def __init__(self, use_endpoints=False):
+        self._api = k8s_client.CoreV1Api()
+        self._request_timeout = None
+        self._use_endpoints = use_endpoints
+
+    def set_timeout(self, timeout):
+        self._request_timeout = (1, timeout / 3.0)
+
+    def __getattr__(self, func):
+        if func.endswith('_kind'):
+            func = func[:-4] + ('endpoints' if self._use_endpoints else 'config_map')
+
+        def wrapper(*args, **kwargs):
+            if '_request_timeout' not in kwargs:
+                kwargs['_request_timeout'] = self._request_timeout
+            try:
+                return getattr(self._api, func)(*args, **kwargs)
+            except k8s_client.rest.ApiException as e:
+                if e.status in (502, 503, 504):  # XXX
+                    raise KubernetesRetriableException(e)
+                raise
+        return wrapper
+
+
+def catch_kubernetes_errors(func):
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except k8s_client.rest.ApiException as e:
+            if e.status == 403:
+                logger.exception('Permission denied')
+            elif e.status != 409:  # Object exists or conflict in resource_version
+                logger.exception('Unexpected error from Kubernetes API')
+            return False
+        except (RetryFailedError, HTTPException, HTTPError, socket.error, socket.timeout):
+            return False
+    return wrapper
+
+
+class Kubernetes(AbstractDCS):
+
+    def __init__(self, config):
+        self._labels = config['labels']
+        self._labels[config.get('scope_label', 'cluster-name')] = config['scope']
+        self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
+        self._namespace = config.get('namespace') or 'default'
+        self._role_label = config.get('role_label', 'role')
+        config['namespace'] = ''
+        super(Kubernetes, self).__init__(config)
+        self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
+                            retry_exceptions=(KubernetesRetriableException, HTTPException,
+                                              HTTPError, socket.error, socket.timeout))
+        self._ttl = None
+        try:
+            k8s_config.load_incluster_config()
+        except k8s_config.ConfigException:
+            k8s_config.load_kube_config(context=config.get('context', 'local'))
+
+        self.__subsets = None
+        use_endpoints = config.get('use_endpoints') and (config.get('patronictl') or 'pod_ip' in config)
+        if use_endpoints:
+            addresses = [k8s_client.V1EndpointAddress(ip=config['pod_ip'])]
+            ports = []
+            for p in config.get('ports', [{}]):
+                port = {'port': int(p.get('port', '5432'))}
+                port.update({n: p[n] for n in ('name', 'protocol') if p.get(n)})
+                ports.append(k8s_client.V1EndpointPort(**port))
+            self.__subsets = [k8s_client.V1EndpointSubset(addresses=addresses, ports=ports)]
+        self._api = CoreV1ApiProxy(use_endpoints)
+        self.set_retry_timeout(config['retry_timeout'])
+        self.set_ttl(config.get('ttl') or 30)
+        self._leader_observed_record = {}
+        self._leader_observed_time = None
+        self._leader_resource_version = None
+        self._leader_observed_subsets = []
+        self.__do_not_watch = False
+
+    def retry(self, *args, **kwargs):
+        return self._retry.copy()(*args, **kwargs)
+
+    def client_path(self, path):
+        return super(Kubernetes, self).client_path(path)[1:].replace('/', '-')
+
+    @property
+    def leader_path(self):
+        return self._base_path[1:] if self.__subsets else super(Kubernetes, self).leader_path
+
+    def set_ttl(self, ttl):
+        ttl = int(ttl)
+        self.__do_not_watch = self._ttl != ttl
+        self._ttl = ttl
+
+    def set_retry_timeout(self, retry_timeout):
+        self._retry.deadline = retry_timeout
+        self._api.set_timeout(retry_timeout)
+
+    @staticmethod
+    def member(pod):
+        annotations = pod.metadata.annotations or {}
+        member = Member.from_node(pod.metadata.resource_version, pod.metadata.name, None, annotations.get('status', ''))
+        member.data['pod_labels'] = pod.metadata.labels
+        return member
+
+    def _load_cluster(self):
+        try:
+            # get list of members
+            response = self.retry(self._api.list_namespaced_pod, self._namespace, label_selector=self._label_selector)
+            members = [self.member(pod) for pod in response.items]
+
+            response = self.retry(self._api.list_namespaced_kind, self._namespace, label_selector=self._label_selector)
+            nodes = {item.metadata.name: item for item in response.items}
+
+            config = nodes.get(self.config_path)
+            metadata = config and config.metadata
+            annotations = metadata and metadata.annotations or {}
+
+            # get initialize flag
+            initialize = annotations.get(self._INITIALIZE)
+
+            # get global dynamic configuration
+            config = ClusterConfig.from_node(metadata and metadata.resource_version,
+                                             annotations.get(self._CONFIG) or '{}')
+
+            # get timeline history
+            history = TimelineHistory.from_node(metadata and metadata.resource_version,
+                                                annotations.get(self._HISTORY) or '[]')
+
+            leader = nodes.get(self.leader_path)
+            metadata = leader and leader.metadata
+            self._leader_resource_version = metadata.resource_version if metadata else None
+            self._leader_observed_subsets = leader.subsets if self.__subsets and leader else []
+            annotations = metadata and metadata.annotations or {}
+
+            # get last leader operation
+            last_leader_operation = annotations.get(self._OPTIME)
+            last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation)
+
+            # get leader
+            leader_record = {n: annotations.get(n) for n in (self._LEADER, 'acquireTime',
+                             'ttl', 'renewTime', 'transitions') if n in annotations}
+            if (leader_record or self._leader_observed_record) and leader_record != self._leader_observed_record:
+                self._leader_observed_record = leader_record
+                self._leader_observed_time = time.time()
+
+            leader = leader_record.get(self._LEADER)
+            try:
+                ttl = int(leader_record.get('ttl')) or self._ttl
+            except (TypeError, ValueError):
+                ttl = self._ttl
+
+            if not metadata or not self._leader_observed_time or self._leader_observed_time + ttl < time.time():
+                leader = None
+
+            if metadata:
+                member = Member(-1, leader, None, {})
+                member = ([m for m in members if m.name == leader] or [member])[0]
+                leader = Leader(response.metadata.resource_version, None, member)
+
+            # failover key
+            failover = nodes.get(self.failover_path)
+            metadata = failover and failover.metadata
+            failover = Failover.from_node(metadata and metadata.resource_version, metadata and metadata.annotations)
+
+            # get synchronization state
+            sync = nodes.get(self.sync_path)
+            metadata = sync and sync.metadata
+            sync = SyncState.from_node(metadata and metadata.resource_version,  metadata and metadata.annotations)
+
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
+        except Exception:
+            logger.exception('get_cluster')
+            raise KubernetesError('Kubernetes API is not responding properly')
+
+    @staticmethod
+    def compare_ports(p1, p2):
+        return p1.name == p2.name and p1.port == p2.port and (p1.protocol or 'TCP') == (p2.protocol or 'TCP')
+
+    @staticmethod
+    def subsets_changed(last_observed_subsets, subsets):
+        """
+        >>> Kubernetes.subsets_changed([], [])
+        False
+        >>> Kubernetes.subsets_changed([], [k8s_client.V1EndpointSubset()])
+        True
+        >>> s1 = [k8s_client.V1EndpointSubset(addresses=[k8s_client.V1EndpointAddress(ip='1.2.3.4')])]
+        >>> s2 = [k8s_client.V1EndpointSubset(addresses=[k8s_client.V1EndpointAddress(ip='1.2.3.5')])]
+        >>> Kubernetes.subsets_changed(s1, s2)
+        True
+        >>> a = [k8s_client.V1EndpointAddress(ip='1.2.3.4')]
+        >>> s1 = [k8s_client.V1EndpointSubset(addresses=a, ports=[k8s_client.V1EndpointPort(protocol='TCP', port=1)])]
+        >>> s2 = [k8s_client.V1EndpointSubset(addresses=a, ports=[k8s_client.V1EndpointPort(port=5432)])]
+        >>> Kubernetes.subsets_changed(s1, s2)
+        True
+        >>> p1 = k8s_client.V1EndpointPort(name='port1', port=1)
+        >>> p2 = k8s_client.V1EndpointPort(name='port2', port=2)
+        >>> p3 = k8s_client.V1EndpointPort(name='port3', port=3)
+        >>> s1 = [k8s_client.V1EndpointSubset(addresses=a, ports=[p1, p2])]
+        >>> s2 = [k8s_client.V1EndpointSubset(addresses=a, ports=[p2, p3])]
+        >>> Kubernetes.subsets_changed(s1, s2)
+        True
+        >>> s2 = [k8s_client.V1EndpointSubset(addresses=a, ports=[p2, p1])]
+        >>> Kubernetes.subsets_changed(s1, s2)
+        False
+        """
+        if len(last_observed_subsets) != len(subsets):
+            return True
+        if subsets == []:
+            return False
+        if len(last_observed_subsets[0].addresses or []) != 1 or \
+                last_observed_subsets[0].addresses[0].ip != subsets[0].addresses[0].ip or \
+                len(last_observed_subsets[0].ports) != len(subsets[0].ports):
+            return True
+        if len(subsets[0].ports) == 1:
+            return not Kubernetes.compare_ports(last_observed_subsets[0].ports[0], subsets[0].ports[0])
+        observed_ports = {p.name: p for p in last_observed_subsets[0].ports}
+        for p in subsets[0].ports:
+            if p.name not in observed_ports or not Kubernetes.compare_ports(p, observed_ports.pop(p.name)):
+                return True
+        return False
+
+    @catch_kubernetes_errors
+    def patch_or_create(self, name, annotations, resource_version=None, patch=False, retry=True, subsets=None):
+        metadata = {'namespace': self._namespace, 'name': name, 'labels': self._labels, 'annotations': annotations}
+        if patch or resource_version:
+            if resource_version is not None:
+                metadata['resource_version'] = resource_version
+            func = functools.partial(self._api.patch_namespaced_kind, name)
+        else:
+            func = functools.partial(self._api.create_namespaced_kind)
+            # skip annotations with null values
+            metadata['annotations'] = {k: v for k, v in metadata['annotations'].items() if v is not None}
+
+        metadata = k8s_client.V1ObjectMeta(**metadata)
+        if subsets is not None and self.__subsets:
+            endpoints = {'metadata': metadata}
+            if self.subsets_changed(self._leader_observed_subsets, subsets):
+                endpoints['subsets'] = subsets
+            body = k8s_client.V1Endpoints(**endpoints)
+        else:
+            body = k8s_client.V1ConfigMap(metadata=metadata)
+        return self.retry(func, self._namespace, body) if retry else func(self._namespace, body)
+
+    def _write_leader_optime(self, last_operation):
+        """Unused"""
+
+    def _update_leader(self):
+        """Unused"""
+
+    def update_leader(self, last_operation):
+        now = datetime.datetime.now(tzutc).isoformat()
+        annotations = {self._LEADER: self._name, 'ttl': str(self._ttl), 'renewTime': now,
+                       'acquireTime': self._leader_observed_record.get('acquireTime') or now,
+                       'transitions': self._leader_observed_record.get('transitions') or '0'}
+        if last_operation:
+            annotations[self._OPTIME] = last_operation
+
+        ret = self.patch_or_create(self.leader_path, annotations, self._leader_resource_version, subsets=self.__subsets)
+        if ret:
+            self._leader_resource_version = ret.metadata.resource_version
+        return ret
+
+    def attempt_to_acquire_leader(self, permanent=False):
+        now = datetime.datetime.now(tzutc).isoformat()
+        annotations = {self._LEADER: self._name, 'ttl': str(sys.maxsize if permanent else self._ttl),
+                       'renewTime': now, 'acquireTime': now, 'transitions': '0'}
+        if self._leader_observed_record:
+            try:
+                transitions = int(self._leader_observed_record.get('transitions'))
+            except (TypeError, ValueError):
+                transitions = 0
+
+            if self._leader_observed_record.get(self._LEADER) != self._name:
+                transitions += 1
+            else:
+                annotations['acquireTime'] = self._leader_observed_record.get('acquireTime') or now
+            annotations['transitions'] = str(transitions)
+        ret = self.patch_or_create(self.leader_path, annotations, self._leader_resource_version, subsets=self.__subsets)
+        if ret:
+            self._leader_resource_version = ret.metadata.resource_version
+        else:
+            logger.info('Could not take out TTL lock')
+        return ret
+
+    def take_leader(self):
+        return self.attempt_to_acquire_leader()
+
+    def set_failover_value(self, value, index=None):
+        """Unused"""
+
+    def manual_failover(self, leader, candidate, scheduled_at=None, index=None):
+        annotations = {'leader': leader or None, 'member': candidate or None, 'scheduled_at': scheduled_at}
+        patch = bool(self.cluster and isinstance(self.cluster.failover, Failover) and self.cluster.failover.index)
+        return self.patch_or_create(self.failover_path, annotations, index, bool(index or patch), False)
+
+    def set_config_value(self, value, index=None):
+        patch = bool(index or self.cluster and self.cluster.config and self.cluster.config.index)
+        return self.patch_or_create(self.config_path, {self._CONFIG: value}, index, patch, False)
+
+    @catch_kubernetes_errors
+    def touch_member(self, data, ttl=None, permanent=False):
+        cluster = self.cluster
+        if cluster and cluster.leader and cluster.leader.name == self._name:
+            role = 'master'
+        elif data['state'] == 'running' and data['role'] != 'master':
+            role = data['role']
+        else:
+            role = None
+
+        member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
+        pod_labels = member and member.data.pop('pod_labels', None)
+        ret = pod_labels is not None and pod_labels.get(self._role_label) == role and deep_compare(data, member.data)
+
+        if not ret:
+            metadata = {'namespace': self._namespace, 'name': self._name, 'labels': {self._role_label: role},
+                        'annotations': {'status': json.dumps(data, separators=(',', ':'))}}
+            body = k8s_client.V1Pod(metadata=k8s_client.V1ObjectMeta(**metadata))
+            ret = self._api.patch_namespaced_pod(self._name, self._namespace, body)
+        return ret
+
+    def initialize(self, create_new=True, sysid=""):
+        cluster = self.cluster
+        resource_version = cluster.config.index if cluster and cluster.config and cluster.config.index else None
+        return self.patch_or_create(self.config_path, {self._INITIALIZE: sysid}, resource_version)
+
+    def delete_leader(self):
+        if self.cluster and isinstance(self.cluster.leader, Leader) and self.cluster.leader.name == self._name:
+            self.patch_or_create(self.leader_path, {self._LEADER: None}, self._leader_resource_version, True, False, [])
+            self.reset_cluster()
+
+    def cancel_initialization(self):
+        self.patch_or_create(self.config_path, {self._INITIALIZE: None}, self.cluster.config.index, True)
+
+    @catch_kubernetes_errors
+    def delete_cluster(self):
+        self.retry(self._api.delete_collection_namespaced_kind, self._namespace, label_selector=self._label_selector)
+
+    def set_history_value(self, value):
+        patch = bool(self.cluster and self.cluster.config and self.cluster.config.index)
+        return self.patch_or_create(self.config_path, {self._HISTORY: value}, None, patch, False)
+
+    def set_sync_state_value(self, value, index=None):
+        """Unused"""
+
+    def write_sync_state(self, leader, sync_standby, index=None):
+        return self.patch_or_create(self.sync_path, self.sync_state(leader, sync_standby), index, False)
+
+    def delete_sync_state(self, index=None):
+        return self.write_sync_state(None, None, index)
+
+    def watch(self, leader_index, timeout):
+        if self.__do_not_watch:
+            self.__do_not_watch = False
+            return True
+
+        if leader_index:
+            end_time = time.time() + timeout
+            w = k8s_watch.Watch()
+            while timeout >= 1:
+                try:
+                    for event in w.stream(self._api.list_namespaced_kind, self._namespace,
+                                          resource_version=leader_index, timeout_seconds=int(timeout + 0.5),
+                                          field_selector='metadata.name=' + self.leader_path,
+                                          _request_timeout=(1, timeout + 1)):
+                        return event['raw_object'].get('metadata', {}).get('resourceVersion') != leader_index
+                    return False
+                except KeyboardInterrupt:
+                    raise
+                except Exception:
+                    logging.exception('watch')
+
+                timeout = end_time - time.time()
+
+        try:
+            return super(Kubernetes, self).watch(None, timeout)
+        finally:
+            self.event.clear()

patroni/dcs/zookeeper.py

@@ -1,11 +1,13 @@
+import json
 import logging
 import time
 
 from kazoo.client import KazooClient, KazooState, KazooRetry
 from kazoo.exceptions import NoNodeError, NodeExistsError
 from kazoo.handlers.threading import SequentialThreadingHandler
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
 from patroni.exceptions import DCSError
+from patroni.utils import deep_compare
 
 logger = logging.getLogger(__name__)
 
@@ -57,7 +59,7 @@ class ZooKeeper(AbstractDCS):
                                    max_delay=1, max_tries=-1, sleep_func=time.sleep))
         self._client.add_listener(self.session_listener)
 
-        self._my_member_data = None
+        self._my_member_data = {}
         self._fetch_cluster = True
 
         self._orig_kazoo_connect = self._client._connection._connect
@@ -159,6 +161,10 @@ class ZooKeeper(AbstractDCS):
         config = self.get_node(self.config_path, watch=self.cluster_watcher) if self._CONFIG in nodes else None
         config = config and ClusterConfig.from_node(config[1].version, config[0], config[1].mzxid)
 
+        # get timeline history
+        history = self.get_node(self.history_path, watch=self.cluster_watcher) if self._HISTORY in nodes else None
+        history = history and TimelineHistory.from_node(history[1].mzxid, history[0])
+
         # get last leader operation
         last_leader_operation = self._OPTIME in nodes and self._fetch_cluster and self.get_node(self.leader_optime_path)
         last_leader_operation = last_leader_operation and int(last_leader_operation[0]) or 0
@@ -191,7 +197,7 @@ class ZooKeeper(AbstractDCS):
         failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
         failover = failover and Failover.from_node(failover[1].version, failover[0])
 
-        self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
+        self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
 
     def _load_cluster(self):
         if self._fetch_cluster or self._cluster is None:
@@ -215,16 +221,19 @@ class ZooKeeper(AbstractDCS):
             logger.info('Could not take out TTL lock')
         return ret
 
-    def set_failover_value(self, value, index=None):
+    def __set_failover_or_sync_state_value(self, key, value, index=None):
         try:
-            self._client.retry(self._client.set, self.failover_path, value.encode('utf-8'), version=index or -1)
+            self._client.retry(self._client.set, key, value.encode('utf-8'), version=index or -1)
             return True
         except NoNodeError:
-            return value == '' or (index is None and self._create(self.failover_path, value))
+            return value == '' or (index is None and self._create(key, value))
         except Exception:
             logging.exception('set_failover_value')
             return False
 
+    def set_failover_value(self, value, index=None):
+        return self.__set_failover_or_sync_state_value(self.failover_path, value, index)
+
     def set_config_value(self, value, index=None):
         try:
             self._client.retry(self._client.set, self.config_path, value.encode('utf-8'), version=index or -1)
@@ -242,7 +251,7 @@ class ZooKeeper(AbstractDCS):
     def touch_member(self, data, ttl=None, permanent=False):
         cluster = self.cluster
         member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
-        data = data.encode('utf-8')
+        encoded_data = json.dumps(data, separators=(',', ':')).encode('utf-8')
         if member and self._client.client_id is not None and member.session != self._client.client_id[0]:
             try:
                 self._client.delete_async(self.member_path).get(timeout=1)
@@ -253,11 +262,12 @@ class ZooKeeper(AbstractDCS):
             member = None
 
         if member:
-            if data == self._my_member_data:
+            if deep_compare(data, self._my_member_data):
                 return True
         else:
             try:
-                self._client.create_async(self.member_path, data, makepath=True, ephemeral=not permanent).get(timeout=1)
+                self._client.create_async(self.member_path, encoded_data, makepath=True,
+                                          ephemeral=not permanent).get(timeout=1)
                 self._my_member_data = data
                 return True
             except Exception as e:
@@ -265,7 +275,7 @@ class ZooKeeper(AbstractDCS):
                     logger.exception('touch_member')
                     return False
         try:
-            self._client.set_async(self.member_path, data).get(timeout=1)
+            self._client.set_async(self.member_path, encoded_data).get(timeout=1)
             self._my_member_data = data
             return True
         except Exception:
@@ -276,22 +286,25 @@ class ZooKeeper(AbstractDCS):
     def take_leader(self):
         return self.attempt_to_acquire_leader()
 
-    def _write_leader_optime(self, last_operation):
-        last_operation = last_operation.encode('utf-8')
+    def __write_leader_optime_or_history_value(self, key, value):
+        value = value.encode('utf-8')
         try:
-            self._client.set_async(self.leader_optime_path, last_operation).get(timeout=1)
+            self._client.set_async(key, value).get(timeout=1)
             return True
         except NoNodeError:
             try:
-                self._client.create_async(self.leader_optime_path, last_operation, makepath=True).get(timeout=1)
+                self._client.create_async(key, value, makepath=True).get(timeout=1)
                 return True
             except Exception:
-                logger.exception('Failed to create %s', self.leader_optime_path)
+                logger.exception('Failed to create %s', key)
         except Exception:
-            logger.exception('Failed to update %s', self.leader_optime_path)
+            logger.exception('Failed to update %s', key)
         return False
 
-    def update_leader(self):
+    def _write_leader_optime(self, last_operation):
+        return self.__write_leader_optime_or_history_value(self.leader_optime_path, last_operation)
+
+    def _update_leader(self):
         return True
 
     def delete_leader(self):
@@ -316,15 +329,11 @@ class ZooKeeper(AbstractDCS):
         except NoNodeError:
             return True
 
+    def set_history_value(self, value):
+        return self.__write_leader_optime_or_history_value(self.history_path, value)
+
     def set_sync_state_value(self, value, index=None):
-        try:
-            self._client.retry(self._client.set, self.sync_path, value.encode('utf-8'), version=index or -1)
-            return True
-        except NoNodeError:
-            return value == '' or (index is None and self._create(self.sync_path, value))
-        except Exception:
-            logging.exception('set_sync_state_value')
-            return False
+        return self.__set_failover_or_sync_state_value(self.sync_path, value, index)
 
     def delete_sync_state(self, index=None):
         return self.set_sync_state_value("{}", index)

patroni/ha.py

@@ -57,11 +57,12 @@ class Ha(object):
         self.dcs = patroni.dcs
         self.cluster = None
         self.old_cluster = None
+        self._leader_timeline = None
         self.recovering = False
         self._post_bootstrap_task = None
         self._crash_recovery_executed = False
         self._start_timeout = None
-        self._async_executor = AsyncExecutor(self.wakeup)
+        self._async_executor = AsyncExecutor(self.state_handler, self.wakeup)
         self.watchdog = patroni.watchdog
 
         # Each member publishes various pieces of information to the DCS using touch_member. This lock protects
@@ -82,18 +83,21 @@ class Ha(object):
             self.old_cluster = cluster
         self.cluster = cluster
 
+        self._leader_timeline = None if cluster.is_unlocked() else cluster.leader.timeline
+
     def acquire_lock(self):
         return self.dcs.attempt_to_acquire_leader()
 
     def update_lock(self, write_leader_optime=False):
-        ret = self.dcs.update_leader()
+        last_operation = None
+        if write_leader_optime:
+            try:
+                last_operation = self.state_handler.last_operation()
+            except Exception:
+                logger.exception('Exception when called state_handler.last_operation()')
+        ret = self.dcs.update_leader(last_operation)
         if ret:
             self.watchdog.keepalive()
-            if write_leader_optime:
-                try:
-                    self.dcs.write_leader_optime(self.state_handler.last_operation())
-                except Exception:
-                    pass
         return ret
 
     def has_lock(self):
@@ -122,9 +126,15 @@ class Ha(object):
                 data['tags'] = tags
             if self.state_handler.pending_restart:
                 data['pending_restart'] = True
-            if not self._async_executor.busy and data['state'] in ['running', 'restarting', 'starting']:
+            if self._async_executor.scheduled_action in (None, 'promote') \
+                    and data['state'] in ['running', 'restarting', 'starting']:
                 try:
-                    data['xlog_location'] = self.state_handler.wal_position(retry=False)
+                    timeline, wal_position = self.state_handler.timeline_wal_position()
+                    data['xlog_location'] = wal_position
+                    if not timeline:
+                        timeline = self.state_handler.replica_cached_timeline(self._leader_timeline)
+                    if timeline:
+                        data['timeline'] = timeline
                 except Exception:
                     pass
             if self.patroni.scheduled_restart:
@@ -132,7 +142,10 @@ class Ha(object):
                 scheduled_restart_data['schedule'] = scheduled_restart_data['schedule'].isoformat()
                 data['scheduled_restart'] = scheduled_restart_data
 
-            return self.dcs.touch_member(json.dumps(data, separators=(',', ':')))
+            if self.is_paused():
+                data['pause'] = True
+
+            return self.dcs.touch_member(data)
 
     def clone(self, clone_member=None, msg='(without leader)'):
         if self.state_handler.clone(clone_member):
@@ -362,6 +375,24 @@ class Ha(object):
             with self._member_state_lock:
                 self._disable_sync -= 1
 
+    def update_cluster_history(self):
+        master_timeline = self.state_handler.get_master_timeline()
+        cluster_history = self.cluster.history and self.cluster.history.lines
+        if master_timeline == 1:
+            if cluster_history:
+                self.dcs.set_history_value('[]')
+        elif not cluster_history or cluster_history[-1][0] != master_timeline - 1 or len(cluster_history[-1]) != 4:
+            cluster_history = {l[0]: l for l in cluster_history or []}
+            history = self.state_handler.get_history(master_timeline)
+            if history:
+                for line in history:
+                    # enrich current history with promotion timestamps stored in DCS
+                    if len(line) == 3 and line[0] in cluster_history \
+                            and len(cluster_history[line[0]]) == 4 \
+                            and cluster_history[line[0]][1] == line[1]:
+                        line.append(cluster_history[line[0]][3])
+                self.dcs.set_history_value(json.dumps(history, separators=(',', ':')))
+
     def enforce_master_role(self, message, promote_message):
         if not self.is_paused() and not self.watchdog.is_running and not self.watchdog.activate():
             if self.state_handler.is_leader():
@@ -371,10 +402,14 @@ class Ha(object):
                 self.release_leader_key_voluntarily()
                 return 'Not promoting self because watchdog could not be activated'
 
-        if self.state_handler.is_leader() or self.state_handler.role == 'master':
+        if self.state_handler.is_leader():
             # Inform the state handler about its master role.
             # It may be unaware of it if postgres is promoted manually.
             self.state_handler.set_role('master')
+            self.process_sync_replication()
+            self.update_cluster_history()
+            return message
+        elif self.state_handler.role == 'master':
             self.process_sync_replication()
             return message
         else:
@@ -386,7 +421,9 @@ class Ha(object):
                     # promotion until next cycle. TODO: trigger immediate retry of run_cycle
                     return 'Postponing promotion because synchronous replication state was updated by somebody else'
                 self.state_handler.set_synchronous_standby('*' if self.is_synchronous_mode_strict() else None)
-            self.state_handler.promote()
+            if self.state_handler.role != 'master':
+                self._async_executor.schedule('promote')
+                self._async_executor.run_async(self.state_handler.promote, args=(self.dcs.loop_wait,))
             return promote_message
 
     @staticmethod
@@ -422,7 +459,7 @@ class Ha(object):
     def _is_healthiest_node(self, members, check_replication_lag=True):
         """This method tries to determine whether I am healthy enough to became a new leader candidate or not."""
 
-        my_wal_position = self.state_handler.wal_position()
+        _, my_wal_position = self.state_handler.timeline_wal_position()
         if check_replication_lag and self.is_lagging(my_wal_position):
             return False  # Too far behind last reported wal position on master
 
@@ -852,7 +889,7 @@ class Ha(object):
         member_role = 'leader' if clone_member == self.cluster.leader else 'replica'
         return self.clone(clone_member, "from {0} '{1}'".format(member_role, clone_member.name))
 
-    def reinitialize(self):
+    def reinitialize(self, force=False):
         with self._async_executor:
             self.load_cluster_from_dcs()
 
@@ -862,7 +899,11 @@ class Ha(object):
             if self.cluster.leader.name == self.state_handler.name:
                 return 'I am the leader, can not reinitialize'
 
-            action = self._async_executor.schedule('reinitialize', immediately=True)
+        if force:
+            self._async_executor.cancel()
+
+        with self._async_executor:
+            action = self._async_executor.schedule('reinitialize')
             if action is not None:
                 return '{0} already in progress'.format(action)
 
@@ -880,7 +921,7 @@ class Ha(object):
                     # background thread and has not even written a pid file yet.
                     with self._async_executor.critical_task as task:
                         if not task.cancel():
-                            self.state_handler.terminate_starting_postmaster(pid=task.result)
+                            self.state_handler.terminate_starting_postmaster(postmaster=task.result)
                 self.demote('immediate-nolock')
                 return 'lost leader lock during ' + self._async_executor.scheduled_action
 
@@ -988,6 +1029,7 @@ class Ha(object):
     def _run_cycle(self):
         dcs_failed = False
         try:
+            self.state_handler.reset_cluster_info_state()
             self.load_cluster_from_dcs()
 
             if self.is_paused():
@@ -1025,8 +1067,8 @@ class Ha(object):
             # is data directory empty?
             if self.state_handler.data_directory_empty():
                 self.state_handler.set_role('uninitialized')
-                self.state_handler.stop()
-                # In case datadir went away while we were master. TODO: check for this and try to stop postgresql.
+                self.state_handler.stop('immediate')
+                # In case datadir went away while we were master.
                 self.watchdog.disable()
 
                 # is this instance the leader?
@@ -1104,7 +1146,8 @@ class Ha(object):
             disable_wd = self.watchdog.disable if self.watchdog.is_running else None
             self.while_not_sync_standby(lambda: self.state_handler.stop(checkpoint=False, on_safepoint=disable_wd))
             if not self.state_handler.is_running():
-                self.dcs.delete_leader()
+                if self.has_lock():
+                    self.dcs.delete_leader()
             else:
                 # XXX: what about when Patroni is started as the wrong user that has access to the watchdog device
                 # but cannot shut down PostgreSQL. Root would be the obvious example. Would be nice to not kill the

patroni/postgresql.py

@@ -1,12 +1,9 @@
 import logging
-import errno
 import os
 import psycopg2
-import psutil
 import re
 import shlex
 import shutil
-import signal
 import socket
 import subprocess
 import tempfile
@@ -14,10 +11,10 @@ import time
 
 from collections import defaultdict
 from contextlib import contextmanager
-from patroni import call_self
 from patroni.callback_executor import CallbackExecutor
-from patroni.exceptions import PostgresConnectionException
-from patroni.utils import compare_values, parse_bool, parse_int, Retry, RetryFailedError, polling_loop, int_or_none
+from patroni.exceptions import PostgresConnectionException, PostgresException
+from patroni.utils import compare_values, parse_bool, parse_int, Retry, RetryFailedError, polling_loop, split_host_port
+from patroni.postmaster import PostmasterProcess
 from six import string_types
 from six.moves.urllib.parse import quote_plus
 from threading import current_thread, Lock
@@ -35,15 +32,17 @@ STATE_REJECT = 'rejecting connections'
 STATE_NO_RESPONSE = 'not responding'
 STATE_UNKNOWN = 'unknown'
 
-STOP_SIGNALS = {
-    'smart': signal.SIGTERM,
-    'fast': signal.SIGINT,
-    'immediate': signal.SIGQUIT,
-}
 STOP_POLLING_INTERVAL = 1
 REWIND_STATUS = type('Enum', (), {'INITIAL': 0, 'CHECK': 1, 'NEED': 2, 'NOT_NEED': 3, 'SUCCESS': 4, 'FAILED': 5})
 sync_standby_name_re = re.compile('^[A-Za-z_][A-Za-z_0-9\$]*$')
 
+cluster_info_query = ("SELECT CASE WHEN pg_is_in_recovery() THEN 0 "
+                      "ELSE ('x' || SUBSTR(pg_{0}file_name(pg_current_{0}_{1}()), 1, 8))::bit(32)::int END, "
+                      "CASE WHEN pg_is_in_recovery() THEN GREATEST("
+                      " pg_{0}_{1}_diff(COALESCE(pg_last_{0}_receive_{1}(), '0/0'), '0/0')::bigint,"
+                      " pg_{0}_{1}_diff(pg_last_{0}_replay_{1}(), '0/0')::bigint)"
+                      "ELSE pg_{0}_{1}_diff(pg_current_{0}_{1}(), '0/0')::bigint END")
+
 
 def quote_ident(value):
     """Very simplified version of quote_ident"""
@@ -71,25 +70,6 @@ def null_context():
     yield
 
 
-def _update_postmaster_cached_info(func):
-    def wrapper(self):
-        ret = func(self)
-        if ret and 'pid' in ret and 'start_time' in ret:
-            old_pid = self._postmaster_cached_info.get('pid', 0)
-            old_start_time = self._postmaster_cached_info.get('start_time', 0)
-            try:
-                pmpid = int(ret['pid'])
-                pmstart = int(ret['start_time'])
-                if pmpid != old_pid or pmstart != old_start_time:  # this check removes repeating messages from logs
-                    self._postmaster_cached_info = {'pid': pmpid, 'start_time': pmstart}
-                    logger.info("Updated postmaster info: %s .", self._postmaster_cached_info)
-            except ValueError:
-                logger.warning('Cannot update postmaster info with data due garbage in pid file: %s', ret)
-        return ret
-
-    return wrapper
-
-
 class Postgresql(object):
 
     # List of parameters which must be always passed to postmaster as command line options
@@ -113,12 +93,12 @@ class Postgresql(object):
         'wal_level': ('hot_standby', lambda v: v.lower() in ('hot_standby', 'replica', 'logical'), 90100),
         'hot_standby': ('on', lambda _: False, 90100),
         'max_connections': (100, lambda v: int(v) >= 100, 90100),
-        'max_wal_senders': (5, lambda v: int(v) >= 5, 90100),
+        'max_wal_senders': (10, lambda v: int(v) >= 10, 90100),
         'wal_keep_segments': (8, lambda v: int(v) >= 8, 90100),
         'max_prepared_transactions': (0, lambda v: int(v) >= 0, 90100),
         'max_locks_per_transaction': (64, lambda v: int(v) >= 64, 90100),
         'track_commit_timestamp': ('off', lambda v: parse_bool(v) is not None, 90500),
-        'max_replication_slots': (5, lambda v: int(v) >= 5, 90400),
+        'max_replication_slots': (10, lambda v: int(v) >= 10, 90400),
         'max_worker_processes': (8, lambda v: int(v) >= 8, 90400),
         'wal_log_hints': ('on', lambda _: False, 90400)
     }
@@ -161,10 +141,13 @@ class Postgresql(object):
         self._pg_hba_conf = os.path.join(self._config_dir, 'pg_hba.conf')
         self._recovery_conf = os.path.join(self._data_dir, 'recovery.conf')
         self._postmaster_pid = os.path.join(self._data_dir, 'postmaster.pid')
-        self._postmaster_cached_info = {'pid': 0, 'start_time': 0}
         self._trigger_file = config.get('recovery_conf', {}).get('trigger_file') or 'promote'
         self._trigger_file = os.path.abspath(os.path.join(self._data_dir, self._trigger_file))
 
+        self._is_cancelled = False
+        self._cancellable = None
+        self._cancellable_lock = Lock()
+
         self._connection_lock = Lock()
         self._connection = None
         self._cursor_holder = None
@@ -184,6 +167,12 @@ class Postgresql(object):
 
         self._state_entry_timestamp = None
 
+        self._cluster_info_state = {}
+        self._cached_replica_timeline = None
+
+        # Last known running process
+        self._postmaster_proc = None
+
         if self.is_running():
             self.set_state('running')
             self.set_role('master' if self.is_leader() else 'replica')
@@ -240,8 +229,8 @@ class Postgresql(object):
 
     def get_server_parameters(self, config):
         parameters = config['parameters'].copy()
-        listen_addresses, port = (config['listen'] + ':5432').split(':')[:2]
-        parameters.update({'cluster_name': self.scope, 'listen_addresses': listen_addresses, 'port': port})
+        listen_addresses, port = split_host_port(config['listen'], 5432)
+        parameters.update({'cluster_name': self.scope, 'listen_addresses': listen_addresses, 'port': str(port)})
         if config.get('synchronous_mode', False):
             if self._synchronous_standby_names is None:
                 if config.get('synchronous_mode_strict', False):
@@ -548,27 +537,22 @@ class Postgresql(object):
         ret = self.pg_ctl('initdb', *options)
         if pwfile:
             os.remove(pwfile)
-        if ret:
-            if not self._server_parameters.get('hba_file') and not self.config.get('pg_hba'):
-                self.write_pg_hba(config.get('pg_hba', []))
-            self._major_version = self.get_major_version()
-            self._server_parameters = self.get_server_parameters(self.config)
-        else:
+        if not ret:
             self.set_state('initdb failed')
         return ret
 
     def _custom_bootstrap(self, config):
+        self.set_state('running custom bootstrap script')
         params = ['--scope=' + self.scope, '--datadir=' + self._data_dir]
         try:
             logger.info('Running custom bootstrap script: %s', config['command'])
-            if subprocess.call(shlex.split(config['command']) + params) != 0:
+            if self.cancellable_subprocess_call(shlex.split(config['command']) + params) != 0:
                 self.set_state('custom bootstrap failed')
                 return False
         except Exception:
             logger.exception('Exception during custom bootstrap')
             return False
         self._post_restore()
-        self.save_configuration_files()
 
         if 'recovery_conf' in config:
             self.write_recovery_conf(config['recovery_conf'])
@@ -607,7 +591,7 @@ class Postgresql(object):
             env = self.write_pgpass(r) if 'password' in r else None
 
             try:
-                ret = subprocess.call(shlex.split(cmd) + [connstring], env=env)
+                ret = self.cancellable_subprocess_call(shlex.split(cmd) + [connstring], env=env)
             except OSError:
                 logger.error('post_init script %s failed', cmd)
                 return False
@@ -671,6 +655,9 @@ class Postgresql(object):
         # go through them in priority order
         ret = 1
         for replica_method in replica_methods:
+            with self._cancellable_lock:
+                if self._is_cancelled:
+                    break
             # if the method is basebackup, then use the built-in
             if replica_method == "basebackup":
                 ret = self.basebackup(connstring, env)
@@ -700,7 +687,7 @@ class Postgresql(object):
                 params = ["--{0}={1}".format(arg, val) for arg, val in method_config.items()]
                 try:
                     # call script with the full set of parameters
-                    ret = subprocess.call(shlex.split(cmd) + params, env=env)
+                    ret = self.cancellable_subprocess_call(shlex.split(cmd) + params, env=env)
                     # if we succeeded, stop
                     if ret == 0:
                         logger.info('replica has been created using %s', replica_method)
@@ -715,25 +702,40 @@ class Postgresql(object):
         self.set_state('stopped')
         return ret
 
+    def reset_cluster_info_state(self):
+        self._cluster_info_state = {}
+
+    def _cluster_info_state_get(self, name):
+        if not self._cluster_info_state:
+            stmt = cluster_info_query.format(self.wal_name, self.lsn_name)
+            try:
+                result = self._is_leader_retry(self._query, stmt).fetchone()
+                self._cluster_info_state = dict(zip(['timeline', 'wal_position'], result))
+            except RetryFailedError as e:  # SELECT failed two times
+                self._cluster_info_state = {'error': str(e)}
+                if not self.is_starting() and self.pg_isready() == STATE_REJECT:
+                    self.set_state('starting')
+
+        if 'error' in self._cluster_info_state:
+            raise PostgresConnectionException(self._cluster_info_state['error'])
+
+        return self._cluster_info_state.get(name)
+
     def is_leader(self):
-        try:
-            return not self._is_leader_retry(self._query, 'SELECT pg_is_in_recovery()').fetchone()[0]
-        except RetryFailedError as e:  # SELECT pg_is_in_recovery() failed two times
-            if not self.is_starting() and self.pg_isready() == STATE_REJECT:
-                self.set_state('starting')
-            raise PostgresConnectionException(str(e))
+        return bool(self._cluster_info_state_get('timeline'))
 
     def is_running(self):
-        if not (self._version_file_exists() and os.path.isfile(self._postmaster_pid)):
-            # XXX: This is dangerous in case somebody deletes the data directory while PostgreSQL is still running.
-            return False
+        """Returns PostmasterProcess if one is running on the data directory or None. If most recently seen process
+        is running udpates the cached process based on pid file."""
+        if self._postmaster_proc:
+            if self._postmaster_proc.is_running():
+                return self._postmaster_proc
+            self._postmaster_proc = None
 
-        pidfile = self.read_pid_file()
-        return self._is_postmaster_pid_running(int_or_none(pidfile.get('pid')),
-                                               start_time=int_or_none(pidfile.get('start_time')))
+        self._postmaster_proc = PostmasterProcess.from_pidfile(self._read_pid_file())
+        return self._postmaster_proc
 
-    @_update_postmaster_cached_info
-    def read_pid_file(self):
+    def _read_pid_file(self):
         """Reads and parses postmaster.pid from the data directory
 
         :returns dictionary of values if successful, empty dictionary otherwise
@@ -745,61 +747,6 @@ class Postgresql(object):
         except IOError:
             return {}
 
-    def get_pid(self):
-        """Fetches pid value from postmaster.pid using read_pid_file
-
-        :returns pid if successful, 0 if pid file is not present"""
-        # TODO: figure out what to do on permission errors
-        pid = self.read_pid_file().get('pid', 0)
-        try:
-            return int(pid)
-        except ValueError:
-            logger.warning("Garbage pid in postmaster.pid: {0!r}".format(pid))
-            return 0
-
-    def get_pid_with_lost_data_dir(self):
-        logger.info("Trying to check if process running without directory "
-                    "with cached postmaster info: %s .", self._postmaster_cached_info)
-        try:
-            process = psutil.Process(self._postmaster_cached_info['pid'])
-            # check difference instead of values because of rounding issues
-            if abs(self._postmaster_cached_info["start_time"] - process.create_time()) < 2:
-                return process.pid
-            else:
-                logger.info("Process with pid %s was started at different time %s .",
-                            process.pid, process.create_time())
-        except psutil.NoSuchProcess:
-            logger.info("Cannot find process %s .", self._postmaster_cached_info['pid'])
-        return 0
-
-    def clean_postmaster_cached_info(self):
-        self._postmaster_cached_info = {'pid': 0, 'start_time': 0}
-        logger.info("postmaster info was cleaned.")
-
-    @staticmethod
-    def _is_postmaster_pid_running(pid, start_time=None):
-        # Normalize pid handling missing values and negative pids from postmaster.pid
-        if not pid:
-            return False
-        if pid < 0:
-            pid = -pid
-
-        try:
-            proc = psutil.Process(pid)
-        except psutil.NoSuchProcess:
-            return False
-
-        # If the process is Patroni or Patronis host process or Patronis child process then it's a false positive
-        my_pid = os.getpid()
-        if pid == my_pid or pid == os.getppid() or proc.parent() == my_pid:
-            return False
-
-        # If process start time differs by more than 3 seconds it's a false positive
-        if start_time is not None and abs(proc.create_time() - start_time) > 3:
-            return False
-
-        return True
-
     @property
     def cb_called(self):
         return self.__cb_called
@@ -844,30 +791,24 @@ class Postgresql(object):
     def is_starting(self):
         return self.state == 'starting'
 
-    def wait_for_port_open(self, pid, initiated, timeout):
+    def wait_for_port_open(self, postmaster, timeout):
         """Waits until PostgreSQL opens ports."""
         for _ in polling_loop(timeout):
-            pid_file = self.read_pid_file()
-            if len(pid_file) > 5:
-                try:
-                    pmpid = int(pid_file['pid'])
-                    pmstart = int(pid_file['start_time'])
+            with self._cancellable_lock:
+                if self._is_cancelled:
+                    return False
 
-                    if pmstart >= initiated - 2 and pmpid == pid:
-                        isready = self.pg_isready()
-                        if isready != STATE_NO_RESPONSE:
-                            if isready not in [STATE_REJECT, STATE_RUNNING]:
-                                logger.warning("Can't determine PostgreSQL startup status, assuming running")
-                            return True
-                except ValueError:
-                    # Garbage in the pid file
-                    pass
-
-            if not self._is_postmaster_pid_running(pid, start_time=initiated):
+            if not postmaster.is_running():
                 logger.error('postmaster is not running')
                 self.set_state('start failed')
                 return False
 
+            isready = self.pg_isready()
+            if isready != STATE_NO_RESPONSE:
+                if isready not in [STATE_REJECT, STATE_RUNNING]:
+                    logger.warning("Can't determine PostgreSQL startup status, assuming running")
+                return True
+
         logger.warning("Timed out waiting for PostgreSQL to start")
         return False
 
@@ -903,33 +844,22 @@ class Postgresql(object):
         options = ['--{0}={1}'.format(p, self._server_parameters[p]) for p in self.CMDLINE_OPTIONS
                    if p in self._server_parameters and p != 'wal_keep_segments']
 
-        # Unfortunately `pg_ctl start` does not return postmaster pid to us. Without this information
-        # it is hard to know the current state of postgres startup, so we had to reimplement pg_ctl start
-        # in python. It will start postgres, wait for port to be open and wait until postgres will start
-        # accepting connections.
-        # Important!!! We can't just start postgres using subprocess.Popen, because in this case it
-        # will be our child for the rest of our live and we will have to take care of it (`waitpid`).
-        # So we will use the same approach as pg_ctl uses: start a new process, which will start postgres.
-        # This process will write postmaster pid to stdout and exit immediately. Now it's responsibility
-        # of init process to take care about postmaster.
-        # In order to make everything portable we can't use fork&exec approach here, so  we will call
-        # ourselves and pass list of arguments which must be used to start postgres.
+        with self._cancellable_lock:
+            if self._is_cancelled:
+                return False
+
         with task or null_context():
             if task and task.is_cancelled:
                 logger.info("PostgreSQL start cancelled.")
                 return False
 
-            start_initiated = time.time()
-            proc = call_self(['pg_ctl_start', self._pgcommand('postgres'), '-D', self._data_dir,
-                             '--config-file={}'.format(self._postgresql_conf)] + options, close_fds=True,
-                             preexec_fn=os.setsid, stdout=subprocess.PIPE,
-                             env={p: os.environ[p] for p in ('PATH', 'LC_ALL', 'LANG') if p in os.environ})
-            pid = int(proc.stdout.readline().strip())
-            proc.wait()
-            logger.info('postmaster pid=%s', pid)
+            self._postmaster_proc = PostmasterProcess.start(self._pgcommand('postgres'),
+                                                            self._data_dir,
+                                                            self._postgresql_conf,
+                                                            options)
 
             if task:
-                task.complete(pid)
+                task.complete(self._postmaster_proc)
 
         start_timeout = timeout
         if not start_timeout:
@@ -939,7 +869,7 @@ class Postgresql(object):
                 start_timeout = 60
 
         # We want postmaster to open ports before we continue
-        if not self.wait_for_port_open(pid, start_initiated, start_timeout):
+        if not self._postmaster_proc or not self.wait_for_port_open(self._postmaster_proc, start_timeout):
             return False
 
         ret = self.wait_for_startup(start_timeout)
@@ -967,7 +897,7 @@ class Postgresql(object):
             logging.exception('Exception during CHECKPOINT')
             return 'not accessible or not healty'
 
-    def stop(self, mode='fast', block_callbacks=False, checkpoint=True, on_safepoint=None):
+    def stop(self, mode='fast', block_callbacks=False, checkpoint=None, on_safepoint=None):
         """Stop PostgreSQL
 
         Supports a callback when a safepoint is reached. A safepoint is when no user backend can return a successful
@@ -976,6 +906,9 @@ class Postgresql(object):
 
         :param on_safepoint: This callback is called when no user backends are running.
         """
+        if checkpoint is None:
+            checkpoint = False if mode == 'immediate' else True
+
         success, pg_signaled = self._do_stop(mode, block_callbacks, checkpoint, on_safepoint)
         if success:
             # block_callbacks is used during restart to avoid
@@ -990,13 +923,8 @@ class Postgresql(object):
         return success
 
     def _do_stop(self, mode, block_callbacks, checkpoint, on_safepoint):
-        if not self.is_running():
-            if self.data_directory_empty() and self._postmaster_cached_info['pid']:
-                pid = self.get_pid_with_lost_data_dir()
-                if pid > 0:
-                    self.terminate_starting_postmaster(pid)
-                    self.clean_postmaster_cached_info()
-                    return True, True
+        postmaster = self.is_running()
+        if not postmaster:
             if on_safepoint:
                 on_safepoint()
             return True, False
@@ -1008,86 +936,39 @@ class Postgresql(object):
             self.set_state('stopping')
 
         # Send signal to postmaster to stop
-        pid, result = self._signal_postmaster_stop(mode)
-        if result is not None:
-            if result and on_safepoint:
+        success = postmaster.signal_stop(mode)
+        if success is not None:
+            if success and on_safepoint:
                 on_safepoint()
-            return result, True
+            return success, True
 
         # We can skip safepoint detection if we don't have a callback
         if on_safepoint:
             # Wait for our connection to terminate so we can be sure that no new connections are being initiated
-            self._wait_for_connection_close(pid)
-            self._wait_for_user_backends_to_close(pid)
+            self._wait_for_connection_close(postmaster)
+            postmaster.wait_for_user_backends_to_close()
             on_safepoint()
 
-        self._wait_for_postmaster_stop(pid)
-        self.clean_postmaster_cached_info()
+        postmaster.wait()
 
         return True, True
 
-    def _wait_for_postmaster_stop(self, pid):
-        # This wait loop differs subtly from pg_ctl as we check for both the pid file going
-        # away and if the pid is running. This seems safer.
-        while pid == self.get_pid() and self._is_postmaster_pid_running(pid):
-            time.sleep(STOP_POLLING_INTERVAL)
-
-    def _signal_postmaster_stop(self, mode):
-        pid = self.get_pid()
-        if pid == 0:
-            return None, True
-        elif pid < 0:
-            logger.warning("Cannot stop server; single-user server is running (PID: {0})".format(-pid))
-            return None, False
-        try:
-            os.kill(pid, STOP_SIGNALS[mode])
-        except OSError as e:
-            if e.errno == errno.ESRCH:
-                return None, True
-            else:
-                logger.warning("Could not send stop signal to PostgreSQL (error: {0})".format(e.errno))
-                return None, False
-        return pid, None
-
-    def terminate_starting_postmaster(self, pid):
+    @staticmethod
+    def terminate_starting_postmaster(postmaster):
         """Terminates a postmaster that has not yet opened ports or possibly even written a pid file. Blocks
         until the process goes away."""
-        try:
-            os.kill(pid, STOP_SIGNALS['immediate'])
-        except OSError as e:
-            if e.errno == errno.ESRCH:
-                return
-            logger.warning("Could not send stop signal to PostgreSQL (error: {0})".format(e.errno))
+        postmaster.signal_stop('immediate')
+        postmaster.wait()
 
-        while self._is_postmaster_pid_running(pid):
-            time.sleep(STOP_POLLING_INTERVAL)
-
-    def _wait_for_connection_close(self, pid):
+    def _wait_for_connection_close(self, postmaster):
         try:
             with self.connection().cursor() as cur:
-                while pid == self.get_pid() and self._is_postmaster_pid_running(pid):  # Need a timeout here?
+                while postmaster.is_running():  # Need a timeout here?
                     cur.execute("SELECT 1")
                     time.sleep(STOP_POLLING_INTERVAL)
         except psycopg2.Error:
             pass
 
-    @staticmethod
-    def _wait_for_user_backends_to_close(postmaster_pid):
-        # These regexps are cross checked against versions PostgreSQL 9.1 .. 9.6
-        aux_proc_re = re.compile("(?:postgres:)( .*:)? (?:""(?:startup|logger|checkpointer|writer|wal writer|"
-                                 "autovacuum launcher|autovacuum worker|stats collector|wal receiver|archiver|"
-                                 "wal sender) process|bgworker: )")
-
-        try:
-            postmaster = psutil.Process(postmaster_pid)
-            user_backends = [p for p in postmaster.children() if not aux_proc_re.match(p.cmdline()[0])]
-            logger.debug("Waiting for user backends {0} to close".format(
-                ",".join(p.cmdline()[0] for p in user_backends)))
-            psutil.wait_procs(user_backends)
-            logger.debug("Backends closed")
-        except psutil.NoSuchProcess:
-            return
-
     def reload(self):
         ret = self.pg_ctl('reload')
         if ret:
@@ -1142,6 +1023,9 @@ class Postgresql(object):
             logger.warning("wait_for_startup() called when not in starting state")
 
         while not self.check_startup_state_changed():
+            with self._cancellable_lock:
+                if self._is_cancelled:
+                    return None
             if timeout and self.time_in_state() > timeout:
                 return None
             time.sleep(1)
@@ -1190,8 +1074,10 @@ class Postgresql(object):
         return True
 
     def write_pg_hba(self, config):
-        with open(self._pg_hba_conf, 'a') as f:
-            f.write('\n{}\n'.format('\n'.join(config)))
+        if not self._server_parameters.get('hba_file') and not self.config.get('pg_hba'):
+            with open(self._pg_hba_conf, 'a') as f:
+                f.write('\n{}\n'.format('\n'.join(config)))
+        return True
 
     def _replace_pg_hba(self):
         """
@@ -1262,10 +1148,9 @@ class Postgresql(object):
         dsn = " ".join("{0}={1}".format(k, v) for k, v in dsn_attrs if v is not None)
         logger.info('running pg_rewind from %s', dsn)
         try:
-            return subprocess.call([self._pgcommand('pg_rewind'),
-                                    '-D', self._data_dir,
-                                    '--source-server', dsn,
-                                    ], env=env) == 0
+            return self.cancellable_subprocess_call([self._pgcommand('pg_rewind'),
+                                                     '-D', self._data_dir,
+                                                     '--source-server', dsn], env=env) == 0
         except OSError:
             return False
 
@@ -1273,7 +1158,7 @@ class Postgresql(object):
         """ return the contents of pg_controldata, or non-True value if pg_controldata call failed """
         result = {}
         # Don't try to call pg_controldata during backup restore
-        if self._version_file_exists() and self.state != 'creating replica':
+        if not self.bootstrapping and self._version_file_exists() and self.state != 'creating replica':
             try:
                 data = subprocess.check_output([self._pgcommand('pg_controldata'), self._data_dir],
                                                env={'LANG': 'C', 'LC_ALL': 'C', 'PATH': os.environ['PATH']})
@@ -1314,31 +1199,57 @@ class Postgresql(object):
         except Exception:
             return logger.exception('Exception when working with leader')
 
-    def _get_local_timeline_lsn(self):
+    def _get_local_timeline_lsn_from_replication_connection(self):
         timeline = lsn = None
+        try:
+            with self._get_replication_connection_cursor(**self._local_replication_address) as cur:
+                cur.execute('IDENTIFY_SYSTEM')
+                timeline, lsn = cur.fetchone()[1:3]
+        except Exception:
+            logger.exception('Can not fetch local timeline and lsn from replication connection')
+        return timeline, lsn
+
+    def _get_local_timeline_lsn_from_controldata(self):
+        timeline = lsn = None
+        data = self.controldata()
+        try:
+            if data.get('Database cluster state') == 'shut down in recovery':
+                lsn = data.get('Minimum recovery ending location')
+                timeline = int(data.get("Min recovery ending loc's timeline"))
+                if lsn == '0/0' or timeline == 0:  # it was a master when it crashed
+                    data['Database cluster state'] = 'shut down'
+            if data.get('Database cluster state') == 'shut down':
+                lsn = data.get('Latest checkpoint location')
+                timeline = int(data.get("Latest checkpoint's TimeLineID"))
+        except (TypeError, ValueError):
+            logger.exception('Failed to get local timeline and lsn from pg_controldata output')
+        return timeline, lsn
+
+    def _get_local_timeline_lsn(self):
         if self.is_running():  # if postgres is running - get timeline and lsn from replication connection
-            try:
-                with self._get_replication_connection_cursor(**self._local_replication_address) as cur:
-                    cur.execute('IDENTIFY_SYSTEM')
-                    timeline, lsn = cur.fetchone()[1:3]
-            except Exception:
-                logger.exception('Can not fetch local timeline and lsn from replication connection')
+            timeline, lsn = self._get_local_timeline_lsn_from_replication_connection()
         else:  # otherwise analyze pg_controldata output
-            data = self.controldata()
-            try:
-                if data.get('Database cluster state') == 'shut down in recovery':
-                    lsn = data.get('Minimum recovery ending location')
-                    timeline = int(data.get("Min recovery ending loc's timeline"))
-                    if lsn == '0/0' or timeline == 0:  # it was a master when it crashed
-                        data['Database cluster state'] = 'shut down'
-                if data.get('Database cluster state') == 'shut down':
-                    lsn = data.get('Latest checkpoint location')
-                    timeline = int(data.get("Latest checkpoint's TimeLineID"))
-            except (TypeError, ValueError):
-                logger.exception('Failed to get local timeline and lsn from pg_controldata output')
+            timeline, lsn = self._get_local_timeline_lsn_from_controldata()
         logger.info('Local timeline=%s lsn=%s', timeline, lsn)
         return timeline, lsn
 
+    @staticmethod
+    def parse_lsn(lsn):
+        t = lsn.split('/')
+        return int(t[0], 16) * 0x100000000 + int(t[1], 16)
+
+    @staticmethod
+    def parse_history(data):
+        for line in data.split('\n'):
+            values = line.strip().split('\t')
+            if len(values) == 3:
+                try:
+                    values[0] = int(values[0])
+                    values[1] = Postgresql.parse_lsn(values[1])
+                    yield values
+                except (IndexError, ValueError):
+                    logger.exception('Exception when parsing timeline history line "%s"', values)
+
     def _check_timeline_and_lsn(self, leader):
         local_timeline, local_lsn = self._get_local_timeline_lsn()
         if local_timeline is None or local_lsn is None:
@@ -1365,28 +1276,44 @@ class Postgresql(object):
             return logger.exception('Exception when working with master via replication connection')
 
         if history is not None:
-            def parse_lsn(lsn):
-                t = lsn.split('/')
-                return int(t[0], 16) * 0x100000000 + int(t[1], 16)
-
-            for line in history.split('\n'):
-                line = line.strip().split('\t')
-                if len(line) == 3:
+            for parent_timeline, switchpoint, _ in self.parse_history(history):
+                if parent_timeline == local_timeline:
                     try:
-                        timeline = int(line[0])
-                        if timeline == local_timeline:
-                            try:
-                                need_rewind = parse_lsn(local_lsn) >= parse_lsn(line[1])
-                            except ValueError:
-                                logger.exception('Exception when parsing lsn')
-                            break
-                        elif timeline > local_timeline:
-                            break
-                    except ValueError:
-                        continue
+                        need_rewind = self.parse_lsn(local_lsn) >= switchpoint
+                    except (IndexError, ValueError):
+                        logger.exception('Exception when parsing lsn')
+                    break
+                elif parent_timeline > local_timeline:
+                    break
 
         self._rewind_state = need_rewind and REWIND_STATUS.NEED or REWIND_STATUS.NOT_NEED
 
+    def get_replica_timeline(self):
+        return self._get_local_timeline_lsn_from_replication_connection()[0]
+
+    def replica_cached_timeline(self, master_timeline):
+        if not self._cached_replica_timeline or not master_timeline or self._cached_replica_timeline != master_timeline:
+            self._cached_replica_timeline = self.get_replica_timeline()
+        return self._cached_replica_timeline
+
+    def get_master_timeline(self):
+        return self._cluster_info_state_get('timeline')
+
+    def get_history(self, timeline):
+        history_path = 'pg_{0}/{1:08X}.history'.format(self.wal_name, timeline)
+        try:
+            cursor = self._cursor()
+            cursor.execute('SELECT isdir, modification FROM pg_stat_file(%s)', (history_path,))
+            isdir, modification = cursor.fetchone()
+            if not isdir:
+                cursor.execute('SELECT pg_read_file(%s)', (history_path,))
+                history = list(self.parse_history(cursor.fetchone()[0]))
+                if history[-1][0] == timeline - 1:
+                    history[-1].append(modification.isoformat())
+                return history
+        except Exception:
+            logger.exception('Failed to read and parse %s', (history_path,))
+
     def rewind(self, leader):
         if self.is_running() and not self.stop(checkpoint=False):
             return logger.warning('Can not run pg_rewind because postgres is still running')
@@ -1467,6 +1394,7 @@ class Postgresql(object):
                     shutil.copy(config_file, backup_file)
         except IOError:
             logger.exception('unable to create backup copies of configuration files')
+        return True
 
     def restore_configuration_files(self):
         """ restore a previously saved postgresql.conf """
@@ -1483,15 +1411,22 @@ class Postgresql(object):
         except IOError:
             logger.exception('unable to restore configuration files from backup')
 
-    def promote(self):
+    def _wait_promote(self, wait_seconds):
+        for _ in polling_loop(wait_seconds - 1):
+            data = self.controldata()
+            if data.get('Database cluster state') == 'in production':
+                return True
+
+    def promote(self, wait_seconds):
         if self.role == 'master':
             return True
-        ret = self.pg_ctl('promote')
+        ret = self.pg_ctl('promote', '-W')
         if ret:
             self.set_role('master')
             logger.info("cleared rewind state after becoming the leader")
             self._rewind_state = REWIND_STATUS.INITIAL
             self.call_nowait(ACTION_ON_ROLE_CHANGE)
+            ret = self._wait_promote(wait_seconds)
         return ret
 
     def create_or_update_role(self, name, password, options):
@@ -1511,22 +1446,15 @@ BEGIN
 END;
 $$""".format(name, ' '.join(options)), name, password, password)
 
-    def wal_position(self, retry=True):
-        stmt = """SELECT CASE WHEN pg_is_in_recovery()
-                              THEN GREATEST(pg_{0}_{1}_diff(COALESCE(pg_last_{0}_receive_{1}(), '0/0'),
-                                                                  '0/0')::bigint,
-                                            pg_{0}_{1}_diff(pg_last_{0}_replay_{1}(), '0/0')::bigint)
-                              ELSE pg_{0}_{1}_diff(pg_current_{0}_{1}(), '0/0')::bigint
-                          END""".format(self.wal_name, self.lsn_name)
-
+    def timeline_wal_position(self):
         # This method could be called from different threads (simultaneously with some other `_query` calls).
         # If it is called not from main thread we will create a new cursor to execute statement.
         if current_thread().ident == self.__thread_ident:
-            return (self.query(stmt) if retry else self._query(stmt)).fetchone()[0]
+            return self._cluster_info_state_get('timeline'), self._cluster_info_state_get('wal_position')
 
         with self.connection().cursor() as cursor:
-            cursor.execute(stmt)
-            return cursor.fetchone()[0]
+            cursor.execute(cluster_info_query.format(self.wal_name, self.lsn_name))
+            return cursor.fetchone()[:2]
 
     def load_replication_slots(self):
         if self.use_slots and self._schedule_load_slots:
@@ -1589,7 +1517,7 @@ $$""".format(name, ' '.join(options)), name, password, password)
                 self._schedule_load_slots = True
 
     def last_operation(self):
-        return str(self.wal_position())
+        return str(self._cluster_info_state_get('wal_position'))
 
     def _post_restore(self):
         self.delete_trigger_file()
@@ -1616,6 +1544,7 @@ $$""".format(name, ' '.join(options)), name, password, password)
 
     def bootstrap(self, config):
         """ Initialize a new node from scratch and start it. """
+        pg_hba = config.get('pg_hba', [])
         method = config.get('method') or 'initdb'
         self._running_custom_bootstrap = method != 'initdb' and method in config and 'command' in config[method]
         if self._running_custom_bootstrap:
@@ -1623,7 +1552,8 @@ $$""".format(name, ' '.join(options)), name, password, password)
             config = config[method]
         else:
             do_initialize = self._initdb
-        return do_initialize(config) and self._configure_server_parameters() and self.start()
+        return do_initialize(config) and self.write_pg_hba(pg_hba) and self.save_configuration_files() \
+            and self._configure_server_parameters() and self.start()
 
     def post_bootstrap(self, config, task):
         try:
@@ -1685,11 +1615,6 @@ $$""".format(name, ' '.join(options)), name, password, password)
             self.move_data_directory()
 
     def basebackup(self, conn_url, env):
-        # save environ to restore it later
-        old_env = os.environ.copy()
-        os.environ.clear()
-        os.environ.update(env)
-
         # creates a replica data dir using pg_basebackup.
         # this is the default, built-in create_replica_method
         # tries twice, then returns failure (as 1)
@@ -1697,12 +1622,15 @@ $$""".format(name, ' '.join(options)), name, password, password)
         maxfailures = 2
         ret = 1
         for bbfailures in range(0, maxfailures):
+            with self._cancellable_lock:
+                if self._is_cancelled:
+                    break
             if not self.data_directory_empty():
                 self.remove_data_directory()
 
             try:
-                ret = subprocess.call([self._pgcommand('pg_basebackup'), '--pgdata=' + self._data_dir,
-                                       '-X', 'stream', '--dbname=' + conn_url])
+                ret = self.cancellable_subprocess_call([self._pgcommand('pg_basebackup'), '--pgdata=' + self._data_dir,
+                                                       '-X', 'stream', '--dbname=' + conn_url], env=env)
                 if ret == 0:
                     break
                 else:
@@ -1715,10 +1643,6 @@ $$""".format(name, ' '.join(options)), name, password, password)
                 logger.warning('Trying again in 5 seconds')
                 time.sleep(5)
 
-        # restore environ
-        os.environ.clear()
-        os.environ.update(old_env)
-
         return ret
 
     def pick_synchronous_standby(self, cluster):
@@ -1770,7 +1694,7 @@ $$""".format(name, ' '.join(options)), name, password, password)
 
     @staticmethod
     def postgres_version_to_int(pg_version):
-        """ Convert the server_version to integer
+        """Convert the server_version to integer
 
         >>> Postgresql.postgres_version_to_int('9.5.3')
         90503
@@ -1778,29 +1702,34 @@ $$""".format(name, ' '.join(options)), name, password, password)
         90313
         >>> Postgresql.postgres_version_to_int('10.1')
         100001
-        >>> Postgresql.postgres_version_to_int('10')
+        >>> Postgresql.postgres_version_to_int('10')  # doctest: +IGNORE_EXCEPTION_DETAIL
         Traceback (most recent call last):
             ...
-        Exception: Invalid PostgreSQL format: X.Y or X.Y.Z is accepted: 10
-        >>> Postgresql.postgres_version_to_int('a.b.c')
+        PostgresException: 'Invalid PostgreSQL version format: X.Y or X.Y.Z is accepted: 10'
+        >>> Postgresql.postgres_version_to_int('9.6')  # doctest: +IGNORE_EXCEPTION_DETAIL
         Traceback (most recent call last):
             ...
-        Exception: Invalid PostgreSQL version: a.b.c
+        PostgresException: 'Invalid PostgreSQL version format: X.Y or X.Y.Z is accepted: 9.6'
+        >>> Postgresql.postgres_version_to_int('a.b.c')  # doctest: +IGNORE_EXCEPTION_DETAIL
+        Traceback (most recent call last):
+            ...
+        PostgresException: 'Invalid PostgreSQL version: a.b.c'
         """
-        components = pg_version.split('.')
 
-        result = []
-        if len(components) < 2 or len(components) > 3:
-            raise Exception("Invalid PostgreSQL format: X.Y or X.Y.Z is accepted: {0}".format(pg_version))
+        try:
+            components = list(map(int, pg_version.split('.')))
+        except ValueError:
+            raise PostgresException('Invalid PostgreSQL version: {0}'.format(pg_version))
+
+        if len(components) < 2 or len(components) == 2 and components[0] < 10 or len(components) > 3:
+            raise PostgresException('Invalid PostgreSQL version format: X.Y or X.Y.Z is accepted: {0}'
+                                    .format(pg_version))
+
         if len(components) == 2:
             # new style verion numbers, i.e. 10.1 becomes 100001
-            components.insert(1, '0')
-        try:
-            result = [c if int(c) > 10 else '0{0}'.format(c) for c in components]
-            result = int(''.join(result))
-        except ValueError:
-            raise Exception("Invalid PostgreSQL version: {0}".format(pg_version))
-        return result
+            components.insert(1, 0)
+
+        return int(''.join('{0:02d}'.format(c) for c in components))
 
     @staticmethod
     def postgres_major_version_to_int(pg_version):
@@ -1833,13 +1762,7 @@ $$""".format(name, ' '.join(options)), name, password, password)
             cmd.extend(['-c', '{0}={1}'.format(opt, val)])
         # need a database name to connect
         cmd.append(self._database)
-        p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=open(os.devnull, 'w'), stderr=subprocess.STDOUT)
-        if p:
-            if command:
-                p.communicate('{0}\n'.format(command))
-            p.stdin.close()
-            return p.wait()
-        return 1
+        return self.cancellable_subprocess_call(cmd, communicate_input=command)
 
     def cleanup_archive_status(self):
         status_dir = os.path.join(self._data_dir, 'pg_' + self.wal_name, 'archive_status')
@@ -1865,3 +1788,56 @@ $$""".format(name, ' '.join(options)), name, password, password)
         if os.path.isfile(self._recovery_conf) or os.path.islink(self._recovery_conf):
             os.unlink(self._recovery_conf)
         return self.single_user_mode(options=opts) == 0 or None
+
+    def cancellable_subprocess_call(self, *args, **kwargs):
+        for s in ('stdin', 'stdout', 'stderr'):
+            kwargs.pop(s, None)
+
+        communicate_input = 'communicate_input' in kwargs
+        if communicate_input:
+            input_data = kwargs.pop('communicate_input', None)
+            if not isinstance(input_data, string_types):
+                input_data = ''
+            if input_data and input_data[-1] != '\n':
+                input_data += '\n'
+            kwargs['stdin'] = subprocess.PIPE
+            kwargs['stdout'] = open(os.devnull, 'w')
+            kwargs['stderr'] = subprocess.STDOUT
+
+        try:
+            with self._cancellable_lock:
+                if self._is_cancelled:
+                    raise PostgresException('cancelled')
+
+                self._is_cancelled = False
+                self._cancellable = subprocess.Popen(*args, **kwargs)
+
+            if communicate_input:
+                if input_data:
+                    self._cancellable.communicate(input_data)
+                self._cancellable.stdin.close()
+
+            return self._cancellable.wait()
+        finally:
+            with self._cancellable_lock:
+                self._cancellable = None
+
+    def reset_is_cancelled(self):
+        with self._cancellable_lock:
+            self._is_cancelled = False
+
+    def cancel(self):
+        with self._cancellable_lock:
+            self._is_cancelled = True
+            if self._cancellable is None or self._cancellable.returncode is not None:
+                return
+            self._cancellable.terminate()
+
+        for _ in polling_loop(10):
+            with self._cancellable_lock:
+                if self._cancellable is None or self._cancellable.returncode is not None:
+                    return
+
+        with self._cancellable_lock:
+            if self._cancellable is not None and self._cancellable.returncode is None:
+                self._cancellable.kill()

patroni/postmaster.py

@@ -0,0 +1,127 @@
+import logging
+import os
+import psutil
+import re
+import signal
+import subprocess
+
+from patroni import call_self
+
+logger = logging.getLogger(__name__)
+
+STOP_SIGNALS = {
+    'smart': signal.SIGTERM,
+    'fast': signal.SIGINT,
+    'immediate': signal.SIGQUIT,
+}
+
+
+class PostmasterProcess(psutil.Process):
+    def __init__(self, pid):
+        self.is_single_user = False
+        if pid < 0:
+            pid = -pid
+            self.is_single_user = True
+        super(PostmasterProcess, self).__init__(pid)
+
+    @classmethod
+    def from_pidfile(cls, pidfile):
+        try:
+            pid = int(pidfile.get('pid', 0))
+            if not pid:
+                return None
+        except ValueError:
+            return None
+
+        try:
+            proc = cls(pid)
+        except psutil.NoSuchProcess:
+            return None
+
+        try:
+            start_time = int(pidfile.get('start_time', 0))
+            if start_time and abs(proc.create_time() - start_time) > 3:
+                return None
+        except ValueError:
+            logger.warning("Garbage start time value in pid file: %r", pidfile.get('start_time'))
+
+        # Extra safety check. The process can't be ourselves, our parent or our direct child.
+        if proc.pid == os.getpid() or proc.pid == os.getppid() or proc.parent() == os.getpid():
+            return None
+
+        return proc
+
+    @classmethod
+    def from_pid(cls, pid):
+        try:
+            return cls(pid)
+        except psutil.NoSuchProcess:
+            return None
+
+    def signal_stop(self, mode):
+        """Signal postmaster process to stop
+
+        :returns None if signaled, True if process is already gone, False if error
+        """
+        if self.is_single_user:
+            logger.warning("Cannot stop server; single-user server is running (PID: {0})".format(self.pid))
+            return False
+        try:
+            self.send_signal(STOP_SIGNALS[mode])
+        except psutil.NoSuchProcess:
+            return True
+        except psutil.AccessDenied as e:
+            logger.warning("Could not send stop signal to PostgreSQL (error: {0})".format(e))
+            return False
+
+        return None
+
+    def wait_for_user_backends_to_close(self):
+        # These regexps are cross checked against versions PostgreSQL 9.1 .. 9.6
+        aux_proc_re = re.compile("(?:postgres:)( .*:)? (?:""(?:startup|logger|checkpointer|writer|wal writer|"
+                                 "autovacuum launcher|autovacuum worker|stats collector|wal receiver|archiver|"
+                                 "wal sender) process|bgworker: )")
+
+        try:
+            user_backends = []
+            user_backends_cmdlines = []
+            for child in self.children():
+                try:
+                    cmdline = child.cmdline()[0]
+                    if not aux_proc_re.match(cmdline):
+                        user_backends.append(child)
+                        user_backends_cmdlines.append(cmdline)
+                except psutil.NoSuchProcess:
+                    pass
+            if user_backends:
+                logger.debug('Waiting for user backends %s to close', ', '.join(user_backends_cmdlines))
+                psutil.wait_procs(user_backends)
+            logger.debug("Backends closed")
+        except psutil.Error:
+            logger.exception('wait_for_user_backends_to_close')
+
+    @classmethod
+    def start(cls, pgcommand, data_dir, conf, options):
+        # Unfortunately `pg_ctl start` does not return postmaster pid to us. Without this information
+        # it is hard to know the current state of postgres startup, so we had to reimplement pg_ctl start
+        # in python. It will start postgres, wait for port to be open and wait until postgres will start
+        # accepting connections.
+        # Important!!! We can't just start postgres using subprocess.Popen, because in this case it
+        # will be our child for the rest of our live and we will have to take care of it (`waitpid`).
+        # So we will use the same approach as pg_ctl uses: start a new process, which will start postgres.
+        # This process will write postmaster pid to stdout and exit immediately. Now it's responsibility
+        # of init process to take care about postmaster.
+        # In order to make everything portable we can't use fork&exec approach here, so  we will call
+        # ourselves and pass list of arguments which must be used to start postgres.
+        proc = call_self(['pg_ctl_start', pgcommand, '-D', data_dir,
+                          '--config-file={}'.format(conf)] + options, close_fds=True,
+                         preexec_fn=os.setsid, stdout=subprocess.PIPE,
+                         env={p: os.environ[p] for p in ('PATH', 'LC_ALL', 'LANG') if p in os.environ})
+        pid = int(proc.stdout.readline().strip())
+        proc.wait()
+        logger.info('postmaster pid=%s', pid)
+
+        # TODO: In an extremely unlikely case, the process could have exited and the pid reassigned. The start
+        # initiation time is not accurate enough to compare to create time as start time would also likely
+        # be relatively close. We need the subprocess extract pid+start_time in a race free manner.
+        return PostmasterProcess.from_pid(pid)

patroni/utils.py

@@ -1,6 +1,5 @@
 import random
 import time
-import re
 
 from dateutil import tz
 from patroni.exceptions import PatroniException
@@ -194,10 +193,6 @@ def _sleep(interval):
     time.sleep(interval)
 
 
-def is_valid_pg_version(version):
-    return re.match(r'[1-9][0-9]?(\.(0|([1-9][0-9]?))){2}$', version)
-
-
 class RetryFailedError(PatroniException):
 
     """Raised when retrying an operation ultimately failed, after retrying the maximum number of attempts."""
@@ -282,9 +277,7 @@ def polling_loop(timeout, interval=1):
         time.sleep(interval)
 
 
-def int_or_none(val):
-    """Returns integer value of the parameter if convertible to int, None otherwise."""
-    try:
-        return int(val)
-    except (ValueError, TypeError):
-        return None
+def split_host_port(value, default_port):
+    t = value.rsplit(':', 1)
+    t.append(default_port)
+    return t[0], int(t[1])

patroni/version.py

@@ -1 +1 @@
-__version__ = '1.3.6'
+__version__ = '1.4.3'

requirements.txt

@@ -1,10 +1,10 @@
-urllib3>=1.9
+urllib3>=1.19.1,!=1.21
 boto
-psycopg2>=2.6.1
+psycopg2>=2.5.4
 PyYAML
 requests
 six >= 1.7
-kazoo==2.2.1
+kazoo>=1.3.1
 python-etcd>=0.4.3,<0.5
 python-consul>=0.7.0
 click>=4.1
@@ -13,3 +13,4 @@ tzlocal
 python-dateutil
 psutil
 cdiff
+kubernetes==3.0.0

setup.py

@@ -115,13 +115,23 @@ def read(fname):
 
 def setup_package():
     # Assemble additional setup commands
-    cmdclass = {}
-    cmdclass['test'] = PyTest
+    cmdclass = {'test': PyTest}
 
     # Some helper variables
     version = os.getenv('GO_PIPELINE_LABEL', VERSION)
 
-    install_reqs = get_install_requirements('requirements.txt')
+    install_requires = []
+    extras_require = {'aws': ['boto'], 'etcd': ['python-etcd'], 'consul': ['python-consul'],
+                      'exhibitor': ['kazoo'], 'zookeeper': ['kazoo'], 'kubernetes': ['kubernetes']}
+
+    for r in get_install_requirements('requirements.txt'):
+        extra = False
+        for e, v in extras_require.items():
+            if r.startswith(v[0]):
+                extras_require[e] = [r]
+                extra = True
+        if not extra:
+            install_requires.append(r)
 
     command_options = {'test': {'test_suite': ('setup.py', 'tests')}}
     if JUNIT_XML:
@@ -145,10 +155,10 @@ def setup_package():
         test_suite='tests',
         packages=find_packages(exclude=['tests', 'tests.*']),
         package_data={MAIN_PACKAGE: ["*.json"]},
-        install_requires=install_reqs,
-        setup_requires=['flake8'],
+        install_requires=install_requires,
+        extras_require=extras_require,
         cmdclass=cmdclass,
-        tests_require=['mock>=2.0.0', 'pytest-cov', 'pytest'],
+        tests_require=['flake8', 'mock>=2.0.0', 'pytest-cov', 'pytest'],
         command_options=command_options,
         entry_points={'console_scripts': CONSOLE_SCRIPTS},
     )

tests/test_api.py

@@ -37,6 +37,10 @@ class MockPostgresql(object):
     def postmaster_start_time():
         return str(postmaster_start_time)
 
+    @staticmethod
+    def replica_cached_timeline(_):
+        return 2
+
 
 class MockWatchdog(object):
     is_healthy = False
@@ -48,7 +52,7 @@ class MockHa(object):
     watchdog = MockWatchdog()
 
     @staticmethod
-    def reinitialize():
+    def reinitialize(_):
         return 'reinitialize'
 
     @staticmethod
@@ -83,6 +87,10 @@ class MockHa(object):
     def wakeup():
         pass
 
+    @staticmethod
+    def is_paused():
+        return True
+
 
 class MockPatroni(object):
 
@@ -92,7 +100,7 @@ class MockPatroni(object):
     dcs = Mock()
     tags = {}
     version = '0.00'
-    noloadbalance = Mock(return_value=False)
+    noloadbalance = PropertyMock(return_value=False)
     scheduled_restart = {'schedule': future_restart_time,
                          'postmaster_start_time': postgresql.postmaster_start_time()}
 
@@ -140,6 +148,12 @@ class TestRestApiHandler(unittest.TestCase):
         with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'master'})):
             MockRestApiServer(RestApiHandler, 'GET /replica')
         MockRestApiServer(RestApiHandler, 'GET /master')
+        MockPatroni.dcs.cluster.sync.sync_standby = MockPostgresql.name
+        MockPatroni.dcs.cluster.is_synchronous_mode = Mock(return_value=True)
+        with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'replica'})):
+            MockRestApiServer(RestApiHandler, 'GET /synchronous')
+        with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'replica'})):
+            MockRestApiServer(RestApiHandler, 'GET /asynchronous')
         MockPatroni.dcs.cluster.leader.name = MockPostgresql.name
         MockRestApiServer(RestApiHandler, 'GET /replica')
         MockPatroni.dcs.cluster = None
@@ -148,6 +162,8 @@ class TestRestApiHandler(unittest.TestCase):
         with patch.object(MockHa, 'restart_scheduled', Mock(return_value=True)):
             MockRestApiServer(RestApiHandler, 'GET /master')
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /master'))
+        with patch.object(RestApiServer, 'query', Mock(return_value=[('', 1, '', '', '', '', False, '')])):
+            self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
 
     def test_do_OPTIONS(self):
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'OPTIONS / HTTP/1.0'))
@@ -264,7 +280,7 @@ class TestRestApiHandler(unittest.TestCase):
     def test_do_POST_reinitialize(self, mock_dcs):
         cluster = mock_dcs.get_cluster.return_value
         cluster.is_paused.return_value = False
-        request = 'POST /reinitialize HTTP/1.0' + self._authorization
+        request = 'POST /reinitialize HTTP/1.0' + self._authorization + '\nContent-Length: 15\n\n{"force": true}'
         MockRestApiServer(RestApiHandler, request)
         with patch.object(MockHa, 'reinitialize', Mock(return_value=None)):
             MockRestApiServer(RestApiHandler, request)
@@ -278,12 +294,13 @@ class TestRestApiHandler(unittest.TestCase):
 
     @patch('time.sleep', Mock())
     @patch.object(MockPatroni, 'dcs')
-    def test_do_POST_failover(self, dcs):
+    def test_do_POST_switchover(self, dcs):
         dcs.loop_wait = 10
         cluster = dcs.get_cluster.return_value
         cluster.is_synchronous_mode.return_value = False
+        cluster.is_paused.return_value = False
 
-        post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
+        post = 'POST /switchover HTTP/1.0' + self._authorization + '\nContent-Length: '
 
         MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}')
 
@@ -293,8 +310,14 @@ class TestRestApiHandler(unittest.TestCase):
         cluster.leader.name = 'postgresql1'
         MockRestApiServer(RestApiHandler, request)
 
+        request = post + '25\n\n{"leader": "postgresql1"}'
+
+        cluster.is_paused.return_value = True
+        MockRestApiServer(RestApiHandler, request)
+
+        cluster.is_paused.return_value = False
         for cluster.is_synchronous_mode.return_value in (True, False):
-            MockRestApiServer(RestApiHandler, post + '25\n\n{"leader": "postgresql1"}')
+            MockRestApiServer(RestApiHandler, request)
 
         cluster.leader.name = 'postgresql2'
         request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
@@ -350,3 +373,9 @@ class TestRestApiHandler(unittest.TestCase):
 
         # Invalid date
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}'))
+
+    @patch.object(MockPatroni, 'dcs', Mock())
+    def test_do_POST_failover(self):
+        post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
+        MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}')
+        MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}')

tests/test_async_executor.py

@@ -8,7 +8,7 @@ from threading import Thread
 class TestAsyncExecutor(unittest.TestCase):
 
     def setUp(self):
-        self.a = AsyncExecutor(Mock())
+        self.a = AsyncExecutor(Mock(), Mock())
 
     @patch.object(Thread, 'start', Mock())
     def test_run_async(self):
@@ -17,6 +17,12 @@ class TestAsyncExecutor(unittest.TestCase):
     def test_run(self):
         self.a.run(Mock(side_effect=Exception()))
 
+    def test_cancel(self):
+        self.a.cancel()
+        self.a.schedule('foo')
+        self.a.cancel()
+        self.a.run(Mock())
+
 
 class TestCriticalTask(unittest.TestCase):
 

tests/test_config.py

@@ -49,6 +49,9 @@ class TestConfig(unittest.TestCase):
             'PATRONI_ETCD_CERT': '/cert',
             'PATRONI_ETCD_KEY': '/key',
             'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
+            'PATRONI_KUBERNETES_LABELS': 'a:b:c',
+            'PATRONI_KUBERNETES_SCOPE_LABEL': 'a',
+            'PATRONI_KUBERNETES_PORTS': '[{"name": "postgresql"}]',
             'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
             'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',
             'PATRONI_EXHIBITOR_PORT': '8181',

tests/test_consul.py

@@ -112,11 +112,11 @@ class TestConsul(unittest.TestCase):
     @patch.object(consul.Consul.KV, 'put', Mock(side_effect=[True, ConsulException]))
     def test_touch_member(self):
         self.c.refresh_session = Mock(return_value=True)
-        self.c.touch_member('balbla')
-        self.c.touch_member('balbla')
-        self.c.touch_member('balbla')
+        self.c.touch_member({'balbla': 'blabla'})
+        self.c.touch_member({'balbla': 'blabla'})
+        self.c.touch_member({'balbla': 'blabla'})
         self.c.refresh_session = Mock(return_value=False)
-        self.c.touch_member('balbla')
+        self.c.touch_member({'balbla': 'blabla'})
 
     @patch.object(consul.Consul.KV, 'put', Mock(return_value=False))
     def test_take_leader(self):
@@ -138,7 +138,7 @@ class TestConsul(unittest.TestCase):
 
     @patch.object(consul.Consul.Session, 'renew', Mock())
     def test_update_leader(self):
-        self.c.update_leader()
+        self.c.update_leader(None)
 
     @patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
     def test_delete_leader(self):
@@ -172,3 +172,7 @@ class TestConsul(unittest.TestCase):
     def test_sync_state(self):
         self.assertTrue(self.c.set_sync_state_value('{}'))
         self.assertTrue(self.c.delete_sync_state())
+
+    @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
+    def test_set_history_value(self):
+        self.assertTrue(self.c.set_history_value('{}'))

tests/test_ctl.py

@@ -5,15 +5,17 @@ import sys
 import unittest
 
 from click.testing import CliRunner
+from datetime import datetime, timedelta
 from mock import patch, Mock
-from patroni.ctl import ctl, members, store_config, load_config, output_members, request_patroni, get_dcs, parse_dcs, \
+from patroni.ctl import ctl, store_config, load_config, output_members, request_patroni, get_dcs, parse_dcs, \
     get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException, apply_config_changes, \
-    format_config_for_editing, show_diff, invoke_editor
-from patroni.dcs.etcd import Client
+    format_config_for_editing, show_diff, invoke_editor, format_pg_version
+from patroni.dcs.etcd import Client, Failover
+from patroni.utils import tzutc
 from psycopg2 import OperationalError
 from test_etcd import etcd_read, requests_get, socket_getaddrinfo, MockResponse
 from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \
-    get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader
+    get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader, get_cluster, Member
 from test_postgresql import MockConnect, psycopg2_connect
 
 CONFIG_FILE_PATH = './test-ctl.yaml'
@@ -31,7 +33,7 @@ def test_rw_config():
 
 
 @patch('patroni.ctl.load_config',
-       Mock(return_value={'postgresql': {'data_dir': '.', 'parameters': {}, 'retry_timeout': 5},
+       Mock(return_value={'scope': 'alpha', 'postgresql': {'data_dir': '.', 'parameters': {}, 'retry_timeout': 5},
                           'restapi': {'auth': 'u:p', 'listen': ''}, 'etcd': {'host': 'localhost:2379'}}))
 class TestCtl(unittest.TestCase):
 
@@ -64,76 +66,92 @@ class TestCtl(unittest.TestCase):
         self.assertRaises(PatroniCtlException, parse_dcs, 'invalid://test')
 
     def test_output_members(self):
-        cluster = get_cluster_initialized_with_leader()
+        scheduled_at = datetime.now(tzutc) + timedelta(seconds=600)
+        cluster = get_cluster_initialized_with_leader(Failover(1, 'foo', 'bar', scheduled_at))
         self.assertIsNone(output_members(cluster, name='abc', fmt='pretty'))
         self.assertIsNone(output_members(cluster, name='abc', fmt='json'))
+        self.assertIsNone(output_members(cluster, name='abc', fmt='yaml'))
         self.assertIsNone(output_members(cluster, name='abc', fmt='tsv'))
 
+    @patch('patroni.ctl.get_dcs')
+    @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
+    def test_switchover(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        mock_get_dcs.return_value.set_failover_value = Mock()
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+        assert 'leader' in result.output
+
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
+        assert result.exit_code == 0
+
+        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+            result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00'])
+            assert result.exit_code == 1
+
+        # Aborting switchover, as we anser NO to the confirmation
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\nN')
+        assert result.exit_code == 1
+
+        # Target and source are equal
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nleader\n\ny')
+        assert result.exit_code == 1
+
+        # Reality is not part of this cluster
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nReality\n\ny')
+        assert result.exit_code == 1
+
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force'])
+        assert 'Member' in result.output
+
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
+        assert result.exit_code == 0
+
+        # Invalid timestamp
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', 'invalid'])
+        assert result.exit_code != 0
+
+        # Invalid timestamp
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', '2115-02-30T12:00:00+01:00'])
+        assert result.exit_code != 0
+
+        # Specifying wrong leader
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='dummy')
+        assert result.exit_code == 1
+
+        with patch('patroni.ctl.request_patroni', Mock(side_effect=Exception)):
+            # Non-responding patroni
+            result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
+            assert 'falling back to DCS' in result.output
+
+        with patch('patroni.ctl.request_patroni') as mocked:
+            mocked.return_value.status_code = 500
+            result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+            assert 'Switchover failed' in result.output
+
+            mocked.return_value.status_code = 501
+            mocked.return_value.text = 'Server does not support this operation'
+            result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+            assert 'Switchover failed' in result.output
+
+        # No members available
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+        assert result.exit_code == 1
+
+        # No master available
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+        assert result.exit_code == 1
+
     @patch('patroni.ctl.get_dcs')
     @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
     def test_failover(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
         mock_get_dcs.return_value.set_failover_value = Mock()
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
-        assert 'leader' in result.output
-
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
-        assert result.exit_code == 0
-
-        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00'])
-            assert result.exit_code == 1
-
-        # Aborting failover,as we anser NO to the confirmation
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\nN')
-        assert result.exit_code == 1
-
-        # Target and source are equal
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nleader\n\ny')
-        assert result.exit_code == 1
-
-        # Reality is not part of this cluster
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nReality\n\ny')
-        assert result.exit_code == 1
-
-        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force'])
-        assert 'Member' in result.output
-
-        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
-        assert result.exit_code == 0
-
-        # Invalid timestamp
-        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', 'invalid'])
-        assert result.exit_code != 0
-
-        # Invalid timestamp
-        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2115-02-30T12:00:00+01:00'])
-        assert result.exit_code != 0
-
-        # Specifying wrong leader
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='dummy')
-        assert result.exit_code == 1
-
-        with patch('patroni.ctl.request_patroni', Mock(side_effect=Exception)):
-            # Non-responding patroni
-            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
-            assert 'falling back to DCS' in result.output
-
-        with patch('patroni.ctl.request_patroni') as mocked:
-            mocked.return_value.status_code = 500
-            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
-            assert 'Failover failed' in result.output
-
-        # No members available
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
-        assert result.exit_code == 1
-
-        # No master available
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
-        assert result.exit_code == 1
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='\n')
+        assert 'Failover could be performed only to a specific candidate' in result.output
 
     def test_get_dcs(self):
         self.assertRaises(PatroniCtlException, get_dcs, {'dummy': {}}, 'dummy')
@@ -219,7 +237,7 @@ class TestCtl(unittest.TestCase):
         assert result.exit_code == 1
 
         # successful reinit
-        result = self.runner.invoke(ctl, ['reinit', 'alpha', 'other'], input='y')
+        result = self.runner.invoke(ctl, ['reinit', 'alpha', 'other'], input='y\ny')
         assert result.exit_code == 0
 
         # Aborted restart
@@ -235,7 +253,7 @@ class TestCtl(unittest.TestCase):
 
         # Wrong pg version
         result = self.runner.invoke(ctl, ['restart', 'alpha', '--any', '--pg-version', '9.1'], input='y')
-        assert 'Error: PostgreSQL version' in result.output
+        assert 'Error: Invalid PostgreSQL version format' in result.output
         assert result.exit_code == 1
 
         result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force', '--timeout', '10min'])
@@ -261,7 +279,7 @@ class TestCtl(unittest.TestCase):
         with patch('requests.post', Mock(return_value=MockResponse(204))):
             # get restart with the non-200 return code
             # normal restart, the schedule is actually parsed, but not validated in patronictl
-            result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0.0',
+            result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0',
                                         '--scheduled', '2300-10-01T14:30'], input='y')
             assert result.exit_code == 0
 
@@ -342,9 +360,11 @@ class TestCtl(unittest.TestCase):
     @patch('patroni.ctl.get_dcs')
     def test_members(self, mock_get_dcs):
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
-        result = self.runner.invoke(members, ['alpha'])
+        result = self.runner.invoke(ctl, ['list'])
         assert '127.0.0.1' in result.output
         assert result.exit_code == 0
+        with patch('patroni.ctl.load_config', Mock(return_value={})):
+            self.runner.invoke(ctl, ['list'])
 
     def test_configure(self):
         result = self.runner.invoke(configure, ['--dcs', 'abc', '-c', 'dummy', '-n', 'bla'])
@@ -380,7 +400,7 @@ class TestCtl(unittest.TestCase):
         cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
         mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster)
 
-        result = self.runner.invoke(ctl, ['list', 'dummy', '--extended'])
+        result = self.runner.invoke(ctl, ['list', 'dummy', '--extended', '--timestamp'])
         assert '2100' in result.output
         assert 'Scheduled restart' in result.output
 
@@ -400,14 +420,11 @@ class TestCtl(unittest.TestCase):
             assert 'Failed: flush scheduled restart' in result.output
 
     @patch('patroni.ctl.get_dcs')
+    @patch('patroni.ctl.polling_loop', Mock(return_value=[1]))
     def test_pause_cluster(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
 
-        with patch('requests.patch', Mock(return_value=MockResponse(200))):
-            result = self.runner.invoke(ctl, ['pause', 'dummy'])
-            assert 'Success' in result.output
-
         with patch('requests.patch', Mock(return_value=MockResponse(500))):
             result = self.runner.invoke(ctl, ['pause', 'dummy'])
             assert 'Failed' in result.output
@@ -417,6 +434,17 @@ class TestCtl(unittest.TestCase):
             result = self.runner.invoke(ctl, ['pause', 'dummy'])
             assert 'Cluster is already paused' in result.output
 
+        with patch('requests.patch', Mock(return_value=MockResponse(200))):
+            result = self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
+            assert "'pause' request sent" in result.output
+            mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
+                                                                      get_cluster(None, None, [], None, None)])
+            self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
+            member = Member(1, 'other', 28, {})
+            mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
+                                                                      get_cluster(None, None, [member], None, None)])
+            self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
+
     @patch('patroni.ctl.get_dcs')
     def test_resume_cluster(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
@@ -506,3 +534,21 @@ class TestCtl(unittest.TestCase):
         self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar')
         mock_get_dcs.return_value.set_config_value = Mock(return_value=True)
         self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar')
+
+    @patch('patroni.ctl.get_dcs')
+    def test_version(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        with patch('patroni.ctl.request_patroni') as mocked:
+            result = self.runner.invoke(ctl, ['version'])
+            assert 'patronictl version' in result.output
+            mocked.return_value.json = lambda: {'patroni': {'version': '1.2.3'}, 'server_version': 100001}
+            result = self.runner.invoke(ctl, ['version', 'dummy'])
+            assert '1.2.3' in result.output
+        with patch('requests.get', Mock(side_effect=Exception)):
+            result = self.runner.invoke(ctl, ['version', 'dummy'])
+            assert 'failed to get version' in result.output
+
+    def test_format_pg_version(self):
+        self.assertEquals(format_pg_version(100001), '10.1')
+        self.assertEquals(format_pg_version(90605), '9.6.5')

tests/test_etcd.py

@@ -18,7 +18,6 @@ class MockResponse(object):
         self.status_code = status_code
         self.content = '{}'
         self.ok = True
-        self.text = ''
 
     def json(self):
         return json.loads(self.content)
@@ -27,6 +26,10 @@ class MockResponse(object):
     def data(self):
         return self.content.encode('utf-8')
 
+    @property
+    def text(self):
+        return self.content
+
     @property
     def status(self):
         return self.status_code
@@ -48,6 +51,12 @@ def requests_get(url, **kwargs):
         response.content = '[{}]' if url.startswith('http://error') else members
     elif url.startswith('http://exhibitor'):
         response.content = '{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}'
+    elif url.endswith(':8011/reinitialize'):
+        data = kwargs.get('data', '')
+        if ' false}' in data:
+            response.status_code = 503
+            response.ok = False
+            response.content = 'restarting after failure already in progress'
     else:
         response.status_code = 404
         response.ok = False
@@ -198,6 +207,7 @@ class TestClient(unittest.TestCase):
         mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
         self.client._machines_cache_updated = 0
         self.client.api_execute('/', 'POST', timeout=0)
+        self.client._machines_cache = [self.client._base_uri]
         self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
         self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '')
         self.client._update_machines_cache = True
@@ -262,6 +272,8 @@ class TestEtcd(unittest.TestCase):
                                   {'url': 'https://test:2379', 'retry_timeout': 10})
                 self.assertRaises(SleepException, self.etcd.get_etcd_client,
                                   {'proxy': 'https://user:password@test:2379', 'retry_timeout': 10})
+                self.assertRaises(SleepException, self.etcd.get_etcd_client,
+                                  {'hosts': 'foo:4001,bar', 'retry_timeout': 10})
 
     def test_get_cluster(self):
         self.assertIsInstance(self.etcd.get_cluster(), Cluster)
@@ -288,7 +300,7 @@ class TestEtcd(unittest.TestCase):
         self.etcd.write_leader_optime('0')
 
     def test_update_leader(self):
-        self.assertTrue(self.etcd.update_leader())
+        self.assertTrue(self.etcd.update_leader(None))
 
     def test_initialize(self):
         self.assertFalse(self.etcd.initialize())
@@ -324,3 +336,6 @@ class TestEtcd(unittest.TestCase):
     def test_sync_state(self):
         self.assertFalse(self.etcd.write_sync_state('leader', None))
         self.assertFalse(self.etcd.delete_sync_state())
+
+    def test_set_history_value(self):
+        self.assertFalse(self.etcd.set_history_value('{}'))

tests/test_ha.py

@@ -5,7 +5,7 @@ import unittest
 
 from mock import Mock, MagicMock, PropertyMock, patch
 from patroni.config import Config
-from patroni.dcs import Cluster, ClusterConfig, Failover, Leader, Member, get_dcs, SyncState
+from patroni.dcs import Cluster, ClusterConfig, Failover, Leader, Member, get_dcs, SyncState, TimelineHistory
 from patroni.dcs.etcd import Client
 from patroni.exceptions import DCSError, PostgresConnectionException, PatroniException
 from patroni.ha import Ha, _MemberStatus
@@ -13,7 +13,7 @@ from patroni.postgresql import Postgresql
 from patroni.watchdog import Watchdog
 from patroni.utils import tzutc
 from test_etcd import socket_getaddrinfo, etcd_read, etcd_write, requests_get
-from test_postgresql import psycopg2_connect
+from test_postgresql import psycopg2_connect, MockPostmaster
 
 
 def true(*args, **kwargs):
@@ -25,7 +25,8 @@ def false(*args, **kwargs):
 
 
 def get_cluster(initialize, leader, members, failover, sync):
-    return Cluster(initialize, ClusterConfig(1, {1: 2}, 1), leader, 10, members, failover, sync)
+    history = TimelineHistory(1, [(1, 67197376, 'no recovery target specified', datetime.datetime.now().isoformat())])
+    return Cluster(initialize, ClusterConfig(1, {1: 2}, 1), leader, 10, members, failover, sync, history)
 
 
 def get_cluster_not_initialized_without_leader():
@@ -39,6 +40,7 @@ def get_cluster_initialized_without_leader(leader=False, failover=None, sync=Non
     m2 = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
                                  'api_url': 'http://127.0.0.1:8011/patroni',
                                  'state': 'running',
+                                 'pause': True,
                                  'tags': {'clonefrom': True},
                                  'scheduled_restart': {'schedule': "2100-01-01 10:53:07.560445+00:00",
                                                        'postgres_version': '99.0.0'}})
@@ -112,9 +114,10 @@ def run_async(self, func, args=()):
     return func(*args) if args else func()
 
 
-@patch.object(Postgresql, 'is_running', Mock(return_value=True))
+@patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
 @patch.object(Postgresql, 'is_leader', Mock(return_value=True))
-@patch.object(Postgresql, 'wal_position', Mock(return_value=10))
+@patch.object(Postgresql, 'timeline_wal_position', Mock(return_value=(1, 10)))
+@patch.object(Postgresql, '_cluster_info_state_get', Mock(return_value=3))
 @patch.object(Postgresql, 'call_nowait', Mock(return_value=True))
 @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
 @patch.object(Postgresql, 'controldata', Mock(return_value={'Database system identifier': '1234567890'}))
@@ -125,9 +128,11 @@ def run_async(self, func, args=()):
 @patch.object(Postgresql, 'query', Mock())
 @patch.object(Postgresql, 'checkpoint', Mock())
 @patch.object(Postgresql, 'call_nowait', Mock())
+@patch.object(Postgresql, 'cancellable_subprocess_call', Mock(return_value=0))
 @patch.object(etcd.Client, 'write', etcd_write)
 @patch.object(etcd.Client, 'read', etcd_read)
 @patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
+@patch('patroni.postgresql.polling_loop', Mock(return_value=range(1)))
 @patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=False))
 @patch('patroni.async_executor.AsyncExecutor.run_async', run_async)
 @patch('subprocess.call', Mock(return_value=0))
@@ -164,7 +169,8 @@ class TestHa(unittest.TestCase):
         self.assertTrue(self.ha.update_lock(True))
 
     def test_touch_member(self):
-        self.p.wal_position = Mock(side_effect=Exception)
+        self.p.timeline_wal_position = Mock(return_value=(0, 1))
+        self.p.replica_cached_timeline = Mock(side_effect=Exception)
         self.ha.touch_member()
 
     def test_start_as_replica(self):
@@ -219,8 +225,10 @@ class TestHa(unittest.TestCase):
         self.p.is_leader = false
         self.p.is_healthy = true
         self.ha.has_lock = true
+        self.p.controldata = lambda: {'Database cluster state': 'in production'}
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader because i had the session lock')
 
+    @patch('psycopg2.connect', psycopg2_connect)
     def test_acquire_lock_as_master(self):
         self.assertEquals(self.ha.run_cycle(), 'acquired session lock as a leader')
 
@@ -229,6 +237,13 @@ class TestHa(unittest.TestCase):
         self.p.is_leader = false
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
 
+    def test_long_promote(self):
+        self.ha.cluster.is_unlocked = false
+        self.ha.has_lock = true
+        self.p.is_leader = false
+        self.p.set_role('master')
+        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
+
     def test_demote_after_failing_to_obtain_lock(self):
         self.ha.acquire_lock = false
         self.assertEquals(self.ha.run_cycle(), 'demoted self after trying and failing to obtain lock')
@@ -345,7 +360,7 @@ class TestHa(unittest.TestCase):
         self.ha.cluster = get_cluster_not_initialized_without_leader()
         self.e.initialize = true
         self.ha.bootstrap()
-        self.p.is_running = true
+        self.p.is_running.return_value = MockPostmaster()
         self.p.is_leader = true
         with patch.object(Watchdog, 'activate', Mock(return_value=False)):
             self.assertEquals(self.ha.post_bootstrap(), 'running post_bootstrap')
@@ -356,7 +371,7 @@ class TestHa(unittest.TestCase):
         self.assertIsNotNone(self.ha.reinitialize())
 
         self.ha.cluster = get_cluster_initialized_with_leader()
-        self.assertIsNone(self.ha.reinitialize())
+        self.assertIsNone(self.ha.reinitialize(True))
 
         self.assertIsNotNone(self.ha.reinitialize())
 
@@ -392,9 +407,9 @@ class TestHa(unittest.TestCase):
             self.ha.update_lock = false
             self.p.set_role('master')
             with patch('patroni.async_executor.CriticalTask.cancel', Mock(return_value=False)):
-                with patch('patroni.postgresql.Postgresql.stop') as stop_mock:
+                with patch('patroni.postgresql.Postgresql.terminate_starting_postmaster') as mock_terminate:
                     self.assertEquals(self.ha.run_cycle(), 'lost leader lock during restart')
-                    stop_mock.assert_called()
+                    mock_terminate.assert_called()
 
     @patch('requests.get', requests_get)
     def test_manual_failover_from_leader(self):
@@ -524,7 +539,7 @@ class TestHa(unittest.TestCase):
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.fetch_node_status = get_node_status(wal_position=11)  # accessible, in_recovery, wal position ahead
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
-        with patch('patroni.postgresql.Postgresql.wal_position', return_value=1):
+        with patch('patroni.postgresql.Postgresql.timeline_wal_position', return_value=(1, 1)):
             self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.patroni.nofailover = True
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
@@ -860,6 +875,7 @@ class TestHa(unittest.TestCase):
 
     def test_shutdown(self):
         self.p.is_running = false
+        self.ha.has_lock = true
         self.ha.shutdown()
 
     @patch('time.sleep', Mock())
@@ -874,3 +890,9 @@ class TestHa(unittest.TestCase):
         self.ha.has_lock = false
         # will not say bootstrap from leader as replica can't self elect
         self.assertEquals(self.ha.run_cycle(), "trying to bootstrap from replica 'other'")
+
+    def test_update_cluster_history(self):
+        self.p.get_master_timeline = Mock(return_value=1)
+        self.ha.has_lock = true
+        self.ha.cluster.is_unlocked = false
+        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')

tests/test_kubernetes.py

@@ -0,0 +1,107 @@
+import unittest
+
+from mock import Mock, patch
+from patroni.dcs.kubernetes import Kubernetes, KubernetesError, k8s_client, k8s_watch, RetryFailedError
+
+
+def mock_list_namespaced_config_map(self, *args, **kwargs):
+    metadata = {'resource_version': '1', 'labels': {'f': 'b'}, 'name': 'test-config',
+                'annotations': {'initialize': '123', 'config': '{}'}}
+    items = [k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata))]
+    metadata.update({'name': 'test-leader', 'annotations': {'optime': '1234', 'leader': 'p-0', 'ttl': '30s'}})
+    items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
+    metadata.update({'name': 'test-failover', 'annotations': {'leader': 'p-0'}})
+    items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
+    metadata.update({'name': 'test-sync', 'annotations': {'leader': 'p-0'}})
+    items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
+    metadata = k8s_client.V1ObjectMeta(resource_version='1')
+    return k8s_client.V1ConfigMapList(metadata=metadata, items=items)
+
+
+def mock_list_namespaced_pod(self, *args, **kwargs):
+    metadata = k8s_client.V1ObjectMeta(resource_version='1', name='p-0', annotations={'status': '{}'})
+    items = [k8s_client.V1Pod(metadata=metadata)]
+    return k8s_client.V1PodList(items=items)
+
+
+@patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', Mock())
+@patch.object(k8s_client.CoreV1Api, 'create_namespaced_config_map', Mock())
+class TestKubernetes(unittest.TestCase):
+
+    @patch('kubernetes.config.load_kube_config', Mock())
+    @patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map)
+    @patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', mock_list_namespaced_pod)
+    def setUp(self):
+        self.k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10, 'labels': {'f': 'b'}})
+        with patch('time.time', Mock(return_value=1)):
+            self.k.get_cluster()
+
+    @patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map)
+    @patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', mock_list_namespaced_pod)
+    def test_get_cluster(self):
+        self.k.get_cluster()
+        with patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', Mock(side_effect=Exception)):
+            self.assertRaises(KubernetesError, self.k.get_cluster)
+
+    @patch('kubernetes.config.load_kube_config', Mock())
+    @patch.object(k8s_client.CoreV1Api, 'create_namespaced_endpoints', Mock())
+    def test_update_leader(self):
+        k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10,
+                        'labels': {'f': 'b'}, 'use_endpoints': True, 'pod_ip': '10.0.0.0'})
+        self.assertIsNotNone(k.update_leader('123'))
+
+    def test_take_leader(self):
+        self.k.take_leader()
+        self.k._leader_observed_record['leader'] = 'test'
+        self.k.patch_or_create = Mock(return_value=False)
+        self.k.take_leader()
+
+    def test_manual_failover(self):
+        with patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', Mock(side_effect=RetryFailedError(''))):
+            self.k.manual_failover('foo', 'bar')
+
+    def test_set_config_value(self):
+        self.k.set_config_value('{}')
+
+    @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_pod', Mock(return_value=True))
+    def test_touch_member(self):
+        self.k.touch_member({})
+        self.k._name = 'p-1'
+        self.k.touch_member({'state': 'running', 'role': 'replica'})
+        self.k.touch_member({'state': 'stopped', 'role': 'master'})
+
+    def test_initialize(self):
+        self.k.initialize()
+
+    def test_delete_leader(self):
+        self.k.delete_leader()
+
+    def test_cancel_initialization(self):
+        self.k.cancel_initialization()
+
+    @patch.object(k8s_client.CoreV1Api, 'delete_collection_namespaced_config_map',
+                  Mock(side_effect=k8s_client.rest.ApiException(403, '')))
+    def test_delete_cluster(self):
+        self.k.delete_cluster()
+
+    @patch('kubernetes.config.load_kube_config', Mock())
+    @patch.object(k8s_client.CoreV1Api, 'create_namespaced_endpoints',
+                  Mock(side_effect=[k8s_client.rest.ApiException(502, ''), k8s_client.rest.ApiException(500, '')]))
+    def test_delete_sync_state(self):
+        k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10,
+                        'labels': {'f': 'b'}, 'use_endpoints': True, 'pod_ip': '10.0.0.0'})
+        self.assertFalse(k.delete_sync_state())
+
+    def test_watch(self):
+        self.k.set_ttl(10)
+        self.k.watch(None, 0)
+        self.k.watch(None, 0)
+        with patch.object(k8s_watch.Watch, 'stream',
+                          Mock(side_effect=[Exception, [], KeyboardInterrupt,
+                                            [{'raw_object': {'metadata': {'resourceVersion': '2'}}}]])):
+            self.assertFalse(self.k.watch('1', 2))
+            self.assertRaises(KeyboardInterrupt, self.k.watch, '1', 2)
+            self.assertTrue(self.k.watch('1', 2))
+
+    def test_set_history_value(self):
+        self.k.set_history_value('{}')

tests/test_patroni.py

@@ -12,7 +12,7 @@ from patroni.exceptions import DCSError
 from patroni import Patroni, main as _main, patroni_main
 from six.moves import BaseHTTPServer
 from test_etcd import SleepException, etcd_read, etcd_write
-from test_postgresql import Postgresql, psycopg2_connect
+from test_postgresql import Postgresql, psycopg2_connect, MockPostmaster
 
 
 class MockFrozenImporter(object):
@@ -26,7 +26,7 @@ class MockFrozenImporter(object):
 @patch.object(Postgresql, 'write_pg_hba', Mock())
 @patch.object(Postgresql, '_write_postgresql_conf', Mock())
 @patch.object(Postgresql, 'write_recovery_conf', Mock())
-@patch.object(Postgresql, 'is_running', Mock(return_value=True))
+@patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
 @patch.object(Postgresql, 'call_nowait', Mock())
 @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
 @patch.object(AsyncExecutor, 'run', Mock())

tests/test_postgresql.py

@@ -1,8 +1,7 @@
-import errno
+import datetime
 import mock  # for the mock.call method, importing it without a namespace breaks python3
 import os
 import psycopg2
-import psutil
 import shutil
 import subprocess
 import unittest
@@ -10,8 +9,9 @@ import unittest
 from mock import Mock, MagicMock, PropertyMock, patch, mock_open
 from patroni.async_executor import CriticalTask
 from patroni.dcs import Cluster, Leader, Member, SyncState
-from patroni.exceptions import PostgresConnectionException
+from patroni.exceptions import PostgresConnectionException, PostgresException
 from patroni.postgresql import Postgresql, STATE_REJECT, STATE_NO_RESPONSE
+from patroni.postmaster import PostmasterProcess
 from patroni.utils import RetryFailedError
 from six.moves import builtins
 from threading import Thread
@@ -35,13 +35,13 @@ class MockCursor(object):
         elif sql.startswith('SELECT slot_name'):
             self.results = [('blabla',), ('foobar',)]
         elif sql.startswith('SELECT CASE WHEN pg_is_in_recovery()'):
-            self.results = [(2,)]
-        elif sql == 'SELECT pg_is_in_recovery()':
-            self.results = [(False, )]
+            self.results = [(1, 2)]
+        elif sql.startswith('SELECT pg_is_in_recovery()'):
+            self.results = [(False, 2)]
         elif sql.startswith('WITH replication_info AS ('):
             replication_info = '[{"application_name":"walreceiver","client_addr":"1.2.3.4",' +\
                                '"state":"streaming","sync_state":"async","sync_priority":0}]'
-            self.results = [('', True, '', '', '', '', False, replication_info)]
+            self.results = [('', 0, '', '', '', '', False, replication_info)]
         elif sql.startswith('SELECT name, setting'):
             self.results = [('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
                             ('search_path', 'public', None, 'string', 'user'),
@@ -51,6 +51,11 @@ class MockCursor(object):
                             ('unix_socket_directories', '/tmp', None, 'string', 'postmaster')]
         elif sql.startswith('IDENTIFY_SYSTEM'):
             self.results = [('1', 2, '0/402EEC0', '')]
+        elif sql.startswith('SELECT isdir, modification'):
+            self.results = [(False, datetime.datetime.now())]
+        elif sql.startswith('SELECT pg_read_file'):
+            self.results = [('1\t0/40159C0\tno recovery target specified\n\n' +
+                             '2\t1/40159C0\tno recovery target specified\n',)]
         elif sql.startswith('TIMELINE_HISTORY '):
             self.results = [('', b'x\t0/40159C0\tno recovery target specified\n\n' +
                                  b'1\t0/40159C0\tno recovery target specified\n\n' +
@@ -96,6 +101,15 @@ class MockConnect(object):
         pass
 
 
+class MockPostmaster(object):
+    def __init__(self, is_running=True, is_single_master=False):
+        self.is_running = Mock(return_value=is_running)
+        self.is_single_master = Mock(return_value=is_single_master)
+        self.wait_for_user_backends_to_close = Mock()
+        self.signal_stop = Mock(return_value=None)
+        self.wait = Mock()
+
+
 def pg_controldata_string(*args, **kwargs):
     return b"""
 pg_control version number:            942
@@ -213,100 +227,84 @@ class TestPostgresql(unittest.TestCase):
     @patch.object(Postgresql, 'wait_for_port_open')
     @patch.object(Postgresql, 'is_running')
     def test_start(self, mock_is_running, mock_wait_for_port_open, mock_wait_for_startup, mock_popen):
-        mock_is_running.return_value = True
+        mock_is_running.return_value = MockPostmaster()
         mock_wait_for_port_open.return_value = True
         mock_wait_for_startup.return_value = False
         mock_popen.return_value.stdout.readline.return_value = '123'
         self.assertTrue(self.p.start())
-        mock_is_running.return_value = False
-        open(os.path.join(self.data_dir, 'postmaster.pid'), 'w').close()
-        pg_conf = os.path.join(self.data_dir, 'postgresql.conf')
-        open(pg_conf, 'w').close()
-        self.assertFalse(self.p.start(task=CriticalTask()))
-        with open(pg_conf) as f:
-            lines = f.readlines()
-            self.assertTrue("f.oo = 'bar'\n" in lines)
+        mock_is_running.return_value = None
 
-        mock_wait_for_startup.return_value = None
-        self.assertFalse(self.p.start(10))
-        self.assertIsNone(self.p.start())
+        mock_postmaster = MockPostmaster()
+        with patch.object(PostmasterProcess, 'start', return_value=mock_postmaster):
+            pg_conf = os.path.join(self.data_dir, 'postgresql.conf')
+            open(pg_conf, 'w').close()
+            self.assertFalse(self.p.start(task=CriticalTask()))
 
-        mock_wait_for_port_open.return_value = False
+            with open(pg_conf) as f:
+                lines = f.readlines()
+                self.assertTrue("f.oo = 'bar'\n" in lines)
+
+            mock_wait_for_startup.return_value = None
+            self.assertFalse(self.p.start(10))
+            self.assertIsNone(self.p.start())
+
+            mock_wait_for_port_open.return_value = False
+            self.assertFalse(self.p.start())
+            task = CriticalTask()
+            task.cancel()
+            self.assertFalse(self.p.start(task=task))
+
+        self.p.cancel()
         self.assertFalse(self.p.start())
-        task = CriticalTask()
-        task.cancel()
-        self.assertFalse(self.p.start(task=task))
 
     @patch.object(Postgresql, 'pg_isready')
-    @patch.object(Postgresql, 'read_pid_file')
-    @patch.object(Postgresql, '_is_postmaster_pid_running')
     @patch('patroni.postgresql.polling_loop', Mock(return_value=range(1)))
-    def test_wait_for_port_open(self, mock_is_postmaster_pid_running, mock_read_pid_file, mock_pg_isready):
-        mock_is_postmaster_pid_running.return_value = False
+    def test_wait_for_port_open(self, mock_pg_isready):
         mock_pg_isready.return_value = STATE_NO_RESPONSE
+        mock_postmaster = MockPostmaster(is_running=False)
 
         # No pid file and postmaster death
-        mock_read_pid_file.return_value = {}
-        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
+        self.assertFalse(self.p.wait_for_port_open(mock_postmaster, 1))
 
-        mock_is_postmaster_pid_running.return_value = True
+        mock_postmaster.is_running.return_value = True
 
         # timeout
-        mock_read_pid_file.return_value = {'pid', 1}
-        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
-
-        # Garbage pid
-        mock_read_pid_file.return_value = {'pid': 'garbage', 'start_time': '101', 'data_dir': '',
-                                           'socket_dir': '', 'port': '', 'listen_addr': ''}
-        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
-
-        # Not ready
-        mock_read_pid_file.return_value = {'pid': '42', 'start_time': '101', 'data_dir': '',
-                                           'socket_dir': '', 'port': '', 'listen_addr': ''}
-        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
+        self.assertFalse(self.p.wait_for_port_open(mock_postmaster, 1))
 
         # pg_isready failure
         mock_pg_isready.return_value = 'garbage'
-        self.assertTrue(self.p.wait_for_port_open(42, 100., 1))
+        self.assertTrue(self.p.wait_for_port_open(mock_postmaster, 1))
+
+        # cancelled
+        self.p.cancel()
+        self.assertFalse(self.p.wait_for_port_open(mock_postmaster, 1))
 
     @patch('time.sleep', Mock())
     @patch.object(Postgresql, 'is_running')
-    @patch.object(Postgresql, 'get_pid')
-    def test_stop(self, mock_get_pid, mock_is_running):
+    @patch.object(Postgresql, '_wait_for_connection_close', Mock())
+    def test_stop(self, mock_is_running):
+        # Postmaster is not running
         mock_callback = Mock()
-        mock_is_running.return_value = False
+        mock_is_running.return_value = None
         self.assertTrue(self.p.stop(on_safepoint=mock_callback))
         mock_callback.assert_called()
 
-        with patch.object(Postgresql, '_is_postmaster_pid_running', Mock(return_value=False)), \
-                patch.object(Postgresql, 'data_directory_empty', Mock(return_value=True)):
-            with patch('psutil.Process') as mock_psutil:
-                self.p._postmaster_cached_info = {'pid': 1, 'start_time': 1}
-                mock_psutil.return_value.pid = 1
-                mock_psutil.return_value.create_time.return_value = 1
-                self.assertTrue(self.p.stop())
-                self.p._postmaster_cached_info = {'pid': 1, 'start_time': 1}
-                mock_psutil.return_value.create_time.return_value = 100
-                self.assertTrue(self.p.stop())
-                self.p._postmaster_cached_info = {'pid': 1, 'start_time': 1}
-                mock_psutil.side_effect = psutil.NoSuchProcess('')
-                self.assertTrue(self.p.stop())
-
-        mock_is_running.return_value = True
-        mock_get_pid.return_value = 0
+        # Is running, stopped successfully
+        mock_is_running.return_value = mock_postmaster = MockPostmaster()
         mock_callback.reset_mock()
         self.assertTrue(self.p.stop(on_safepoint=mock_callback))
         mock_callback.assert_called()
-        mock_get_pid.return_value = -1
+        mock_postmaster.signal_stop.assert_called()
+
+        # Stop signal failed
+        mock_postmaster.signal_stop.return_value = False
         self.assertFalse(self.p.stop())
-        mock_get_pid.return_value = 123
-        with patch('os.kill', Mock(side_effect=[OSError(errno.ESRCH, ''), OSError, None])):
-            self.assertTrue(self.p.stop())
-            self.assertFalse(self.p.stop())
-            self.assertTrue(self.p.stop())
-        with patch.object(Postgresql, '_signal_postmaster_stop', Mock(return_value=(123, None))):
-            with patch.object(Postgresql, '_is_postmaster_pid_running', Mock(side_effect=[True, False, False])):
-                self.assertTrue(self.p.stop())
+
+        # Stop signal failed to find process
+        mock_postmaster.signal_stop.return_value = True
+        mock_callback.reset_mock()
+        self.assertTrue(self.p.stop(on_safepoint=mock_callback))
+        mock_callback.assert_called()
 
     def test_restart(self):
         self.p.start = Mock(return_value=False)
@@ -325,12 +323,13 @@ class TestPostgresql(unittest.TestCase):
             self.assertIsNone(self.p.checkpoint())
         self.assertEquals(self.p.checkpoint(), 'not accessible or not healty')
 
-    @patch('subprocess.call', side_effect=OSError)
+    @patch.object(Postgresql, 'cancellable_subprocess_call')
     @patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
-    def test_pg_rewind(self, mock_call):
+    def test_pg_rewind(self, mock_cancellable_subprocess_call):
         r = {'user': '', 'host': '', 'port': '', 'database': '', 'password': ''}
+        mock_cancellable_subprocess_call.return_value = 0
         self.assertTrue(self.p.pg_rewind(r))
-        subprocess.call = mock_call
+        mock_cancellable_subprocess_call.side_effect = OSError
         self.assertFalse(self.p.pg_rewind(r))
 
     def test_check_recovery_conf(self):
@@ -354,7 +353,7 @@ class TestPostgresql(unittest.TestCase):
 
     @patch.object(Postgresql, 'start', Mock())
     @patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
-    @patch.object(Postgresql, '_get_local_timeline_lsn', Mock(return_value=(2, '0/40159C1')))
+    @patch.object(Postgresql, '_get_local_timeline_lsn', Mock(return_value=(2, '40159C1')))
     @patch.object(Postgresql, 'check_leader_is_not_in_recovery')
     def test__check_timeline_and_lsn(self, mock_check_leader_is_not_in_recovery):
         mock_check_leader_is_not_in_recovery.return_value = False
@@ -365,12 +364,8 @@ class TestPostgresql(unittest.TestCase):
         self.p.trigger_check_diverged_lsn()
         with patch('psycopg2.connect', Mock(side_effect=Exception)):
             self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
-        with patch.object(MockCursor, 'fetchone',
-                          Mock(side_effect=[('', 2, '0/0'), ('', b'2\tG/40159C0\tno recovery target specified\n\n')])):
-            self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
         self.p.trigger_check_diverged_lsn()
-        with patch.object(MockCursor, 'fetchone',
-                          Mock(side_effect=[('', 2, '0/0'), ('', b'3\t040159C0\tno recovery target specified\n')])):
+        with patch.object(MockCursor, 'fetchone', Mock(side_effect=[('', 2, '0/0'), ('', b'3\t0/40159C0\tn\n')])):
             self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
         self.p.trigger_check_diverged_lsn()
         with patch.object(MockCursor, 'fetchone', Mock(return_value=('', 1, '0/0'))):
@@ -384,6 +379,7 @@ class TestPostgresql(unittest.TestCase):
         self.p.check_leader_is_not_in_recovery()
         self.p.check_leader_is_not_in_recovery()
 
+    @patch.object(Postgresql, 'cancellable_subprocess_call', Mock(return_value=0))
     @patch.object(Postgresql, 'checkpoint', side_effect=['', '1'])
     @patch.object(Postgresql, 'stop', Mock(return_value=False))
     @patch.object(Postgresql, 'start', Mock())
@@ -420,31 +416,41 @@ class TestPostgresql(unittest.TestCase):
         self.assertFalse(self.p.can_rewind)
 
     @patch('time.sleep', Mock())
+    @patch.object(Postgresql, 'cancellable_subprocess_call')
     @patch.object(Postgresql, 'remove_data_directory', Mock(return_value=True))
-    def test_create_replica(self):
+    def test_create_replica(self, mock_cancellable_subprocess_call):
         self.p.delete_trigger_file = Mock(side_effect=OSError)
-        with patch('subprocess.call', Mock(side_effect=[1, 0])):
-            self.assertEquals(self.p.create_replica(self.leader), 0)
-        with patch('subprocess.call', Mock(side_effect=[Exception(), 0])):
-            self.assertEquals(self.p.create_replica(self.leader), 0)
 
         self.p.config['create_replica_method'] = ['wale', 'basebackup']
         self.p.config['wale'] = {'command': 'foo'}
-        with patch('subprocess.call', Mock(return_value=0)):
-            self.assertEquals(self.p.create_replica(self.leader), 0)
-            del self.p.config['wale']
-            self.assertEquals(self.p.create_replica(self.leader), 0)
+        mock_cancellable_subprocess_call.return_value = 0
+        self.assertEquals(self.p.create_replica(self.leader), 0)
+        del self.p.config['wale']
+        self.assertEquals(self.p.create_replica(self.leader), 0)
 
-        with patch('subprocess.call', Mock(side_effect=Exception("foo"))):
-            self.assertEquals(self.p.create_replica(self.leader), 1)
+        mock_cancellable_subprocess_call.return_value = 1
+        self.assertEquals(self.p.create_replica(self.leader), 1)
 
-        with patch('subprocess.call', Mock(return_value=1)):
-            self.assertEquals(self.p.create_replica(self.leader), 1)
+        mock_cancellable_subprocess_call.side_effect = Exception('foo')
+        self.assertEquals(self.p.create_replica(self.leader), 1)
+
+        mock_cancellable_subprocess_call.side_effect = [1, 0]
+        self.assertEquals(self.p.create_replica(self.leader), 0)
+
+        mock_cancellable_subprocess_call.side_effect = [Exception(), 0]
+        self.assertEquals(self.p.create_replica(self.leader), 0)
+
+        self.p.cancel()
+        self.assertEquals(self.p.create_replica(self.leader), 1)
+
+    def test_basebackup(self):
+        self.p.cancel()
+        self.p.basebackup(None, None)
 
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def test_sync_replication_slots(self):
         self.p.start()
-        cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None, None)
+        cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None, None, None)
         with mock.patch('patroni.postgresql.Postgresql._query', Mock(side_effect=psycopg2.OperationalError)):
             self.p.sync_replication_slots(cluster)
         self.p.sync_replication_slots(cluster)
@@ -474,6 +480,7 @@ class TestPostgresql(unittest.TestCase):
     @patch.object(Postgresql, 'pg_isready', Mock(return_value=STATE_REJECT))
     def test_is_leader(self):
         self.assertTrue(self.p.is_leader())
+        self.p.reset_cluster_info_state()
         with patch.object(Postgresql, '_query', Mock(side_effect=RetryFailedError(''))):
             self.assertRaises(PostgresConnectionException, self.p.is_leader)
 
@@ -489,21 +496,29 @@ class TestPostgresql(unittest.TestCase):
 
     def test_promote(self):
         self.p.set_role('replica')
-        self.assertTrue(self.p.promote())
-        self.assertTrue(self.p.promote())
+        self.assertIsNone(self.p.promote(0))
+        self.assertTrue(self.p.promote(0))
 
-    def test_last_operation(self):
-        self.assertEquals(self.p.last_operation(), '2')
-        Thread(target=self.p.last_operation).start()
+    def test_timeline_wal_position(self):
+        self.assertEquals(self.p.timeline_wal_position(), (1, 2))
+        Thread(target=self.p.timeline_wal_position).start()
 
-    @patch('os.path.isfile', Mock(return_value=True))
-    @patch('os.kill', Mock(side_effect=Exception))
-    @patch('os.getpid', Mock(return_value=2))
-    @patch('os.getppid', Mock(return_value=2))
-    @patch.object(builtins, 'open', mock_open(read_data='-1'))
-    @patch.object(Postgresql, '_version_file_exists', Mock(return_value=True))
-    def test_is_running(self):
-        self.assertFalse(self.p.is_running())
+    @patch.object(PostmasterProcess, 'from_pidfile')
+    def test_is_running(self, mock_frompidfile):
+        # Cached postmaster running
+        mock_postmaster = self.p._postmaster_proc = MockPostmaster()
+        self.assertEquals(self.p.is_running(), mock_postmaster)
+
+        # Cached postmaster not running, no postmaster running
+        mock_postmaster.is_running.return_value = False
+        mock_frompidfile.return_value = None
+        self.assertEquals(self.p.is_running(), None)
+        self.assertEquals(self.p._postmaster_proc, None)
+
+        # No cached postmaster, postmaster running
+        mock_frompidfile.return_value = mock_postmaster2 = MockPostmaster()
+        self.assertEquals(self.p.is_running(), mock_postmaster2)
+        self.assertEquals(self.p._postmaster_proc, mock_postmaster2)
 
     @patch('shlex.split', Mock(side_effect=OSError))
     def test_call_nowait(self):
@@ -515,7 +530,7 @@ class TestPostgresql(unittest.TestCase):
     def test_non_existing_callback(self):
         self.assertFalse(self.p.call_nowait('foobar'))
 
-    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
+    @patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
     def test_is_leader_exception(self):
         self.p.start()
         self.p.query = Mock(side_effect=psycopg2.OperationalError("not supported"))
@@ -550,14 +565,16 @@ class TestPostgresql(unittest.TestCase):
             lines = f.readlines()
             self.assertTrue('host replication replicator 127.0.0.1/32 md5\n' in lines)
 
-    def test_custom_bootstrap(self):
+    @patch.object(Postgresql, 'cancellable_subprocess_call')
+    def test_custom_bootstrap(self, mock_cancellable_subprocess_call):
+        self.p.config.pop('pg_hba')
         config = {'method': 'foo', 'foo': {'command': 'bar'}}
-        with patch('subprocess.call', Mock(return_value=1)):
-            self.assertFalse(self.p.bootstrap(config))
-        with patch('subprocess.call', Mock(side_effect=Exception)):
-            self.assertFalse(self.p.bootstrap(config))
-        with patch('subprocess.call', Mock(return_value=0)),\
-                patch('subprocess.Popen', Mock(side_effect=Exception("42"))),\
+
+        mock_cancellable_subprocess_call.return_value = 1
+        self.assertFalse(self.p.bootstrap(config))
+
+        mock_cancellable_subprocess_call.return_value = 0
+        with patch('subprocess.Popen', Mock(side_effect=Exception("42"))),\
                 patch('os.path.isfile', Mock(return_value=True)),\
                 patch('os.unlink', Mock()),\
                 patch.object(Postgresql, 'save_configuration_files', Mock()),\
@@ -573,6 +590,9 @@ class TestPostgresql(unittest.TestCase):
                 self.p.bootstrap(config)
             self.assertEqual(str(e.exception), '42')
 
+        mock_cancellable_subprocess_call.side_effect = Exception
+        self.assertFalse(self.p.bootstrap(config))
+
     @patch('time.sleep', Mock())
     @patch('os.unlink', Mock())
     @patch.object(Postgresql, 'run_bootstrap_post_init', Mock(return_value=True))
@@ -600,26 +620,27 @@ class TestPostgresql(unittest.TestCase):
             self.p.post_bootstrap({}, task)
             mock_restart.assert_called_once()
 
-    def test_run_bootstrap_post_init(self):
-        with patch('subprocess.call', Mock(return_value=1)):
-            self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+    @patch.object(Postgresql, 'cancellable_subprocess_call')
+    def test_run_bootstrap_post_init(self, mock_cancellable_subprocess_call):
+        mock_cancellable_subprocess_call.return_value = 1
+        self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
 
-        with patch('subprocess.call', Mock(side_effect=OSError)):
-            self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+        mock_cancellable_subprocess_call.return_value = 0
+        self.p._superuser.pop('username')
+        self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+        mock_cancellable_subprocess_call.assert_called()
+        args, kwargs = mock_cancellable_subprocess_call.call_args
+        self.assertTrue('PGPASSFILE' in kwargs['env'])
+        self.assertEquals(args[0], ['/bin/false', 'postgres://127.0.0.2:5432/postgres'])
 
-        with patch('subprocess.call', Mock(return_value=0)) as mock_method:
-            self.p._superuser.pop('username')
-            self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
-            mock_method.assert_called()
-            args, kwargs = mock_method.call_args
-            self.assertTrue('PGPASSFILE' in kwargs['env'])
-            self.assertEquals(args[0], ['/bin/false', 'postgres://127.0.0.2:5432/postgres'])
+        mock_cancellable_subprocess_call.reset_mock()
+        self.p._local_address.pop('host')
+        self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+        mock_cancellable_subprocess_call.assert_called()
+        self.assertEquals(mock_cancellable_subprocess_call.call_args[0][0], ['/bin/false', 'postgres://:5432/postgres'])
 
-            mock_method.reset_mock()
-            self.p._local_address.pop('host')
-            self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
-            mock_method.assert_called()
-            self.assertEquals(mock_method.call_args[0][0], ['/bin/false', 'postgres://:5432/postgres'])
+        mock_cancellable_subprocess_call.side_effect = OSError
+        self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
 
     @patch('patroni.postgresql.Postgresql.create_replica', Mock(return_value=0))
     def test_clone(self):
@@ -782,29 +803,24 @@ class TestPostgresql(unittest.TestCase):
                 self.assertFalse(self.p.wait_for_startup(timeout=2))
                 self.assertEquals(state['sleeps'], 3)
 
+        with patch.object(Postgresql, 'check_startup_state_changed', Mock(return_value=False)):
+            self.p.cancel()
+            self.p._state = 'starting'
+            self.assertIsNone(self.p.wait_for_startup())
+
     def test_read_pid_file(self):
         pidfile = os.path.join(self.data_dir, 'postmaster.pid')
         if os.path.exists(pidfile):
             os.remove(pidfile)
-        self.assertEquals(self.p.read_pid_file(), {})
-
-    @patch.object(Postgresql, '_version_file_exists', Mock(return_value=True))
-    @patch('os.path.isfile', Mock(return_value=True))
-    @patch.object(Postgresql, 'read_pid_file')
-    @patch('psutil.Process')
-    def test_is_postmaster_pid_running(self, mock_psutil, mock_read_pid_file):
-        mock_psutil.return_value.create_time.return_value = 1
-        mock_read_pid_file.return_value = {'pid': -100, 'start_time': 1}
-        self.assertTrue(self.p.is_running())
-        with patch('os.getpid', Mock(return_value=100)):
-            mock_read_pid_file.return_value = {'pid': 100, 'start_time': 1}
-            self.assertFalse(self.p.is_running())
-        mock_read_pid_file.return_value = {'pid': 100, 'start_time': 100}
-        self.assertFalse(self.p.is_running())
+        self.assertEquals(self.p._read_pid_file(), {})
+        with open(pidfile, 'w') as fd:
+            fd.write("123\n/foo/bar\n123456789\n5432")
+        self.assertEquals(self.p._read_pid_file(), {"pid": "123", "data_dir": "/foo/bar",
+                                                    "start_time": "123456789", "port": "5432"})
 
     def test_pick_sync_standby(self):
         cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None,
-                          SyncState(0, self.me.name, self.leadermem.name))
+                          SyncState(0, self.me.name, self.leadermem.name), None)
 
         with patch.object(Postgresql, "query", return_value=[
                     (self.leadermem.name, 'streaming', 'sync'),
@@ -870,43 +886,23 @@ class TestPostgresql(unittest.TestCase):
         self.p.set_synchronous_standby('foo')
         self.p.get_server_parameters(config)
 
-    @patch.object(Postgresql, 'read_pid_file', Mock(return_value={'pid': 'z'}))
-    def test_get_pid(self):
-        self.p.get_pid()
-
-    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
-    @patch.object(Postgresql, '_signal_postmaster_stop', Mock(return_value=(123, None)))
-    @patch.object(Postgresql, 'get_pid', Mock(return_value=123))
     @patch('time.sleep', Mock())
-    @patch.object(Postgresql, '_is_postmaster_pid_running')
-    def test__wait_for_connection_close(self, mock_is_postmaster_pid_running):
-        mock_is_postmaster_pid_running.side_effect = [True, False, False]
-        mock_callback = Mock()
-        self.p.stop(on_safepoint=mock_callback)
-
-        mock_is_postmaster_pid_running.side_effect = [True, False, False]
-        with patch.object(MockCursor, "execute", Mock(side_effect=psycopg2.Error)):
+    def test__wait_for_connection_close(self):
+        mock_postmaster = MockPostmaster()
+        with patch.object(Postgresql, 'is_running', Mock(return_value=mock_postmaster)):
+            mock_postmaster.is_running.side_effect = [True, False, False]
+            mock_callback = Mock()
             self.p.stop(on_safepoint=mock_callback)
 
-    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
-    @patch.object(Postgresql, '_signal_postmaster_stop', Mock(return_value=(123, None)))
-    @patch.object(Postgresql, 'get_pid', Mock(return_value=123))
-    @patch.object(Postgresql, '_is_postmaster_pid_running', Mock(return_value=False))
-    @patch('psutil.Process')
-    def test__wait_for_user_backends_to_close(self, mock_psutil):
-        child = Mock()
-        child.cmdline.return_value = ['foo']
-        mock_psutil.return_value.children.return_value = [child]
-        mock_callback = Mock()
-        self.p.stop(on_safepoint=mock_callback)
+            mock_postmaster.is_running.side_effect = [True, False, False]
+            with patch.object(MockCursor, "execute", Mock(side_effect=psycopg2.Error)):
+                self.p.stop(on_safepoint=mock_callback)
 
-    @patch('os.kill', Mock(side_effect=[OSError(errno.ESRCH, ''), OSError]))
-    @patch('psutil.Process', Mock(side_effect=[psutil.NoSuchProcess]))
-    @patch('time.sleep', Mock())
-    @patch.object(Postgresql, '_is_postmaster_pid_running', Mock(side_effect=[True, False]))
     def test_terminate_starting_postmaster(self):
-        self.p.terminate_starting_postmaster(123)
-        self.p.terminate_starting_postmaster(123)
+        mock_postmaster = MockPostmaster()
+        self.p.terminate_starting_postmaster(mock_postmaster)
+        mock_postmaster.signal_stop.assert_called()
+        mock_postmaster.wait.assert_called()
 
     def test_read_postmaster_opts(self):
         m = mock_open(read_data='/usr/lib/postgres/9.6/bin/postgres "-D" "data/postgresql0" \
@@ -923,13 +919,9 @@ class TestPostgresql(unittest.TestCase):
             self.assertEqual(data, dict())
 
     @patch('subprocess.Popen')
-    @patch.object(builtins, 'open', Mock(return_value=42))
     def test_single_user_mode(self, subprocess_popen_mock):
         subprocess_popen_mock.return_value.wait.return_value = 0
-        self.assertEquals(self.p.single_user_mode(command="CHECKPOINT"), 0)
-        subprocess_popen_mock.return_value = None
-        self.assertEquals(self.p.single_user_mode(), 1)
-        self.assertEquals(self.p.single_user_mode(options={'archive_mode': 'on'}), 1)
+        self.assertEquals(self.p.single_user_mode('CHECKPOINT', {'archive_mode': 'on'}), 0)
 
     @patch('os.listdir', Mock(side_effect=[OSError, ['a', 'b']]))
     @patch('os.unlink', Mock(side_effect=OSError))
@@ -946,10 +938,20 @@ class TestPostgresql(unittest.TestCase):
     def test_fix_cluster_state(self):
         self.assertTrue(self.p.fix_cluster_state())
 
-    def test__update_postmaster_cached_info(self):
-        with open(os.path.join(self.data_dir, 'postmaster.pid'), 'w') as f:
-            f.write('1\n\n1\n')
-        self.p.read_pid_file()
-        with open(os.path.join(self.data_dir, 'postmaster.pid'), 'w') as f:
-            f.write('a\n\n1\n')
-        self.p.read_pid_file()
+    def test_replica_cached_timeline(self):
+        self.assertEquals(self.p.replica_cached_timeline(1), 2)
+
+    def test_get_master_timeline(self):
+        self.assertEquals(self.p.get_master_timeline(), 1)
+
+    def test_cancellable_subprocess_call(self):
+        self.p.cancel()
+        self.assertRaises(PostgresException, self.p.cancellable_subprocess_call, communicate_input=None)
+
+    @patch('patroni.postgresql.polling_loop', Mock(return_value=[0, 0]))
+    def test_cancel(self):
+        self.p._cancellable = Mock()
+        self.p._cancellable.returncode = None
+        self.p.cancel()
+        type(self.p._cancellable).returncode = PropertyMock(side_effect=[None, -15])
+        self.p.cancel()

tests/test_postmaster.py

@@ -0,0 +1,85 @@
+import unittest
+
+from mock import Mock, patch
+from patroni.postmaster import PostmasterProcess
+import psutil
+
+
+class TestPostmasterProcess(unittest.TestCase):
+    @patch('psutil.Process.__init__', Mock())
+    def test_init(self):
+        proc = PostmasterProcess(-123)
+        self.assertTrue(proc.is_single_user)
+
+    @patch('psutil.Process.create_time')
+    @patch('psutil.Process.__init__')
+    def test_from_pidfile(self, mock_init, mock_create_time):
+        mock_init.side_effect = psutil.NoSuchProcess(123)
+        self.assertEquals(PostmasterProcess.from_pidfile({}), None)
+        self.assertEquals(PostmasterProcess.from_pidfile({"pid": "foo"}), None)
+        self.assertEquals(PostmasterProcess.from_pidfile({"pid": "123"}), None)
+
+        mock_init.side_effect = None
+        with patch.object(psutil.Process, 'pid', 123), \
+                patch.object(psutil.Process, 'parent', return_value=124), \
+                patch('os.getpid', return_value=125) as mock_ospid, \
+                patch('os.getppid', return_value=126):
+
+            self.assertNotEquals(PostmasterProcess.from_pidfile({"pid": "123"}), None)
+
+            mock_create_time.return_value = 100000
+            self.assertEquals(PostmasterProcess.from_pidfile({"pid": "123", "start_time": "200000"}), None)
+            self.assertNotEquals(PostmasterProcess.from_pidfile({"pid": "123", "start_time": "foobar"}), None)
+
+            mock_ospid.return_value = 123
+            self.assertEquals(PostmasterProcess.from_pidfile({"pid": "123", "start_time": "100000"}), None)
+
+    @patch('psutil.Process.__init__')
+    def test_from_pid(self, mock_init):
+        mock_init.side_effect = psutil.NoSuchProcess(123)
+        self.assertEquals(PostmasterProcess.from_pid(123), None)
+        mock_init.side_effect = None
+        self.assertNotEquals(PostmasterProcess.from_pid(123), None)
+
+    @patch('psutil.Process.__init__', Mock())
+    @patch('psutil.Process.send_signal')
+    @patch('psutil.Process.pid', Mock(return_value=123))
+    def test_signal_stop(self, mock_send_signal):
+        proc = PostmasterProcess(-123)
+        self.assertEquals(proc.signal_stop('immediate'), False)
+
+        mock_send_signal.side_effect = [None, psutil.NoSuchProcess(123), psutil.AccessDenied()]
+        proc = PostmasterProcess(123)
+        self.assertEquals(proc.signal_stop('immediate'), None)
+        self.assertEquals(proc.signal_stop('immediate'), True)
+        self.assertEquals(proc.signal_stop('immediate'), False)
+
+    @patch('psutil.Process.__init__', Mock())
+    @patch('psutil.wait_procs')
+    def test_wait_for_user_backends_to_close(self, mock_wait):
+        c1 = Mock()
+        c1.cmdline = Mock(return_value=["postgres: startup process"])
+        c2 = Mock()
+        c2.cmdline = Mock(return_value=["postgres: postgres postgres [local] idle"])
+        c3 = Mock()
+        c3.cmdline = Mock(side_effect=psutil.NoSuchProcess(123))
+        with patch('psutil.Process.children', Mock(return_value=[c1, c2, c3])):
+            proc = PostmasterProcess(123)
+            self.assertIsNone(proc.wait_for_user_backends_to_close())
+            mock_wait.assert_called_with([c2])
+
+        c3.cmdline = Mock(side_effect=psutil.AccessDenied(123))
+        with patch('psutil.Process.children', Mock(return_value=[c3])):
+            proc = PostmasterProcess(123)
+            self.assertIsNone(proc.wait_for_user_backends_to_close())
+
+    @patch('subprocess.Popen')
+    @patch.object(PostmasterProcess, 'from_pid')
+    def test_start(self, mock_frompid, mock_popen):
+        mock_frompid.return_value = "proc 123"
+        mock_popen.return_value.stdout.readline.return_value = '123'
+        self.assertEquals(
+            PostmasterProcess.start('/bin/true', '/tmp/', '/tmp/test.conf', ['--foo=bar', '--bar=baz']),
+            "proc 123"
+        )
+        mock_frompid.assert_called_with(123)

tests/test_zookeeper.py

@@ -58,11 +58,11 @@ class MockKazooClient(Mock):
             raise TypeError("Invalid type for 'path' (string expected)")
         if not isinstance(value, (six.binary_type,)):
             raise TypeError("Invalid type for 'value' (must be a byte string)")
-        if value == b'Exception':
+        if b'Exception' in value:
             raise Exception
         if path.endswith('/initialize') or path == '/service/test/optime/leader':
             raise Exception
-        elif value == b'retry' or (value == b'exists' and self.exists):
+        elif b'retry' in value or (b'exists' in value and self.exists):
             raise NodeExistsError
 
     def create_async(self, path, value=b"", acl=None, ephemeral=False, sequence=False, makepath=False):
@@ -76,10 +76,10 @@ class MockKazooClient(Mock):
             raise TypeError("Invalid type for 'value' (must be a byte string)")
         if path == '/service/bla/optime/leader':
             raise Exception
-        if path == '/service/test/members/bar' and value == b'retry':
+        if path == '/service/test/members/bar' and b'retry' in value:
             return
         if path in ('/service/test/failover', '/service/test/config', '/service/test/sync'):
-            if value == b'Exception':
+            if b'Exception' in value:
                 raise Exception
             elif value == b'ok':
                 return
@@ -145,7 +145,7 @@ class TestZooKeeper(unittest.TestCase):
         self.assertRaises(ZooKeeperError, self.zk.get_cluster)
         cluster = self.zk.get_cluster()
         self.assertIsInstance(cluster.leader, Leader)
-        self.zk.touch_member('foo')
+        self.zk.touch_member({'foo': 'foo'})
 
     def test_delete_leader(self):
         self.assertTrue(self.zk.delete_leader())
@@ -169,17 +169,17 @@ class TestZooKeeper(unittest.TestCase):
     def test_touch_member(self):
         self.zk._name = 'buzz'
         self.zk.get_cluster()
-        self.zk.touch_member('new')
+        self.zk.touch_member({'new': 'new'})
         self.zk._name = 'bar'
-        self.zk.touch_member('new')
+        self.zk.touch_member({'new': 'new'})
         self.zk._name = 'na'
         self.zk._client.exists = 1
-        self.zk.touch_member('Exception')
+        self.zk.touch_member({'Exception': 'Exception'})
         self.zk._name = 'bar'
-        self.zk.touch_member('retry')
+        self.zk.touch_member({'retry': 'retry'})
         self.zk._fetch_cluster = True
         self.zk.get_cluster()
-        self.zk.touch_member('retry')
+        self.zk.touch_member({'retry': 'retry'})
 
     def test_take_leader(self):
         self.zk.take_leader()
@@ -187,7 +187,7 @@ class TestZooKeeper(unittest.TestCase):
             self.zk.take_leader()
 
     def test_update_leader(self):
-        self.assertTrue(self.zk.update_leader())
+        self.assertTrue(self.zk.update_leader(None))
 
     def test_write_leader_optime(self):
         self.zk.last_leader_operation = '0'
@@ -217,3 +217,6 @@ class TestZooKeeper(unittest.TestCase):
         self.zk.set_sync_state_value('ok')
         self.zk.set_sync_state_value('Exception')
         self.zk.delete_sync_state()
+
+    def test_set_history_value(self):
+        self.zk.set_history_value('{}')