Reset is_leader status on demote (#777)

Make sure demoted cluster member stops responding with code 200 on the  /master API call.

Issue a new minor release.

Fixes https://github.com/zalando/patroni/issues/776

.travis.yml

@@ -1,17 +1,28 @@
-sudo: false
+sudo: true
 dist: trusty
 language: python
-python:
-- "3.4"  # 2.7 and 3.5 are preinstalled by default
 env:
   global:
-  - ETCDVERSION=3.0.17 ZKVERSION=3.4.9 CONSULVERSION=0.7.4
-  - PYVERSIONS="2.7 3.4 3.5"
-  matrix:
-  - TEST_SUITE="python setup.py"
-  - DCS="etcd" TEST_SUITE="behave"
-  - DCS="exhibitor" TEST_SUITE="behave"
-  - DCS="consul" TEST_SUITE="behave"
+  - ETCDVERSION=3.0.17 ZKVERSION=3.4.11 CONSULVERSION=0.7.4
+  - PYVERSIONS="2.7 3.5 3.6"
+  - EXCLUDE_BEHAVE="3.5"
+  - BOTO_CONFIG=/doesnotexist
+matrix:
+  include:
+  - python: "3.5"
+    env: TEST_SUITE="python setup.py"
+  - python: "3.6"
+    env: DCS="etcd" TEST_SUITE="behave"
+  - python: "3.6"
+    env: DCS="exhibitor" TEST_SUITE="behave"
+  - python: "3.6"
+    env: DCS="consul" TEST_SUITE="behave"
+  - python: "3.6"
+    env: DCS="kubernetes" TEST_SUITE="behave"
+branches:
+  only:
+  - master
+  - /^v\d+\.\d+(\.\d+)?$/
 cache:
   directories:
   - $HOME/mycache
@@ -19,7 +30,7 @@ before_cache:
 - |
   rm -fr $HOME/mycache/python*
   for pv in $PYVERSIONS; do
-      if [[ $TEST_SUITE != "behave" || $pv != "3.4" ]]; then
+      if [[ $TEST_SUITE != "behave" || $pv != $EXCLUDE_BEHAVE ]]; then
           fpv=$(basename $(readlink $HOME/virtualenv/python${pv}))
           mv $HOME/virtualenv/${fpv} $HOME/mycache/${fpv}
       fi
@@ -51,6 +62,26 @@ install:
             ln -s $EC etcd
         }
 
+        function get_kubernetes() {
+            wget -O localkube "https://storage.googleapis.com/minikube/k8sReleases/v1.7.0/localkube-linux-amd64"
+            chmod +x localkube
+            sudo nohup ./localkube --logtostderr=true --enable-dns=false > localkube.log 2>&1 &
+
+            echo "Waiting for localkube to start..."
+            if ! timeout 120 sh -c "while ! curl -ks http://127.0.0.1:8080/ >/dev/null; do sleep 1; done"; then
+                sudo cat localkube.log
+                echo "localkube did not start"
+                exit 1
+            fi
+            echo "Check certificate permissions"
+            sudo chmod 644 /var/lib/localkube/certs/*
+            sudo ls -altr /var/lib/localkube/certs/
+
+            echo "Set up .kube/config"
+            mkdir ~/.kube
+            echo -e "apiVersion: v1\nclusters:\n- cluster:\n    certificate-authority: /var/lib/localkube/certs/ca.crt\n    server: https://127.0.0.1:8443\n  name: local\ncontexts:\n- context:\n    cluster: local\n    user: myself\n  name: local\ncurrent-context: local\nkind: Config\npreferences: {}\nusers:\n- name: myself\n  user:\n    client-certificate: /var/lib/localkube/certs/apiserver.crt\n    client-key: /var/lib/localkube/certs/apiserver.key\n" > ~/.kube/config
+        }
+
         function get_exhibitor() {
             ZC=~/mycache/zookeeper-${ZKVERSION}
             if [[ ! -d $ZC ]]; then
@@ -65,7 +96,6 @@ install:
                 echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \
                     | nc -l 8181 &> /dev/null
             done&
-            ZK_PID=$!
         }
 
         attempt_num=1
@@ -77,7 +107,7 @@ install:
     fi
 
     for pv in $PYVERSIONS; do
-        if [[ $TEST_SUITE != "behave" || $pv != "3.4" ]]; then
+        if [[ $TEST_SUITE != "behave" || $pv != $EXCLUDE_BEHAVE ]]; then
             fpv=$(basename $(readlink $HOME/virtualenv/python$pv))
             if [[ -d ~/mycache/${fpv} ]]; then
                 mv ~/virtualenv/${fpv} ~/virtualenv/${fpv}.bckp
@@ -93,13 +123,17 @@ install:
 script:
   - |
     for pv in $PYVERSIONS; do
+        if [[ $TEST_SUITE == "behave" && $pv == $EXCLUDE_BEHAVE ]]; then
+            continue
+        fi
+
         source ~/virtualenv/python${pv}/bin/activate
 
         if [[ $TEST_SUITE != "behave" ]]; then
             echo Running unit tests using python${pv}
             $TEST_SUITE test
             $TEST_SUITE flake8
-        elif [[ $pv != "3.4" ]]; then
+        elif [[ $pv != $EXCLUDE_BEHAVE ]]; then
             echo Running acceptance tests using python${pv}
             if ! PATH=.:/usr/lib/postgresql/9.6/bin:$PATH $TEST_SUITE; then
                 # output all log files when tests are failing
@@ -112,7 +146,8 @@ script:
     set +e
 after_success:
   # before_cache is executed earlier than after_success, so we need to restore one of virtualenv directories
-  - fpv=$(basename $(readlink $HOME/virtualenv/python3.5)) && mv $HOME/mycache/${fpv} $HOME/virtualenv/${fpv}
+  - fpv=$(basename $(readlink $HOME/virtualenv/python3.6)) && mv $HOME/mycache/${fpv} $HOME/virtualenv/${fpv}
   - coveralls
   - if [[ $TEST_SUITE != "behave" ]]; then python-codacy-coverage -r coverage.xml; fi
-  - if [[ $DCS == "exhibitor" ]]; then ~/mycache/zookeeper-${ZKVERSION}/bin/zkServer.sh stop; kill -9 $ZK_PID; fi
+  - if [[ $DCS == "exhibitor" ]]; then ~/mycache/zookeeper-${ZKVERSION}/bin/zkServer.sh stop; fi
+  - sudo kill $(jobs -p)

Dockerfile

@@ -1,45 +1,42 @@
 ## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine
 ## It has all the necessary components to play/debug with a single node appliance, running etcd
-FROM postgres:9.6
+FROM postgres:10
 MAINTAINER Alexander Kukushkin <[email protected]>
 
 RUN export DEBIAN_FRONTEND=noninteractive \
     && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
     && apt-get update -y \
     && apt-get upgrade -y \
-    && apt-get install -y curl jq haproxy python-psycopg2 python-yaml python-requests python-six python-pysocks \
-        python-dateutil python-pip python-setuptools python-prettytable python-wheel python-psutil python locales \
+    # postgres:10 is based on debian, which has patroni package. We will install all required dependencies
+    && apt-get install -s patroni | sed -n -e '/^Inst patroni /d' -e 's/^Inst \([^ ]\+\) .*$/\1/p' \
+            | xargs apt-get install -y curl jq haproxy locales python3-etcd python3-kazoo \
 
     ## Make sure we have a en_US.UTF-8 locale available
     && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
 
-    && pip install 'python-etcd>=0.4.3,<0.5' click tzlocal cdiff \
-
     && mkdir -p /home/postgres \
     && chown postgres:postgres /home/postgres \
 
     # Clean up
-    && apt-get remove -y python-pip python-setuptools \
+    && apt-get purge -y libpython2.7-stdlib libpython2.7-minimal \
     && apt-get autoremove -y \
     && apt-get clean -y \
     && rm -rf /var/lib/apt/lists/* /root/.cache
 
-ENV ETCDVERSION 3.2.3
+ENV ETCDVERSION 3.2.23
 RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
     | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl
 
-ENV CONFDVERSION 0.11.0
+ENV CONFDVERSION 0.16.0
 RUN curl -L https://github.com/kelseyhightower/confd/releases/download/v${CONFDVERSION}/confd-${CONFDVERSION}-linux-amd64 > /usr/local/bin/confd \
     && chmod +x /usr/local/bin/confd
 
 ADD patronictl.py patroni.py docker/entrypoint.sh /
 ADD patroni /patroni/
 ADD extras/confd /etc/confd
-RUN ln -s /patronictl.py /usr/local/bin/patronictl
 
-### Setting up a simple script that will serve as an entrypoint
-RUN mkdir /data/ && touch /pgpass /patroni.yml \
-    && chown postgres:postgres -R /patroni/ /data/ /pgpass /patroni.yml /etc/haproxy /var/run/ /var/lib/ /var/log/
+RUN sed -i 's/env python/&3/' patroni*.py && ln -s /patronictl.py /usr/local/bin/patronictl && mkdir /data/ /run/haproxy \
+    && touch /pgpass /patroni.yml && chown postgres:postgres -R /patroni/ /data/ /pgpass /patroni.yml /etc/haproxy /var/run/ /var/lib/ /var/log/
 
 EXPOSE 2379 5432 8008
 

README.rst

@@ -1,27 +1,27 @@
 |Build Status| |Coverage Status|
 
 Patroni: A Template for PostgreSQL HA with ZooKeeper, etcd or Consul
-------------------------------------------------------------
+--------------------------------------------------------------------
 
 You can find a version of this documentation that is searchable and also easier to navigate at `patroni.readthedocs.io <https://patroni.readthedocs.io>`__.
 
 
 There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
 
-Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__ or `Consul <https://github.com/hashicorp/consul>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
+Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__, `Consul <https://github.com/hashicorp/consul>`__ or `Kubernetes <https://kubernetes.io>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
 
 We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely.
 
-**Note to Kubernetes users**: We're currently developing Patroni to be as useful as possible for teams running Kubernetes on top of Google Compute Engine; Patroni can be the HA solution for Postgres in such an environment. To this end, there is a `Helm chart <https://github.com/kubernetes/charts/tree/master/incubator/patroni>`__ that uses Patroni and  `Spilo  <https://github.com/zalando/spilo/>`__ to provision a five-node PostgreSQL HA cluster in a Kubernetes+GCE environment. (The Helm chart deploys Spilo Docker images, not just "bare" Patroni.)
+**Note to Kubernetes users**: Patroni can run natively on top of Kubernetes. Take a look at the `Kubernetes <https://github.com/zalando/patroni/blob/master/docs/kubernetes.rst>`__ chapter of the Patroni documentation.
 
 .. contents::
     :local:
     :depth: 1
     :backlinks: none
 
-==============
+=================
 How Patroni Works
-==============
+=================
 
 Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features.
 
@@ -29,34 +29,70 @@ For an example of a Docker-based deployment with Patroni, see `Spilo <https://gi
 
 For additional background info, see:
 
-* `Elephants on Automatic: HA Clustered PostgreSQL with Helm <https://www.youtube.com/watch?v=CftcVhFMGSY>`_, talk by Josh Berkus and Oleksii Kliukin at KubeCon Berlin 2017 
+* `Elephants on Automatic: HA Clustered PostgreSQL with Helm <https://www.youtube.com/watch?v=CftcVhFMGSY>`_, talk by Josh Berkus and Oleksii Kliukin at KubeCon Berlin 2017
 * `PostgreSQL HA with Kubernetes and Patroni <https://www.youtube.com/watch?v=iruaCgeG7qs>`__, talk by Josh Berkus at KubeCon 2016 (video)
 * `Feb. 2016 Zalando Tech blog post <https://tech.zalando.de/blog/zalandos-patroni-a-template-for-high-availability-postgresql/>`__
 
-================
+==================
 Development Status
-================
+==================
 
 Patroni is in active development and accepts contributions. See our `Contributing <https://github.com/zalando/patroni/blob/master/docs/CONTRIBUTING.rst>`__ section below for more details.
 
 We report new releases information `here <https://github.com/zalando/patroni/releases>`__.
 
-===========================
-Technical Requirements/Installation
-===========================
+=========
+Community
+=========
 
-**For Mac**
+There are two places to connect with the Patroni community: `on github <https://github.com/zalando/patroni>`__, via Issues and PRs, and on channel #patroni in the `PostgreSQL Slack <https://postgres-slack.herokuapp.com/>`__.  If you're using Patroni, or just interested, please join us.
+
+===================================
+Technical Requirements/Installation
+===================================
+
+**Pre-requirements for Mac OS**
 
 To install requirements on a Mac, run the following:
 
 ::
 
     brew install postgresql etcd haproxy libyaml python
-    pip install psycopg2 pyyaml
 
-===================
+**General installation for pip**
+
+Patroni can be installed with pip:
+
+::
+
+    pip install patroni[dependencies]
+
+where dependencies can be either empty, or consist of one or more of the following:
+
+etcd
+    `python-etcd` module in order to use Etcd as DCS
+consul
+    `python-consul` module in order to use Consul as DCS
+zookeeper
+    `kazoo` module in order to use Zookeeper as DCS
+exhibitor
+    `kazoo` module in order to use Exhibitor as DCS (same dependencies as for Zookeeper)
+kubernetes
+    `kubernetes` module in order to use Kubernetes as DCS in Patroni
+aws
+    `boto` in order to use AWS callbacks
+
+For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is:
+
+::
+
+    pip install patroni[etcd,aws]
+
+Note that external tools to call in the replica creation or custom bootstap scripts (i.e. WAL-E) should be installed independently of Patroni.
+
+=======================
 Running and Configuring
-===================
+=======================
 
 To get started, do the following from different terminals:
 ::
@@ -80,9 +116,9 @@ run:
 
     > psql --host 127.0.0.1 --port 5000 postgres
 
-===============
+==================
 YAML Configuration
-===============
+==================
 
 Go `here <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
 
@@ -92,15 +128,15 @@ Environment Configuration
 
 Go `here <https://github.com/zalando/patroni/blob/master/docs/ENVIRONMENT.rst>`__ for comprehensive information about configuring(overriding) settings via environment variables.
 
-===============
+===================
 Replication Choices
-===============
+===================
 
 Patroni uses Postgres' streaming replication, which is asynchronous by default. Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the leader. This setting should be increased or decreased based on business requirements. It's also possible to use synchronous replication for better durability guarantees. See `replication modes documentation <https://github.com/zalando/patroni/blob/master/docs/replication_modes.rst>`__ for details.
 
-===============================
+======================================
 Applications Should Not Use Superusers
-===============================
+======================================
 
 When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
 

docker-compose.yml

@@ -15,7 +15,7 @@ dbnode1:
         - ./patroni:/patroni
     env_file: docker/patroni-secrets.env
     environment:
-        PATRONI_ETCD_HOST: patroni_etcd:2379
+        PATRONI_ETCD_URL: http://patroni_etcd:2379
         PATRONI_NAME: dbnode1
         PATRONI_SCOPE: testcluster
 
@@ -28,7 +28,7 @@ dbnode2:
         - ./patroni:/patroni
     env_file: docker/patroni-secrets.env
     environment:
-        PATRONI_ETCD_HOST: patroni_etcd:2379
+        PATRONI_ETCD_URL: http://patroni_etcd:2379
         PATRONI_NAME: dbnode2
         PATRONI_SCOPE: testcluster
 
@@ -41,7 +41,7 @@ dbnode3:
         - ./patroni:/patroni
     env_file: docker/patroni-secrets.env
     environment:
-        PATRONI_ETCD_HOST: patroni_etcd:2379
+        PATRONI_ETCD_URL: http://patroni_etcd:2379
         PATRONI_NAME: dbnode3
         PATRONI_SCOPE: testcluster
 
@@ -53,6 +53,6 @@ haproxy:
         - "5000:5000"
         - "5001:5001"
     environment:
-        PATRONI_ETCD_HOST: patroni_etcd:2379
+        PATRONI_ETCD_URL: http://patroni_etcd:2379
         PATRONI_SCOPE: testcluster
     command: --confd

docker/dev_patroni_cluster.sh

@@ -84,7 +84,7 @@ function docker_run()
 ETCD_CONTAINER="${PATRONI_SCOPE}_etcd"
 docker_run ${ETCD_CONTAINER} ${DOCKER_IMAGE} --etcd
 
-DOCKER_ARGS="--link=${ETCD_CONTAINER}:${ETCD_CONTAINER} -e PATRONI_SCOPE=${PATRONI_SCOPE} -e PATRONI_ETCD_HOST=${ETCD_CONTAINER}:2379"
+DOCKER_ARGS="--link=${ETCD_CONTAINER}:${ETCD_CONTAINER} -e PATRONI_SCOPE=${PATRONI_SCOPE} -e PATRONI_ETCD_URL=http://${ETCD_CONTAINER}:2379"
 PATRONI_ENV=$(sed 's/#.*//g' docker/patroni-secrets.env | sed -n 's/^PATRONI_.*$/-e &/p' | tr '\n' ' ')
 PATRONI_VOLUME="-v $(dirname $(dirname $(realpath $0)))/patroni:/patroni"
 

docker/entrypoint.sh

@@ -38,10 +38,10 @@ while getopts "$optspec" optchar; do
                         done
                         exec $CONFD zookeeper -node ${PATRONI_ZOOKEEPER_HOSTS}
                     else
-                        while ! curl -s ${PATRONI_ETCD_HOST}/v2/members | jq -r '.members[0].clientURLs[0]' | grep -q http; do
+                        while ! curl -s ${PATRONI_ETCD_URL}/v2/members | jq -r '.members[0].clientURLs[0]' | grep -q http; do
                             sleep 1
                         done
-                        exec $CONFD etcd -node $PATRONI_ETCD_HOST
+                        exec $CONFD etcd -node $PATRONI_ETCD_URL
                     fi
                     ;;
                 etcd)
@@ -74,9 +74,9 @@ while getopts "$optspec" optchar; do
 done
 
 ## We start an etcd
-if [[ -z ${PATRONI_ETCD_HOST} && -z ${PATRONI_ZOOKEEPER_HOSTS} ]]; then
+if [[ -z ${PATRONI_ETCD_URL} && -z ${PATRONI_ZOOKEEPER_HOSTS} ]]; then
     etcd $ETCD_ARGS > /var/log/etcd.log 2> /var/log/etcd.err &
-    export PATRONI_ETCD_HOST="127.0.0.1:2379"
+    export PATRONI_ETCD_URL="http://127.0.0.1:2379"
 fi
 
 export PATRONI_SCOPE
@@ -108,7 +108,7 @@ __EOF__
 mkdir -p "$HOME/.config/patroni"
 [ -h "$HOME/.config/patroni/patronictl.yaml" ] || ln -s /patroni.yml "$HOME/.config/patroni/patronictl.yaml"
 
-[ -z $CHEAT ] && exec python /patroni.py /patroni.yml
+[ -z $CHEAT ] && exec python3 /patroni.py /patroni.yml
 
 while true; do
     sleep 60

docs/CONTRIBUTING.rst

@@ -5,6 +5,11 @@ Contributing guidelines
 
 Wanna contribute to Patroni? Yay - here is how!
 
+Chatting
+--------
+
+Just want to chat with other Patroni users?  Looking for interactive troubleshooting help? Join us on channel #patroni in the `PostgreSQL Slack <https://postgres-slack.herokuapp.com/>`__.
+
 Reporting issues
 ----------------
 

docs/ENVIRONMENT.rst

@@ -1,6 +1,5 @@
 .. _environment:
 
-==================================
 Environment Configuration Settings
 ==================================
 
@@ -12,6 +11,8 @@ Global/Universal
 -  **PATRONI\_NAME**: name of the node where the current instance of Patroni is running. Must be unique for the cluster.
 -  **PATRONI\_NAMESPACE**: path within the configuration store where Patroni will keep information about the cluster. Default value: "/service"
 -  **PATRONI\_SCOPE**: cluster name
+-  **PATRONI\_LOGLEVEL**: sets the general logging level (see `the docs for Python logging <https://docs.python.org/3.6/library/logging.html#levels>`_)
+-  **PATRONI\_REQUESTS_LOGLEVEL**: sets the logging level for all HTTP requests e.g. Kubernetes API calls (see `the docs for Python logging <https://docs.python.org/3.6/library/logging.html#levels>`_)
 
 Bootstrap configuration
 -----------------------
@@ -30,7 +31,7 @@ Consul
 -  **PATRONI\_CONSUL\_SCHEME**: (optional) **http** or **https**, defaults to **http**
 -  **PATRONI\_CONSUL\_TOKEN**: (optional) ACL token
 -  **PATRONI\_CONSUL\_VERIFY**: (optional) whether to verify the SSL certificate for HTTPS requests
--  **PATRONI\_CONSUL\_CACERT**: (optional) The ca certificate. If pressent it will enable validation.
+-  **PATRONI\_CONSUL\_CACERT**: (optional) The ca certificate. If present it will enable validation.
 -  **PATRONI\_CONSUL\_CERT**: (optional) File with the client certificate
 -  **PATRONI\_CONSUL\_KEY**: (optional) File with the client key. Can be empty if the key is part of certificate.
 -  **PATRONI\_CONSUL\_DC**: (optional) Datacenter to communicate with. By default the datacenter of the host is used.
@@ -39,10 +40,11 @@ Consul
 Etcd
 ----
 -  **PATRONI\_ETCD\_HOST**: the host:port for the etcd endpoint.
+-  **PATRONI\_ETCD\_HOSTS**: list of etcd endpoints in format host1:port1,host2:port2,etc...
 -  **PATRONI\_ETCD\_URL**: url for the etcd, in format: http(s)://(username:password@)host:port
 -  **PATRONI\_ETCD\_PROXY**: proxy url for the etcd. If you are connecting to the etcd using proxy, use this parameter instead of **PATRONI\_ETCD\_URL**
 -  **PATRONI\_ETCD\_SRV**: Domain to search the SRV record(s) for cluster autodiscovery.
--  **PATRONI\_ETCD\_CACERT**: The ca certificate. If pressent it will enable validation.
+-  **PATRONI\_ETCD\_CACERT**: The ca certificate. If present it will enable validation.
 -  **PATRONI\_ETCD\_CERT**: File with the client certificate
 -  **PATRONI\_ETCD\_KEY**: File with the client key. Can be empty if the key is part of certificate.
 
@@ -51,6 +53,18 @@ Exhibitor
 -  **PATRONI\_EXHIBITOR\_HOSTS**: initial list of Exhibitor (ZooKeeper) nodes in format: 'host1,host2,etc...'. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes.
 -  **PATRONI\_EXHIBITOR\_PORT**: Exhibitor port.
 
+.. _kubernetes_environment:
+
+Kubernetes
+----------
+-  **PATRONI\_KUBERNETES\_NAMESPACE**: (optional) Kubernetes namespace where the Patroni pod is running. Default value is `default`.
+-  **PATRONI\_KUBERNETES\_LABELS**: Labels in format ``{label1: value1, label2: value2}``. These labels will be used to find existing objects (Pods and either Endpoints or ConfigMaps) associated with the current cluster. Also Patroni will set them on every object (Endpoint or ConfigMap) it creates.
+-  **PATRONI\_KUBERNETES\_SCOPE\_LABEL**: (optional) name of the label containing cluster name. Default value is `cluster-name`.
+-  **PATRONI\_KUBERNETES\_ROLE\_LABEL**: (optional) name of the label containing Postgres role (`master` or `replica`). Patroni will set this label on the pod it is running in. Default value is `role`.
+-  **PATRONI\_KUBERNETES\_USE\_ENDPOINTS**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
+-  **PATRONI\_KUBERNETES\_POD\_IP**: (optional) IP address of the pod Patroni is running in. This value is required when `PATRONI_KUBERNETES_USE_ENDPOINTS` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
+-  **PATRONI\_KUBERNETES\_PORTS**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``PATRONI_KUBERNETES_PORTS='{[{"name": "postgresql", "port": 5432}]}'`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `PATRONI_KUBERNETES_USE_ENDPOINTS` is set.
+
 PostgreSQL
 ----------
 -  **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.

docs/README.rst

@@ -1,8 +1,8 @@
 .. _readme:
 
-=================
-How Patroni Works
-=================
+============
+Introduction
+============
 
 Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features.
 
@@ -13,30 +13,65 @@ For additional background info, see:
 * `PostgreSQL HA with Kubernetes and Patroni <https://www.youtube.com/watch?v=iruaCgeG7qs>`__, talk by Josh Berkus at KubeCon 2016 (video)
 * `Feb. 2016 Zalando Tech blog post <https://tech.zalando.de/blog/zalandos-patroni-a-template-for-high-availability-postgresql/>`__
 
-==================
+
 Development Status
-==================
+------------------
 
 Patroni is in active development and accepts contributions. See our :ref:`Contributing <contributing>` section below for more details.
 
 We report new releases information :ref:`here <releases>`.
 
-===================================
-Technical Requirements/Installation
-===================================
 
-**For Mac**
+Technical Requirements/Installation
+-----------------------------------
+
+**Pre-requirements for Mac OS**
 
 To install requirements on a Mac, run the following:
 
 ::
 
     brew install postgresql etcd haproxy libyaml python
-    pip install psycopg2 pyyaml
 
-=======================
+**General installation for pip**
+
+Patroni can be installed with pip:
+
+::
+
+    pip install patroni[dependencies]
+
+where dependencies can be either empty, or consist of one or more of the following:
+
+etcd
+    `python-etcd` module in order to use Etcd as DCS
+consul
+    `python-consul` module in order to use Consul as DCS
+zookeeper
+    `kazoo` module in order to use Zookeeper as DCS
+exhibitor
+    `kazoo` module in order to use Exhibitor as DCS (same dependencies as for Zookeeper)
+kubernetes
+    `kubernetes` module in order to use Kubernetes as DCS in Patroni
+aws
+    `boto` in order to use AWS callbacks
+
+For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is:
+
+::
+
+    pip install patroni[etcd,aws]
+
+Note that external tools to call in the replica creation or custom bootstrap scripts (i.e. WAL-E) should be installed
+independently of Patroni.
+
+
 Running and Configuring
-=======================
+-----------------------
+
+The following section assumes Patroni repository as being cloned from https://github.com/zalando/patroni. Namely, you
+will need example configuration files `postgres0.yml` and `postgres1.yml`. If you installed Patroni with pip, you can
+obtain those files from the git repository and replace `./patroni.py` below with `patroni` command.
 
 To get started, do the following from different terminals:
 ::
@@ -60,27 +95,27 @@ run:
 
     > psql --host 127.0.0.1 --port 5000 postgres
 
-==================
+
 YAML Configuration
-==================
+------------------
 
 Go :ref:`here <settings>` for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
 
-=========================
+
 Environment Configuration
-=========================
+-------------------------
 
 Go :ref:`here <environment>` for comprehensive information about configuring(overriding) settings via environment variables.
 
-===================
+
 Replication Choices
-===================
+-------------------
 
 Patroni uses Postgres' streaming replication, which is asynchronous by default. Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the leader. This setting should be increased or decreased based on business requirements. It's also possible to use synchronous replication for better durability guarantees. See :ref:`replication modes documentation <replication_modes>` for details.
 
-======================================
+
 Applications Should Not Use Superusers
-======================================
+--------------------------------------
 
 When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
 

docs/SETTINGS.rst

@@ -18,13 +18,14 @@ Bootstrap configuration
     -  **retry\_timeout**: timeout for DCS and PostgreSQL operation retries. DCS or network issues shorter than this will not cause Patroni to demote the leader. Default value: 10
     -  **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election.
     -  **master\_start\_timeout**: the amount of time a master is allowed to recover from failures before failover is triggered. Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Best worst case failover time for master failure is: loop\_wait + master\_start\_timeout + loop\_wait, unless master\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff.
-    -  **synchronous\_mode**: turns on synchronous replication mode. In this mode a replica will be chosen as synchronous and only the latest leader and synchronous replica are able to participate in leader election. Synchronous mode makes sure that succesfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See `replication modes documentation <https://github.com/zalando/patroni/blob/master/docs/replication_modes.rst>`__ for details.
+    -  **synchronous\_mode**: turns on synchronous replication mode. In this mode a replica will be chosen as synchronous and only the latest leader and synchronous replica are able to participate in leader election. Synchronous mode makes sure that successfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See :ref:`replication modes documentation <replication_modes>` for details.
+    -  **synchronous\_mode\_strict**: prevents disabling synchronous replication if no synchronous replicas are available, blocking all client writes to the master. See :ref:`replication modes documentation <replication_modes>` for details.
     -  **postgresql**:
-        -  **use\_pg\_rewind**:whether or not to use pg_rewind
+        -  **use\_pg\_rewind**: whether or not to use pg_rewind
         -  **use\_slots**: whether or not to use replication_slots. Must be False for PostgreSQL 9.3. You should comment out max_replication_slots before it becomes ineligible for leader status.
         -  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower. 
         -  **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work.
--  **method**: custom script to use for bootstrpapping this cluster.
+-  **method**: custom script to use for bootstrapping this cluster.
    See :ref:`custom bootstrap methods documentation <custom_bootstrap>` for details.
    When ``initdb`` is specified revert to the default ``initdb`` command. ``initdb`` is also triggered when no ``method``
    parameter is present in the configuration file.
@@ -35,7 +36,7 @@ Bootstrap configuration
 -  **pg\_hba**: list of lines that you should add to pg\_hba.conf.
         -  **- host all all 0.0.0.0/0 md5**.
         -  **- host replication replicator 127.0.0.1/32 md5**: A line like this is required for replication.
--  **users**: Some additional users users which needs to be created after initializing new cluster
+-  **users**: Some additional users which need to be created after initializing new cluster
     -  **admin**: the name of user
         -  **password: zalando**:
         -  **options**: list of options for CREATE USER statement
@@ -54,8 +55,8 @@ Most of the parameters are optional, but you have to specify one of the **host**
 -  **port**: (optional) Consul port
 -  **scheme**: (optional) **http** or **https**, defaults to **http**
 -  **token**: (optional) ACL token
--  **verify** (optional) whether to verify the SSL certificate for HTTPS requests
--  **cacert**: (optional) The ca certificate. If pressent it will enable validation.
+-  **verify**: (optional) whether to verify the SSL certificate for HTTPS requests
+-  **cacert**: (optional) The ca certificate. If present it will enable validation.
 -  **cert**: (optional) file with the client certificate
 -  **key**: (optional) file with the client key. Can be empty if the key is part of **cert**.
 -  **dc**: (optional) Datacenter to communicate with. By default the datacenter of the host is used.
@@ -63,16 +64,17 @@ Most of the parameters are optional, but you have to specify one of the **host**
 
 Etcd
 ----
-Most of the parameters are optional, but you have to specify one of the **host**, **url**, **proxy** or **srv**
+Most of the parameters are optional, but you have to specify one of the **host**, **hosts**, **url**, **proxy** or **srv**
 
 -  **host**: the host:port for the etcd endpoint.
+-  **hosts**: list of etcd endpoint in format host1:port1,host2:port2,etc... Could be a comma separated string or an actual yaml list.
 -  **url**: url for the etcd
 -  **proxy**: proxy url for the etcd. If you are connecting to the etcd using proxy, use this parameter instead of **url**
 -  **srv**: Domain to search the SRV record(s) for cluster autodiscovery.
 -  **protocol**: (optional) http or https, if not specified http is used. If the **url** or **proxy** is specified - will take protocol from them.
 -  **username**: (optional) username for etcd authentication
 -  **password**: (optional) password for etcd authentication.
--  **cacert**: (optional) The ca certificate. If pressent it will enable validation.
+-  **cacert**: (optional) The ca certificate. If present it will enable validation.
 -  **cert**: (optional) file with the client certificate
 -  **key**: (optional) file with the client key. Can be empty if the key is part of **cert**.
 
@@ -82,6 +84,18 @@ Exhibitor
 -  **poll\_interval**: how often the list of ZooKeeper and Exhibitor nodes should be updated from Exhibitor
 -  **port**: Exhibitor port.
 
+.. _kubernetes_settings:
+
+Kubernetes
+----------
+-  **namespace**: (optional) Kubernetes namespace where Patroni pod is running. Default value is `default`.
+-  **labels**: Labels in format ``{label1: value1, label2: value2}``. These labels will be used to find existing objects (Pods and either Endpoints or ConfigMaps) associated with the current cluster. Also Patroni will set them on every object (Endpoint or ConfigMap) it creates.
+-  **scope\_label**: (optional) name of the label containing cluster name. Default value is `cluster-name`.
+-  **role\_label**: (optional) name of the label containing role (master or replica). Patroni will set this label on the pod it runs in. Default value is ``role``.
+-  **use\_endpoints**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
+-  **pod\_ip**: (optional) IP address of the pod Patroni is running in. This value is required when `use_endpoints` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
+-  **ports**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``kubernetes.ports: {[{"name": "postgresql", "port": 5432}]}`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `kubernetes.use_endpoints` is set.
+
 .. _postgresql_settings:
 
 PostgreSQL
@@ -100,17 +114,17 @@ PostgreSQL
         -  **on\_start**: run this script when the cluster starts.
         -  **on\_stop**: run this script when the cluster stops.
 -  **connect\_address**: IP address + port through which Postgres is accessible from other nodes and applications.
--  **create\_replica\_method**: an ordered list of the create methods for turning a Patroni node into a new replica.
+-  **create\_replica\_methods**: an ordered list of the create methods for turning a Patroni node into a new replica.
    "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured as its
    own config item. See :ref:`custom replica creation methods documentation <custom_replica_creation>` for further explanation.
 -  **data\_dir**: The location of the Postgres data directory, either existing or to be initialized by Patroni.
 -  **config\_dir**: The location of the Postgres configuration directory, defaults to the data directory. Must be writable by Patroni.
--  **bin\_dir**: Path to PostgreSQL binaries. (pg_ctl, pg_rewind, pg_basebackup, postgres) The  default value is an empty string meaning that PATH environment variable will be used to find the executables.
+-  **bin\_dir**: Path to PostgreSQL binaries (pg_ctl, pg_rewind, pg_basebackup, postgres). The default value is an empty string meaning that PATH environment variable will be used to find the executables.
 -  **listen**: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
 -  **use\_unix\_socket**: specifies that Patroni should prefer to use unix sockets to connect to the cluster. Default value is ``false``. If ``unix_socket_directories`` is definded, Patroni will use first suitable value from it to connect to the cluster and fallback to tcp if nothing is suitable. If ``unix_socket_directories`` is not specified in ``postgresql.parameters``, Patroni will assume that default value should be used and omit ``host`` from connection parameters.
 -  **pgpass**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup, the post_init script and under some other circumstances. The location must be writable by Patroni.
 -  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower.
--  **custom\_conf** : path to an optional custom ``postgresql.conf`` file, that will be used in place of ``postgresql.base.conf``. The file must exist on all cluster nodes, be readable by PostgreSQL and will be included from its location on the real ``postgresql.conf``. Note that Patroni will not monitor this file for changes, nor backup it. However, its settings can still be overriden by Patroni's own configuration facilities - see `dynamic configuration <https://github.com/zalando/patroni/blob/master/docs/dynamic_configuration.rst>`__ for details.
+-  **custom\_conf** : path to an optional custom ``postgresql.conf`` file, that will be used in place of ``postgresql.base.conf``. The file must exist on all cluster nodes, be readable by PostgreSQL and will be included from its location on the real ``postgresql.conf``. Note that Patroni will not monitor this file for changes, nor backup it. However, its settings can still be overridden by Patroni's own configuration facilities - see :ref:`dynamic configuration <dynamic_configuration>` for details.
 -  **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work.
 -  **pg\_hba**: list of lines that Patroni will use to generate ``pg_hba.conf``. This parameter has higher priority than ``bootstrap.pg_hba``. Together with :ref:`dynamic configuration <dynamic_configuration>` it simplifies management of ``pg_hba.conf``.
         -  **- host all all 0.0.0.0/0 md5**.
@@ -118,12 +132,12 @@ PostgreSQL
 -  **pg\_ctl\_timeout**: How long should pg_ctl wait when doing ``start``, ``stop`` or ``restart``. Default value is 60 seconds.
 -  **use\_pg\_rewind**: try to use pg\_rewind on the former leader when it joins cluster as a replica.
 -  **remove\_data\_directory\_on\_rewind\_failure**: If this option is enabled, Patroni will remove postgres data directory and recreate replica. Otherwise it will try to follow the new leader. Default value is **false**.
--  **replica\_method** for each create_replica_method other than basebackup, you would add a configuration section of the same name. At a minimum, this should include "command" with a full path to the actual script to be executed.  Other configuration parameters will be passed along to the script in the form "parameter=value".
+-  **replica\_method**: for each create_replica_methods other than basebackup, you would add a configuration section of the same name. At a minimum, this should include "command" with a full path to the actual script to be executed. Other configuration parameters will be passed along to the script in the form "parameter=value".
 
 REST API
 -------- 
--  **connect\_address**: IP address and port to access the REST API.
--  **listen**: IP address and port that Patroni will listen to, to provide health-check information for HAProxy.
+-  **connect\_address**: IP address (or hostname) and port, to access the Patroni's REST API. It can serve as a endpoint for HTTP health checks (read below about the "listen" REST API parameter), and also for user queries (either directly or via the REST API), as well as for the health checks done by the cluster members during leader elections (for example, to determine whether the master is still running, or if there is a node which has a WAL position that is ahead of the one doing the query; etc.) The connect_address is put in the member key in DCS, making it possible to translate the member name into the address to connect to its REST API.
+-  **listen**: IP address (or hostname) and port that Patroni will listen to for the REST API - to provide also the same health checks and cluster messaging between the participating nodes, as described above. to provide health-check information for HAProxy (or any other load balancer capable of doing a HTTP "OPTION" or "GET" checks)  
 -  **Optional**:
         -  **authentication**:
             -  **username**: Basic-auth username to protect unsafe REST API endpoints.
@@ -138,6 +152,6 @@ ZooKeeper
 
 Watchdog
 --------
-- **mode**: ``off``, ``automatic`` or ``required``. When ``off`` watchdog is disabled. When ``automatic`` watchdog will be used if available, but ignored if it is not. When ``required`` the node will not become a leader unless watchdog can be succesfully enabled.
+- **mode**: ``off``, ``automatic`` or ``required``. When ``off`` watchdog is disabled. When ``automatic`` watchdog will be used if available, but ignored if it is not. When ``required`` the node will not become a leader unless watchdog can be successfully enabled.
 - **device**: Path to watchdog device. Defaults to ``/dev/watchdog``.
 - **safety_margin**: Number of seconds of safety margin between watchdog triggering and leader key expiration.

docs/conf.py

@@ -18,9 +18,11 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
 
+import sys
+sys.path.insert(0, os.path.abspath('..'))
+
+from patroni.version import __version__
 
 # -- General configuration ------------------------------------------------
 
@@ -51,7 +53,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'Patroni'
-copyright = '2016, Zalando SE'
+copyright = '2015 Compose, Zalando SE'
 author = 'Zalando SE'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -59,9 +61,9 @@ author = 'Zalando SE'
 # built documents.
 #
 # The short X.Y version.
-version = '1.2'
+version = __version__[:__version__.rfind('.')]
 # The full version, including alpha/beta/rc tags.
-release = '1.2.2'
+release = __version__
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/dynamic_configuration.rst

@@ -16,9 +16,9 @@ Patroni configuration is stored in the DCS (Distributed Configuration Store). Th
 	These options are defined in the configuration file and take precedence over dynamic configuration.
 	patroni.yml could be changed and reload in runtime (without restart of Patroni) by sending SIGHUP to the Patroni process or by performing ``POST /reload`` REST-API request.
 
-- Environment :ref:`configuration <environment>` .
+- Environment :ref:`configuration <environment>`.
 	It is possible to set/override some of the "Local" configuration parameters with environment variables.
-	Environment configuration is very useful when you are running in a dynamic environment and you don't know some of the parameters in advance (for example it's not possible to know you external IP address when you are running inside ``docker``).
+	Environment configuration is very useful when you are running in a dynamic environment and you don't know some of the parameters in advance (for example it's not possible to know your external IP address when you are running inside ``docker``).
 
 Some of the PostgreSQL parameters must hold the same values on the master and the replicas. For those, values set either in the local patroni configuration files or via the environment variables take no effect. To alter or set their values one must change the shared configuration in the DCS. Below is the actual list of such parameters together with the default values:
 
@@ -42,7 +42,7 @@ There are some other Postgres parameters controlled by Patroni:
 
 - listen_addresses - is set either from ``postgresql.listen`` or from ``PATRONI_POSTGRESQL_LISTEN`` environment variable
 - port - is set either from ``postgresql.listen`` or from ``PATRONI_POSTGRESQL_LISTEN`` environment variable
-- cluster_name - is set either from ``scope`` or from ``PATRRONI_SCOPE`` environment variable
+- cluster_name - is set either from ``scope`` or from ``PATRONI_SCOPE`` environment variable
 - hot_standby: on
 
 To be on the safe side parameters from the above lists are not written into ``postgresql.conf``, but passed as a list of arguments to the ``pg_ctl start`` which gives them the highest precedence, even above `ALTER SYSTEM <https://www.postgresql.org/docs/current/static/sql-altersystem.html>`__

docs/index.rst

@@ -6,11 +6,11 @@
 Introduction
 ============
 
-Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__ or `Consul <https://github.com/hashicorp/consul>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
+Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__, `Consul <https://github.com/hashicorp/consul>`__ or `Kubernetes <https://kubernetes.io>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
 
 We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely. There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
 
-**Note to Kubernetes users**: We're currently developing Patroni to be as useful as possible for teams running Kubernetes on top of Google Compute Engine; Patroni can be the HA solution for Postgres in such an environment. To this end, we've created a `Helm Chart <https://github.com/kubernetes/charts/tree/master/incubator/patroni>`__ that enables you to deploy a five-node Patroni cluster using a Kubernetes PetSet.
+**Note to Kubernetes users**: Patroni can run natively on top of Kubernetes. Take a look at the :ref:`Kubernetes <kubernetes>` chapter of the Patroni documentation.
 
 
 .. toctree::
@@ -24,6 +24,8 @@ We call Patroni a "template" because it is far from being a one-size-fits-all or
    replica_bootstrap
    replication_modes
    pause
+   kubernetes
+   watchdog
    releases
    CONTRIBUTING
 
@@ -33,5 +35,3 @@ Indices and tables
 * :ref:`genindex`
 * :ref:`modindex`
 * :ref:`search`
-
-

docs/kubernetes.rst

@@ -0,0 +1,53 @@
+.. _kubernetes:
+
+Using Patroni with Kubernetes
+=============================
+
+Patroni can use Kubernetes objects in order to store the state of the cluster and manage the leader key. That makes it
+capable of operating Postgres in Kubernetes environment without any consistency store, namely, one doesn't
+need to run an extra Etcd deployment. There are two different type of Kubernetes objects Patroni can use to store the
+leader and the configuration keys, they are configured with the `kubernetes.use_endpoints` or `PATRONI_KUBERNETES_USE_ENDPOINTS`
+environment variable.
+
+Use Endpoints
+-------------
+
+Despite the fact that this is the recommended mode, it is turned off by default for compatibility reasons. When it is on, Patroni stores
+the cluster configuration and the leader key in the `metadata: annotations` fields of the respective `Endpoints` it creates.
+Changing the leader is safer than when using `ConfigMaps`, since both the annotations, containing the leader information, and the actual addresses
+pointing to the running leader pod are updated simultaneously in one go.
+
+Use ConfigMaps
+--------------
+
+In this mode, Patroni will create ConfigMaps instead of Endpoints and store keys inside meta-data of those ConfigMaps.
+Changing the leader takes at least two updates, one to the leader ConfigMap and another to the respective Endpoint.
+
+There are two ways to direct the traffic to the Postgres master:
+
+- use the `callback script <https://github.com/zalando/patroni/blob/master/kubernetes/callback.py>`_ provided by Patroni
+- configure the Kubernetes Postgres service to use the label selector with the `role_label` (configured in patroni configuration).
+
+Note that in some cases, for instance, when running on OpenShift, there is no alternative to using ConfigMaps.
+
+Configuration
+-------------
+
+Patroni Kubernetes :ref:`settings <kubernetes_settings>` and :ref:`environment variables <kubernetes_environment>` are described in the general chapters of the documentation.
+
+Examples
+--------
+
+- The `kubernetes <https://github.com/zalando/patroni/tree/master/kubernetes>`__ folder of the Patroni repository contains
+  examples of the Docker image, the Kubernetes manifest and the callback script in order to test Patroni Kubernetes setup.
+  Note that in the current state it will not be able to use PersistentVolumes because of permission issues.
+
+- You can find the full-featured Docker image that can use Persistent Volumes in the
+  `Spilo Project <https://github.com/zalando/spilo>`_.
+
+- There is also a `Helm chart <https://github.com/kubernetes/charts/tree/master/incubator/patroni>`_
+  to deploy the Spilo image configured with Patroni running using Kubernetes.
+
+- In order to run your database clusters at scale using Patroni and Spilo, take a look at the
+  `postgres-operator <https://github.com/zalando-incubator/postgres-operator>`_ project. It implements the operator pattern
+  to manage Spilo clusters.

docs/pause.rst

@@ -25,7 +25,7 @@ When Patroni runs in a paused mode, it does not change the state of PostgreSQL,
 
 - If there is no leader lock in the cluster, the running master acquires the lock. If there is more than one master node, then the first master to acquire the lock wins. If there are no masters altogether, Patroni does not try to promote any replicas. There is an exception in this rule: if there is no leader lock because the old master has demoted itself due to the manual promotion, then only the candidate node mentioned in the promotion request may take the leader lock. When the new leader lock is granted (i.e. after promoting a replica manually), Patroni makes sure the replicas that were streaming from the previous leader will switch to the new one.
 
-- When Postgres is stopped, Patroni does not try to start it. When Patroni is stopped, it does not to stop Postgres instance it is managing.
+- When Postgres is stopped, Patroni does not try to start it. When Patroni is stopped, it does not try to stop the Postgres instance it is managing.
 
 User guide
 ----------

docs/releases.rst

@@ -3,6 +3,314 @@
 Release notes
 =============
 
+Version 1.4.6
+-------------
+
+**Bug fixes and stability improvements**
+
+This release fixes a critical issue with Patroni API /master endpoint returning 200 for the non-master node. This is a
+reporting issue, no actual split-brain, but under certain circumstances clients might be directed to the read-only node.
+
+- Reset is_leader status on demote (Alexander Kukushkin, Oleksii Kliukin)
+
+  Make sure demoted cluster member stops responding with code 200 on the /master API call.
+
+- Add new "cluster_unlocked" field to the API output (Dmitry Dolgov)
+
+  This field indicates whether the cluster has the master running. It can be used when it is not possible to query any
+  other node but one of the replicas.
+
+Version 1.4.5
+-------------
+
+**New features**
+
+- Improve logging when applying new postgres configuration (Don Seiler)
+
+  Patroni logs changed parameter names and values.
+
+- Python 3.7 compatibility (Christoph Berg)
+
+  async is a reserved keyword in python3.7
+
+- Set state to "stopped" in the DCS when a member is shut down (Tony Sorrentino)
+
+  This shows the member state as "stopped" in "patronictl list" command.
+
+- Improve the message logged when stale postmaster.pid matches a running process (Ants Aasma)
+
+  The previous one was beyond confusing.
+
+- Implement patronictl reload functionality (Don Seiler)
+
+  Before that it was only possible to reload configuration by either calling REST API or by sending SIGHUP signal to the Patroni process.
+
+- Take and apply some parameters from controldata when starting as a replica (Alexander Kukushkin)
+
+  The value of `max_connections` and some other parameters set in the global configuration may be lower than the one actually used by the primary; when this happens, the replica cannot start and should be fixed manually. Patroni takes care of that now by reading and applying the value from  `pg_controldata`, starting postgres and setting `pending_restart` flag.
+
+- If set, use LD_LIBRARY_PATH when starting postgres (Chris Fraser)
+
+  When starting up Postgres, Patroni was passing along PATH, LC_ALL and LANG env vars if they are set. Now it is doing the same with LD_LIBRARY_PATH. It should help if somebody installed PostgreSQL to non-standard place.
+
+- Rename create_replica_method to create_replica_methods (Dmitry Dolgov)
+
+  To make it clear that it's actually an array. The old name is still supported for backward compatibility.
+
+**Bug fixes and stability improvements**
+
+- Fix condition for the replica start due to pg_rewind in paused state (Oleksii  Kliukin)
+
+  Avoid starting the replica that had already executed pg_rewind before.
+
+- Respond 200 to the master health-check only if update_lock has been successful (Alexander)
+
+  Prevent Patroni from reporting itself a master on the former (demoted) master if DCS is partitioned.
+
+- Fix compatibility with the new consul module (Alexander)
+
+  Starting from v1.1.0 python-consul changed internal API and started using `list` instead of `dict` to pass query parameters.
+
+- Catch exceptions from Patroni REST API thread during shutdown (Alexander)
+
+  Those uncaught exceptions kept PostgreSQL running at shutdown.
+
+- Do crash recovery only when Postgres runs as the master (Alexander)
+
+  Require `pg_controldata` to report  'in production' or 'shutting down' or 'in crash recovery'. In all other cases no crash recovery is necessary.
+
+- Improve handling of configuration errors (Henning Jacobs, Alexander)
+
+  It is possible to change a lot of parameters in runtime (including `restapi.listen`) by updating Patroni config file and sending SIGHUP to Patroni process. This fix eliminates obscure exceptions from the 'restapi' thread when some of the parameters receive invalid values.
+
+
+Version 1.4.4
+-------------
+
+**Stability improvements**
+
+- Fix race condition in poll_failover_result (Alexander Kukushkin)
+
+  It didn't affect directly neither failover nor switchover, but in some rare cases it was reporting success too early, when the former leader released the lock, producing a 'Failed over to "None"' instead of 'Failed over to "desired-node"' message.
+
+- Treat Postgres parameter names as case insensitive (Alexander)
+
+  Most of the Postgres parameters have snake_case names, but there are three exceptions from this rule: DateStyle, IntervalStyle and TimeZone. Postgres accepts those parameters when written in a different case (e.g. timezone = 'some/tzn'); however, Patroni was unable to find case-insensitive matches of those parameter names in pg_settings and ignored such parameters as a result.
+
+- Abort start if attaching to running postgres and cluster not initialized (Alexander)
+
+  Patroni can attach itself to an already running Postgres instance. It is imperative to start running Patroni on the master node before getting to the replicas.
+
+- Fix behavior of patronictl scaffold (Alexander)
+
+  Pass dict object to touch_member instead of json encoded string, DCS implementation will take care of encoding it.
+
+- Don't demote master if failed to update leader key in pause (Alexander)
+
+  During maintenance a DCS may start failing write requests while continuing to responds to read ones. In that case, Patroni used to put the Postgres master node to a read-only mode after failing to update the leader lock in DCS.
+
+- Sync replication slots when Patroni notices a new postmaster process (Alexander)
+
+  If Postgres has been restarted, Patroni has to make sure that list of replication slots matches its expectations.
+
+- Verify sysid and sync replication slots after coming out of pause (Alexander)
+
+  During the `maintenance` mode it may happen that data directory was completely rewritten and therefore we have to make sure that `Database system identifier` still belongs to our cluster and replication slots are in sync with Patroni expectations.
+
+- Fix a possible failure to start not running Postgres on a data directory with postmaster lock file present (Alexander)
+
+  Detect reuse of PID from the postmaster lock file. More likely to hit such problem if you run Patroni and Postgres in the docker container.
+
+- Improve protection of DCS being accidentally wiped (Alexander)
+
+  Patroni has a lot of logic in place to prevent failover in such case; it can also restore all keys back; however, until this change an accidental removal of /config key was switching off pause mode for 1 cycle of HA loop.
+
+- Do not exit when encountering invalid system ID (Oleksii Kliukin)
+
+  Do not exit when the cluster system ID is empty or the one that doesn't pass the validation check. In that case, the cluster most likely needs a reinit; mention it in the result message. Avoid terminating Patroni, as otherwise reinit cannot happen.
+
+**Compatibility with Kubernetes 1.10+**
+
+- Added check for empty subsets (Cody Coons)
+
+  Kubernetes 1.10.0+ started returning `Endpoints.subsets` set to `None` instead of `[]`.
+
+**Bootstrap improvements**
+
+- Make deleting recovery.conf optional (Brad Nicholson)
+
+  If `bootstrap.<custom_bootstrap_method_name>.keep_existing_recovery_conf` is defined and set to ``True``, Patroni will not remove the existing ``recovery.conf`` file. This is useful when bootstrapping from a backup with tools like pgBackRest that generate the appropriate `recovery.conf` for you.
+
+- Allow options to the basebackup built-in method (Oleksii)
+
+  It is now possible to supply options to the built-in basebackup method by defining the `basebackup` section in the configuration, similar to how those are defined for custom replica creation methods. The difference is in the format accepted by the `basebackup` section: since pg_basebackup accepts both `--key=value` and `--key` options, the contents of the section could be either a dictionary of key-value pairs, or a list of either one-element dictionaries or just keys (for the options that don't accept values). See :ref:`replica creation method <custom_replica_creation>` section for additional examples.
+
+
+Version 1.4.3
+-------------
+
+**Improvements in logging**
+
+- Make log level configurable from environment variables (Andy Newton, Keyvan Hedayati)
+
+  `PATRONI_LOGLEVEL` - sets the general logging level
+  `PATRONI_REQUESTS_LOGLEVEL` - sets the logging level for all HTTP requests e.g. Kubernetes API calls
+  See `the docs for Python logging <https://docs.python.org/3.6/library/logging.html#levels>` to get the names of possible log levels
+
+**Stability improvements and bug fixes**
+
+- Don't rediscover etcd cluster topology when watch timed out (Alexander Kukushkin)
+
+  If we have only one host in etcd configuration and exactly this host is not accessible, Patroni was starting discovery of cluster topology and never succeeding. Instead it should just switch to the next available node.
+
+- Write content of bootstrap.pg_hba into a pg_hba.conf after custom bootstrap (Alexander)
+
+  Now it behaves similarly to the usual bootstrap with `initdb`
+
+- Single user mode was waiting for user input and never finish (Alexander)
+
+  Regression was introduced in https://github.com/zalando/patroni/pull/576
+
+
+Version 1.4.2
+-------------
+
+**Improvements in patronictl**
+
+- Rename scheduled failover to scheduled switchover (Alexander Kukushkin)
+
+  Failover and switchover functions were separated in version 1.4, but `patronictl list` was still reporting `Scheduled failover` instead of `Scheduled switchover`.
+
+- Show information about pending restarts (Alexander)
+
+  In order to apply some configuration changes sometimes it is necessary to restart postgres. Patroni was already giving a hint about that in the REST API and when writing node status into DCS, but there were no easy way to display it.
+
+- Make show-config to work with cluster_name from config file (Alexander)
+
+  It works similar to the `patronictl edit-config`
+
+**Stability improvements**
+
+- Avoid calling pg_controldata during bootstrap (Alexander)
+
+  During initdb or custom bootstrap there is a time window when pgdata is not empty but pg_controldata has not been written yet. In such case pg_controldata call was failing with error messages.
+
+- Handle exceptions raised from psutil (Alexander)
+
+  cmdline is read and parsed every time when `cmdline()` method is called. It could happen that the process being examined
+  has already disappeared, in that case `NoSuchProcess` is raised.
+
+**Kubernetes support improvements**
+
+- Don't swallow errors from k8s API (Alexander)
+
+  A call to Kubernetes API could fail for a different number of reasons. In some cases such call should be retried, in some other cases we should log the error message and the exception stack trace. The change here will help debug Kubernetes permission issues.
+
+- Update Kubernetes example Dockerfile to install Patroni from the master branch (Maciej Szulik)
+
+  Before that it was using `feature/k8s`, which became outdated.
+
+- Add proper RBAC to run patroni on k8s (Maciej)
+
+  Add the Service account that is assigned to the pods of the cluster, the role that holds only the necessary permissions, and the rolebinding that connects the Service account and the Role.
+
+
+Version 1.4.1
+-------------
+
+**Fixes in patronictl**
+
+- Don't show current leader in suggested list of members to failover to. (Alexander Kukushkin)
+
+  patronictl failover could still work when there is leader in the cluster and it should be excluded from the list of member where it is possible to failover to.
+
+- Make patronictl switchover compatible with the old Patroni api (Alexander)
+
+  In case if POST /switchover REST API call has failed with status code 501 it will do it once again, but to /failover endpoint.
+
+
+Version 1.4
+-----------
+
+This version adds support for using Kubernetes as a DCS, allowing to run Patroni as a cloud-native agent in Kubernetes without any additional deployments of Etcd, Zookeeper or Consul.
+
+**Upgrade notice**
+
+Installing Patroni via pip will no longer bring in dependencies for (such as libraries for Etcd, Zookeper, Consul or Kubernetes, or support for AWS). In order to enable them one need to list them in pip install command explicitely, for instance `pip install patroni[etcd,kubernetes]`.
+
+**Kubernetes support**
+
+Implement Kubernetes-based DCS. The endpoints meta-data is used in order to store the configuration and the leader key. The meta-data field inside the pods definition is used to store the member-related data.
+In addition to using Endpoints, Patroni supports ConfigMaps. You can find more information about this feature in the :ref:`Kubernetes chapter of the documentation <kubernetes>`
+
+**Stability improvements**
+
+- Factor out postmaster process into a separate object (Ants Aasma)
+
+  This object identifies a running postmaster process via pid and start time and simplifies detection (and resolution) of situations when the postmaster was restarted behind our back or when postgres directory disappeared from the file system.
+
+- Minimize the amount of SELECT's issued by Patroni on every loop of HA cylce (Alexander Kukushkin)
+
+  On every iteration of HA loop Patroni needs to know recovery status and absolute wal position. From now on Patroni will run only single SELECT to get this information instead of two on the replica and three on the master. 
+  
+- Remove leader key on shutdown only when we have the lock (Ants)
+
+  Unconditional removal was generating unnecessary and missleading exceptions.
+
+**Improvements in patronictl**
+
+- Add version command to patronictl (Ants)
+
+  It will show the version of installed Patroni and versions of running Patroni instances (if the cluster name is specified).
+  
+- Make optional specifying cluster_name argument for some of patronictl commands (Alexander, Ants)
+
+  It will work if patronictl is using usual Patroni configuration file with the ``scope`` defined.
+
+- Show information about scheduled switchover and maintenance mode (Alexander)
+
+  Before that it was possible to get this information only from Patroni logs or directly from DCS.
+  
+- Improve ``patronictl reinit`` (Alexander)
+
+  Sometimes ``patronictl reinit`` refused to proceed when Patroni was busy with other actions, namely trying to start postgres. `patronictl` didn't provide any commands to cancel such long running actions and the only (dangerous) workarond was removing a data directory manually. The new implementation of `reinit` forcefully cancells other long-running actions before proceeding with reinit.
+  
+- Implement ``--wait`` flag in ``patronictl pause`` and ``patronictl resume`` (Alexander)
+
+  It will make ``patronictl`` wait until the requested action is acknowledged by all nodes in the cluster.
+  Such behaviour is achieved by exposing the ``pause`` flag for every node in DCS and via the REST API.
+
+- Rename ``patronictl failover`` into ``patronictl switchover`` (Alexander)
+
+  The previous ``failover`` was actually only capable of doing a switchover; it refused to proceed in a cluster without the leader.
+
+- Alter the behavior of ``patronictl failover`` (Alexander)
+
+  It will work even if there is no leader, but in that case you will have to explicitely specify a node which should become the new leader.
+
+**Expose information about timeline and history**
+
+- Expose current timeline in DCS and via API (Alexander)
+
+  Store information about the current timeline for each member of the cluster. This information is accessible via the API and is stored in the DCS
+
+- Store promotion history in the /history key in DCS (Alexander)
+
+  In addition, store the timeline history enriched with the timestamp of the corresponding promotion in the /history key in DCS and update it with each promote.
+
+**Add endpoints for getting synchronous and asynchronous replicas**
+
+- Add new /sync and /async endpoints (Alexander, Oleksii Kliukin)
+
+ Those endpoints (also accessible as /synchronous and /asynchronous) return 200 only for synchronous and asynchornous replicas correspondingly (exclusing those marked as `noloadbalance`).
+
+**Allow multiple hosts for Etcd**
+
+- Add a new `hosts` parameter to Etcd configuration (Alexander)
+
+  This parameter should contain the initial list of hosts that will be used to discover and populate the list of the running etcd cluster members. If for some reason during work this list of discovered hosts is exhausted (no available hosts from that list), Patroni will return to the initial list from the `hosts` parameter.
+
+
 Version 1.3.6
 -------------
 
@@ -24,7 +332,7 @@ Version 1.3.6
 
 **Consul improvements**
 
-- Make it possible to provide datacenter configuration for Consul (DeathBorn, Alexander)
+- Make it possible to provide datacenter configuration for Consul (Vilius Okockis, Alexander)
 
   Before that Patroni was always communicating with datacenter of the host it runs on.
 

docs/replica_bootstrap.rst

@@ -3,7 +3,7 @@ Replica imaging and bootstrap
 
 Patroni allows customizing creation of a new replica. It also supports defining what happens when the new empty cluster
 is being bootstrapped. The distinction between two is well defined: Patroni creates replicas only if the ``initialize``
-key is present in Etcd for the cluster. If there is no ``initialize`` key - Patroni calls bootstrap exclusively on the
+key is present in DCS for the cluster. If there is no ``initialize`` key - Patroni calls bootstrap exclusively on the
 first node that takes the initialize key lock.
 
 .. _custom_bootstrap:
@@ -23,6 +23,7 @@ arguments to them, i.e. the name of the cluster and the path to the data directo
         method: <custom_bootstrap_method_name>
         <custom_bootstrap_method_name>:
             command: <path_to_custom_bootstrap_script> [param1 [, ...]]
+            keep_existing_recovery_conf: False
             recovery_conf:
                 recovery_target_action: promote
                 recovery_target_timeline: latest
@@ -47,6 +48,9 @@ If a ``recovery_conf`` block is defined in the same section as the custom bootst
 ``recovery.conf`` before starting the newly bootstrapped instance. Typically, such recovery.conf should contain at least
 one of the ``recovery_target_*`` parameters, together with the ``recovery_target_timeline`` set to ``promote``.
 
+If ``keep_existing_recovery_conf`` is defined and set to ``True``, Patroni will not remove the existing ``recovery.conf`` file if it exists.
+This is useful when bootstrapping from a backup with tools like pgBackRest that generate the appropriate ``recovery.conf`` for you.
+
  .. note:: Bootstrap methods are neither chained, nor fallen-back to the default one in case the primary one fails
 
 
@@ -64,7 +68,7 @@ scripts to clone a new replica. Those are configured in the ``postgresql`` confi
 .. code:: YAML
 
     postgresql:
-        create_replica_method:
+        create_replica_methods:
             - wal_e
             - basebackup
         wal_e:
@@ -72,13 +76,14 @@ scripts to clone a new replica. Those are configured in the ``postgresql`` confi
             no_master: 1
             envdir: {{WALE_ENV_DIR}}
             use_iam: 1
+        basebackup:
+            max-rate: '100M'
 
 
-The ``create_replica_method`` defines available replica creation methods and the order of executing them. Patroni will
-stop on the first one that returns 0. The basebackup is the built-in method and doesn't require any configuration. The
-rest of the methods should define a separate section in the configuration file, listing the command to execute and any
-custom parameters that should be passed to that command. All parameters will be passed in a ``--name=value`` format.
-Besides user-defined parameters, Patroni supplies a couple of cluster-specific ones:
+The ``create_replica_methods`` defines available replica creation methods and the order of executing them. Patroni will
+stop on the first one that returns 0. Each method should define a separate section in the configuration file, listing the command
+to execute and any custom parameters that should be passed to that command. All parameters will be passed in a
+``--name=value`` format. Besides user-defined parameters, Patroni supplies a couple of cluster-specific ones:
 
 --scope
     Which cluster this replica belongs to
@@ -94,5 +99,33 @@ A special ``no_master`` parameter, if defined, allows Patroni to call the replic
 running master or replicas. In that case, an empty string will be passed in a connection string. This is useful for
 restoring the formerly running cluster from the binary backup.
 
-If all replica creation methods fail, Patroni will try again all methods in order during the next event loop cycle.
+A ``basebackup`` method is a special case: it will be used if
+``create_replica_methods`` is empty, although it is possible
+to list it explicitly among the ``create_replica_methods`` methods. This method initializes a new replica with the
+``pg_basebackup``, the base backup is taken from the master unless there are replicas with ``clonefrom`` tag, in which case one
+of such replicas will be used as the origin for pg_basebackup. It works without any configuration; however, it is
+possible to specify a ``basebackup`` configuration section. Same rules as with the other method configuration apply,
+namely, only long (with --) options should be specified there. Not all parameters make sense, if you override a connection
+string or provide an option to created tar-ed or compressed base backups, patroni won't be able to make a replica out
+of it. There is no validation performed on the names or values of the parameters passed to the ``basebackup`` section.
+You can specify basebackup parameters as either a map (key-value pairs) or a list of elements, where each element
+could be either a key-value pair or a single key (for options that does not receive any values, for instance, ``--verbose``).
+Consider those 2 examples:
 
+.. code:: YAML
+
+    postgresql:
+        basebackup:
+            max-rate: '100M'
+            checkpoint: 'fast'
+
+and
+
+.. code:: YAML
+
+    postgresql:
+        basebackup:
+            - verbose
+            - max-rate: '100M'
+
+If all replica creation methods fail, Patroni will try again all methods in order during the next event loop cycle.

docs/replication_modes.rst

@@ -9,9 +9,9 @@ Patroni uses PostgreSQL streaming replication. For more information about stream
 Asynchronous mode durability
 ----------------------------
 
-In asynchronous mode the cluster is allowed to lose some committed transactions to ensure availability. When master server fails or becomes unavailable for any other reason Patroni will automatically promote a sufficiently healthy standby to master. Any transactions that have not been replicated to that standby remain in a "forked timeline" on the master, and are effectively unrecoverable [1]_.
+In asynchronous mode the cluster is allowed to lose some committed transactions to ensure availability. When the primary server fails or becomes unavailable for any other reason Patroni will automatically promote a sufficiently healthy standby to primary. Any transactions that have not been replicated to that standby remain in a "forked timeline" on the primary, and are effectively unrecoverable [1]_.
 
-The amount of transactions that can be lost is controlled via ``maximum_lag_on_failover`` parameter. Because master transaction log position is not sampled in real time, in reality the amount of lost data on failover is worst case bounded by  ``maximum_lag_on_failover`` bytes of transaction log plus the amount that is written in the last ``ttl`` seconds (``loop_wait``/2 seconds in the average case). However typical steady state replication delay is well under a second.
+The amount of transactions that can be lost is controlled via ``maximum_lag_on_failover`` parameter. Because the primary transaction log position is not sampled in real time, in reality the amount of lost data on failover is worst case bounded by  ``maximum_lag_on_failover`` bytes of transaction log plus the amount that is written in the last ``ttl`` seconds (``loop_wait``/2 seconds in the average case). However typical steady state replication delay is well under a second.
 
 PostgreSQL synchronous replication
 ----------------------------------
@@ -20,7 +20,7 @@ You can use Postgres's `synchronous replication <http://www.postgresql.org/docs/
 
 In hosted datacenter environments (like AWS, Rackspace, or any network you do not control), synchronous replication significantly increases the variability of write performance. If followers become inaccessible from the leader, the leader effectively becomes read-only.
 
-To enable a simple synchronous replication test, add the follow lines to the ``parameters`` section of your YAML configuration files:
+To enable a simple synchronous replication test, add the following lines to the ``parameters`` section of your YAML configuration files:
 
 .. code:: YAML
 
@@ -29,18 +29,27 @@ To enable a simple synchronous replication test, add the follow lines to the ``p
 
 When using PostgreSQL synchronous replication, use at least three Postgres data nodes to ensure write availability if one host fails.
 
-Using PostgreSQL synchronous replication does not guarantee zero lost transactions under all circumstances. When master and standby that is currently acting as synchronous fail simultaneously a third node that might not contain all transactions will be promoted.
+Using PostgreSQL synchronous replication does not guarantee zero lost transactions under all circumstances. When the primary and the secondary that is currently acting as a synchronous replica fail simultaneously a third node that might not contain all transactions will be promoted.
 
 .. _synchronous_mode:
 
 Synchronous mode
 ----------------
 
-For use cases where losing committed transactions is not permissible you can turn on Patronis ``synchronous_mode``. When ``synchronous_mode`` is turned on Patroni will not promote a standby unless it is certain that the standby contains all transactions that may have returned a successful commit status to client [2]_. This means that the system may be unavailable for writes even though some servers are available. System administrators can still use manual failover commmands to promote a standby even if it results in transaction loss.
+For use cases where losing committed transactions is not permissible you can turn on Patroni's ``synchronous_mode``. When ``synchronous_mode`` is turned on Patroni will not promote a standby unless it is certain that the standby contains all transactions that may have returned a successful commit status to client [2]_. This means that the system may be unavailable for writes even though some servers are available. System administrators can still use manual failover commands to promote a standby even if it results in transaction loss.
 
-Turning on ``synchronous_mode`` does not guarantee multi node durability of commits under all circumstances. When no suitable standby is available, master server will still accept writes, but does not guarantee their replication. When the master fails in this mode no standby will be promote.  When the host that used to be master comes back it will get promoted automatically, unless system administrator performed a manual failover. This behavior makes synchronous mode usable with 2 node clusters.
+Turning on ``synchronous_mode`` does not guarantee multi node durability of commits under all circumstances. When no suitable standby is available, primary server will still accept writes, but does not guarantee their replication. When the primary fails in this mode no standby will be promoted. When the host that used to be the primary comes back it will get promoted automatically, unless system administrator performed a manual failover. This behavior makes synchronous mode usable with 2 node clusters.
 
-When ``synchronous_mode`` is on and a standby crashes, commits will block until next iteration of Patroni runs and switches master to standalone mode (worst case delay for writes ``ttl`` seconds, average case ``loop_wait``/2 seconds). Manually shutting down or restarting a standby will not cause a commit service interruption. Standby will signal the master to release itself from synchronous standby duties before PostgreSQL shutdown is initiated.
+When ``synchronous_mode`` is on and a standby crashes, commits will block until next iteration of Patroni runs and switches the primary to standalone mode (worst case delay for writes ``ttl`` seconds, average case ``loop_wait``/2 seconds). Manually shutting down or restarting a standby will not cause a commit service interruption. Standby will signal the primary to release itself from synchronous standby duties before PostgreSQL shutdown is initiated.
+
+When it is absolutely necessary to guarantee that each write is stored durably
+on at least two nodes, enable ``synchronous_mode_strict`` in addition to the
+``synchronous_node``. This parameter prevents Patroni from switching off the
+synchronous replication on the primary when no synchronous standby candidates
+are available. As a downside, the primary is not be available for writes
+(unless the Postgres transaction explicitly turns of ``synchronous_mode``),
+blocking all client write requests until at least one synchronous replica comes
+up.
 
 You can ensure that a standby never becomes the synchronous standby by setting ``nosync`` tag to true. This is recommended to set for standbys that are behind slow network connections and would cause performance degradation when becoming a synchronous standby.
 
@@ -50,7 +59,7 @@ Synchronous mode can be switched on and off via Patroni REST interface. See :ref
 Synchronous mode implementation
 -------------------------------
 
-When in synchronous mode Patroni maintains synchronization state in the DCS, containing the latest master and current synchronous standby. This state is updated with strict ordering constraints to ensure the following invariants:
+When in synchronous mode Patroni maintains synchronization state in the DCS, containing the latest primary and current synchronous standby. This state is updated with strict ordering constraints to ensure the following invariants:
 
 - A node must be marked as the latest leader whenever it can accept write transactions. Patroni crashing or PostgreSQL not shutting down can cause violations of this invariant.
 
@@ -60,9 +69,9 @@ When in synchronous mode Patroni maintains synchronization state in the DCS, con
 
 Patroni will only ever assign one standby to ``synchronous_standby_names`` because with multiple candidates it is not possible to know which node was acting as synchronous during the failure.
 
-On each HA loop iteration Patroni re-evaluates synchronous standby choice. If the current synchronous standby is connected and has not requested its synchronous status to be removed it remains picked. Otherwise the cluster member avaiable for sync that is furthest ahead in replication is picked.
+On each HA loop iteration Patroni re-evaluates synchronous standby choice. If the current synchronous standby is connected and has not requested its synchronous status to be removed it remains picked. Otherwise the cluster member available for sync that is furthest ahead in replication is picked.
 
 
-.. [1] The data is still there, but recovering it requires a manual recovery effort by data recovery specialists. When Patroni is allowed to rewind with ``use_pg_rewind`` the forked timeline will be automatically erased to rejoin the failed master with the cluster.
+.. [1] The data is still there, but recovering it requires a manual recovery effort by data recovery specialists. When Patroni is allowed to rewind with ``use_pg_rewind`` the forked timeline will be automatically erased to rejoin the failed primary with the cluster.
 
-.. [2] Clients can change the behavior per transaction using PostgreSQL's ``synchronous_commit`` setting. Transactions with ``synchronous_commit`` values of ``off`` and ``local`` may be lost on fail over, but will not be blocked by replication delays.
+.. [2] Clients can change the behavior per transaction using PostgreSQL's ``synchronous_commit`` setting. Transactions with ``synchronous_commit`` values of ``off`` and ``local`` may be lost on fail over, but will not be blocked by replication delays.

docs/watchdog.rst

@@ -1,6 +1,5 @@
 .. _watchdog:
 
-================
 Watchdog support
 ================
 
@@ -10,7 +9,7 @@ Having multiple PostgreSQL servers running as master can result in transactions
 
 - Shutting down PostgreSQL is too slow.
 
-- Patroni does not get to run due to high load on the system, th VM being paused by the hypervisor, or other infrastructure issues.
+- Patroni does not get to run due to high load on the system, the VM being paused by the hypervisor, or other infrastructure issues.
 
 To guarantee correct behavior under these conditions Patroni supports watchdog devices. Watchdog devices are software or hardware mechanisms that will reset the whole system when they do not get a keepalive heartbeat within a specified timeframe. This adds an additional layer of fail safe in case usual Patroni split-brain protection mechanisms fail.
 

extras/README.md

@@ -3,7 +3,7 @@
 `confd` directory contains haproxy template files for the [confd](https://github.com/kelseyhightower/confd) -- lightweight configuration management tool
 You need to copy content of `confd` directory into /etcd/confd and run confd service:
 ```bash
-$ confd -prefix=/service/$PATRONI_SCOPE -backend etcd -node $PATRONI_ETCD_HOST -interval=10
+$ confd -prefix=/service/$PATRONI_SCOPE -backend etcd -node $PATRONI_ETCD_URL -interval=10
 ```
 It will periodically update haproxy.cfg with the actual list of Patroni nodes from `etcd` and "reload" haproxy when it is necessary.
 

extras/confd/templates/haproxy.tmpl

@@ -10,19 +10,23 @@ defaults
 	timeout server 30m
 	timeout check 5s
 
-frontend master_postgresql
+listen stats
+	mode http
+	bind *:7000
+	stats enable
+	stats uri /
+
+listen master
 	bind *:5000
-	default_backend backend_master
-
-frontend replicas_postgresql
-	bind *:5001
-	default_backend backend_replicas
-
-backend backend_master
 	option httpchk OPTIONS /master
+	http-check expect status 200
+	default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
 {{range gets "/members/*"}}	server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}}
 {{end}}
-backend backend_replicas
+listen replicas
+	bind *:5001
 	option httpchk OPTIONS /replica
+	http-check expect status 200
+	default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
 {{range gets "/members/*"}}	server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}}
 {{end}}

extras/startup-scripts/patroni.service

@@ -11,12 +11,20 @@ Type=simple
 User=postgres
 Group=postgres
 
+# Read in configuration file if it exists, otherwise proceed
+EnvironmentFile=-/etc/patroni_env.conf
+
+WorkingDirectory=~
+
 # Where to send early-startup messages from the server
 # This is normally controlled by the global default set by systemd
 # StandardOutput=syslog
 
 ExecStart=/bin/patroni /etc/patroni.yml
 
+# Send HUP to reload from patroni.yml
+ExecReload=/bin/kill -s HUP $MAINPID
+
 # only kill the patroni process, not it's children, so it will gracefully stop postgres
 KillMode=process
 

features/basic_replication.feature

@@ -19,11 +19,21 @@ Feature: basic replication
     And I run patronictl.py restart batman postgres1 --force
     Then I receive a response returncode 0
     And "sync" key in DCS has sync_standby=postgres2 after 10 seconds
+    And I sleep for 2 seconds
+    When I issue a GET request to http://127.0.0.1:8010/sync
+    Then I receive a response code 200
+    When I issue a GET request to http://127.0.0.1:8009/async
+    Then I receive a response code 200
 
   Scenario: check the basic failover in synchronous mode
-    When I kill postgres0
-    Then postgres2 role is the primary after 22 seconds
-    When I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_mode": null, "master_start_timeout": 0}
+    Given I run patronictl.py pause batman
+    Then I receive a response returncode 0
+    When I sleep for 2 seconds
+    And I shut down postgres0
+    And I run patronictl.py resume batman 
+    Then I receive a response returncode 0
+    And postgres2 role is the primary after 24 seconds
+    When I issue a PATCH request to http://127.0.0.1:8010/config with {"synchronous_mode": null, "master_start_timeout": 0}
     Then I receive a response code 200
     When I add the table bar to postgres2
     Then table bar is present on postgres1 after 20 seconds

features/environment.py

@@ -7,6 +7,7 @@ import kazoo.exceptions
 import os
 import psutil
 import psycopg2
+import json
 import shutil
 import signal
 import six
@@ -98,6 +99,7 @@ class PatroniController(AbstractController):
         else:
             self.watchdog = None
 
+        self._scope = (custom_config or {}).get('scope', 'batman')
         self._config = self._make_patroni_test_config(name, custom_config)
         self._closables = []
 
@@ -125,6 +127,9 @@ class PatroniController(AbstractController):
     def _start(self):
         if self.watchdog:
             self.watchdog.start()
+        if isinstance(self._context.dcs_ctl, KubernetesController):
+            self._context.dcs_ctl.create_pod(self._name[8:], self._scope)
+            os.environ['PATRONI_KUBERNETES_POD_IP'] = '10.0.0.' + self._name[-1]
         return subprocess.Popen(['coverage', 'run', '--source=patroni', '-p', 'patroni.py', self._config],
                                 stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory)
 
@@ -132,6 +137,8 @@ class PatroniController(AbstractController):
         if postgres:
             return subprocess.call(['pg_ctl', '-D', self._data_dir, 'stop', '-mi', '-w'])
         super(PatroniController, self).stop(kill, timeout)
+        if isinstance(self._context.dcs_ctl, KubernetesController):
+            self._context.dcs_ctl.delete_pod(self._name[8:])
         if self.watchdog:
             self.watchdog.stop()
 
@@ -147,7 +154,7 @@ class PatroniController(AbstractController):
 
         with open(patroni_config_name) as f:
             config = yaml.safe_load(f)
-            config.pop('etcd')
+            config.pop('etcd', None)
 
         host = config['postgresql']['listen'].split(':')[0]
 
@@ -335,10 +342,6 @@ class AbstractDcsController(AbstractController):
     def query(self, key, scope='batman'):
         """ query for a value of a given key """
 
-    @abc.abstractmethod
-    def set(self, key, value):
-        """ set a value to a given key """
-
     @abc.abstractmethod
     def cleanup_service_tree(self):
         """ clean all contents stored in the tree used for the tests """
@@ -388,9 +391,6 @@ class ConsulController(AbstractDcsController):
         _, value = self._client.kv.get(self.path(key, scope))
         return value and value['Value'].decode('utf-8')
 
-    def set(self, key, value):
-        self._client.kv.put(self.path(key), value)
-
     def cleanup_service_tree(self):
         self._client.kv.delete(self.path(scope=''), recurse=True)
 
@@ -417,9 +417,6 @@ class EtcdController(AbstractDcsController):
         except etcd.EtcdKeyNotFound:
             return None
 
-    def set(self, key, value):
-        self._client.set(self.path(key), value)
-
     def cleanup_service_tree(self):
         try:
             self._client.delete(self.path(scope=''), recursive=True)
@@ -436,6 +433,76 @@ class EtcdController(AbstractDcsController):
             return False
 
 
+class KubernetesController(AbstractDcsController):
+
+    def __init__(self, context):
+        super(KubernetesController, self).__init__(context)
+        self._namespace = 'default'
+        self._labels = {"application": "patroni"}
+        self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
+        os.environ['PATRONI_KUBERNETES_LABELS'] = json.dumps(self._labels)
+        os.environ['PATRONI_KUBERNETES_USE_ENDPOINTS'] = 'true'
+
+        from kubernetes import client as k8s_client, config as k8s_config
+        k8s_config.load_kube_config(context='local')
+        self._client = k8s_client
+        self._api = self._client.CoreV1Api()
+
+    def _start(self):
+        pass
+
+    def create_pod(self, name, scope):
+        labels = self._labels.copy()
+        labels['cluster-name'] = scope
+        metadata = self._client.V1ObjectMeta(namespace=self._namespace, name=name, labels=labels)
+        spec = self._client.V1PodSpec(containers=[self._client.V1Container(name=name, image='empty')])
+        body = self._client.V1Pod(metadata=metadata, spec=spec)
+        self._api.create_namespaced_pod(self._namespace, body)
+
+    def delete_pod(self, name):
+        try:
+            self._api.delete_namespaced_pod(name, self._namespace, self._client.V1DeleteOptions())
+        except:
+            pass
+        while True:
+            try:
+                self._api.read_namespaced_pod(name, self._namespace)
+            except:
+                break
+
+    def query(self, key, scope='batman'):
+        if key.startswith('members/'):
+            pod = self._api.read_namespaced_pod(key[8:], self._namespace)
+            return (pod.metadata.annotations or {}).get('status', '')
+        else:
+            try:
+                e = self._api.read_namespaced_endpoints(scope + ('' if key == 'leader' else '-' + key), self._namespace)
+                if key == 'leader':
+                    return e.metadata.annotations[key]
+                else:
+                    return json.dumps(e.metadata.annotations)
+            except:
+                return None
+
+    def cleanup_service_tree(self):
+        try:
+            self._api.delete_collection_namespaced_pod(self._namespace, label_selector=self._label_selector)
+        except:
+            pass
+        try:
+            self._api.delete_collection_namespaced_endpoints(self._namespace, label_selector=self._label_selector)
+        except:
+            pass
+
+        while True:
+            result = self._api.list_namespaced_pod(self._namespace, label_selector=self._label_selector)
+            if len(result.items) < 1:
+                break
+
+    def _is_running(self):
+        return True
+
+
 class ZooKeeperController(AbstractDcsController):
 
     """ handles all zookeeper related tasks, used for the tests setup and cleanup """
@@ -455,9 +522,6 @@ class ZooKeeperController(AbstractDcsController):
         except kazoo.exceptions.NoNodeError:
             return None
 
-    def set(self, key, value):
-        self._client.set(self.path(key), value.encode('utf-8'))
-
     def cleanup_service_tree(self):
         try:
             self._client.delete(self.path(scope=''), recursive=True)
@@ -548,6 +612,13 @@ class PatroniPoolController(object):
                 'method': 'pg_basebackup',
                 'pg_basebackup': {
                     'command': self.BACKUP_SCRIPT + ' --walmethod=stream --dbname=' + f.backup_source
+                },
+                'dcs': {
+                    'postgresql': {
+                        'parameters': {
+                            'max_connections': 101
+                        }
+                    }
                 }
             },
             'postgresql': {
@@ -713,6 +784,7 @@ class WatchdogMonitor(object):
 
 # actions to execute on start/stop of the tests and before running invidual features
 def before_all(context):
+    os.environ.update({'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password'})
     context.ci = 'TRAVIS_BUILD_NUMBER' in os.environ or 'BUILD_NUMBER' in os.environ
     context.timeout_multiplier = 2 if context.ci else 1
     context.pctl = PatroniPoolController(context)

features/patroni_api.feature

@@ -13,17 +13,17 @@ Scenario: check API requests on a stand-alone server
 	When I run patronictl.py reinit batman postgres0 --force
 	Then I receive a response returncode 0
 	And I receive a response output "Failed: reinitialize for member postgres0, status code=503, (I am the leader, can not reinitialize)"
-	When I run patronictl.py failover batman --master postgres0 --force
+	When I run patronictl.py switchover batman --master postgres0 --force
 	Then I receive a response returncode 1
-	And I receive a response output "Error: No candidates found to failover to"
-	When I issue a POST request to http://127.0.0.1:8008/failover with {"leader": "postgres0"}
-	Then I receive a response code 500
-	And I receive a response text failover is not possible: cluster does not have members except leader
+	And I receive a response output "Error: No candidates found to switchover to"
+	When I issue a POST request to http://127.0.0.1:8008/switchover with {"leader": "postgres0"}
+	Then I receive a response code 412
+	And I receive a response text switchover is not possible: cluster does not have members except leader
 	When I issue an empty POST request to http://127.0.0.1:8008/failover
 	Then I receive a response code 400
 	When I issue a POST request to http://127.0.0.1:8008/failover with {"foo": "bar"}
 	Then I receive a response code 400
-	And I receive a response text "No values given for required parameters leader and candidate"
+	And I receive a response text "Failover could be performed only to a specific candidate"
 
 Scenario: check local configuration reload
 	Given I issue an empty POST request to http://127.0.0.1:8008/reload
@@ -64,21 +64,21 @@ Scenario: check API requests for the primary-replica pair in the pause mode
 	When I sleep for 10 seconds
 	Then postgres1 role is the secondary after 15 seconds
 
-Scenario: check the failover via the API in the pause mode
-	Given I issue a POST request to http://127.0.0.1:8008/failover with {"leader": "postgres0", "candidate": "postgres1"}
+Scenario: check the switchover via the API in the pause mode
+	Given I issue a POST request to http://127.0.0.1:8008/switchover with {"leader": "postgres0", "candidate": "postgres1"}
 	Then I receive a response code 200
 	And postgres1 is a leader after 5 seconds
 	And postgres1 role is the primary after 10 seconds
 	And postgres0 role is the secondary after 10 seconds
 	And replication works from postgres1 to postgres0 after 20 seconds
 
-Scenario: check the scheduled failover
-	Given I issue a scheduled failover from postgres1 to postgres0 in 3 seconds
+Scenario: check the scheduled switchover
+	Given I issue a scheduled switchover from postgres1 to postgres0 in 3 seconds
 	Then I receive a response returncode 1
-	And I receive a response output "Can't schedule failover in the paused state"
+	And I receive a response output "Can't schedule switchover in the paused state"
 	When I run patronictl.py resume batman
 	Then I receive a response returncode 0
-	Given I issue a scheduled failover from postgres1 to postgres0 in 3 seconds
+	Given I issue a scheduled switchover from postgres1 to postgres0 in 3 seconds
 	Then I receive a response returncode 0
 	And postgres0 is a leader after 20 seconds
 	And postgres0 role is the primary after 10 seconds

features/steps/patroni_api.py

@@ -1,3 +1,4 @@
+import base64
 import json
 import os
 import parse
@@ -78,11 +79,13 @@ def do_post_empty(context, url):
 @step('I issue a {request_method:w} request to {url:url} with {data}')
 def do_request(context, request_method, url, data):
     data = data and json.loads(data) or {}
+    headers = {'Authorization': 'Basic ' + base64.b64encode('username:password'.encode('utf-8')).decode('utf-8'),
+               'Content-Type': 'application/json'}
     try:
         if request_method == 'PATCH':
-            r = requests.patch(url, json=data)
+            r = requests.patch(url, headers=headers, json=data)
         else:
-            r = requests.post(url, json=data)
+            r = requests.post(url, headers=headers, json=data)
     except requests.exceptions.RequestException:
         context.status_code = None
         context.response = None
@@ -122,10 +125,10 @@ def check_response(context, component, data):
         assert str(context.response[component]) == str(data), "{0} does not contain {1}".format(component, data)
 
 
-@step('I issue a scheduled failover from {from_host:w} to {to_host:w} in {in_seconds:d} seconds')
-def scheduled_failover(context, from_host, to_host, in_seconds):
+@step('I issue a scheduled switchover from {from_host:w} to {to_host:w} in {in_seconds:d} seconds')
+def scheduled_switchover(context, from_host, to_host, in_seconds):
     context.execute_steps(u"""
-        Given I run patronictl.py failover batman --master {0} --candidate {1} --scheduled "{2}" --force
+        Given I run patronictl.py switchover batman --master {0} --candidate {1} --scheduled "{2}" --force
     """.format(from_host, to_host, datetime.now(tzutc) + timedelta(seconds=int(in_seconds))))
 
 

kubernetes/Dockerfile

@@ -0,0 +1,32 @@
+FROM postgres:9.6
+MAINTAINER Alexander Kukushkin <[email protected]>
+
+RUN export DEBIAN_FRONTEND=noninteractive \
+    && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
+    && apt-get update -y \
+    && apt-get upgrade -y \
+    && apt-get install -y git curl jq python-psycopg2 python-yaml python-requests python-six python-pysocks \
+        python-dateutil python-pip python-prettytable python-wheel python-psutil python locales \
+
+    ## Make sure we have a en_US.UTF-8 locale available
+    && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
+
+    && pip install setuptools pip --upgrade \
+    && pip install 'git+https://github.com/zalando/patroni.git#egg=patroni[kubernetes]' \
+
+    && mkdir -p /home/postgres \
+    && chown postgres:postgres /home/postgres \
+
+    # Clean up
+    && apt-get remove -y git python-pip python-setuptools \
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/* /root/.cache
+
+ADD entrypoint.sh callback.py /
+
+EXPOSE 5432 8008
+ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
+USER postgres
+WORKDIR /home/postgres
+CMD ["/bin/bash", "/entrypoint.sh"]

kubernetes/callback.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+
+import logging
+import os
+import socket
+import sys
+import time
+
+from kubernetes import client as k8s_client, config as k8s_config
+from urllib3.exceptions import HTTPError
+from six.moves.http_client import HTTPException
+
+logger = logging.getLogger(__name__)
+
+
+class CoreV1Api(k8s_client.CoreV1Api):
+
+    def retry(func):
+        def wrapped(*args, **kwargs):
+            count = 0
+            while True:
+                try:
+                    return func(*args, **kwargs)
+                except (HTTPException, HTTPError, socket.error, socket.timeout):
+                    if count >= 10:
+                        raise
+                    logger.info('Throttling API requests...')
+                    time.sleep(2 ** count * 0.5)
+                    count += 1
+        return wrapped
+
+    @retry
+    def patch_namespaced_endpoints(self, *args, **kwargs):
+        return super(CoreV1Api, self).patch_namespaced_endpoints(*args, **kwargs)
+
+
+def patch_master_endpoint(api, namespace, cluster):
+    addresses = [k8s_client.V1EndpointAddress(ip=os.environ['POD_IP'])]
+    ports = [k8s_client.V1EndpointPort(port=5432)]
+    subsets = [k8s_client.V1EndpointSubset(addresses=addresses, ports=ports)]
+    body = k8s_client.V1Endpoints(subsets=subsets)
+    return api.patch_namespaced_endpoints(cluster, namespace, body)
+
+
+def main():
+    logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
+    if len(sys.argv) != 4 or sys.argv[1] not in ('on_start', 'on_stop', 'on_role_change'):
+        sys.exit('Usage: %s <action> <role> <cluster_name>', sys.argv[0])
+
+    action, role, cluster = sys.argv[1:4]
+
+    k8s_config.load_incluster_config()
+    k8s_api = CoreV1Api()
+
+    namespace = os.environ['KUBERNETES_NAMESPACE']
+
+    if role == 'master' and action in ('on_start', 'on_role_change'):
+        patch_master_endpoint(k8s_api, namespace, cluster)
+
+
+if __name__ == '__main__':
+    main()

kubernetes/entrypoint.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+cat > /home/postgres/patroni.yml <<__EOF__
+bootstrap:
+  dcs:
+    postgresql:
+      use_pg_rewind: true
+  initdb:
+  - auth-host: md5
+  - auth-local: trust
+  - encoding: UTF8
+  - locale: en_US.UTF-8
+  - data-checksums
+  pg_hba:
+  - host all all 0.0.0.0/0 md5
+  - host replication ${PATRONI_REPLICATION_USERNAME} ${POD_IP}/16    md5
+restapi:
+  connect_address: '${POD_IP}:8008'
+postgresql:
+  connect_address: '${POD_IP}:5432'
+  authentication:
+    superuser:
+      password: '${PATRONI_SUPERUSER_PASSWORD}'
+    replication:
+      password: '${PATRONI_REPLICATION_PASSWORD}'
+  callbacks:
+    on_start: /callback.py
+    on_stop: /callback.py
+    on_role_change: /callback.py
+__EOF__
+
+unset PATRONI_SUPERUSER_PASSWORD PATRONI_REPLICATION_PASSWORD
+export KUBERNETES_NAMESPACE=$PATRONI_KUBERNETES_NAMESPACE
+export POD_NAME=$PATRONI_NAME
+
+exec /usr/bin/python /usr/local/bin/patroni /home/postgres/patroni.yml

kubernetes/patroni_k8s.yaml

@@ -0,0 +1,186 @@
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  name: &cluster_name patronidemo
+  labels:
+    application: patroni
+    cluster-name: *cluster_name
+spec:
+  replicas: 3
+  serviceName: *cluster_name
+  template:
+    metadata:
+      labels:
+        application: patroni
+        cluster-name: *cluster_name
+    spec:
+      serviceAccountName: patronidemo
+      containers:
+      - name: *cluster_name
+        image: patroni  # docker build -t patroni .
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 8008
+          protocol: TCP
+        - containerPort: 5432
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /home/postgres/pgdata
+          name: pgdata
+        env:
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        - name: PATRONI_KUBERNETES_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: PATRONI_KUBERNETES_LABELS
+          value: '{application: patroni, cluster-name: patronidemo}'
+        - name: PATRONI_SUPERUSER_USERNAME
+          value: postgres
+        - name: PATRONI_SUPERUSER_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: *cluster_name
+              key: superuser-password
+        - name: PATRONI_REPLICATION_USERNAME
+          value: standby
+        - name: PATRONI_REPLICATION_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: *cluster_name
+              key: replication-password
+        - name: PATRONI_SCOPE
+          value: *cluster_name
+        - name: PATRONI_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: PATRONI_POSTGRESQL_DATA_DIR
+          value: /home/postgres/pgdata/pgroot/data
+        - name: PATRONI_POSTGRESQL_PGPASS
+          value: /tmp/pgpass
+        - name: PATRONI_POSTGRESQL_LISTEN
+          value: '0.0.0.0:5432'
+        - name: PATRONI_RESTAPI_LISTEN
+          value: '0.0.0.0:8008'
+      terminationGracePeriodSeconds: 0
+      volumes:
+      - name: pgdata
+        emptyDir: {}
+#  volumeClaimTemplates:
+#  - metadata:
+#      labels:
+#        application: spilo
+#        spilo-cluster: *cluster_name
+#      annotations:
+#        volume.alpha.kubernetes.io/storage-class: anything
+#      name: pgdata
+#    spec:
+#      accessModes:
+#      - ReadWriteOnce
+#      resources:
+#        requests:
+#          storage: 5Gi
+
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: &cluster_name patronidemo
+  labels:
+    application: patroni
+    cluster-name: *cluster_name
+subsets: []
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: &cluster_name patronidemo
+  labels:
+    application: patroni
+    cluster-name: *cluster_name
+spec:
+  type: ClusterIP
+  ports:
+  - port: 5432
+    targetPort: 5432
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: &cluster_name patronidemo
+  labels:
+    application: patroni
+    cluster-name: *cluster_name
+type: Opaque
+data:
+  superuser-password: emFsYW5kbw==
+  replication-password: cmVwLXBhc3M=
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: patronidemo
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: patronidemo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
+  # delete is required only for 'patronictl remove'
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - patch
+  - update
+  # the following three privileges are necessary only when using endpoints
+  - create
+  - list
+  - watch
+  # delete is required only for for 'patronictl remove'
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: patronidemo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: patronidemo
+subjects:
+- kind: ServiceAccount
+  name: patronidemo

patroni/__init__.py

@@ -39,7 +39,7 @@ class Patroni(object):
         while True:
             try:
                 cluster = self.dcs.get_cluster()
-                if cluster and cluster.config:
+                if cluster and cluster.config and cluster.config.data:
                     if self.config.set_dynamic_configuration(cluster.config):
                         self.dcs.reload_config(self.config)
                         self.watchdog.reload_config(self.config)
@@ -113,8 +113,8 @@ class Patroni(object):
 
             logger.info(self.ha.run_cycle())
 
-            cluster = self.dcs.cluster
-            if cluster and cluster.config and self.config.set_dynamic_configuration(cluster.config):
+            if self.dcs.cluster and self.dcs.cluster.config and self.dcs.cluster.config.data \
+                    and self.config.set_dynamic_configuration(self.dcs.cluster.config):
                 self.reload_config()
 
             if self.postgresql.role != 'uninitialized':
@@ -129,14 +129,19 @@ class Patroni(object):
         signal.signal(signal.SIGTERM, self.sigterm_handler)
 
     def shutdown(self):
-        self.api.shutdown()
+        try:
+            self.api.shutdown()
+        except Exception:
+            logger.exception('Exception during RestApi.shutdown')
         self.ha.shutdown()
 
 
 def patroni_main():
     logformat = os.environ.get('PATRONI_LOGFORMAT', '%(asctime)s %(levelname)s: %(message)s')
-    logging.basicConfig(format=logformat, level=logging.INFO)
-    logging.getLogger('requests').setLevel(logging.WARNING)
+    loglevel = os.environ.get('PATRONI_LOGLEVEL', 'INFO')
+    requests_loglevel = os.environ.get('PATRONI_REQUESTS_LOGLEVEL', 'WARNING')
+    logging.basicConfig(format=logformat, level=loglevel)
+    logging.getLogger('requests').setLevel(requests_loglevel)
 
     patroni = Patroni()
     try:

patroni/api.py

@@ -7,8 +7,9 @@ import time
 import dateutil.parser
 import datetime
 
-from patroni.exceptions import PostgresConnectionException
-from patroni.utils import deep_compare, patch_config, Retry, RetryFailedError, is_valid_pg_version, parse_int, tzutc
+from patroni.postgresql import PostgresConnectionException, PostgresException, Postgresql
+from patroni.utils import deep_compare, parse_bool, patch_config, Retry, \
+    RetryFailedError, parse_int, split_host_port, tzutc
 from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 from six.moves.socketserver import ThreadingMixIn
 from threading import Thread
@@ -24,9 +25,9 @@ def check_auth(func):
     def do_PUT_foo():
         pass
     """
-    def wrapper(handler):
+    def wrapper(handler, *args, **kwargs):
         if handler.check_auth_header():
-            return func(handler)
+            return func(handler, *args, **kwargs)
     return wrapper
 
 
@@ -70,6 +71,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
             response['scheduled_restart']['schedule'] = (response['scheduled_restart']['schedule']).isoformat()
         if not patroni.ha.watchdog.is_healthy:
             response['watchdog_failed'] = True
+        if patroni.ha.is_paused():
+            response['pause'] = True
         self._write_json_response(status_code, response)
 
     def do_GET(self, write_status_code_only=False):
@@ -80,13 +83,25 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         patroni = self.server.patroni
         cluster = patroni.dcs.cluster
+
+        def is_synchronous():
+            return (cluster.is_synchronous_mode() and cluster.sync
+                    and cluster.sync.sync_standby == patroni.postgresql.name)
+
+        def is_balanceable_replica():
+            return response.get('role') == 'replica' and not patroni.noloadbalance
+
         if cluster:  # dcs available
-            if cluster.leader and cluster.leader.name == patroni.postgresql.name:  # is_leader
+            if patroni.ha.is_leader():
                 status_code = 200 if 'master' in path else 503
             elif 'role' not in response:
                 status_code = 503
             elif response['role'] == 'master':  # running as master but without leader lock!!!!
                 status_code = 503
+            elif path in ('/sync', '/synchronous'):
+                status_code = 200 if is_balanceable_replica() and is_synchronous() else 503
+            elif path in ('/async', '/asynchronous'):
+                status_code = 200 if is_balanceable_replica() and not is_synchronous() else 503
             elif response['role'] in path:  # response['role'] != 'master'
                 status_code = 503 if patroni.noloadbalance else 200
             else:
@@ -221,9 +236,11 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     data = "PostgreSQL role should be either master or replica"
                     break
             elif k == 'postgres_version':
-                if not is_valid_pg_version(request[k]):
+                try:
+                    Postgresql.postgres_version_to_int(request[k])
+                except PostgresException as e:
                     status_code = 400
-                    data = "PostgreSQL version should be in the first.major.minor format"
+                    data = e.value
                     break
             elif k == 'timeout':
                 request[k] = parse_int(request[k], 's')
@@ -264,7 +281,14 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
     @check_auth
     def do_POST_reinitialize(self):
-        data = self.server.patroni.ha.reinitialize()
+        request = self._read_json_content(body_is_optional=True)
+
+        if request:
+            logger.debug('received reinitialize request: %s', request)
+
+        force = isinstance(request, dict) and parse_bool(request.get('force')) or False
+
+        data = self.server.patroni.ha.reinitialize(force)
         if data is None:
             status_code = 200
             data = 'reinitialize started'
@@ -272,28 +296,29 @@ class RestApiHandler(BaseHTTPRequestHandler):
             status_code = 503
         self._write_response(status_code, data)
 
-    def poll_failover_result(self, leader, candidate):
+    def poll_failover_result(self, leader, candidate, action):
         timeout = max(10, self.server.patroni.dcs.loop_wait)
         for _ in range(0, timeout*2):
             time.sleep(1)
             try:
                 cluster = self.server.patroni.dcs.get_cluster()
-                if cluster.leader and cluster.leader.name != leader:
+                if not cluster.is_unlocked() and cluster.leader.name != leader:
                     if not candidate or candidate == cluster.leader.name:
-                        return 200, 'Successfully failed over to "{0}"'.format(cluster.leader.name)
+                        return 200, 'Successfully {0}ed over to "{1}"'.format(action[:-4], cluster.leader.name)
                     else:
-                        return 200, 'Failed over to "{0}" instead of "{1}"'.format(cluster.leader.name, candidate)
+                        return 200, '{0}ed over to "{1}" instead of "{2}"'.format(action[:-4].title(),
+                                                                                  cluster.leader.name, candidate)
                 if not cluster.failover:
-                    return 503, 'Failover failed'
+                    return 503, action.title() + ' failed'
             except Exception as e:
-                logger.debug('Exception occured during polling failover result: %s', e)
-        return 503, 'Failover status unknown'
+                logger.debug('Exception occured during polling %s result: %s', action, e)
+        return 503, action.title() + ' status unknown'
 
-    def is_failover_possible(self, cluster, leader, candidate):
+    def is_failover_possible(self, cluster, leader, candidate, action):
         if leader and (not cluster.leader or cluster.leader.name != leader):
             return 'leader name does not match'
         if candidate:
-            if cluster.is_synchronous_mode() and cluster.sync.sync_standby != candidate:
+            if action == 'switchover' and cluster.is_synchronous_mode() and cluster.sync.sync_standby != candidate:
                 return 'candidate name does not match with sync_standby'
             members = [m for m in cluster.members if m.name == candidate]
             if not members:
@@ -301,20 +326,20 @@ class RestApiHandler(BaseHTTPRequestHandler):
         elif cluster.is_synchronous_mode():
             members = [m for m in cluster.members if m.name == cluster.sync.sync_standby]
             if not members:
-                return 'failover is not possible: can not find sync_standby'
+                return action + ' is not possible: can not find sync_standby'
         else:
             members = [m for m in cluster.members if m.name != cluster.leader.name and m.api_url]
             if not members:
-                return 'failover is not possible: cluster does not have members except leader'
+                return action + ' is not possible: cluster does not have members except leader'
         for st in self.server.patroni.ha.fetch_nodes_statuses(members):
             if st.failover_limitation() is None:
                 return None
-        return 'failover is not possible: no good candidates have been found'
+        return action + ' is not possible: no good candidates have been found'
 
     @check_auth
-    def do_POST_failover(self):
+    def do_POST_failover(self, action='failover'):
         request = self._read_json_content()
-        status_code = 500
+        (status_code, data) = (400, '')
         if not request:
             return
 
@@ -323,39 +348,47 @@ class RestApiHandler(BaseHTTPRequestHandler):
         scheduled_at = request.get('scheduled_at')
         cluster = self.server.patroni.dcs.get_cluster()
 
-        if scheduled_at and cluster.is_paused():
-            self._write_response(status_code, "Can't schedule failover in the paused state")
+        logger.info("received %s request with leader=%s candidate=%s scheduled_at=%s",
+                    action, leader, candidate, scheduled_at)
 
-        logger.info("received failover request with leader=%s candidate=%s scheduled_at=%s",
-                    leader, candidate, scheduled_at)
+        if action == 'failover' and not candidate:
+            data = 'Failover could be performed only to a specific candidate'
+        elif action == 'switchover' and not leader:
+            data = 'Switchover could be performed only from a specific leader'
 
-        data = ''
-        if leader or candidate:
-            if scheduled_at:
-                (_, data, scheduled_at) = self.parse_schedule(scheduled_at, "failover")
-                if _:
-                    status_code = _
-                elif self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
-                    self.server.patroni.ha.wakeup()
-                    data = 'Failover scheduled'
+        if not data and scheduled_at:
+            if not leader:
+                data = 'Scheduled {0} is possible only from a specific leader'.format(action)
+            if not data and cluster.is_paused():
+                data = "Can't schedule {0} in the paused state".format(action)
+            if not data:
+                (status_code, data, scheduled_at) = self.parse_schedule(scheduled_at, action)
+
+        if not data and cluster.is_paused() and not candidate:
+            data = action.title() + ' is possible only to a specific candidate in a paused state'
+
+        if not data and not scheduled_at:
+            data = self.is_failover_possible(cluster, leader, candidate, action)
+            if data:
+                status_code = 412
+
+        if not data:
+            if self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
+                self.server.patroni.ha.wakeup()
+                if scheduled_at:
+                    data = action.title() + ' scheduled'
                     status_code = 202
                 else:
-                    data = 'failed to write failover key into DCS'
-                    status_code = 503
+                    status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name,
+                                                                  candidate, action)
             else:
-                data = self.is_failover_possible(cluster, leader, candidate)
-                if not data:
-                    if self.server.patroni.dcs.manual_failover(leader, candidate):
-                        self.server.patroni.ha.wakeup()
-                        status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name, candidate)
-                    else:
-                        data = 'failed to write failover key into DCS'
-                        status_code = 503
-        else:
-            status_code = 400
-            data = 'No values given for required parameters leader and candidate'
+                data = 'failed to write {0} key into DCS'.format(action)
+                status_code = 503
         self._write_response(status_code, data)
 
+    def do_POST_switchover(self):
+        self.do_POST_failover(action='switchover')
+
     def parse_request(self):
         """Override parse_request method to enrich basic functionality of `BaseHTTPRequestHandler` class
 
@@ -382,42 +415,48 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
     def get_postgresql_status(self, retry=False):
         try:
+            cluster = self.server.patroni.dcs.cluster
+
             if self.server.patroni.postgresql.state not in ('running', 'restarting', 'starting'):
                 raise RetryFailedError('')
-            row = self.query("""WITH replication_info AS (
-                                    SELECT usename, application_name, client_addr, state, sync_state, sync_priority
-                                      FROM pg_stat_replication
-                                )
-                                SELECT to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
-                                       pg_is_in_recovery(),
-                                       CASE WHEN pg_is_in_recovery()
-                                            THEN 0
-                                            ELSE pg_{0}_{1}_diff(pg_current_{0}_{1}(), '0/0')::bigint
-                                       END,
-                                       pg_{0}_{1}_diff(COALESCE(pg_last_{0}_receive_{1}(),
-                                                                      pg_last_{0}_replay_{1}()), '0/0')::bigint,
-                                       pg_{0}_{1}_diff(pg_last_{0}_replay_{1}(), '0/0')::bigint,
-                                       to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
-                                       pg_is_in_recovery() AND pg_is_{0}_replay_paused(),
-                                       (SELECT array_to_json(array_agg(row_to_json(ri)))
-                                          FROM replication_info ri)""".format(self.server.patroni.postgresql.wal_name,
-                                                                              self.server.patroni.postgresql.lsn_name),
-                             retry=retry)[0]
+            stmt = ("WITH replication_info AS ("
+                    "SELECT usename, application_name, client_addr, state, sync_state, sync_priority"
+                    " FROM pg_stat_replication) SELECT"
+                    " to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),"
+                    " CASE WHEN pg_is_in_recovery() THEN 0"
+                    " ELSE ('x' || SUBSTR(pg_{0}file_name(pg_current_{0}_{1}()), 1, 8))::bit(32)::int END,"
+                    " CASE WHEN pg_is_in_recovery() THEN 0"
+                    " ELSE pg_{0}_{1}_diff(pg_current_{0}_{1}(), '0/0')::bigint END,"
+                    " pg_{0}_{1}_diff(COALESCE(pg_last_{0}_receive_{1}(), pg_last_{0}_replay_{1}()), '0/0')::bigint,"
+                    " pg_{0}_{1}_diff(pg_last_{0}_replay_{1}(), '0/0')::bigint,"
+                    " to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),"
+                    " pg_is_in_recovery() AND pg_is_{0}_replay_paused(),"
+                    " (SELECT array_to_json(array_agg(row_to_json(ri))) FROM replication_info ri)")
+
+            row = self.query(stmt.format(self.server.patroni.postgresql.wal_name,
+                                         self.server.patroni.postgresql.lsn_name), retry=retry)[0]
 
             result = {
                 'state': self.server.patroni.postgresql.state,
                 'postmaster_start_time': row[0],
-                'role': 'replica' if row[1] else 'master',
+                'role': 'replica' if row[1] == 0 else 'master',
                 'server_version': self.server.patroni.postgresql.server_version,
+                'cluster_unlocked': bool(not cluster or cluster.is_unlocked()),
                 'xlog': ({
                     'received_location': row[3],
                     'replayed_location': row[4],
                     'replayed_timestamp': row[5],
-                    'paused': row[6]} if row[1] else {
+                    'paused': row[6]} if row[1] == 0 else {
                     'location': row[2]
                 })
             }
 
+            if row[1] > 0:
+                result['timeline'] = row[1]
+            else:
+                leader_timeline = None if not cluster or cluster.is_unlocked() else cluster.leader.timeline
+                result['timeline'] = self.server.patroni.postgresql.replica_cached_timeline(leader_timeline)
+
             if row[7]:
                 result['replication'] = row[7]
 
@@ -437,6 +476,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
 
     def __init__(self, patroni, config):
         self.patroni = patroni
+        self.__listen = None
         self.__initialize(config)
         self.__set_config_parameters(config)
         self.daemon = True
@@ -471,18 +511,25 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
     def __get_ssl_options(config):
         return {option: config[option] for option in ['certfile', 'keyfile'] if option in config}
 
-    def __set_connection_string(self, connect_address):
-        self.connection_string = '{0}://{1}/patroni'.format(self.__protocol, connect_address or self.__listen)
-
     def __set_config_parameters(self, config):
         self.__auth_key = base64.b64encode(config['auth'].encode('utf-8')).decode('utf-8') if 'auth' in config else None
-        self.__set_connection_string(config.get('connect_address'))
+        self.connection_string = '{0}://{1}/patroni'.format(self.__protocol,
+                                                            config.get('connect_address') or self.__listen)
 
     def __initialize(self, config):
-        self.__ssl_options = self.__get_ssl_options(config)
+        try:
+            host, port = split_host_port(config['listen'], None)
+        except Exception:
+            raise ValueError('Invalid "restapi" config: expected <HOST>:<PORT> for "listen", but got "{0}"'
+                             .format(config['listen']))
+
+        if self.__listen is not None:  # changing config in runtime
+            self.shutdown()
+
         self.__listen = config['listen']
-        host, port = config['listen'].split(':')
-        HTTPServer.__init__(self, (host, int(port)), RestApiHandler)
+        self.__ssl_options = self.__get_ssl_options(config)
+
+        HTTPServer.__init__(self, (host, port), RestApiHandler)
         Thread.__init__(self, target=self.serve_forever)
         self._set_fd_cloexec(self.socket)
 
@@ -494,11 +541,13 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
             import ssl
             self.socket = ssl.wrap_socket(self.socket, server_side=True, **self.__ssl_options)
             self.__protocol = 'https'
-        self.__set_connection_string(config.get('connect_address'))
+        return True
 
     def reload_config(self, config):
-        self.__set_config_parameters(config)
-        if self.__listen != config['listen'] or self.__ssl_options != self.__get_ssl_options(config):
-            self.shutdown()
-            self.__initialize(config)
+        if 'listen' not in config:  # changing config in runtime
+            raise ValueError('Can not find "restapi.listen" config')
+
+        elif (self.__listen != config['listen'] or self.__ssl_options != self.__get_ssl_options(config)) \
+                and self.__initialize(config):
             self.start()
+        self.__set_config_parameters(config)

patroni/async_executor.py

@@ -1,5 +1,5 @@
 import logging
-from threading import Lock, RLock, Thread
+from threading import Event, Lock, RLock, Thread
 
 logger = logging.getLogger(__name__)
 
@@ -52,22 +52,27 @@ class CriticalTask(object):
 
 class AsyncExecutor(object):
 
-    def __init__(self, ha_wakeup):
+    def __init__(self, state_handler, ha_wakeup):
+        self.state_handler = state_handler
         self._ha_wakeup = ha_wakeup
         self._thread_lock = RLock()
         self._scheduled_action = None
         self._scheduled_action_lock = RLock()
+        self._is_cancelled = False
+        self._finish_event = Event()
         self.critical_task = CriticalTask()
 
     @property
     def busy(self):
         return self.scheduled_action is not None
 
-    def schedule(self, action, immediately=False):
+    def schedule(self, action):
         with self._scheduled_action_lock:
             if self._scheduled_action is not None:
                 return self._scheduled_action
             self._scheduled_action = action
+            self._is_cancelled = False
+            self._finish_event.set()
         return None
 
     @property
@@ -82,6 +87,12 @@ class AsyncExecutor(object):
     def run(self, func, args=()):
         wakeup = False
         try:
+            with self:
+                if self._is_cancelled:
+                    return
+                self._finish_event.clear()
+
+            self.state_handler.reset_is_cancelled()
             # if the func returned something (not None) - wake up main HA loop
             wakeup = func(*args) if args else func()
             return wakeup
@@ -90,6 +101,7 @@ class AsyncExecutor(object):
         finally:
             with self:
                 self.reset_scheduled_action()
+                self._finish_event.set()
                 with self.critical_task:
                     self.critical_task.reset()
             if wakeup is not None:
@@ -98,6 +110,20 @@ class AsyncExecutor(object):
     def run_async(self, func, args=()):
         Thread(target=self.run, args=(func, args)).start()
 
+    def cancel(self):
+        with self:
+            with self._scheduled_action_lock:
+                if self._scheduled_action is None:
+                    return
+                logger.warning('Cancelling long running task %s', self._scheduled_action)
+            self._is_cancelled = True
+
+        self.state_handler.cancel()
+        self._finish_event.wait()
+
+        with self:
+            self.reset_scheduled_action()
+
     def __enter__(self):
         self._thread_lock.acquire()
 

patroni/config.py

@@ -9,7 +9,8 @@ from collections import defaultdict
 from copy import deepcopy
 from patroni.dcs import ClusterConfig
 from patroni.postgresql import Postgresql
-from patroni.utils import deep_compare, parse_int, patch_config
+from patroni.utils import deep_compare, parse_bool, parse_int, patch_config
+from requests.structures import CaseInsensitiveDict
 
 logger = logging.getLogger(__name__)
 
@@ -47,7 +48,7 @@ class Config(object):
         'postgresql': {
             'bin_dir': '',
             'use_slots': True,
-            'parameters': {p: v[0] for p, v in Postgresql.CMDLINE_OPTIONS.items()}
+            'parameters': CaseInsensitiveDict({p: v[0] for p, v in Postgresql.CMDLINE_OPTIONS.items()})
         },
         'watchdog': {
             'mode': 'automatic',
@@ -87,6 +88,9 @@ class Config(object):
     def dynamic_configuration(self):
         return deepcopy(self._dynamic_configuration)
 
+    def check_mode(self, mode):
+        return bool(parse_bool(self._dynamic_configuration.get(mode)))
+
     def _load_config_file(self):
         """Loads config.yaml from filesystem and applies some values which were set via ENV"""
         with open(self._config_file) as f:
@@ -154,6 +158,8 @@ class Config(object):
                     self._local_configuration = configuration
                     self.__effective_configuration = new_configuration
                     return True
+                else:
+                    logger.info('No configuration items changed, nothing to reload.')
             except Exception:
                 logger.exception('Exception when reloading local configuration from %s', self.config_file)
                 if dry_run:
@@ -239,16 +245,25 @@ class Config(object):
 
         for param in list(os.environ.keys()):
             if param.startswith(Config.PATRONI_ENV_PREFIX):
-                name, suffix = (param[8:].rsplit('_', 1) + [''])[:2]
+                name, suffix = (param[8:].split('_', 1) + [''])[:2]
                 if name and suffix:
                     # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..)
-                    if suffix in ('HOST', 'HOSTS', 'PORT', 'SRV', 'URL', 'PROXY', 'CACERT', 'CERT', 'KEY',
-                                  'VERIFY', 'TOKEN', 'CHECKS', 'DC') and '_' not in name:
+                    if suffix in ('HOST', 'HOSTS', 'PORT', 'SRV', 'URL', 'PROXY', 'CACERT', 'CERT',
+                                  'KEY', 'VERIFY', 'TOKEN', 'CHECKS', 'DC', 'NAMESPACE', 'CONTEXT',
+                                  'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL', 'POD_IP', 'PORTS', 'LABELS'):
                         value = os.environ.pop(param)
                         if suffix == 'PORT':
                             value = value and parse_int(value)
-                        elif suffix in ('HOSTS', 'CHECKS'):
+                        elif suffix in ('HOSTS', 'PORTS', 'CHECKS'):
                             value = value and _parse_list(value)
+                        elif suffix == 'LABELS':
+                            if not value.strip().startswith('{'):
+                                value = '{{{0}}}'.format(value)
+                            try:
+                                value = yaml.safe_load(value)
+                            except Exception:
+                                logger.exception('Exception when parsing dict %s', value)
+                                value = None
                         if value:
                             ret[name.lower()][suffix.lower()] = value
                     # PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>

patroni/ctl.py

@@ -30,7 +30,8 @@ from patroni.config import Config
 from patroni.dcs import get_dcs as _get_dcs
 from patroni.exceptions import PatroniException
 from patroni.postgresql import Postgresql
-from patroni.utils import is_valid_pg_version, patch_config
+from patroni.utils import patch_config, polling_loop
+from patroni.version import __version__
 from prettytable import PrettyTable
 from six.moves.urllib_parse import urlparse
 from six import text_type
@@ -96,10 +97,12 @@ def store_config(config, path):
         yaml.dump(config, fd)
 
 
-option_format = click.option('--format', '-f', 'fmt', help='Output format (pretty, json)', default='pretty')
+option_format = click.option('--format', '-f', 'fmt', help='Output format (pretty, json, yaml)', default='pretty')
 option_watchrefresh = click.option('-w', '--watch', type=float, help='Auto update the screen every X seconds')
 option_watch = click.option('-W', is_flag=True, help='Auto update the screen every 2 seconds')
 option_force = click.option('--force', is_flag=True, help='Do not ask for confirmation at any point')
+arg_cluster_name = click.argument('cluster_name', required=False,
+                                  default=lambda: click.get_current_context().obj.get('scope'))
 
 
 @click.group()
@@ -149,9 +152,12 @@ def print_output(columns, rows=None, alignment=None, fmt='pretty', header=True,
         click.echo(t)
         return
 
-    if fmt == 'json':
+    if fmt in ['json', 'yaml', 'yml']:
         elements = [dict(zip(columns, r)) for r in rows]
-        click.echo(json.dumps(elements))
+        if fmt == 'json':
+            click.echo(json.dumps(elements))
+        elif fmt in ('yaml', 'yml'):
+            click.echo(yaml.safe_dump(elements, encoding=None, default_flow_style=False, allow_unicode=True, width=200))
 
     if fmt == 'tsv':
         if columns is not None and header:
@@ -257,9 +263,9 @@ def get_members(cluster, cluster_name, member_names, role, force, action):
         member_names = [click.prompt('Which member do you want to {0} [{1}]?'.format(action,
                         ', '.join(candidates.keys())), type=str, default='')]
 
-    for mn in member_names:
-        if mn not in candidates:
-            raise PatroniCtlException('{0} is not a member of cluster'.format(mn))
+    for member_name in member_names:
+        if member_name not in candidates:
+            raise PatroniCtlException('{0} is not a member of cluster'.format(member_name))
 
     if not force:
         confirm = click.confirm('Are you sure you want to {0} members {1}?'.format(action, ', '.join(member_names)))
@@ -273,7 +279,7 @@ def get_members(cluster, cluster_name, member_names, role, force, action):
 @click.option('--role', '-r', help='Give a dsn of any member with this role', type=click.Choice(['master', 'replica',
               'any']), default=None)
 @click.option('--member', '-m', help='Generate a dsn for this member', type=str)
-@click.argument('cluster_name')
+@arg_cluster_name
 @click.pass_obj
 def dsn(obj, cluster_name, role, member):
     if role is not None and member is not None:
@@ -291,7 +297,7 @@ def dsn(obj, cluster_name, role, member):
 
 
 @ctl.command('query', help='Query a Patroni PostgreSQL member')
-@click.argument('cluster_name')
+@arg_cluster_name
 @option_format
 @click.option('--format', 'fmt', help='Output format (pretty, json)', default='tsv')
 @click.option('--file', '-f', 'p_file', help='Execute the SQL commands from this file', type=click.File('rb'))
@@ -406,7 +412,7 @@ def remove(obj, cluster_name, fmt):
     if message != confirm:
         raise PatroniCtlException('You did not exactly type "{0}"'.format(message))
 
-    if cluster.leader:
+    if cluster.leader and cluster.leader.name:
         confirm = click.prompt('This cluster currently is healthy. Please specify the master name to continue')
         if confirm != cluster.leader.name:
             raise PatroniCtlException('You did not specify the current master of the cluster')
@@ -419,8 +425,10 @@ def check_response(response, member_name, action_name, silent_success=False):
         click.echo('Failed: {0} for member {1}, status code={2}, ({3})'.format(
             action_name, member_name, response.status_code, response.text
         ))
+        return False
     elif not silent_success:
         click.echo('Success: {0} for member {1}'.format(action_name, member_name))
+    return True
 
 
 def parse_scheduled(scheduled):
@@ -437,6 +445,33 @@ def parse_scheduled(scheduled):
     return None
 
 
+@ctl.command('reload', help='Reload cluster member configuration')
+@click.argument('cluster_name')
+@click.argument('member_names', nargs=-1)
+@click.option('--role', '-r', help='Reload only members with this role', default='any',
+              type=click.Choice(['master', 'replica', 'any']))
+@option_force
+@click.pass_obj
+def reload(obj, cluster_name, member_names, force, role):
+    cluster = get_dcs(obj, cluster_name).get_cluster()
+
+    members = get_members(cluster, cluster_name, member_names, role, force, 'reload')
+
+    content = {}
+    for member in members:
+        r = request_patroni(member, 'post', 'reload', content, auth_header(obj))
+        if r.status_code == 200:
+            click.echo('No changes to apply on member {0}'.format(member.name))
+        elif r.status_code == 202:
+            click.echo('Reload request received for member {0} and will be processed within {1} seconds'.format(
+                member.name, cluster.config.data.get('loop_wait'))
+            )
+        else:
+            click.echo('Failed: reload for member {0}, status code={1}, ({2})'.format(
+                member.name, r.status_code, r.text)
+            )
+
+
 @ctl.command('restart', help='Restart cluster member')
 @click.argument('cluster_name')
 @click.argument('member_names', nargs=-1)
@@ -469,11 +504,12 @@ def restart(obj, cluster_name, member_names, force, role, p_any, scheduled, vers
         content['restart_pending'] = True
 
     if version:
-        if not is_valid_pg_version(version):
-            message = 'PostgreSQL version should be in the first.major.minor format'
-            raise PatroniCtlException(message)
-        else:
-            content['postgres_version'] = version
+        try:
+            Postgresql.postgres_version_to_int(version)
+        except PatroniException as e:
+            raise PatroniCtlException(e.value)
+
+        content['postgres_version'] = version
 
     if scheduled is None and not force:
         scheduled = click.prompt('When should the restart take place (e.g. 2015-10-01T14:30) ', type=str, default='now')
@@ -516,35 +552,33 @@ def reinit(obj, cluster_name, member_names, force):
     members = get_members(cluster, cluster_name, member_names, None, force, 'reinitialize')
 
     for member in members:
-        r = request_patroni(member, 'post', 'reinitialize', headers=auth_header(obj))
-        check_response(r, member.name, 'reinitialize')
+        body = {'force': force}
+        while True:
+            r = request_patroni(member, 'post', 'reinitialize', body, auth_header(obj))
+            if not check_response(r, member.name, 'reinitialize') and r.text.endswith(' already in progress') \
+                    and not force and click.confirm('Do you want to cancel it and reinitialize anyway?'):
+                body['force'] = True
+                continue
+            break
 
 
-@ctl.command('failover', help='Failover to a replica')
-@click.argument('cluster_name')
-@click.option('--master', help='The name of the current master', default=None)
-@click.option('--candidate', help='The name of the candidate', default=None)
-@click.option('--scheduled', help='Timestamp of a scheduled failover in unambiguous format (e.g. ISO 8601)',
-              default=None)
-@option_force
-@click.pass_obj
-def failover(obj, cluster_name, master, candidate, force, scheduled):
+def _do_failover_or_switchover(obj, action, cluster_name, master, candidate, force, scheduled=None):
     """
-        We want to trigger a failover for the specified cluster name.
+        We want to trigger a failover or switchover for the specified cluster name.
 
         We verify that the cluster name, master name and candidate name are correct.
-        If so, we trigger a failover and keep the client up to date.
+        If so, we trigger an action and keep the client up to date.
     """
 
     dcs = get_dcs(obj, cluster_name)
     cluster = dcs.get_cluster()
 
-    if cluster.leader is None and not cluster.is_paused():
+    if action == 'switchover' and cluster.leader is None:
         raise PatroniCtlException('This cluster has no master')
 
-    if master is None and (not cluster.is_paused() or cluster.leader):
-        if force:
-            master = cluster.leader.member.name
+    if master is None:
+        if force or action == 'failover':
+            master = cluster.leader and cluster.leader.name
         else:
             master = click.prompt('Master', type=str, default=cluster.leader.member.name)
 
@@ -556,28 +590,32 @@ def failover(obj, cluster_name, master, candidate, force, scheduled):
     candidate_names.sort()
 
     if not candidate_names:
-        raise PatroniCtlException('No candidates found to failover to')
+        raise PatroniCtlException('No candidates found to {0} to'.format(action))
 
     if candidate is None and not force:
         candidate = click.prompt('Candidate ' + str(candidate_names), type=str, default='')
 
+    if action == 'failover' and not candidate:
+        raise PatroniCtlException('Failover could be performed only to a specific candidate')
+
     if candidate == master:
-        raise PatroniCtlException('Failover target and source are the same.')
+        raise PatroniCtlException(action.title() + ' target and source are the same.')
 
     if candidate and candidate not in candidate_names:
         raise PatroniCtlException('Member {0} does not exist in cluster {1}'.format(candidate, cluster_name))
 
-    if scheduled is None and not force:
-        scheduled = click.prompt('When should the failover take place (e.g. 2015-10-01T14:30) ', type=str,
-                                 default='now')
-
-    scheduled_at = parse_scheduled(scheduled)
-
     scheduled_at_str = None
-    if scheduled_at:
-        if cluster.is_paused():
-            raise PatroniCtlException("Can't schedule failover in the paused state")
-        scheduled_at_str = scheduled_at.isoformat()
+
+    if action == 'switchover':
+        if scheduled is None and not force:
+            scheduled = click.prompt('When should the switchover take place (e.g. 2015-10-01T14:30) ',
+                                     type=str, default='now')
+
+        scheduled_at = parse_scheduled(scheduled)
+        if scheduled_at:
+            if cluster.is_paused():
+                raise PatroniCtlException("Can't schedule switchover in the paused state")
+            scheduled_at_str = scheduled_at.isoformat()
 
     failover_value = {'leader': master, 'candidate': candidate, 'scheduled_at': scheduled_at_str}
 
@@ -588,46 +626,76 @@ def failover(obj, cluster_name, master, candidate, force, scheduled):
     output_members(dcs.get_cluster(), cluster_name)
 
     if not force:
-        a = \
-            click.confirm('Are you sure you want to failover cluster {0}, demoting current master {1}?'.format(
-                cluster_name, master))
-        if not a:
-            raise PatroniCtlException('Aborting failover')
+        demote_msg = ', demoting current master ' + master if master else ''
+
+        if not click.confirm('Are you sure you want to {0} cluster {1}{2}?'.format(action, cluster_name, demote_msg)):
+            raise PatroniCtlException('Aborting ' + action)
 
     r = None
     try:
-        member = cluster.leader.member if cluster.leader else [m for m in cluster.members if m.name == candidate][0]
+        member = cluster.leader.member if cluster.leader else cluster.get_member(candidate, False)
+
+        r = request_patroni(member, 'post', action, failover_value, auth_header(obj))
+
+        # probably old patroni, which doesn't support switchover yet
+        if r.status_code == 501 and action == 'switchover' and 'Server does not support this operation' in r.text:
+            r = request_patroni(member, 'post', 'failover', failover_value, auth_header(obj))
 
-        r = request_patroni(member, 'post', 'failover', failover_value, auth_header(obj))
         if r.status_code in (200, 202):
             logging.debug(r)
             cluster = dcs.get_cluster()
             logging.debug(cluster)
             click.echo('{0} {1}'.format(timestamp(), r.text))
         else:
-            click.echo('Failover failed, details: {0}, {1}'.format(r.status_code, r.text))
+            click.echo('{0} failed, details: {1}, {2}'.format(action.title(), r.status_code, r.text))
             return
     except Exception:
         logging.exception(r)
         logging.warning('Failing over to DCS')
-        click.echo(timestamp() + ' Could not failover using Patroni api, falling back to DCS')
-        click.echo(timestamp() + ' Initializing failover from master {0}'.format(master))
+        click.echo('{0} Could not {1} using Patroni api, falling back to DCS'.format(timestamp(), action))
         dcs.manual_failover(master, candidate, scheduled_at=scheduled_at)
 
     output_members(cluster, cluster_name)
 
 
+@ctl.command('failover', help='Failover to a replica')
+@arg_cluster_name
+@click.option('--master', help='The name of the current master', default=None)
+@click.option('--candidate', help='The name of the candidate', default=None)
+@option_force
+@click.pass_obj
+def failover(obj, cluster_name, master, candidate, force):
+    action = 'switchover' if master else 'failover'
+    _do_failover_or_switchover(obj, action, cluster_name, master, candidate, force)
+
+
+@ctl.command('switchover', help='Switchover to a replica')
+@arg_cluster_name
+@click.option('--master', help='The name of the current master', default=None)
+@click.option('--candidate', help='The name of the candidate', default=None)
+@click.option('--scheduled', help='Timestamp of a scheduled switchover in unambiguous format (e.g. ISO 8601)',
+              default=None)
+@option_force
+@click.pass_obj
+def switchover(obj, cluster_name, master, candidate, force, scheduled):
+    _do_failover_or_switchover(obj, 'switchover', cluster_name, master, candidate, force, scheduled)
+
+
 def output_members(cluster, name, extended=False, fmt='pretty'):
     rows = []
     logging.debug(cluster)
     leader_name = None
     if cluster.leader:
-        leader_name = cluster.leader.member.name
+        leader_name = cluster.leader.name
 
     xlog_location_cluster = cluster.last_leader_operation or 0
 
     # Mainly for consistent pretty printing and watching we sort the output
     cluster.members.sort(key=lambda x: x.name)
+
+    has_scheduled_restarts = any(m.data.get('scheduled_restart') for m in cluster.members)
+    has_pending_restarts = any(m.data.get('pending_restart') for m in cluster.members)
+
     for m in cluster.members:
         logging.debug(m)
 
@@ -637,22 +705,19 @@ def output_members(cluster, name, extended=False, fmt='pretty'):
         elif m.name == cluster.sync.sync_standby:
             role = 'Sync standby'
 
-        host = m.conn_kwargs()['host']
-
-        xlog_location = m.data.get('xlog_location') or 0
+        xlog_location = m.data.get('xlog_location')
         lag = ''
-        if xlog_location_cluster >= xlog_location:
+        if xlog_location is None:
+            lag = 'unknown'
+        elif xlog_location_cluster >= xlog_location:
             lag = round((xlog_location_cluster - xlog_location)/1024/1024)
 
-        row = [
-            name,
-            m.name,
-            host,
-            role,
-            m.data.get('state', ''),
-            lag,
-        ]
-        if extended:
+        row = [name, m.name, m.conn_kwargs()['host'], role, m.data.get('state', ''), lag]
+
+        if extended or has_pending_restarts:
+            row.append('*' if m.data.get('pending_restart') else '')
+
+        if extended or has_scheduled_restarts:
             value = ''
             scheduled_restart = m.data.get('scheduled_restart')
             if scheduled_restart:
@@ -664,39 +729,55 @@ def output_members(cluster, name, extended=False, fmt='pretty'):
 
         rows.append(row)
 
-    columns = [
-        'Cluster',
-        'Member',
-        'Host',
-        'Role',
-        'State',
-        'Lag in MB',
-    ]
-    alignment = {'Cluster': 'l', 'Member': 'l', 'Host': 'l', 'Lag in MB': 'r'}
+    columns = ['Cluster', 'Member', 'Host', 'Role', 'State', 'Lag in MB']
+    alignment = {'Lag in MB': 'r'}
 
-    if extended:
+    if extended or has_pending_restarts:
+        columns.append('Pending restart')
+
+    if extended or has_scheduled_restarts:
         columns.append('Scheduled restart')
-        alignment['Scheduled restart'] = 'l'
 
     print_output(columns, rows, alignment, fmt)
 
+    service_info = []
+    if cluster.is_paused():
+        service_info.append('Maintenance mode: on')
+
+    if cluster.failover and cluster.failover.scheduled_at:
+        info = 'Switchover scheduled at: ' + cluster.failover.scheduled_at.isoformat()
+        if cluster.failover.leader:
+            info += '\n                    from: ' + cluster.failover.leader
+        if cluster.failover.candidate:
+            info += '\n                      to: ' + cluster.failover.candidate
+        service_info.append(info)
+
+    if service_info:
+        click.echo(' ' + '\n '.join(service_info))
+
 
 @ctl.command('list', help='List the Patroni members for a given Patroni')
 @click.argument('cluster_names', nargs=-1)
 @click.option('--extended', '-e', help='Show some extra information', is_flag=True)
+@click.option('--timestamp', '-t', 'ts', help='Print timestamp', is_flag=True)
 @option_format
 @option_watch
 @option_watchrefresh
 @click.pass_obj
-def members(obj, cluster_names, fmt, watch, w, extended):
+def members(obj, cluster_names, fmt, watch, w, extended, ts):
     if not cluster_names:
-        logging.warning('Listing members: No cluster names were provided')
-        return
+        if 'scope' in obj:
+            cluster_names = [obj['scope']]
+        if not cluster_names:
+            return logging.warning('Listing members: No cluster names were provided')
 
     for cluster_name in cluster_names:
         dcs = get_dcs(obj, cluster_name)
 
         for _ in watching(w, watch):
+            if ts:
+                click.echo(timestamp(0))
+
             cluster = dcs.get_cluster()
             output_members(cluster, cluster_name, extended, fmt)
 
@@ -732,7 +813,7 @@ def touch_member(config, dcs):
         'role': p.role
     }
 
-    return dcs.touch_member(json.dumps(data, separators=(',', ':')), permanent=True)
+    return dcs.touch_member(data, permanent=True)
 
 
 def set_defaults(config, cluster_name):
@@ -789,8 +870,27 @@ def flush(obj, cluster_name, member_names, force, role, target):
                 click.echo('No scheduled restart for member {0}'.format(member.name))
 
 
-def toggle_pause(config, cluster_name, paused):
-    cluster = get_dcs(config, cluster_name).get_cluster()
+def wait_until_pause_is_applied(dcs, paused, old_cluster):
+    click.echo("'{0}' request sent, waiting until it is recognized by all nodes".format(paused and 'pause' or 'resume'))
+    old = {m.name: m.index for m in old_cluster.members if m.api_url}
+    loop_wait = old_cluster.config.data.get('loop_wait', dcs.loop_wait)
+
+    for _ in polling_loop(loop_wait + 1):
+        cluster = dcs.get_cluster()
+        if all(m.data.get('pause', False) == paused for m in cluster.members if m.name in old):
+            break
+    else:
+        remaining = [m.name for m in cluster.members if m.data.get('pause', False) != paused
+                     and m.name in old and old[m.name] != m.index]
+        if remaining:
+            return click.echo("{0} members didn't recognized pause state after {1} seconds"
+                              .format(', '.join(remaining), loop_wait))
+    return click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
+
+
+def toggle_pause(config, cluster_name, paused, wait):
+    dcs = get_dcs(config, cluster_name)
+    cluster = dcs.get_cluster()
     if cluster.is_paused() == paused:
         raise PatroniCtlException('Cluster is {0} paused'.format(paused and 'already' or 'not'))
 
@@ -807,7 +907,10 @@ def toggle_pause(config, cluster_name, paused):
             continue
 
         if r.status_code == 200:
-            click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
+            if wait:
+                wait_until_pause_is_applied(dcs, paused, cluster)
+            else:
+                click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
         else:
             click.echo('Failed: {0} cluster management status code={1}, ({2})'.format(
                        paused and 'pause' or 'resume', r.status_code, r.text))
@@ -817,17 +920,19 @@ def toggle_pause(config, cluster_name, paused):
 
 
 @ctl.command('pause', help='Disable auto failover')
-@click.argument('cluster_name')
+@arg_cluster_name
 @click.pass_obj
-def pause(obj, cluster_name):
-    return toggle_pause(obj, cluster_name, True)
+@click.option('--wait', help='Wait until pause is applied on all nodes', is_flag=True)
+def pause(obj, cluster_name, wait):
+    return toggle_pause(obj, cluster_name, True, wait)
 
 
 @ctl.command('resume', help='Resume auto failover')
-@click.argument('cluster_name')
+@arg_cluster_name
+@click.option('--wait', help='Wait until pause is cleared on all nodes', is_flag=True)
 @click.pass_obj
-def resume(obj, cluster_name):
-    return toggle_pause(obj, cluster_name, False)
+def resume(obj, cluster_name, wait):
+    return toggle_pause(obj, cluster_name, False, wait)
 
 
 @contextmanager
@@ -969,7 +1074,7 @@ def invoke_editor(before_editing, cluster_name):
 
 
 @ctl.command('edit-config', help="Edit cluster configuration")
-@click.argument('cluster_name')
+@arg_cluster_name
 @click.option('--quiet', '-q', is_flag=True, help='Do not show changes')
 @click.option('--set', '-s', 'kvpairs', multiple=True,
               help='Set specific configuration value. Can be specified multiple times')
@@ -1023,9 +1128,42 @@ def edit_config(obj, cluster_name, force, quiet, kvpairs, pgkvpairs, apply_filen
 
 
 @ctl.command('show-config', help="Show cluster configuration")
-@click.argument('cluster_name')
+@arg_cluster_name
 @click.pass_obj
 def show_config(obj, cluster_name):
     cluster = get_dcs(obj, cluster_name).get_cluster()
 
     click.echo(format_config_for_editing(cluster.config.data))
+
+
+@ctl.command('version', help='Output version of patronictl command or a running Patroni instance')
+@click.argument('cluster_name', required=False)
+@click.argument('member_names', nargs=-1)
+@click.pass_obj
+def version(obj, cluster_name, member_names):
+    click.echo("patronictl version {0}".format(__version__))
+
+    if not cluster_name:
+        return
+
+    click.echo("")
+    cluster = get_dcs(obj, cluster_name).get_cluster()
+    for m in cluster.members:
+        if m.api_url:
+            if not member_names or m.name in member_names:
+                try:
+                    response = request_patroni(m, 'get', 'patroni')
+                    data = response.json()
+                    version = data.get('patroni', {}).get('version')
+                    pg_version = data.get('server_version')
+                    pg_version_str = " PostgreSQL {0}".format(format_pg_version(pg_version)) if pg_version else ""
+                    click.echo("{0}: Patroni {1}{2}".format(m.name, version, pg_version_str))
+                except Exception as e:
+                    click.echo("{0}: failed to get version: {1}".format(m.name, e))
+
+
+def format_pg_version(version):
+    if version < 100000:
+        return "{0}.{1}.{2}".format(version // 10000, version // 100 % 100, version % 100)
+    else:
+        return "{0}.{1}".format(version // 10000, version % 100)

patroni/dcs/__init__.py

@@ -11,6 +11,7 @@ import sys
 
 from collections import namedtuple
 from patroni.exceptions import PatroniException
+from patroni.utils import parse_bool
 from random import randint
 from six.moves.urllib_parse import urlparse, urlunparse, parse_qsl
 from threading import Event, Lock
@@ -177,40 +178,48 @@ class Leader(namedtuple('Leader', 'index,session,member')):
     def conn_url(self):
         return self.member.conn_url
 
+    @property
+    def timeline(self):
+        return self.member.data.get('timeline')
+
 
 class Failover(namedtuple('Failover', 'index,leader,candidate,scheduled_at')):
 
     """
     >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader"}'))
     True
+    >>> 'Failover' in str(Failover.from_node(1, {"leader": "cluster_leader"}))
+    True
     >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader", "member": "cluster_candidate"}'))
     True
     >>> Failover.from_node(1, 'null') is None
-    True
+    False
     >>> n = '{"leader": "cluster_leader", "member": "cluster_candidate", "scheduled_at": "2016-01-14T10:09:57.1394Z"}'
     >>> 'tzinfo=' in str(Failover.from_node(1, n))
     True
     >>> Failover.from_node(1, None) is None
-    True
+    False
     >>> Failover.from_node(1, '{}') is None
-    True
+    False
     >>> 'abc' in Failover.from_node(1, 'abc:def')
     True
     """
     @staticmethod
     def from_node(index, value):
-        if not value:
-            return None
-
-        try:
-            data = json.loads(value)
-            if not data:
-                return None
-        except ValueError:
-            t = [a.strip() for a in value.split(':')]
-            leader = t[0]
-            candidate = t[1] if len(t) > 1 else None
-            return Failover(index, leader, candidate, None) if leader or candidate else None
+        if isinstance(value, dict):
+            data = value
+        elif value:
+            try:
+                data = json.loads(value)
+                if not isinstance(data, dict):
+                    data = {}
+            except ValueError:
+                t = [a.strip() for a in value.split(':')]
+                leader = t[0]
+                candidate = t[1] if len(t) > 1 else None
+                return Failover(index, leader, candidate, None) if leader or candidate else None
+        else:
+            data = {}
 
         if data.get('scheduled_at'):
             data['scheduled_at'] = dateutil.parser.parse(data['scheduled_at'])
@@ -258,8 +267,12 @@ class SyncState(namedtuple('SyncState', 'index,leader,sync_standby')):
         True
         >>> SyncState.from_node(1, '{"leader": "leader"}').leader == "leader"
         True
+        >>> SyncState.from_node(1, {"leader": "leader"}).leader == "leader"
+        True
         """
-        if value:
+        if isinstance(value, dict):
+            data = value
+        elif value:
             try:
                 data = json.loads(value)
                 if not isinstance(data, dict):
@@ -289,7 +302,26 @@ class SyncState(namedtuple('SyncState', 'index,leader,sync_standby')):
         return name is not None and name in (self.leader, self.sync_standby)
 
 
-class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operation,members,failover,sync')):
+class TimelineHistory(namedtuple('TimelineHistory', 'index,lines')):
+    """Object representing timeline history file"""
+
+    @staticmethod
+    def from_node(index, value):
+        """
+        >>> h = TimelineHistory.from_node(1, 2)
+        >>> h.lines
+        []
+        """
+        try:
+            lines = json.loads(value)
+        except (TypeError, ValueError):
+            lines = None
+        if not isinstance(lines, list):
+            lines = []
+        return TimelineHistory(index, lines)
+
+
+class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operation,members,failover,sync,history')):
 
     """Immutable object (namedtuple) which represents PostgreSQL cluster.
     Consists of the following fields:
@@ -301,6 +333,7 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operat
     :param members: list of Member object, all PostgreSQL cluster members including leader
     :param failover: reference to `Failover` object
     :param sync: reference to `SyncState` object, last observed synchronous replication state.
+    :param history: reference to `TimelineHistory` object
     """
 
     def is_unlocked(self):
@@ -317,14 +350,14 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operat
         candidates = [m for m in self.members if m.clonefrom and m.is_running and m.name not in exclude]
         return candidates[randint(0, len(candidates) - 1)] if candidates else self.leader
 
+    def check_mode(self, mode):
+        return bool(self.config and parse_bool(self.config.data.get(mode)))
+
     def is_paused(self):
-        return self.config and self.config.data.get('pause', False) or False
+        return self.check_mode('pause')
 
     def is_synchronous_mode(self):
-        return bool(self.config and self.config.data.get('synchronous_mode'))
-
-    def is_synchronous_mode_strict(self):
-        return bool(self.config and self.config.data.get('synchronous_mode_strict'))
+        return self.check_mode('synchronous_mode')
 
 
 @six.add_metaclass(abc.ABCMeta)
@@ -334,6 +367,7 @@ class AbstractDCS(object):
     _CONFIG = 'config'
     _LEADER = 'leader'
     _FAILOVER = 'failover'
+    _HISTORY = 'history'
     _MEMBERS = 'members/'
     _OPTIME = 'optime'
     _LEADER_OPTIME = _OPTIME + '/' + _LEADER
@@ -345,8 +379,7 @@ class AbstractDCS(object):
             i.e.: `zookeeper` for zookeeper, `etcd` for etcd, etc...
         """
         self._name = config['name']
-        self._namespace = '/{0}'.format(config.get('namespace', '/service/').strip('/'))
-        self._base_path = '/'.join([self._namespace, config['scope']])
+        self._base_path = os.path.join('/', config.get('namespace', '/service/').strip('/'), config['scope'])
         self._set_loop_wait(config.get('loop_wait', 10))
 
         self._ctl = bool(config.get('patronictl', False))
@@ -382,6 +415,10 @@ class AbstractDCS(object):
     def failover_path(self):
         return self.client_path(self._FAILOVER)
 
+    @property
+    def history_path(self):
+        return self.client_path(self._HISTORY)
+
     @property
     def leader_optime_path(self):
         return self.client_path(self._LEADER_OPTIME)
@@ -449,7 +486,7 @@ class AbstractDCS(object):
             self._last_leader_operation = last_operation
 
     @abc.abstractmethod
-    def update_leader(self):
+    def _update_leader(self):
         """Update leader key (or session) ttl
 
         :returns: `!True` if leader key (or session) has been updated successfully.
@@ -458,6 +495,18 @@ class AbstractDCS(object):
         You have to use CAS (Compare And Swap) operation in order to update leader key,
         for example for etcd `prevValue` parameter must be used."""
 
+    def update_leader(self, last_operation):
+        """Update leader key (or session) ttl and optime/leader
+
+        :param last_operation: absolute xlog location in bytes
+        :returns: `!True` if leader key (or session) has been updated successfully.
+            If not, `!False` must be returned and current instance would be demoted."""
+
+        ret = self._update_leader()
+        if ret and last_operation:
+            self.write_leader_optime(last_operation)
+        return ret
+
     @abc.abstractmethod
     def attempt_to_acquire_leader(self, permanent=False):
         """Attempt to acquire leader lock
@@ -483,7 +532,6 @@ class AbstractDCS(object):
 
         if scheduled_at:
             failover_value['scheduled_at'] = scheduled_at.isoformat()
-
         return self.set_failover_value(json.dumps(failover_value, separators=(',', ':')), index)
 
     @abc.abstractmethod
@@ -496,7 +544,7 @@ class AbstractDCS(object):
         This method should create or update key with the name = '/members/' + `~self._name`
         and value = data in a given DCS.
 
-        :param data: json serialized information about instance (including connection strings)
+        :param data: information about instance (including connection strings)
         :param ttl: ttl for member key, optional parameter. If it is None `~self.member_ttl will be used`
         :param permanent: if set to `!True`, the member key will never expire.
          Used in patronictl for the external master.
@@ -533,8 +581,18 @@ class AbstractDCS(object):
     def delete_cluster(self):
         """Delete cluster from DCS"""
 
+    @staticmethod
+    def sync_state(leader, sync_standby):
+        """Build sync_state dict"""
+        return {'leader': leader, 'sync_standby': sync_standby}
+
     def write_sync_state(self, leader, sync_standby, index=None):
-        return self.set_sync_state_value(json.dumps({'leader': leader, 'sync_standby': sync_standby}), index=index)
+        sync_value = self.sync_state(leader, sync_standby)
+        return self.set_sync_state_value(json.dumps(sync_value, separators=(',', ':')), index)
+
+    @abc.abstractmethod
+    def set_history_value(self, value):
+        """"""
 
     @abc.abstractmethod
     def set_sync_state_value(self, value, index=None):

patroni/dcs/consul.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+import json
 import logging
 import os
 import socket
@@ -7,9 +8,9 @@ import time
 import urllib3
 
 from consul import ConsulException, NotFound, base
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
 from patroni.exceptions import DCSError
-from patroni.utils import parse_bool, Retry, RetryFailedError
+from patroni.utils import deep_compare, parse_bool, Retry, RetryFailedError, split_host_port
 from urllib3.exceptions import HTTPError
 from six.moves.urllib.parse import urlencode, urlparse
 from six.moves.http_client import HTTPException
@@ -90,6 +91,8 @@ class HTTPClient(object):
                     data = '{' + ttl + '}'
                 else:
                     data = data[:-1] + ', ' + ttl + '}'
+            if isinstance(params, list):  # starting from v1.1.0 python-consul switched from `dict` to `list` for params
+                params = {k: v for k, v in params}
             kwargs = {'retries': 0, 'preload_content': False, 'body': data}
             if method == 'get' and isinstance(params, dict) and 'index' in params:
                 kwargs['timeout'] = (float(params['wait'][:-1]) if 'wait' in params else 300) + 1
@@ -141,13 +144,13 @@ class Consul(AbstractDCS):
                             retry_exceptions=(ConsulInternalError, HTTPException,
                                               HTTPError, socket.error, socket.timeout))
 
-        self._my_member_data = None
+        self._my_member_data = {}
         kwargs = {}
         if 'url' in config:
             r = urlparse(config['url'])
             config.update({'scheme': r.scheme, 'host': r.hostname, 'port': r.port or 8500})
         elif 'host' in config:
-            host, port = (config.get('host', '127.0.0.1:8500') + ':8500').split(':')[:2]
+            host, port = split_host_port(config.get('host', '127.0.0.1:8500'), 8500)
             config['host'] = host
             if 'port' not in config:
                 config['port'] = int(port)
@@ -263,6 +266,10 @@ class Consul(AbstractDCS):
             config = nodes.get(self._CONFIG)
             config = config and ClusterConfig.from_node(config['ModifyIndex'], config['Value'])
 
+            # get timeline history
+            history = nodes.get(self._HISTORY)
+            history = history and TimelineHistory.from_node(history['ModifyIndex'], history['Value'])
+
             # get last leader operation
             last_leader_operation = nodes.get(self._LEADER_OPTIME)
             last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation['Value'])
@@ -292,9 +299,9 @@ class Consul(AbstractDCS):
             sync = nodes.get(self._SYNC)
             sync = SyncState.from_node(sync and sync['ModifyIndex'], sync and sync['Value'])
 
-            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
         except NotFound:
-            self._cluster = Cluster(None, None, None, None, [], None, None)
+            self._cluster = Cluster(None, None, None, None, [], None, None, None)
         except Exception:
             logger.exception('get_cluster')
             raise ConsulError('Consul is not responding properly')
@@ -311,12 +318,12 @@ class Consul(AbstractDCS):
             except Exception:
                 return False
 
-        if not create_member and member and data == self._my_member_data:
+        if not create_member and member and deep_compare(data, self._my_member_data):
             return True
 
         try:
             args = {} if permanent else {'acquire': self._session}
-            self._client.kv.put(self.member_path, data, **args)
+            self._client.kv.put(self.member_path, json.dumps(data, separators=(',', ':')), **args)
             self._my_member_data = data
             return True
         except Exception:
@@ -352,7 +359,7 @@ class Consul(AbstractDCS):
         return self._client.kv.put(self.leader_optime_path, last_operation)
 
     @catch_consul_errors
-    def update_leader(self):
+    def _update_leader(self):
         if self._session:
             self.retry(self._client.session.renew, self._session)
             self._last_session_refresh = time.time()
@@ -371,6 +378,10 @@ class Consul(AbstractDCS):
     def delete_cluster(self):
         return self.retry(self._client.kv.delete, self.client_path(''), recurse=True)
 
+    @catch_consul_errors
+    def set_history_value(self, value):
+        return self._client.kv.put(self.history_path, value)
+
     @catch_consul_errors
     def delete_leader(self):
         cluster = self.cluster
@@ -379,11 +390,11 @@ class Consul(AbstractDCS):
 
     @catch_consul_errors
     def set_sync_state_value(self, value, index=None):
-        return self._client.kv.put(self.sync_path, value, cas=index)
+        return self.retry(self._client.kv.put, self.sync_path, value, cas=index)
 
     @catch_consul_errors
     def delete_sync_state(self, index=None):
-        return self._client.kv.delete(self.sync_path, cas=index)
+        return self.retry(self._client.kv.delete, self.sync_path, cas=index)
 
     def watch(self, leader_index, timeout):
         if self.__do_not_watch:

patroni/dcs/etcd.py

@@ -1,18 +1,20 @@
 from __future__ import absolute_import
 import etcd
+import json
 import logging
 import os
 import urllib3.util.connection
 import random
 import requests
+import six
 import socket
 import time
 
 from dns.exception import DNSException
 from dns import resolver
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
 from patroni.exceptions import DCSError
-from patroni.utils import Retry, RetryFailedError
+from patroni.utils import Retry, RetryFailedError, split_host_port
 from urllib3.exceptions import HTTPError, ReadTimeoutError
 from requests.exceptions import RequestException
 from six.moves.queue import Queue
@@ -23,6 +25,10 @@ from threading import Thread
 logger = logging.getLogger(__name__)
 
 
+def uri(protocol, host, port, endpoint=''):
+    return '{0}://{1}:{2}{3}'.format(protocol, host, port, endpoint)
+
+
 class EtcdError(DCSError):
     pass
 
@@ -204,8 +210,11 @@ class Client(etcd.Client):
                 self._machines_cache = self.machines
                 if self._base_uri in self._machines_cache:
                     self._machines_cache.remove(self._base_uri)
-        except etcd.EtcdConnectionFailed:
-            self._update_machines_cache = True
+        except etcd.EtcdConnectionFailed as e:
+            if isinstance(e, etcd.EtcdWatchTimedOut) and self._machines_cache:
+                self._base_uri = self._next_server()
+            else:
+                self._update_machines_cache = True
             if not response:
                 raise
         return self._handle_server_response(response)
@@ -227,7 +236,7 @@ class Client(etcd.Client):
             protocol = 'https' if '-ssl' in r else 'http'
             endpoint = '/members' if '-server' in r else ''
             for host, port in self.get_srv_record('_etcd{0}._tcp.{1}'.format(r, srv)):
-                url = '{0}://{1}:{2}{3}'.format(protocol, host, port, endpoint)
+                url = uri(protocol, host, port, endpoint)
                 if endpoint:
                     try:
                         response = requests.get(url, timeout=self.read_timeout, verify=False)
@@ -254,10 +263,10 @@ class Client(etcd.Client):
                 host, port = sa[:2]
                 if af == socket.AF_INET6:
                     host = '[{0}]'.format(host)
-                ret.append('{0}://{1}:{2}'.format(self.protocol, host, port))
+                ret.append(uri(self.protocol, host, port))
             if ret:
                 return list(set(ret))
-        return ['{0}://{1}:{2}'.format(self.protocol, host, port)]
+        return [uri(self.protocol, host, port)]
 
     def _load_machines_cache(self):
         """This method should fill up `_machines_cache` from scratch.
@@ -267,17 +276,20 @@ class Client(etcd.Client):
 
         self._update_machines_cache = True
 
-        if 'srv' not in self._config and 'host' not in self._config:
-            raise Exception('Neither srv nor host url are defined in etcd section of config')
+        if 'srv' not in self._config and 'host' not in self._config and 'hosts' not in self._config:
+            raise Exception('Neither srv, hosts, host nor url are defined in etcd section of config')
 
         if self._use_proxies:
-            self._machines_cache = ['{0}://{1}:{2}'.format(self.protocol, self._config['host'], self._config['port'])]
+            self._machines_cache = [uri(self.protocol, self._config['host'], self._config['port'])]
         else:
             self._machines_cache = []
 
             if 'srv' in self._config:
                 self._machines_cache = self._get_machines_cache_from_srv(self._config['srv'])
 
+            if not self._machines_cache and 'hosts' in self._config:
+                self._machines_cache = list(self._config['hosts'])
+
             if not self._machines_cache and 'host' in self._config:
                 self._machines_cache = self._get_machines_cache_from_dns(self._config['host'], self._config['port'])
 
@@ -347,8 +359,20 @@ class Etcd(AbstractDCS):
             r = urlparse(config['url'])
             config.update({'protocol': r.scheme, 'host': r.hostname, 'port': r.port or 2379,
                            'username': r.username, 'password': r.password})
+        elif 'hosts' in config:
+            hosts = config.pop('hosts')
+            default_port = config.pop('port', 2379)
+            protocol = config.get('protocol', 'http')
+
+            if isinstance(hosts, six.string_types):
+                hosts = hosts.split(',')
+
+            config['hosts'] = []
+            for value in hosts:
+                if isinstance(value, six.string_types):
+                    config['hosts'].append(uri(protocol, *split_host_port(value, default_port)))
         elif 'host' in config:
-            host, port = (config['host'] + ':2379').split(':')[:2]
+            host, port = split_host_port(config['host'], 2379)
             config['host'] = host
             if 'port' not in config:
                 config['port'] = int(port)
@@ -434,6 +458,10 @@ class Etcd(AbstractDCS):
             config = nodes.get(self._CONFIG)
             config = config and ClusterConfig.from_node(config.modifiedIndex, config.value)
 
+            # get timeline history
+            history = nodes.get(self._HISTORY)
+            history = history and TimelineHistory.from_node(history.modifiedIndex, history.value)
+
             # get last leader operation
             last_leader_operation = nodes.get(self._LEADER_OPTIME)
             last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation.value)
@@ -458,15 +486,16 @@ class Etcd(AbstractDCS):
             sync = nodes.get(self._SYNC)
             sync = SyncState.from_node(sync and sync.modifiedIndex, sync and sync.value)
 
-            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
         except etcd.EtcdKeyNotFound:
-            self._cluster = Cluster(None, None, None, None, [], None, None)
+            self._cluster = Cluster(None, None, None, None, [], None, None, None)
         except Exception as e:
             self._handle_exception(e, 'get_cluster', raise_ex=EtcdError('Etcd is not responding properly'))
         self._has_failed = False
 
     @catch_etcd_errors
     def touch_member(self, data, ttl=None, permanent=False):
+        data = json.dumps(data, separators=(',', ':'))
         return self.retry(self._client.set, self.member_path, data, None if permanent else ttl or self._ttl)
 
     @catch_etcd_errors
@@ -499,7 +528,7 @@ class Etcd(AbstractDCS):
         return self._client.set(self.leader_optime_path, last_operation)
 
     @catch_etcd_errors
-    def update_leader(self):
+    def _update_leader(self):
         return self.retry(self._client.test_and_set, self.leader_path, self._name, self._name, self._ttl)
 
     @catch_etcd_errors
@@ -518,9 +547,13 @@ class Etcd(AbstractDCS):
     def delete_cluster(self):
         return self.retry(self._client.delete, self.client_path(''), recursive=True)
 
+    @catch_etcd_errors
+    def set_history_value(self, value):
+        return self._client.write(self.history_path, value)
+
     @catch_etcd_errors
     def set_sync_state_value(self, value, index=None):
-        return self._client.write(self.sync_path, value, prevIndex=index or 0)
+        return self.retry(self._client.write, self.sync_path, value, prevIndex=index or 0)
 
     @catch_etcd_errors
     def delete_sync_state(self, index=None):

patroni/dcs/kubernetes.py

@@ -0,0 +1,409 @@
+from __future__ import absolute_import
+import datetime
+import functools
+import json
+import logging
+import socket
+import sys
+import time
+
+from kubernetes import client as k8s_client, config as k8s_config, watch as k8s_watch
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
+from patroni.exceptions import DCSError
+from patroni.utils import deep_compare, tzutc, Retry, RetryFailedError
+from urllib3.exceptions import HTTPError
+from six.moves.http_client import HTTPException
+
+logger = logging.getLogger(__name__)
+
+
+class KubernetesError(DCSError):
+    pass
+
+
+class KubernetesRetriableException(k8s_client.rest.ApiException):
+
+    def __init__(self, orig):
+        super(KubernetesRetriableException, self).__init__(orig.status, orig.reason)
+        self.body = orig.body
+        self.headers = orig.headers
+
+
+class CoreV1ApiProxy(object):
+
+    def __init__(self, use_endpoints=False):
+        self._api = k8s_client.CoreV1Api()
+        self._request_timeout = None
+        self._use_endpoints = use_endpoints
+
+    def set_timeout(self, timeout):
+        self._request_timeout = (1, timeout / 3.0)
+
+    def __getattr__(self, func):
+        if func.endswith('_kind'):
+            func = func[:-4] + ('endpoints' if self._use_endpoints else 'config_map')
+
+        def wrapper(*args, **kwargs):
+            if '_request_timeout' not in kwargs:
+                kwargs['_request_timeout'] = self._request_timeout
+            try:
+                return getattr(self._api, func)(*args, **kwargs)
+            except k8s_client.rest.ApiException as e:
+                if e.status in (502, 503, 504):  # XXX
+                    raise KubernetesRetriableException(e)
+                raise
+        return wrapper
+
+
+def catch_kubernetes_errors(func):
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except k8s_client.rest.ApiException as e:
+            if e.status == 403:
+                logger.exception('Permission denied')
+            elif e.status != 409:  # Object exists or conflict in resource_version
+                logger.exception('Unexpected error from Kubernetes API')
+            return False
+        except (RetryFailedError, HTTPException, HTTPError, socket.error, socket.timeout):
+            return False
+    return wrapper
+
+
+class Kubernetes(AbstractDCS):
+
+    def __init__(self, config):
+        self._labels = config['labels']
+        self._labels[config.get('scope_label', 'cluster-name')] = config['scope']
+        self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
+        self._namespace = config.get('namespace') or 'default'
+        self._role_label = config.get('role_label', 'role')
+        config['namespace'] = ''
+        super(Kubernetes, self).__init__(config)
+        self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
+                            retry_exceptions=(KubernetesRetriableException, HTTPException,
+                                              HTTPError, socket.error, socket.timeout))
+        self._ttl = None
+        try:
+            k8s_config.load_incluster_config()
+        except k8s_config.ConfigException:
+            k8s_config.load_kube_config(context=config.get('context', 'local'))
+
+        self.__subsets = None
+        use_endpoints = config.get('use_endpoints') and (config.get('patronictl') or 'pod_ip' in config)
+        if use_endpoints:
+            addresses = [k8s_client.V1EndpointAddress(ip=config['pod_ip'])]
+            ports = []
+            for p in config.get('ports', [{}]):
+                port = {'port': int(p.get('port', '5432'))}
+                port.update({n: p[n] for n in ('name', 'protocol') if p.get(n)})
+                ports.append(k8s_client.V1EndpointPort(**port))
+            self.__subsets = [k8s_client.V1EndpointSubset(addresses=addresses, ports=ports)]
+        self._api = CoreV1ApiProxy(use_endpoints)
+        self.set_retry_timeout(config['retry_timeout'])
+        self.set_ttl(config.get('ttl') or 30)
+        self._leader_observed_record = {}
+        self._leader_observed_time = None
+        self._leader_resource_version = None
+        self._leader_observed_subsets = []
+        self.__do_not_watch = False
+
+    def retry(self, *args, **kwargs):
+        return self._retry.copy()(*args, **kwargs)
+
+    def client_path(self, path):
+        return super(Kubernetes, self).client_path(path)[1:].replace('/', '-')
+
+    @property
+    def leader_path(self):
+        return self._base_path[1:] if self.__subsets else super(Kubernetes, self).leader_path
+
+    def set_ttl(self, ttl):
+        ttl = int(ttl)
+        self.__do_not_watch = self._ttl != ttl
+        self._ttl = ttl
+
+    def set_retry_timeout(self, retry_timeout):
+        self._retry.deadline = retry_timeout
+        self._api.set_timeout(retry_timeout)
+
+    @staticmethod
+    def member(pod):
+        annotations = pod.metadata.annotations or {}
+        member = Member.from_node(pod.metadata.resource_version, pod.metadata.name, None, annotations.get('status', ''))
+        member.data['pod_labels'] = pod.metadata.labels
+        return member
+
+    def _load_cluster(self):
+        try:
+            # get list of members
+            response = self.retry(self._api.list_namespaced_pod, self._namespace, label_selector=self._label_selector)
+            members = [self.member(pod) for pod in response.items]
+
+            response = self.retry(self._api.list_namespaced_kind, self._namespace, label_selector=self._label_selector)
+            nodes = {item.metadata.name: item for item in response.items}
+
+            config = nodes.get(self.config_path)
+            metadata = config and config.metadata
+            annotations = metadata and metadata.annotations or {}
+
+            # get initialize flag
+            initialize = annotations.get(self._INITIALIZE)
+
+            # get global dynamic configuration
+            config = ClusterConfig.from_node(metadata and metadata.resource_version,
+                                             annotations.get(self._CONFIG) or '{}')
+
+            # get timeline history
+            history = TimelineHistory.from_node(metadata and metadata.resource_version,
+                                                annotations.get(self._HISTORY) or '[]')
+
+            leader = nodes.get(self.leader_path)
+            metadata = leader and leader.metadata
+            self._leader_resource_version = metadata.resource_version if metadata else None
+            self._leader_observed_subsets = leader.subsets if self.__subsets and leader and leader.subsets else []
+            annotations = metadata and metadata.annotations or {}
+
+            # get last leader operation
+            last_leader_operation = annotations.get(self._OPTIME)
+            last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation)
+
+            # get leader
+            leader_record = {n: annotations.get(n) for n in (self._LEADER, 'acquireTime',
+                             'ttl', 'renewTime', 'transitions') if n in annotations}
+            if (leader_record or self._leader_observed_record) and leader_record != self._leader_observed_record:
+                self._leader_observed_record = leader_record
+                self._leader_observed_time = time.time()
+
+            leader = leader_record.get(self._LEADER)
+            try:
+                ttl = int(leader_record.get('ttl')) or self._ttl
+            except (TypeError, ValueError):
+                ttl = self._ttl
+
+            if not metadata or not self._leader_observed_time or self._leader_observed_time + ttl < time.time():
+                leader = None
+
+            if metadata:
+                member = Member(-1, leader, None, {})
+                member = ([m for m in members if m.name == leader] or [member])[0]
+                leader = Leader(response.metadata.resource_version, None, member)
+
+            # failover key
+            failover = nodes.get(self.failover_path)
+            metadata = failover and failover.metadata
+            failover = Failover.from_node(metadata and metadata.resource_version, metadata and metadata.annotations)
+
+            # get synchronization state
+            sync = nodes.get(self.sync_path)
+            metadata = sync and sync.metadata
+            sync = SyncState.from_node(metadata and metadata.resource_version,  metadata and metadata.annotations)
+
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
+        except Exception:
+            logger.exception('get_cluster')
+            raise KubernetesError('Kubernetes API is not responding properly')
+
+    @staticmethod
+    def compare_ports(p1, p2):
+        return p1.name == p2.name and p1.port == p2.port and (p1.protocol or 'TCP') == (p2.protocol or 'TCP')
+
+    @staticmethod
+    def subsets_changed(last_observed_subsets, subsets):
+        """
+        >>> Kubernetes.subsets_changed([], [])
+        False
+        >>> Kubernetes.subsets_changed([], [k8s_client.V1EndpointSubset()])
+        True
+        >>> s1 = [k8s_client.V1EndpointSubset(addresses=[k8s_client.V1EndpointAddress(ip='1.2.3.4')])]
+        >>> s2 = [k8s_client.V1EndpointSubset(addresses=[k8s_client.V1EndpointAddress(ip='1.2.3.5')])]
+        >>> Kubernetes.subsets_changed(s1, s2)
+        True
+        >>> a = [k8s_client.V1EndpointAddress(ip='1.2.3.4')]
+        >>> s1 = [k8s_client.V1EndpointSubset(addresses=a, ports=[k8s_client.V1EndpointPort(protocol='TCP', port=1)])]
+        >>> s2 = [k8s_client.V1EndpointSubset(addresses=a, ports=[k8s_client.V1EndpointPort(port=5432)])]
+        >>> Kubernetes.subsets_changed(s1, s2)
+        True
+        >>> p1 = k8s_client.V1EndpointPort(name='port1', port=1)
+        >>> p2 = k8s_client.V1EndpointPort(name='port2', port=2)
+        >>> p3 = k8s_client.V1EndpointPort(name='port3', port=3)
+        >>> s1 = [k8s_client.V1EndpointSubset(addresses=a, ports=[p1, p2])]
+        >>> s2 = [k8s_client.V1EndpointSubset(addresses=a, ports=[p2, p3])]
+        >>> Kubernetes.subsets_changed(s1, s2)
+        True
+        >>> s2 = [k8s_client.V1EndpointSubset(addresses=a, ports=[p2, p1])]
+        >>> Kubernetes.subsets_changed(s1, s2)
+        False
+        """
+        if len(last_observed_subsets) != len(subsets):
+            return True
+        if subsets == []:
+            return False
+        if len(last_observed_subsets[0].addresses or []) != 1 or \
+                last_observed_subsets[0].addresses[0].ip != subsets[0].addresses[0].ip or \
+                len(last_observed_subsets[0].ports) != len(subsets[0].ports):
+            return True
+        if len(subsets[0].ports) == 1:
+            return not Kubernetes.compare_ports(last_observed_subsets[0].ports[0], subsets[0].ports[0])
+        observed_ports = {p.name: p for p in last_observed_subsets[0].ports}
+        for p in subsets[0].ports:
+            if p.name not in observed_ports or not Kubernetes.compare_ports(p, observed_ports.pop(p.name)):
+                return True
+        return False
+
+    @catch_kubernetes_errors
+    def patch_or_create(self, name, annotations, resource_version=None, patch=False, retry=True, subsets=None):
+        metadata = {'namespace': self._namespace, 'name': name, 'labels': self._labels, 'annotations': annotations}
+        if patch or resource_version:
+            if resource_version is not None:
+                metadata['resource_version'] = resource_version
+            func = functools.partial(self._api.patch_namespaced_kind, name)
+        else:
+            func = functools.partial(self._api.create_namespaced_kind)
+            # skip annotations with null values
+            metadata['annotations'] = {k: v for k, v in metadata['annotations'].items() if v is not None}
+
+        metadata = k8s_client.V1ObjectMeta(**metadata)
+        if subsets is not None and self.__subsets:
+            endpoints = {'metadata': metadata}
+            if self.subsets_changed(self._leader_observed_subsets, subsets):
+                endpoints['subsets'] = subsets
+            body = k8s_client.V1Endpoints(**endpoints)
+        else:
+            body = k8s_client.V1ConfigMap(metadata=metadata)
+        return self.retry(func, self._namespace, body) if retry else func(self._namespace, body)
+
+    def _write_leader_optime(self, last_operation):
+        """Unused"""
+
+    def _update_leader(self):
+        """Unused"""
+
+    def update_leader(self, last_operation):
+        now = datetime.datetime.now(tzutc).isoformat()
+        annotations = {self._LEADER: self._name, 'ttl': str(self._ttl), 'renewTime': now,
+                       'acquireTime': self._leader_observed_record.get('acquireTime') or now,
+                       'transitions': self._leader_observed_record.get('transitions') or '0'}
+        if last_operation:
+            annotations[self._OPTIME] = last_operation
+
+        ret = self.patch_or_create(self.leader_path, annotations, self._leader_resource_version, subsets=self.__subsets)
+        if ret:
+            self._leader_resource_version = ret.metadata.resource_version
+        return ret
+
+    def attempt_to_acquire_leader(self, permanent=False):
+        now = datetime.datetime.now(tzutc).isoformat()
+        annotations = {self._LEADER: self._name, 'ttl': str(sys.maxsize if permanent else self._ttl),
+                       'renewTime': now, 'acquireTime': now, 'transitions': '0'}
+        if self._leader_observed_record:
+            try:
+                transitions = int(self._leader_observed_record.get('transitions'))
+            except (TypeError, ValueError):
+                transitions = 0
+
+            if self._leader_observed_record.get(self._LEADER) != self._name:
+                transitions += 1
+            else:
+                annotations['acquireTime'] = self._leader_observed_record.get('acquireTime') or now
+            annotations['transitions'] = str(transitions)
+        ret = self.patch_or_create(self.leader_path, annotations, self._leader_resource_version, subsets=self.__subsets)
+        if ret:
+            self._leader_resource_version = ret.metadata.resource_version
+        else:
+            logger.info('Could not take out TTL lock')
+        return ret
+
+    def take_leader(self):
+        return self.attempt_to_acquire_leader()
+
+    def set_failover_value(self, value, index=None):
+        """Unused"""
+
+    def manual_failover(self, leader, candidate, scheduled_at=None, index=None):
+        annotations = {'leader': leader or None, 'member': candidate or None, 'scheduled_at': scheduled_at}
+        patch = bool(self.cluster and isinstance(self.cluster.failover, Failover) and self.cluster.failover.index)
+        return self.patch_or_create(self.failover_path, annotations, index, bool(index or patch), False)
+
+    def set_config_value(self, value, index=None):
+        patch = bool(index or self.cluster and self.cluster.config and self.cluster.config.index)
+        return self.patch_or_create(self.config_path, {self._CONFIG: value}, index, patch, False)
+
+    @catch_kubernetes_errors
+    def touch_member(self, data, ttl=None, permanent=False):
+        cluster = self.cluster
+        if cluster and cluster.leader and cluster.leader.name == self._name:
+            role = 'master'
+        elif data['state'] == 'running' and data['role'] != 'master':
+            role = data['role']
+        else:
+            role = None
+
+        member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
+        pod_labels = member and member.data.pop('pod_labels', None)
+        ret = pod_labels is not None and pod_labels.get(self._role_label) == role and deep_compare(data, member.data)
+
+        if not ret:
+            metadata = {'namespace': self._namespace, 'name': self._name, 'labels': {self._role_label: role},
+                        'annotations': {'status': json.dumps(data, separators=(',', ':'))}}
+            body = k8s_client.V1Pod(metadata=k8s_client.V1ObjectMeta(**metadata))
+            ret = self._api.patch_namespaced_pod(self._name, self._namespace, body)
+        return ret
+
+    def initialize(self, create_new=True, sysid=""):
+        cluster = self.cluster
+        resource_version = cluster.config.index if cluster and cluster.config and cluster.config.index else None
+        return self.patch_or_create(self.config_path, {self._INITIALIZE: sysid}, resource_version)
+
+    def delete_leader(self):
+        if self.cluster and isinstance(self.cluster.leader, Leader) and self.cluster.leader.name == self._name:
+            self.patch_or_create(self.leader_path, {self._LEADER: None}, self._leader_resource_version, True, False, [])
+            self.reset_cluster()
+
+    def cancel_initialization(self):
+        self.patch_or_create(self.config_path, {self._INITIALIZE: None}, self.cluster.config.index, True)
+
+    @catch_kubernetes_errors
+    def delete_cluster(self):
+        self.retry(self._api.delete_collection_namespaced_kind, self._namespace, label_selector=self._label_selector)
+
+    def set_history_value(self, value):
+        patch = bool(self.cluster and self.cluster.config and self.cluster.config.index)
+        return self.patch_or_create(self.config_path, {self._HISTORY: value}, None, patch, False)
+
+    def set_sync_state_value(self, value, index=None):
+        """Unused"""
+
+    def write_sync_state(self, leader, sync_standby, index=None):
+        return self.patch_or_create(self.sync_path, self.sync_state(leader, sync_standby), index, False)
+
+    def delete_sync_state(self, index=None):
+        return self.write_sync_state(None, None, index)
+
+    def watch(self, leader_index, timeout):
+        if self.__do_not_watch:
+            self.__do_not_watch = False
+            return True
+
+        if leader_index:
+            end_time = time.time() + timeout
+            w = k8s_watch.Watch()
+            while timeout >= 1:
+                try:
+                    for event in w.stream(self._api.list_namespaced_kind, self._namespace,
+                                          resource_version=leader_index, timeout_seconds=int(timeout + 0.5),
+                                          field_selector='metadata.name=' + self.leader_path,
+                                          _request_timeout=(1, timeout + 1)):
+                        return event['raw_object'].get('metadata', {}).get('resourceVersion') != leader_index
+                    return False
+                except KeyboardInterrupt:
+                    raise
+                except Exception:
+                    logging.exception('watch')
+
+                timeout = end_time - time.time()
+
+        try:
+            return super(Kubernetes, self).watch(None, timeout)
+        finally:
+            self.event.clear()

patroni/dcs/zookeeper.py

@@ -1,11 +1,13 @@
+import json
 import logging
 import time
 
 from kazoo.client import KazooClient, KazooState, KazooRetry
 from kazoo.exceptions import NoNodeError, NodeExistsError
 from kazoo.handlers.threading import SequentialThreadingHandler
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
 from patroni.exceptions import DCSError
+from patroni.utils import deep_compare
 
 logger = logging.getLogger(__name__)
 
@@ -57,7 +59,7 @@ class ZooKeeper(AbstractDCS):
                                    max_delay=1, max_tries=-1, sleep_func=time.sleep))
         self._client.add_listener(self.session_listener)
 
-        self._my_member_data = None
+        self._my_member_data = {}
         self._fetch_cluster = True
 
         self._orig_kazoo_connect = self._client._connection._connect
@@ -159,6 +161,10 @@ class ZooKeeper(AbstractDCS):
         config = self.get_node(self.config_path, watch=self.cluster_watcher) if self._CONFIG in nodes else None
         config = config and ClusterConfig.from_node(config[1].version, config[0], config[1].mzxid)
 
+        # get timeline history
+        history = self.get_node(self.history_path, watch=self.cluster_watcher) if self._HISTORY in nodes else None
+        history = history and TimelineHistory.from_node(history[1].mzxid, history[0])
+
         # get last leader operation
         last_leader_operation = self._OPTIME in nodes and self._fetch_cluster and self.get_node(self.leader_optime_path)
         last_leader_operation = last_leader_operation and int(last_leader_operation[0]) or 0
@@ -191,7 +197,7 @@ class ZooKeeper(AbstractDCS):
         failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
         failover = failover and Failover.from_node(failover[1].version, failover[0])
 
-        self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync)
+        self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
 
     def _load_cluster(self):
         if self._fetch_cluster or self._cluster is None:
@@ -215,16 +221,19 @@ class ZooKeeper(AbstractDCS):
             logger.info('Could not take out TTL lock')
         return ret
 
-    def set_failover_value(self, value, index=None):
+    def __set_failover_or_sync_state_value(self, key, value, index=None):
         try:
-            self._client.retry(self._client.set, self.failover_path, value.encode('utf-8'), version=index or -1)
+            self._client.retry(self._client.set, key, value.encode('utf-8'), version=index or -1)
             return True
         except NoNodeError:
-            return value == '' or (index is None and self._create(self.failover_path, value))
+            return value == '' or (index is None and self._create(key, value))
         except Exception:
             logging.exception('set_failover_value')
             return False
 
+    def set_failover_value(self, value, index=None):
+        return self.__set_failover_or_sync_state_value(self.failover_path, value, index)
+
     def set_config_value(self, value, index=None):
         try:
             self._client.retry(self._client.set, self.config_path, value.encode('utf-8'), version=index or -1)
@@ -242,7 +251,7 @@ class ZooKeeper(AbstractDCS):
     def touch_member(self, data, ttl=None, permanent=False):
         cluster = self.cluster
         member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
-        data = data.encode('utf-8')
+        encoded_data = json.dumps(data, separators=(',', ':')).encode('utf-8')
         if member and self._client.client_id is not None and member.session != self._client.client_id[0]:
             try:
                 self._client.delete_async(self.member_path).get(timeout=1)
@@ -253,11 +262,12 @@ class ZooKeeper(AbstractDCS):
             member = None
 
         if member:
-            if data == self._my_member_data:
+            if deep_compare(data, self._my_member_data):
                 return True
         else:
             try:
-                self._client.create_async(self.member_path, data, makepath=True, ephemeral=not permanent).get(timeout=1)
+                self._client.create_async(self.member_path, encoded_data, makepath=True,
+                                          ephemeral=not permanent).get(timeout=1)
                 self._my_member_data = data
                 return True
             except Exception as e:
@@ -265,7 +275,7 @@ class ZooKeeper(AbstractDCS):
                     logger.exception('touch_member')
                     return False
         try:
-            self._client.set_async(self.member_path, data).get(timeout=1)
+            self._client.set_async(self.member_path, encoded_data).get(timeout=1)
             self._my_member_data = data
             return True
         except Exception:
@@ -276,22 +286,25 @@ class ZooKeeper(AbstractDCS):
     def take_leader(self):
         return self.attempt_to_acquire_leader()
 
-    def _write_leader_optime(self, last_operation):
-        last_operation = last_operation.encode('utf-8')
+    def __write_leader_optime_or_history_value(self, key, value):
+        value = value.encode('utf-8')
         try:
-            self._client.set_async(self.leader_optime_path, last_operation).get(timeout=1)
+            self._client.set_async(key, value).get(timeout=1)
             return True
         except NoNodeError:
             try:
-                self._client.create_async(self.leader_optime_path, last_operation, makepath=True).get(timeout=1)
+                self._client.create_async(key, value, makepath=True).get(timeout=1)
                 return True
             except Exception:
-                logger.exception('Failed to create %s', self.leader_optime_path)
+                logger.exception('Failed to create %s', key)
         except Exception:
-            logger.exception('Failed to update %s', self.leader_optime_path)
+            logger.exception('Failed to update %s', key)
         return False
 
-    def update_leader(self):
+    def _write_leader_optime(self, last_operation):
+        return self.__write_leader_optime_or_history_value(self.leader_optime_path, last_operation)
+
+    def _update_leader(self):
         return True
 
     def delete_leader(self):
@@ -316,15 +329,11 @@ class ZooKeeper(AbstractDCS):
         except NoNodeError:
             return True
 
+    def set_history_value(self, value):
+        return self.__write_leader_optime_or_history_value(self.history_path, value)
+
     def set_sync_state_value(self, value, index=None):
-        try:
-            self._client.retry(self._client.set, self.sync_path, value.encode('utf-8'), version=index or -1)
-            return True
-        except NoNodeError:
-            return value == '' or (index is None and self._create(self.sync_path, value))
-        except Exception:
-            logging.exception('set_sync_state_value')
-            return False
+        return self.__set_failover_or_sync_state_value(self.sync_path, value, index)
 
     def delete_sync_state(self, index=None):
         return self.set_sync_state_value("{}", index)

patroni/ha.py

@@ -57,11 +57,15 @@ class Ha(object):
         self.dcs = patroni.dcs
         self.cluster = None
         self.old_cluster = None
+        self._is_leader = False
+        self._is_leader_lock = RLock()
+        self._was_paused = False
+        self._leader_timeline = None
         self.recovering = False
         self._post_bootstrap_task = None
         self._crash_recovery_executed = False
         self._start_timeout = None
-        self._async_executor = AsyncExecutor(self.wakeup)
+        self._async_executor = AsyncExecutor(self.state_handler, self.wakeup)
         self.watchdog = patroni.watchdog
 
         # Each member publishes various pieces of information to the DCS using touch_member. This lock protects
@@ -71,8 +75,27 @@ class Ha(object):
         # standby. Changes protected by _member_state_lock.
         self._disable_sync = 0
 
+        # We need following property to avoid shutdown of postgres when join of Patroni to the postgres
+        # already running as replica was aborted due to cluster not beeing initialized in DCS.
+        self._join_aborted = False
+
+    def check_mode(self, mode):
+        # Try to protect from the case when DCS was wiped out during pause
+        if self.cluster and self.cluster.config and self.cluster.config.modify_index:
+            return self.cluster.check_mode(mode)
+        else:
+            return self.patroni.config.check_mode(mode)
+
     def is_paused(self):
-        return self.cluster and self.cluster.is_paused()
+        return self.check_mode('pause')
+
+    def is_leader(self):
+        with self._is_leader_lock:
+            return self._is_leader
+
+    def set_is_leader(self, value):
+        with self._is_leader_lock:
+            self._is_leader = value
 
     def load_cluster_from_dcs(self):
         cluster = self.dcs.get_cluster()
@@ -82,18 +105,27 @@ class Ha(object):
             self.old_cluster = cluster
         self.cluster = cluster
 
+        if self.cluster.is_unlocked() or self.cluster.leader.name != self.state_handler.name:
+            self.set_is_leader(False)
+
+        self._leader_timeline = None if cluster.is_unlocked() else cluster.leader.timeline
+
     def acquire_lock(self):
-        return self.dcs.attempt_to_acquire_leader()
+        ret = self.dcs.attempt_to_acquire_leader()
+        self.set_is_leader(ret)
+        return ret
 
     def update_lock(self, write_leader_optime=False):
-        ret = self.dcs.update_leader()
+        last_operation = None
+        if write_leader_optime:
+            try:
+                last_operation = self.state_handler.last_operation()
+            except Exception:
+                logger.exception('Exception when called state_handler.last_operation()')
+        ret = self.dcs.update_leader(last_operation)
+        self.set_is_leader(ret)
         if ret:
             self.watchdog.keepalive()
-            if write_leader_optime:
-                try:
-                    self.dcs.write_leader_optime(self.state_handler.last_operation())
-                except Exception:
-                    pass
         return ret
 
     def has_lock(self):
@@ -122,9 +154,15 @@ class Ha(object):
                 data['tags'] = tags
             if self.state_handler.pending_restart:
                 data['pending_restart'] = True
-            if not self._async_executor.busy and data['state'] in ['running', 'restarting', 'starting']:
+            if self._async_executor.scheduled_action in (None, 'promote') \
+                    and data['state'] in ['running', 'restarting', 'starting']:
                 try:
-                    data['xlog_location'] = self.state_handler.wal_position(retry=False)
+                    timeline, wal_position = self.state_handler.timeline_wal_position()
+                    data['xlog_location'] = wal_position
+                    if not timeline:
+                        timeline = self.state_handler.replica_cached_timeline(self._leader_timeline)
+                    if timeline:
+                        data['timeline'] = timeline
                 except Exception:
                     pass
             if self.patroni.scheduled_restart:
@@ -132,7 +170,10 @@ class Ha(object):
                 scheduled_restart_data['schedule'] = scheduled_restart_data['schedule'].isoformat()
                 data['scheduled_restart'] = scheduled_restart_data
 
-            return self.dcs.touch_member(json.dumps(data, separators=(',', ':')))
+            if self.is_paused():
+                data['pause'] = True
+
+            return self.dcs.touch_member(data)
 
     def clone(self, clone_member=None, msg='(without leader)'):
         if self.state_handler.clone(clone_member):
@@ -176,11 +217,6 @@ class Ha(object):
             self._async_executor.run_async(self.state_handler.rewind, (self.cluster.leader,))
             return True
 
-    def _start_crash_recovery(self, msg):
-        self._async_executor.schedule(msg)
-        self._async_executor.run_async(self.state_handler.fix_cluster_state)
-        return msg
-
     def recover(self):
         # Postgres is not running and we will restart in standby mode. Watchdog is not needed until we promote.
         self.watchdog.disable()
@@ -201,10 +237,12 @@ class Ha(object):
             timeout = None
 
         data = self.state_handler.controldata()
-        if data.get('Database cluster state') == 'in production' and not self._crash_recovery_executed and \
-                (self.cluster.is_unlocked() or self.state_handler.can_rewind):
+        if data.get('Database cluster state') in ('in production', 'shutting down', 'in crash recovery') and \
+                not self._crash_recovery_executed and (self.cluster.is_unlocked() or self.state_handler.can_rewind):
             self._crash_recovery_executed = True
-            return self._start_crash_recovery('doing crash recovery in a single user mode')
+            self._async_executor.schedule('doing crash recovery in a single user mode')
+            self._async_executor.run_async(self.state_handler.fix_cluster_state)
+            return self._async_executor.scheduled_action
 
         self.load_cluster_from_dcs()
 
@@ -219,13 +257,6 @@ class Ha(object):
             msg = "starting as a secondary"
             node_to_follow = self._get_node_to_follow(self.cluster)
 
-        # once we already tried to start postgres but failed, single user mode is a rescue in this case
-        if self.recovering and not self.state_handler.rewind_executed \
-                and not self._crash_recovery_executed and self.state_handler.can_rewind \
-                and data.get('Database cluster state') not in ('shut down', 'shut down in recovery'):
-            self.recovering = False
-            return self._start_crash_recovery('fixing cluster state in a single user mode')
-
         self.recovering = True
 
         self._async_executor.schedule('restarting after failure')
@@ -271,10 +302,10 @@ class Ha(object):
         return follow_reason
 
     def is_synchronous_mode(self):
-        return bool(self.cluster and self.cluster.is_synchronous_mode())
+        return self.check_mode('synchronous_mode')
 
     def is_synchronous_mode_strict(self):
-        return bool(self.cluster and self.cluster.is_synchronous_mode_strict())
+        return self.check_mode('synchronous_mode_strict')
 
     def process_sync_replication(self):
         """Process synchronous standby beahvior.
@@ -362,6 +393,24 @@ class Ha(object):
             with self._member_state_lock:
                 self._disable_sync -= 1
 
+    def update_cluster_history(self):
+        master_timeline = self.state_handler.get_master_timeline()
+        cluster_history = self.cluster.history and self.cluster.history.lines
+        if master_timeline == 1:
+            if cluster_history:
+                self.dcs.set_history_value('[]')
+        elif not cluster_history or cluster_history[-1][0] != master_timeline - 1 or len(cluster_history[-1]) != 4:
+            cluster_history = {l[0]: l for l in cluster_history or []}
+            history = self.state_handler.get_history(master_timeline)
+            if history:
+                for line in history:
+                    # enrich current history with promotion timestamps stored in DCS
+                    if len(line) == 3 and line[0] in cluster_history \
+                            and len(cluster_history[line[0]]) == 4 \
+                            and cluster_history[line[0]][1] == line[1]:
+                        line.append(cluster_history[line[0]][3])
+                self.dcs.set_history_value(json.dumps(history, separators=(',', ':')))
+
     def enforce_master_role(self, message, promote_message):
         if not self.is_paused() and not self.watchdog.is_running and not self.watchdog.activate():
             if self.state_handler.is_leader():
@@ -371,10 +420,14 @@ class Ha(object):
                 self.release_leader_key_voluntarily()
                 return 'Not promoting self because watchdog could not be activated'
 
-        if self.state_handler.is_leader() or self.state_handler.role == 'master':
+        if self.state_handler.is_leader():
             # Inform the state handler about its master role.
             # It may be unaware of it if postgres is promoted manually.
             self.state_handler.set_role('master')
+            self.process_sync_replication()
+            self.update_cluster_history()
+            return message
+        elif self.state_handler.role == 'master':
             self.process_sync_replication()
             return message
         else:
@@ -386,7 +439,9 @@ class Ha(object):
                     # promotion until next cycle. TODO: trigger immediate retry of run_cycle
                     return 'Postponing promotion because synchronous replication state was updated by somebody else'
                 self.state_handler.set_synchronous_standby('*' if self.is_synchronous_mode_strict() else None)
-            self.state_handler.promote()
+            if self.state_handler.role != 'master':
+                self._async_executor.schedule('promote')
+                self._async_executor.run_async(self.state_handler.promote, args=(self.dcs.loop_wait,))
             return promote_message
 
     @staticmethod
@@ -422,7 +477,7 @@ class Ha(object):
     def _is_healthiest_node(self, members, check_replication_lag=True):
         """This method tries to determine whether I am healthy enough to became a new leader candidate or not."""
 
-        my_wal_position = self.state_handler.wal_position()
+        _, my_wal_position = self.state_handler.timeline_wal_position()
         if check_replication_lag and self.is_lagging(my_wal_position):
             return False  # Too far behind last reported wal position on master
 
@@ -558,16 +613,17 @@ class Ha(object):
                 PostgreSQL as quickly as possible without regard for data durability. May only be called synchronously.
         """
         mode_control = {
-            'offline':          dict(stop='fast', checkpoint=False, release=False, offline=True, async=False),
-            'graceful':         dict(stop='fast', checkpoint=True, release=True, offline=False, async=False),
-            'immediate':        dict(stop='immediate', checkpoint=False, release=True, offline=False, async=True),
-            'immediate-nolock': dict(stop='immediate', checkpoint=False, release=False, offline=False, async=True),
+            'offline':          dict(stop='fast', checkpoint=False, release=False, offline=True, async_req=False),
+            'graceful':         dict(stop='fast', checkpoint=True, release=True, offline=False, async_req=False),
+            'immediate':        dict(stop='immediate', checkpoint=False, release=True, offline=False, async_req=True),
+            'immediate-nolock': dict(stop='immediate', checkpoint=False, release=False, offline=False, async_req=True),
         }[mode]
 
         self.state_handler.trigger_check_diverged_lsn()
         self.state_handler.stop(mode_control['stop'], checkpoint=mode_control['checkpoint'],
                                 on_safepoint=self.watchdog.disable if self.watchdog.is_running else None)
         self.state_handler.set_role('demoted')
+        self.set_is_leader(False)
 
         if mode_control['release']:
             self.release_leader_key_voluntarily()
@@ -581,7 +637,7 @@ class Ha(object):
         # FIXME: with mode offline called from DCS exception handler and handle_long_action_in_progress
         # there could be an async action already running, calling follow from here will lead
         # to racy state handler state updates.
-        if mode_control['async']:
+        if mode_control['async_req']:
             self._async_executor.schedule('starting after demotion')
             self._async_executor.run_async(self.state_handler.follow, (node_to_follow,))
         else:
@@ -723,6 +779,8 @@ class Ha(object):
                 # Either there is no connection to DCS or someone else acquired the lock
                 logger.error('failed to update leader lock')
                 if self.state_handler.is_leader():
+                    if self.is_paused():
+                        return 'continue to run as master after failing to update leader lock in DCS'
                     self.demote('immediate-nolock')
                     return 'demoted self because failed to update leader lock in DCS'
                 else:
@@ -852,7 +910,7 @@ class Ha(object):
         member_role = 'leader' if clone_member == self.cluster.leader else 'replica'
         return self.clone(clone_member, "from {0} '{1}'".format(member_role, clone_member.name))
 
-    def reinitialize(self):
+    def reinitialize(self, force=False):
         with self._async_executor:
             self.load_cluster_from_dcs()
 
@@ -862,7 +920,11 @@ class Ha(object):
             if self.cluster.leader.name == self.state_handler.name:
                 return 'I am the leader, can not reinitialize'
 
-            action = self._async_executor.schedule('reinitialize', immediately=True)
+        if force:
+            self._async_executor.cancel()
+
+        with self._async_executor:
+            action = self._async_executor.schedule('reinitialize')
             if action is not None:
                 return '{0} already in progress'.format(action)
 
@@ -880,7 +942,7 @@ class Ha(object):
                     # background thread and has not even written a pid file yet.
                     with self._async_executor.critical_task as task:
                         if not task.cancel():
-                            self.state_handler.terminate_starting_postmaster(pid=task.result)
+                            self.state_handler.terminate_starting_postmaster(postmaster=task.result)
                 self.demote('immediate-nolock')
                 return 'lost leader lock during ' + self._async_executor.scheduled_action
 
@@ -936,6 +998,7 @@ class Ha(object):
             logger.error('Cancelling bootstrap because watchdog activation failed')
             self.cancel_initialization()
         self.dcs.take_leader()
+        self.set_is_leader(True)
         self.state_handler.call_nowait(ACTION_ON_START)
         self.load_cluster_from_dcs()
         return 'initialized a new cluster'
@@ -988,10 +1051,16 @@ class Ha(object):
     def _run_cycle(self):
         dcs_failed = False
         try:
+            self.state_handler.reset_cluster_info_state()
             self.load_cluster_from_dcs()
 
             if self.is_paused():
                 self.watchdog.disable()
+                self._was_paused = True
+            else:
+                if self._was_paused:
+                    self.state_handler.schedule_sanity_checks_after_pause()
+                self._was_paused = False
 
             if not self.cluster.has_member(self.state_handler.name):
                 self.touch_member()
@@ -1025,8 +1094,8 @@ class Ha(object):
             # is data directory empty?
             if self.state_handler.data_directory_empty():
                 self.state_handler.set_role('uninitialized')
-                self.state_handler.stop()
-                # In case datadir went away while we were master. TODO: check for this and try to stop postgresql.
+                self.state_handler.stop('immediate')
+                # In case datadir went away while we were master.
                 self.watchdog.disable()
 
                 # is this instance the leader?
@@ -1037,9 +1106,20 @@ class Ha(object):
                 return self.bootstrap()  # new node
             # "bootstrap", but data directory is not empty
             elif not self.sysid_valid(self.cluster.initialize) and self.cluster.is_unlocked() and not self.is_paused():
+                if not self.state_handler.cb_called and self.state_handler.is_running() \
+                        and not self.state_handler.is_leader():
+                    self._join_aborted = True
+                    logger.error('No initialize key in DCS and PostgreSQL is running as replica, aborting start')
+                    logger.error('Please first start Patroni on the node running as master')
+                    sys.exit(1)
                 self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=self.state_handler.sysid)
             else:
                 # check if we are allowed to join
+                data_sysid = self.state_handler.sysid
+                if not self.sysid_valid(data_sysid):
+                    # data directory is not empty, but no valid sysid, cluster must be broken, suggest reinit
+                    return "data dir for the cluster is not empty, but system ID is invalid; consider doing reinitalize"
+
                 if self.sysid_valid(self.cluster.initialize) and self.cluster.initialize != self.state_handler.sysid:
                     logger.fatal("system ID mismatch, node %s belongs to a different cluster: %s != %s",
                                  self.state_handler.name, self.cluster.initialize, self.state_handler.sysid)
@@ -1051,8 +1131,13 @@ class Ha(object):
                         self.dcs.delete_leader()
                         self.dcs.reset_cluster()
                         return 'removed leader lock because postgres is not running'
-                    elif not (self.state_handler.rewind_executed or
-                              self.state_handler.need_rewind and self.state_handler.can_rewind):
+                    # Normally we don't start Postgres in a paused state. We make an exception for the demoted primary
+                    # that needs to be started after it had been stopped by demote. When there is no need to call rewind
+                    # the demote code follows through to starting Postgres right away, however, in the rewind case
+                    # it returns from demote and reaches this point to start PostgreSQL again after rewind. In that
+                    # case it makes no sense to continue to recover() unless rewind has finished successfully.
+                    elif (self.state_handler.rewind_failed or
+                          not (self.state_handler.need_rewind and self.state_handler.can_rewind)):
                         return 'postgres is not running'
 
                 # try to start dead postgres
@@ -1096,7 +1181,7 @@ class Ha(object):
         if self.is_paused():
             logger.info('Leader key is not deleted and Postgresql is not stopped due paused state')
             self.watchdog.disable()
-        else:
+        elif not self._join_aborted:
             # FIXME: If stop doesn't reach safepoint quickly enough keepalive is triggered. If shutdown checkpoint
             # takes longer than ttl, then leader key is lost and replication might not have sent out all xlog.
             # This might not be the desired behavior of users, as a graceful shutdown of the host can mean lost data.
@@ -1104,7 +1189,9 @@ class Ha(object):
             disable_wd = self.watchdog.disable if self.watchdog.is_running else None
             self.while_not_sync_standby(lambda: self.state_handler.stop(checkpoint=False, on_safepoint=disable_wd))
             if not self.state_handler.is_running():
-                self.dcs.delete_leader()
+                if self.has_lock():
+                    self.dcs.delete_leader()
+                self.touch_member()
             else:
                 # XXX: what about when Patroni is started as the wrong user that has access to the watchdog device
                 # but cannot shut down PostgreSQL. Root would be the obvious example. Would be nice to not kill the

patroni/postmaster.py

@@ -0,0 +1,167 @@
+import logging
+import os
+import psutil
+import re
+import signal
+import subprocess
+
+from patroni import call_self
+
+logger = logging.getLogger(__name__)
+
+STOP_SIGNALS = {
+    'smart': signal.SIGTERM,
+    'fast': signal.SIGINT,
+    'immediate': signal.SIGQUIT,
+}
+
+
+class PostmasterProcess(psutil.Process):
+
+    def __init__(self, pid):
+        self.is_single_user = False
+        if pid < 0:
+            pid = -pid
+            self.is_single_user = True
+        super(PostmasterProcess, self).__init__(pid)
+
+    @staticmethod
+    def _read_postmaster_pidfile(data_dir):
+        """Reads and parses postmaster.pid from the data directory
+
+        :returns dictionary of values if successful, empty dictionary otherwise
+        """
+        pid_line_names = ['pid', 'data_dir', 'start_time', 'port', 'socket_dir', 'listen_addr', 'shmem_key']
+        try:
+            with open(os.path.join(data_dir, 'postmaster.pid')) as f:
+                return {name: line.rstrip('\n') for name, line in zip(pid_line_names, f)}
+        except IOError:
+            return {}
+
+    def _is_postmaster_process(self):
+        try:
+            start_time = int(self._postmaster_pid.get('start_time', 0))
+            if start_time and abs(self.create_time() - start_time) > 3:
+                logger.info('Process %s is not postmaster, too much difference between PID file start time %s and '
+                            'process start time %s', self.pid, self.create_time(), start_time)
+                return False
+        except ValueError:
+            logger.warning('Garbage start time value in pid file: %r', self._postmaster_pid.get('start_time'))
+
+        # Extra safety check. The process can't be ourselves, our parent or our direct child.
+        if self.pid == os.getpid() or self.pid == os.getppid() or self.ppid() == os.getpid():
+            logger.info('Patroni (pid=%s, ppid=%s), "fake postmaster" (pid=%s, ppid=%s)',
+                        os.getpid(), os.getppid(), self.pid, self.ppid())
+            return False
+
+        return True
+
+    @classmethod
+    def _from_pidfile(cls, data_dir):
+        postmaster_pid = PostmasterProcess._read_postmaster_pidfile(data_dir)
+        try:
+            pid = int(postmaster_pid.get('pid', 0))
+            if pid:
+                proc = cls(pid)
+                proc._postmaster_pid = postmaster_pid
+                return proc
+        except ValueError:
+            pass
+
+    @staticmethod
+    def from_pidfile(data_dir):
+        try:
+            proc = PostmasterProcess._from_pidfile(data_dir)
+            return proc if proc and proc._is_postmaster_process() else None
+        except psutil.NoSuchProcess:
+            return None
+
+    @classmethod
+    def from_pid(cls, pid):
+        try:
+            return cls(pid)
+        except psutil.NoSuchProcess:
+            return None
+
+    def signal_stop(self, mode):
+        """Signal postmaster process to stop
+
+        :returns None if signaled, True if process is already gone, False if error
+        """
+        if self.is_single_user:
+            logger.warning("Cannot stop server; single-user server is running (PID: {0})".format(self.pid))
+            return False
+        try:
+            self.send_signal(STOP_SIGNALS[mode])
+        except psutil.NoSuchProcess:
+            return True
+        except psutil.AccessDenied as e:
+            logger.warning("Could not send stop signal to PostgreSQL (error: {0})".format(e))
+            return False
+
+        return None
+
+    def wait_for_user_backends_to_close(self):
+        # These regexps are cross checked against versions PostgreSQL 9.1 .. 9.6
+        aux_proc_re = re.compile("(?:postgres:)( .*:)? (?:""(?:startup|logger|checkpointer|writer|wal writer|"
+                                 "autovacuum launcher|autovacuum worker|stats collector|wal receiver|archiver|"
+                                 "wal sender) process|bgworker: )")
+
+        try:
+            user_backends = []
+            user_backends_cmdlines = []
+            for child in self.children():
+                try:
+                    cmdline = child.cmdline()[0]
+                    if not aux_proc_re.match(cmdline):
+                        user_backends.append(child)
+                        user_backends_cmdlines.append(cmdline)
+                except psutil.NoSuchProcess:
+                    pass
+            if user_backends:
+                logger.debug('Waiting for user backends %s to close', ', '.join(user_backends_cmdlines))
+                psutil.wait_procs(user_backends)
+            logger.debug("Backends closed")
+        except psutil.Error:
+            logger.exception('wait_for_user_backends_to_close')
+
+    @staticmethod
+    def start(pgcommand, data_dir, conf, options):
+        # Unfortunately `pg_ctl start` does not return postmaster pid to us. Without this information
+        # it is hard to know the current state of postgres startup, so we had to reimplement pg_ctl start
+        # in python. It will start postgres, wait for port to be open and wait until postgres will start
+        # accepting connections.
+        # Important!!! We can't just start postgres using subprocess.Popen, because in this case it
+        # will be our child for the rest of our live and we will have to take care of it (`waitpid`).
+        # So we will use the same approach as pg_ctl uses: start a new process, which will start postgres.
+        # This process will write postmaster pid to stdout and exit immediately. Now it's responsibility
+        # of init process to take care about postmaster.
+        # In order to make everything portable we can't use fork&exec approach here, so  we will call
+        # ourselves and pass list of arguments which must be used to start postgres.
+        env = {p: os.environ[p] for p in ('PATH', 'LD_LIBRARY_PATH', 'LC_ALL', 'LANG') if p in os.environ}
+        try:
+            proc = PostmasterProcess._from_pidfile(data_dir)
+            if proc and not proc._is_postmaster_process():
+                # Upon start postmaster process performs various safety checks if there is a postmaster.pid
+                # file in the data directory. Although Patroni already detected that the running process
+                # corresponding to the postmaster.pid is not a postmaster, the new postmaster might fail
+                # to start, because it thinks that postmaster.pid is already locked.
+                # Important!!! Unlink of postmaster.pid isn't an option, because it has a lot of nasty race conditions.
+                # Luckily there is a workaround to this problem, we can pass the pid from postmaster.pid
+                # in the `PG_GRANDPARENT_PID` environment variable and postmaster will ignore it.
+                logger.info("Telling pg_ctl that it is safe to ignore postmaster.pid for process %s", proc.pid)
+                env['PG_GRANDPARENT_PID'] = str(proc.pid)
+        except psutil.NoSuchProcess:
+            pass
+
+        proc = call_self(['pg_ctl_start', pgcommand, '-D', data_dir,
+                          '--config-file={}'.format(conf)] + options, close_fds=True,
+                         preexec_fn=os.setsid, stdout=subprocess.PIPE, env=env)
+        pid = int(proc.stdout.readline().strip())
+        proc.wait()
+        logger.info('postmaster pid=%s', pid)
+
+        # TODO: In an extremely unlikely case, the process could have exited and the pid reassigned. The start
+        # initiation time is not accurate enough to compare to create time as start time would also likely
+        # be relatively close. We need the subprocess extract pid+start_time in a race free manner.
+        return PostmasterProcess.from_pid(pid)

patroni/utils.py

@@ -1,6 +1,5 @@
 import random
 import time
-import re
 
 from dateutil import tz
 from patroni.exceptions import PatroniException
@@ -194,10 +193,6 @@ def _sleep(interval):
     time.sleep(interval)
 
 
-def is_valid_pg_version(version):
-    return re.match(r'[1-9][0-9]?(\.(0|([1-9][0-9]?))){2}$', version)
-
-
 class RetryFailedError(PatroniException):
 
     """Raised when retrying an operation ultimately failed, after retrying the maximum number of attempts."""
@@ -282,9 +277,7 @@ def polling_loop(timeout, interval=1):
         time.sleep(interval)
 
 
-def int_or_none(val):
-    """Returns integer value of the parameter if convertible to int, None otherwise."""
-    try:
-        return int(val)
-    except (ValueError, TypeError):
-        return None
+def split_host_port(value, default_port):
+    t = value.rsplit(':', 1)
+    t.append(default_port)
+    return t[0], int(t[1])

patroni/version.py

@@ -1 +1 @@
-__version__ = '1.3.6'
+__version__ = '1.4.6'

postgres0.yml

@@ -31,8 +31,8 @@ bootstrap:
 #        wal_level: hot_standby
 #        hot_standby: "on"
 #        wal_keep_segments: 8
-#        max_wal_senders: 5
-#        max_replication_slots: 5
+#        max_wal_senders: 10
+#        max_replication_slots: 10
 #        wal_log_hints: "on"
 #        archive_mode: "on"
 #        archive_timeout: 1800s

postgres1.yml

@@ -29,8 +29,8 @@ bootstrap:
 #        wal_level: hot_standby
 #        hot_standby: "on"
 #        wal_keep_segments: 8
-#        max_wal_senders: 5
-#        max_replication_slots: 5
+#        max_wal_senders: 10
+#        max_replication_slots: 10
 #        wal_log_hints: "on"
 #        archive_mode: "on"
 #        archive_timeout: 1800s
@@ -75,6 +75,9 @@ postgresql:
       password: zalando
   parameters:
     unix_socket_directories: '.'
+  basebackup:
+      - verbose
+      - max-rate: 100M
 tags:
     nofailover: false
     noloadbalance: false

postgres2.yml

@@ -29,8 +29,8 @@ bootstrap:
 #        wal_level: hot_standby
 #        hot_standby: "on"
 #        wal_keep_segments: 8
-#        max_wal_senders: 5
-#        max_replication_slots: 5
+#        max_wal_senders: 10
+#        max_replication_slots: 10
 #        wal_log_hints: "on"
 #        archive_mode: "on"
 #        archive_timeout: 1800s

requirements.txt

@@ -1,10 +1,10 @@
-urllib3>=1.9
+urllib3>=1.19.1,!=1.21
 boto
-psycopg2>=2.6.1
+psycopg2>=2.5.4
 PyYAML
 requests
 six >= 1.7
-kazoo==2.2.1
+kazoo>=1.3.1
 python-etcd>=0.4.3,<0.5
 python-consul>=0.7.0
 click>=4.1
@@ -13,3 +13,4 @@ tzlocal
 python-dateutil
 psutil
 cdiff
+kubernetes>=2.0.0,<=7.0.0,!=4.0.*,!=5.0.*

setup.py

@@ -53,6 +53,8 @@ CLASSIFIERS = [
     'Programming Language :: Python :: 2.7',
     'Programming Language :: Python :: 3.4',
     'Programming Language :: Python :: 3.5',
+    'Programming Language :: Python :: 3.6',
+    'Programming Language :: Python :: 3.7',
     'Programming Language :: Python :: Implementation :: CPython',
 ]
 
@@ -115,13 +117,23 @@ def read(fname):
 
 def setup_package():
     # Assemble additional setup commands
-    cmdclass = {}
-    cmdclass['test'] = PyTest
+    cmdclass = {'test': PyTest}
 
     # Some helper variables
     version = os.getenv('GO_PIPELINE_LABEL', VERSION)
 
-    install_reqs = get_install_requirements('requirements.txt')
+    install_requires = []
+    extras_require = {'aws': ['boto'], 'etcd': ['python-etcd'], 'consul': ['python-consul'],
+                      'exhibitor': ['kazoo'], 'zookeeper': ['kazoo'], 'kubernetes': ['kubernetes']}
+
+    for r in get_install_requirements('requirements.txt'):
+        extra = False
+        for e, v in extras_require.items():
+            if r.startswith(v[0]):
+                extras_require[e] = [r]
+                extra = True
+        if not extra:
+            install_requires.append(r)
 
     command_options = {'test': {'test_suite': ('setup.py', 'tests')}}
     if JUNIT_XML:
@@ -145,10 +157,10 @@ def setup_package():
         test_suite='tests',
         packages=find_packages(exclude=['tests', 'tests.*']),
         package_data={MAIN_PACKAGE: ["*.json"]},
-        install_requires=install_reqs,
-        setup_requires=['flake8'],
+        install_requires=install_requires,
+        extras_require=extras_require,
         cmdclass=cmdclass,
-        tests_require=['mock>=2.0.0', 'pytest-cov', 'pytest'],
+        tests_require=['flake8', 'mock>=2.0.0', 'pytest-cov', 'pytest'],
         command_options=command_options,
         entry_points={'console_scripts': CONSOLE_SCRIPTS},
     )

tests/test_api.py

@@ -37,6 +37,10 @@ class MockPostgresql(object):
     def postmaster_start_time():
         return str(postmaster_start_time)
 
+    @staticmethod
+    def replica_cached_timeline(_):
+        return 2
+
 
 class MockWatchdog(object):
     is_healthy = False
@@ -48,7 +52,11 @@ class MockHa(object):
     watchdog = MockWatchdog()
 
     @staticmethod
-    def reinitialize():
+    def is_leader():
+        return False
+
+    @staticmethod
+    def reinitialize(_):
         return 'reinitialize'
 
     @staticmethod
@@ -83,6 +91,10 @@ class MockHa(object):
     def wakeup():
         pass
 
+    @staticmethod
+    def is_paused():
+        return True
+
 
 class MockPatroni(object):
 
@@ -92,7 +104,7 @@ class MockPatroni(object):
     dcs = Mock()
     tags = {}
     version = '0.00'
-    noloadbalance = Mock(return_value=False)
+    noloadbalance = PropertyMock(return_value=False)
     scheduled_restart = {'schedule': future_restart_time,
                          'postmaster_start_time': postgresql.postmaster_start_time()}
 
@@ -115,16 +127,14 @@ class MockRequest(object):
 
 class MockRestApiServer(RestApiServer):
 
-    def __init__(self, Handler, request):
+    def __init__(self, Handler, request, config=None):
         self.socket = 0
         self.serve_forever = Mock()
         BaseHTTPServer.HTTPServer.__init__ = Mock()
         MockRestApiServer._BaseServer__is_shut_down = Mock()
         MockRestApiServer._BaseServer__shutdown_request = True
-        config = {'listen': '127.0.0.1:8008', 'auth': 'test:test'}
+        config = config or {'listen': '127.0.0.1:8008', 'auth': 'test:test', 'certfile': 'dumb'}
         super(MockRestApiServer, self).__init__(MockPatroni(), config)
-        config['certfile'] = 'dumb'
-        self.reload_config(config)
         Handler(MockRequest(request), ('0.0.0.0', 8080), self)
 
 
@@ -140,7 +150,13 @@ class TestRestApiHandler(unittest.TestCase):
         with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'master'})):
             MockRestApiServer(RestApiHandler, 'GET /replica')
         MockRestApiServer(RestApiHandler, 'GET /master')
-        MockPatroni.dcs.cluster.leader.name = MockPostgresql.name
+        MockPatroni.dcs.cluster.sync.sync_standby = MockPostgresql.name
+        MockPatroni.dcs.cluster.is_synchronous_mode = Mock(return_value=True)
+        with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'replica'})):
+            MockRestApiServer(RestApiHandler, 'GET /synchronous')
+        with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'replica'})):
+            MockRestApiServer(RestApiHandler, 'GET /asynchronous')
+        MockPatroni.ha.is_leader = Mock(return_value=True)
         MockRestApiServer(RestApiHandler, 'GET /replica')
         MockPatroni.dcs.cluster = None
         with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'master'})):
@@ -148,6 +164,8 @@ class TestRestApiHandler(unittest.TestCase):
         with patch.object(MockHa, 'restart_scheduled', Mock(return_value=True)):
             MockRestApiServer(RestApiHandler, 'GET /master')
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /master'))
+        with patch.object(RestApiServer, 'query', Mock(return_value=[('', 1, '', '', '', '', False, '')])):
+            self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
 
     def test_do_OPTIONS(self):
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'OPTIONS / HTTP/1.0'))
@@ -264,7 +282,7 @@ class TestRestApiHandler(unittest.TestCase):
     def test_do_POST_reinitialize(self, mock_dcs):
         cluster = mock_dcs.get_cluster.return_value
         cluster.is_paused.return_value = False
-        request = 'POST /reinitialize HTTP/1.0' + self._authorization
+        request = 'POST /reinitialize HTTP/1.0' + self._authorization + '\nContent-Length: 15\n\n{"force": true}'
         MockRestApiServer(RestApiHandler, request)
         with patch.object(MockHa, 'reinitialize', Mock(return_value=None)):
             MockRestApiServer(RestApiHandler, request)
@@ -278,12 +296,13 @@ class TestRestApiHandler(unittest.TestCase):
 
     @patch('time.sleep', Mock())
     @patch.object(MockPatroni, 'dcs')
-    def test_do_POST_failover(self, dcs):
+    def test_do_POST_switchover(self, dcs):
         dcs.loop_wait = 10
         cluster = dcs.get_cluster.return_value
         cluster.is_synchronous_mode.return_value = False
+        cluster.is_paused.return_value = False
 
-        post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
+        post = 'POST /switchover HTTP/1.0' + self._authorization + '\nContent-Length: '
 
         MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}')
 
@@ -293,8 +312,14 @@ class TestRestApiHandler(unittest.TestCase):
         cluster.leader.name = 'postgresql1'
         MockRestApiServer(RestApiHandler, request)
 
+        request = post + '25\n\n{"leader": "postgresql1"}'
+
+        cluster.is_paused.return_value = True
+        MockRestApiServer(RestApiHandler, request)
+
+        cluster.is_paused.return_value = False
         for cluster.is_synchronous_mode.return_value in (True, False):
-            MockRestApiServer(RestApiHandler, post + '25\n\n{"leader": "postgresql1"}')
+            MockRestApiServer(RestApiHandler, request)
 
         cluster.leader.name = 'postgresql2'
         request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
@@ -316,6 +341,7 @@ class TestRestApiHandler(unittest.TestCase):
 
         cluster2 = cluster.copy()
         cluster2.leader.name = 'postgresql0'
+        cluster2.is_unlocked.return_value = False
         dcs.get_cluster.side_effect = [cluster, cluster2]
         MockRestApiServer(RestApiHandler, request)
 
@@ -350,3 +376,21 @@ class TestRestApiHandler(unittest.TestCase):
 
         # Invalid date
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}'))
+
+    @patch.object(MockPatroni, 'dcs', Mock())
+    def test_do_POST_failover(self):
+        post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
+        MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}')
+        MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}')
+
+
+@patch('ssl.wrap_socket', Mock(return_value=0))
+class TestRestApiServer(unittest.TestCase):
+
+    def test_reload_config(self):
+        bad_config = {'listen': 'foo'}
+        self.assertRaises(ValueError, MockRestApiServer, None, '', bad_config)
+        srv = MockRestApiServer(lambda a1, a2, a3: None, '')
+        self.assertRaises(ValueError, srv.reload_config, bad_config)
+        self.assertRaises(ValueError, srv.reload_config, {})
+        srv.reload_config({'listen': '127.0.0.2:8008'})

tests/test_async_executor.py

@@ -8,7 +8,7 @@ from threading import Thread
 class TestAsyncExecutor(unittest.TestCase):
 
     def setUp(self):
-        self.a = AsyncExecutor(Mock())
+        self.a = AsyncExecutor(Mock(), Mock())
 
     @patch.object(Thread, 'start', Mock())
     def test_run_async(self):
@@ -17,6 +17,12 @@ class TestAsyncExecutor(unittest.TestCase):
     def test_run(self):
         self.a.run(Mock(side_effect=Exception()))
 
+    def test_cancel(self):
+        self.a.cancel()
+        self.a.schedule('foo')
+        self.a.cancel()
+        self.a.run(Mock())
+
 
 class TestCriticalTask(unittest.TestCase):
 

tests/test_config.py

@@ -49,6 +49,9 @@ class TestConfig(unittest.TestCase):
             'PATRONI_ETCD_CERT': '/cert',
             'PATRONI_ETCD_KEY': '/key',
             'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
+            'PATRONI_KUBERNETES_LABELS': 'a:b:c',
+            'PATRONI_KUBERNETES_SCOPE_LABEL': 'a',
+            'PATRONI_KUBERNETES_PORTS': '[{"name": "postgresql"}]',
             'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
             'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',
             'PATRONI_EXHIBITOR_PORT': '8181',

tests/test_consul.py

@@ -112,11 +112,11 @@ class TestConsul(unittest.TestCase):
     @patch.object(consul.Consul.KV, 'put', Mock(side_effect=[True, ConsulException]))
     def test_touch_member(self):
         self.c.refresh_session = Mock(return_value=True)
-        self.c.touch_member('balbla')
-        self.c.touch_member('balbla')
-        self.c.touch_member('balbla')
+        self.c.touch_member({'balbla': 'blabla'})
+        self.c.touch_member({'balbla': 'blabla'})
+        self.c.touch_member({'balbla': 'blabla'})
         self.c.refresh_session = Mock(return_value=False)
-        self.c.touch_member('balbla')
+        self.c.touch_member({'balbla': 'blabla'})
 
     @patch.object(consul.Consul.KV, 'put', Mock(return_value=False))
     def test_take_leader(self):
@@ -138,7 +138,7 @@ class TestConsul(unittest.TestCase):
 
     @patch.object(consul.Consul.Session, 'renew', Mock())
     def test_update_leader(self):
-        self.c.update_leader()
+        self.c.update_leader(None)
 
     @patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
     def test_delete_leader(self):
@@ -172,3 +172,7 @@ class TestConsul(unittest.TestCase):
     def test_sync_state(self):
         self.assertTrue(self.c.set_sync_state_value('{}'))
         self.assertTrue(self.c.delete_sync_state())
+
+    @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
+    def test_set_history_value(self):
+        self.assertTrue(self.c.set_history_value('{}'))

tests/test_ctl.py

@@ -5,15 +5,17 @@ import sys
 import unittest
 
 from click.testing import CliRunner
+from datetime import datetime, timedelta
 from mock import patch, Mock
-from patroni.ctl import ctl, members, store_config, load_config, output_members, request_patroni, get_dcs, parse_dcs, \
+from patroni.ctl import ctl, store_config, load_config, output_members, request_patroni, get_dcs, parse_dcs, \
     get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException, apply_config_changes, \
-    format_config_for_editing, show_diff, invoke_editor
-from patroni.dcs.etcd import Client
+    format_config_for_editing, show_diff, invoke_editor, format_pg_version
+from patroni.dcs.etcd import Client, Failover
+from patroni.utils import tzutc
 from psycopg2 import OperationalError
 from test_etcd import etcd_read, requests_get, socket_getaddrinfo, MockResponse
 from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \
-    get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader
+    get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader, get_cluster, Member
 from test_postgresql import MockConnect, psycopg2_connect
 
 CONFIG_FILE_PATH = './test-ctl.yaml'
@@ -31,7 +33,7 @@ def test_rw_config():
 
 
 @patch('patroni.ctl.load_config',
-       Mock(return_value={'postgresql': {'data_dir': '.', 'parameters': {}, 'retry_timeout': 5},
+       Mock(return_value={'scope': 'alpha', 'postgresql': {'data_dir': '.', 'parameters': {}, 'retry_timeout': 5},
                           'restapi': {'auth': 'u:p', 'listen': ''}, 'etcd': {'host': 'localhost:2379'}}))
 class TestCtl(unittest.TestCase):
 
@@ -64,76 +66,92 @@ class TestCtl(unittest.TestCase):
         self.assertRaises(PatroniCtlException, parse_dcs, 'invalid://test')
 
     def test_output_members(self):
-        cluster = get_cluster_initialized_with_leader()
+        scheduled_at = datetime.now(tzutc) + timedelta(seconds=600)
+        cluster = get_cluster_initialized_with_leader(Failover(1, 'foo', 'bar', scheduled_at))
         self.assertIsNone(output_members(cluster, name='abc', fmt='pretty'))
         self.assertIsNone(output_members(cluster, name='abc', fmt='json'))
+        self.assertIsNone(output_members(cluster, name='abc', fmt='yaml'))
         self.assertIsNone(output_members(cluster, name='abc', fmt='tsv'))
 
+    @patch('patroni.ctl.get_dcs')
+    @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
+    def test_switchover(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        mock_get_dcs.return_value.set_failover_value = Mock()
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+        assert 'leader' in result.output
+
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
+        assert result.exit_code == 0
+
+        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+            result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00'])
+            assert result.exit_code == 1
+
+        # Aborting switchover, as we anser NO to the confirmation
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\nN')
+        assert result.exit_code == 1
+
+        # Target and source are equal
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nleader\n\ny')
+        assert result.exit_code == 1
+
+        # Reality is not part of this cluster
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nReality\n\ny')
+        assert result.exit_code == 1
+
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force'])
+        assert 'Member' in result.output
+
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
+        assert result.exit_code == 0
+
+        # Invalid timestamp
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', 'invalid'])
+        assert result.exit_code != 0
+
+        # Invalid timestamp
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', '2115-02-30T12:00:00+01:00'])
+        assert result.exit_code != 0
+
+        # Specifying wrong leader
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='dummy')
+        assert result.exit_code == 1
+
+        with patch('patroni.ctl.request_patroni', Mock(side_effect=Exception)):
+            # Non-responding patroni
+            result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
+            assert 'falling back to DCS' in result.output
+
+        with patch('patroni.ctl.request_patroni') as mocked:
+            mocked.return_value.status_code = 500
+            result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+            assert 'Switchover failed' in result.output
+
+            mocked.return_value.status_code = 501
+            mocked.return_value.text = 'Server does not support this operation'
+            result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+            assert 'Switchover failed' in result.output
+
+        # No members available
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+        assert result.exit_code == 1
+
+        # No master available
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+        assert result.exit_code == 1
+
     @patch('patroni.ctl.get_dcs')
     @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
     def test_failover(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
         mock_get_dcs.return_value.set_failover_value = Mock()
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
-        assert 'leader' in result.output
-
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
-        assert result.exit_code == 0
-
-        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
-            result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00'])
-            assert result.exit_code == 1
-
-        # Aborting failover,as we anser NO to the confirmation
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\nN')
-        assert result.exit_code == 1
-
-        # Target and source are equal
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nleader\n\ny')
-        assert result.exit_code == 1
-
-        # Reality is not part of this cluster
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nReality\n\ny')
-        assert result.exit_code == 1
-
-        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force'])
-        assert 'Member' in result.output
-
-        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
-        assert result.exit_code == 0
-
-        # Invalid timestamp
-        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', 'invalid'])
-        assert result.exit_code != 0
-
-        # Invalid timestamp
-        result = self.runner.invoke(ctl, ['failover', 'dummy', '--force', '--scheduled', '2115-02-30T12:00:00+01:00'])
-        assert result.exit_code != 0
-
-        # Specifying wrong leader
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='dummy')
-        assert result.exit_code == 1
-
-        with patch('patroni.ctl.request_patroni', Mock(side_effect=Exception)):
-            # Non-responding patroni
-            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
-            assert 'falling back to DCS' in result.output
-
-        with patch('patroni.ctl.request_patroni') as mocked:
-            mocked.return_value.status_code = 500
-            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
-            assert 'Failover failed' in result.output
-
-        # No members available
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
-        assert result.exit_code == 1
-
-        # No master available
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
-        assert result.exit_code == 1
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='\n')
+        assert 'Failover could be performed only to a specific candidate' in result.output
 
     def test_get_dcs(self):
         self.assertRaises(PatroniCtlException, get_dcs, {'dummy': {}}, 'dummy')
@@ -219,7 +237,7 @@ class TestCtl(unittest.TestCase):
         assert result.exit_code == 1
 
         # successful reinit
-        result = self.runner.invoke(ctl, ['reinit', 'alpha', 'other'], input='y')
+        result = self.runner.invoke(ctl, ['reinit', 'alpha', 'other'], input='y\ny')
         assert result.exit_code == 0
 
         # Aborted restart
@@ -235,7 +253,7 @@ class TestCtl(unittest.TestCase):
 
         # Wrong pg version
         result = self.runner.invoke(ctl, ['restart', 'alpha', '--any', '--pg-version', '9.1'], input='y')
-        assert 'Error: PostgreSQL version' in result.output
+        assert 'Error: Invalid PostgreSQL version format' in result.output
         assert result.exit_code == 1
 
         result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force', '--timeout', '10min'])
@@ -261,7 +279,7 @@ class TestCtl(unittest.TestCase):
         with patch('requests.post', Mock(return_value=MockResponse(204))):
             # get restart with the non-200 return code
             # normal restart, the schedule is actually parsed, but not validated in patronictl
-            result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0.0',
+            result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0',
                                         '--scheduled', '2300-10-01T14:30'], input='y')
             assert result.exit_code == 0
 
@@ -342,9 +360,11 @@ class TestCtl(unittest.TestCase):
     @patch('patroni.ctl.get_dcs')
     def test_members(self, mock_get_dcs):
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
-        result = self.runner.invoke(members, ['alpha'])
+        result = self.runner.invoke(ctl, ['list'])
         assert '127.0.0.1' in result.output
         assert result.exit_code == 0
+        with patch('patroni.ctl.load_config', Mock(return_value={})):
+            self.runner.invoke(ctl, ['list'])
 
     def test_configure(self):
         result = self.runner.invoke(configure, ['--dcs', 'abc', '-c', 'dummy', '-n', 'bla'])
@@ -380,7 +400,7 @@ class TestCtl(unittest.TestCase):
         cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
         mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster)
 
-        result = self.runner.invoke(ctl, ['list', 'dummy', '--extended'])
+        result = self.runner.invoke(ctl, ['list', 'dummy', '--extended', '--timestamp'])
         assert '2100' in result.output
         assert 'Scheduled restart' in result.output
 
@@ -400,14 +420,11 @@ class TestCtl(unittest.TestCase):
             assert 'Failed: flush scheduled restart' in result.output
 
     @patch('patroni.ctl.get_dcs')
+    @patch('patroni.ctl.polling_loop', Mock(return_value=[1]))
     def test_pause_cluster(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
 
-        with patch('requests.patch', Mock(return_value=MockResponse(200))):
-            result = self.runner.invoke(ctl, ['pause', 'dummy'])
-            assert 'Success' in result.output
-
         with patch('requests.patch', Mock(return_value=MockResponse(500))):
             result = self.runner.invoke(ctl, ['pause', 'dummy'])
             assert 'Failed' in result.output
@@ -417,6 +434,17 @@ class TestCtl(unittest.TestCase):
             result = self.runner.invoke(ctl, ['pause', 'dummy'])
             assert 'Cluster is already paused' in result.output
 
+        with patch('requests.patch', Mock(return_value=MockResponse(200))):
+            result = self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
+            assert "'pause' request sent" in result.output
+            mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
+                                                                      get_cluster(None, None, [], None, None)])
+            self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
+            member = Member(1, 'other', 28, {})
+            mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
+                                                                      get_cluster(None, None, [member], None, None)])
+            self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
+
     @patch('patroni.ctl.get_dcs')
     def test_resume_cluster(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
@@ -506,3 +534,21 @@ class TestCtl(unittest.TestCase):
         self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar')
         mock_get_dcs.return_value.set_config_value = Mock(return_value=True)
         self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar')
+
+    @patch('patroni.ctl.get_dcs')
+    def test_version(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        with patch('patroni.ctl.request_patroni') as mocked:
+            result = self.runner.invoke(ctl, ['version'])
+            assert 'patronictl version' in result.output
+            mocked.return_value.json = lambda: {'patroni': {'version': '1.2.3'}, 'server_version': 100001}
+            result = self.runner.invoke(ctl, ['version', 'dummy'])
+            assert '1.2.3' in result.output
+        with patch('requests.get', Mock(side_effect=Exception)):
+            result = self.runner.invoke(ctl, ['version', 'dummy'])
+            assert 'failed to get version' in result.output
+
+    def test_format_pg_version(self):
+        self.assertEquals(format_pg_version(100001), '10.1')
+        self.assertEquals(format_pg_version(90605), '9.6.5')

tests/test_etcd.py

@@ -18,7 +18,6 @@ class MockResponse(object):
         self.status_code = status_code
         self.content = '{}'
         self.ok = True
-        self.text = ''
 
     def json(self):
         return json.loads(self.content)
@@ -27,6 +26,10 @@ class MockResponse(object):
     def data(self):
         return self.content.encode('utf-8')
 
+    @property
+    def text(self):
+        return self.content
+
     @property
     def status(self):
         return self.status_code
@@ -48,6 +51,12 @@ def requests_get(url, **kwargs):
         response.content = '[{}]' if url.startswith('http://error') else members
     elif url.startswith('http://exhibitor'):
         response.content = '{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}'
+    elif url.endswith(':8011/reinitialize'):
+        data = kwargs.get('data', '')
+        if ' false}' in data:
+            response.status_code = 503
+            response.ok = False
+            response.content = 'restarting after failure already in progress'
     else:
         response.status_code = 404
         response.ok = False
@@ -81,7 +90,7 @@ def etcd_read(self, key, **kwargs):
         raise etcd.EtcdKeyNotFound
 
     response = {"action": "get", "node": {"key": "/service/batman5", "dir": True, "nodes": [
-                {"key": "/service/batman5/config", "value": '{"foo": "bar"}',
+                {"key": "/service/batman5/config", "value": '{"synchronous_mode": 0}',
                  "modifiedIndex": 1582, "createdIndex": 1582},
                 {"key": "/service/batman5/failover", "value": "",
                  "modifiedIndex": 1582, "createdIndex": 1582},
@@ -198,6 +207,7 @@ class TestClient(unittest.TestCase):
         mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
         self.client._machines_cache_updated = 0
         self.client.api_execute('/', 'POST', timeout=0)
+        self.client._machines_cache = [self.client._base_uri]
         self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
         self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '')
         self.client._update_machines_cache = True
@@ -262,9 +272,13 @@ class TestEtcd(unittest.TestCase):
                                   {'url': 'https://test:2379', 'retry_timeout': 10})
                 self.assertRaises(SleepException, self.etcd.get_etcd_client,
                                   {'proxy': 'https://user:password@test:2379', 'retry_timeout': 10})
+                self.assertRaises(SleepException, self.etcd.get_etcd_client,
+                                  {'hosts': 'foo:4001,bar', 'retry_timeout': 10})
 
     def test_get_cluster(self):
-        self.assertIsInstance(self.etcd.get_cluster(), Cluster)
+        cluster = self.etcd.get_cluster()
+        self.assertIsInstance(cluster, Cluster)
+        self.assertFalse(cluster.is_synchronous_mode())
         self.etcd._base_path = '/service/nocluster'
         cluster = self.etcd.get_cluster()
         self.assertIsInstance(cluster, Cluster)
@@ -288,7 +302,7 @@ class TestEtcd(unittest.TestCase):
         self.etcd.write_leader_optime('0')
 
     def test_update_leader(self):
-        self.assertTrue(self.etcd.update_leader())
+        self.assertTrue(self.etcd.update_leader(None))
 
     def test_initialize(self):
         self.assertFalse(self.etcd.initialize())
@@ -324,3 +338,6 @@ class TestEtcd(unittest.TestCase):
     def test_sync_state(self):
         self.assertFalse(self.etcd.write_sync_state('leader', None))
         self.assertFalse(self.etcd.delete_sync_state())
+
+    def test_set_history_value(self):
+        self.assertFalse(self.etcd.set_history_value('{}'))

tests/test_ha.py

@@ -5,7 +5,7 @@ import unittest
 
 from mock import Mock, MagicMock, PropertyMock, patch
 from patroni.config import Config
-from patroni.dcs import Cluster, ClusterConfig, Failover, Leader, Member, get_dcs, SyncState
+from patroni.dcs import Cluster, ClusterConfig, Failover, Leader, Member, get_dcs, SyncState, TimelineHistory
 from patroni.dcs.etcd import Client
 from patroni.exceptions import DCSError, PostgresConnectionException, PatroniException
 from patroni.ha import Ha, _MemberStatus
@@ -13,7 +13,9 @@ from patroni.postgresql import Postgresql
 from patroni.watchdog import Watchdog
 from patroni.utils import tzutc
 from test_etcd import socket_getaddrinfo, etcd_read, etcd_write, requests_get
-from test_postgresql import psycopg2_connect
+from test_postgresql import psycopg2_connect, MockPostmaster
+
+SYSID = '12345678901'
 
 
 def true(*args, **kwargs):
@@ -25,7 +27,8 @@ def false(*args, **kwargs):
 
 
 def get_cluster(initialize, leader, members, failover, sync):
-    return Cluster(initialize, ClusterConfig(1, {1: 2}, 1), leader, 10, members, failover, sync)
+    history = TimelineHistory(1, [(1, 67197376, 'no recovery target specified', datetime.datetime.now().isoformat())])
+    return Cluster(initialize, ClusterConfig(1, {1: 2}, 1), leader, 10, members, failover, sync, history)
 
 
 def get_cluster_not_initialized_without_leader():
@@ -39,11 +42,12 @@ def get_cluster_initialized_without_leader(leader=False, failover=None, sync=Non
     m2 = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
                                  'api_url': 'http://127.0.0.1:8011/patroni',
                                  'state': 'running',
+                                 'pause': True,
                                  'tags': {'clonefrom': True},
                                  'scheduled_restart': {'schedule': "2100-01-01 10:53:07.560445+00:00",
                                                        'postgres_version': '99.0.0'}})
     syncstate = SyncState(0 if sync else None, sync and sync[0], sync and sync[1])
-    return get_cluster(True, leader, [m1, m2], failover, syncstate)
+    return get_cluster(SYSID, leader, [m1, m2], failover, syncstate)
 
 
 def get_cluster_initialized_with_leader(failover=None, sync=None):
@@ -112,12 +116,13 @@ def run_async(self, func, args=()):
     return func(*args) if args else func()
 
 
-@patch.object(Postgresql, 'is_running', Mock(return_value=True))
+@patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
 @patch.object(Postgresql, 'is_leader', Mock(return_value=True))
-@patch.object(Postgresql, 'wal_position', Mock(return_value=10))
+@patch.object(Postgresql, 'timeline_wal_position', Mock(return_value=(1, 10)))
+@patch.object(Postgresql, '_cluster_info_state_get', Mock(return_value=3))
 @patch.object(Postgresql, 'call_nowait', Mock(return_value=True))
 @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
-@patch.object(Postgresql, 'controldata', Mock(return_value={'Database system identifier': '1234567890'}))
+@patch.object(Postgresql, 'controldata', Mock(return_value={'Database system identifier': SYSID}))
 @patch.object(Postgresql, 'sync_replication_slots', Mock())
 @patch.object(Postgresql, 'write_pg_hba', Mock())
 @patch.object(Postgresql, 'write_pgpass', Mock(return_value={}))
@@ -125,9 +130,11 @@ def run_async(self, func, args=()):
 @patch.object(Postgresql, 'query', Mock())
 @patch.object(Postgresql, 'checkpoint', Mock())
 @patch.object(Postgresql, 'call_nowait', Mock())
+@patch.object(Postgresql, 'cancellable_subprocess_call', Mock(return_value=0))
 @patch.object(etcd.Client, 'write', etcd_write)
 @patch.object(etcd.Client, 'read', etcd_read)
 @patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
+@patch('patroni.postgresql.polling_loop', Mock(return_value=range(1)))
 @patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=False))
 @patch('patroni.async_executor.AsyncExecutor.run_async', run_async)
 @patch('subprocess.call', Mock(return_value=0))
@@ -156,7 +163,7 @@ class TestHa(unittest.TestCase):
                                        'name': 'foo', 'retry_timeout': 10}})
             self.ha = Ha(MockPatroni(self.p, self.e))
             self.ha.old_cluster = self.e.get_cluster()
-            self.ha.cluster = get_cluster_not_initialized_without_leader()
+            self.ha.cluster = get_cluster_initialized_without_leader()
             self.ha.load_cluster_from_dcs = Mock()
 
     def test_update_lock(self):
@@ -164,7 +171,8 @@ class TestHa(unittest.TestCase):
         self.assertTrue(self.ha.update_lock(True))
 
     def test_touch_member(self):
-        self.p.wal_position = Mock(side_effect=Exception)
+        self.p.timeline_wal_position = Mock(return_value=(0, 1))
+        self.p.replica_cached_timeline = Mock(side_effect=Exception)
         self.ha.touch_member()
 
     def test_start_as_replica(self):
@@ -172,7 +180,7 @@ class TestHa(unittest.TestCase):
         self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
 
     def test_recover_replica_failed(self):
-        self.p.controldata = lambda: {'Database cluster state': 'in recovery'}
+        self.p.controldata = lambda: {'Database cluster state': 'in recovery', 'Database system identifier': SYSID}
         self.p.is_running = false
         self.p.follow = false
         self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
@@ -183,14 +191,14 @@ class TestHa(unittest.TestCase):
         self.p.is_running = false
         self.p.name = 'leader'
         self.p.set_role('master')
-        self.p.controldata = lambda: {'Database cluster state': 'shut down'}
+        self.p.controldata = lambda: {'Database cluster state': 'shut down', 'Database system identifier': SYSID}
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.assertEquals(self.ha.run_cycle(), 'starting as readonly because i had the session lock')
 
     @patch.object(Postgresql, 'fix_cluster_state', Mock())
     def test_crash_recovery(self):
         self.p.is_running = false
-        self.p.controldata = lambda: {'Database cluster state': 'in production'}
+        self.p.controldata = lambda: {'Database cluster state': 'in production', 'Database system identifier': SYSID}
         self.assertEquals(self.ha.run_cycle(), 'doing crash recovery in a single user mode')
 
     @patch.object(Postgresql, 'rewind_needed_and_possible', Mock(return_value=True))
@@ -199,18 +207,10 @@ class TestHa(unittest.TestCase):
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.assertEquals(self.ha.run_cycle(), 'running pg_rewind from leader')
 
-    @patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
-    @patch.object(Postgresql, 'fix_cluster_state', Mock())
-    def test_single_user_after_recover_failed(self):
-        self.p.controldata = lambda: {'Database cluster state': 'in recovery'}
-        self.p.is_running = false
-        self.p.follow = false
-        self.assertEquals(self.ha.run_cycle(), 'starting as a secondary')
-        self.assertEquals(self.ha.run_cycle(), 'fixing cluster state in a single user mode')
-
     @patch('sys.exit', return_value=1)
     @patch('patroni.ha.Ha.sysid_valid', MagicMock(return_value=True))
     def test_sysid_no_match(self, exit_mock):
+        self.p.controldata = lambda: {'Database cluster state': 'in recovery', 'Database system identifier': '123'}
         self.ha.run_cycle()
         exit_mock.assert_called_once_with(1)
 
@@ -219,8 +219,10 @@ class TestHa(unittest.TestCase):
         self.p.is_leader = false
         self.p.is_healthy = true
         self.ha.has_lock = true
+        self.p.controldata = lambda: {'Database cluster state': 'in production', 'Database system identifier': SYSID}
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader because i had the session lock')
 
+    @patch('psycopg2.connect', psycopg2_connect)
     def test_acquire_lock_as_master(self):
         self.assertEquals(self.ha.run_cycle(), 'acquired session lock as a leader')
 
@@ -229,6 +231,13 @@ class TestHa(unittest.TestCase):
         self.p.is_leader = false
         self.assertEquals(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
 
+    def test_long_promote(self):
+        self.ha.cluster.is_unlocked = false
+        self.ha.has_lock = true
+        self.p.is_leader = false
+        self.p.set_role('master')
+        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
+
     def test_demote_after_failing_to_obtain_lock(self):
         self.ha.acquire_lock = false
         self.assertEquals(self.ha.run_cycle(), 'demoted self after trying and failing to obtain lock')
@@ -264,6 +273,7 @@ class TestHa(unittest.TestCase):
             self.assertEquals(self.ha.run_cycle(), 'Not promoting self because watchdog could not be activated')
 
     def test_leader_with_lock(self):
+        self.ha.cluster = get_cluster_not_initialized_without_leader()
         self.ha.cluster.is_unlocked = false
         self.ha.has_lock = true
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
@@ -345,7 +355,7 @@ class TestHa(unittest.TestCase):
         self.ha.cluster = get_cluster_not_initialized_without_leader()
         self.e.initialize = true
         self.ha.bootstrap()
-        self.p.is_running = true
+        self.p.is_running.return_value = MockPostmaster()
         self.p.is_leader = true
         with patch.object(Watchdog, 'activate', Mock(return_value=False)):
             self.assertEquals(self.ha.post_bootstrap(), 'running post_bootstrap')
@@ -356,7 +366,7 @@ class TestHa(unittest.TestCase):
         self.assertIsNotNone(self.ha.reinitialize())
 
         self.ha.cluster = get_cluster_initialized_with_leader()
-        self.assertIsNone(self.ha.reinitialize())
+        self.assertIsNone(self.ha.reinitialize(True))
 
         self.assertIsNotNone(self.ha.reinitialize())
 
@@ -392,9 +402,9 @@ class TestHa(unittest.TestCase):
             self.ha.update_lock = false
             self.p.set_role('master')
             with patch('patroni.async_executor.CriticalTask.cancel', Mock(return_value=False)):
-                with patch('patroni.postgresql.Postgresql.stop') as stop_mock:
+                with patch('patroni.postgresql.Postgresql.terminate_starting_postmaster') as mock_terminate:
                     self.assertEquals(self.ha.run_cycle(), 'lost leader lock during restart')
-                    stop_mock.assert_called()
+                    mock_terminate.assert_called()
 
     @patch('requests.get', requests_get)
     def test_manual_failover_from_leader(self):
@@ -524,7 +534,7 @@ class TestHa(unittest.TestCase):
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.fetch_node_status = get_node_status(wal_position=11)  # accessible, in_recovery, wal position ahead
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
-        with patch('patroni.postgresql.Postgresql.wal_position', return_value=1):
+        with patch('patroni.postgresql.Postgresql.timeline_wal_position', return_value=(1, 1)):
             self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.patroni.nofailover = True
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
@@ -603,6 +613,14 @@ class TestHa(unittest.TestCase):
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', self.p.name, None))
         self.assertEquals(self.ha.run_cycle(), 'PAUSE: waiting to become master after promote...')
 
+    def test_failed_to_update_lock_in_pause(self):
+        self.ha.update_lock = false
+        self.ha.is_paused = true
+        self.p.name = 'leader'
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.assertEquals(self.ha.run_cycle(),
+                          'PAUSE: continue to run as master after failing to update leader lock in DCS')
+
     def test_postgres_unhealthy_in_pause(self):
         self.ha.is_paused = true
         self.p.is_healthy = false
@@ -860,6 +878,7 @@ class TestHa(unittest.TestCase):
 
     def test_shutdown(self):
         self.p.is_running = false
+        self.ha.has_lock = true
         self.ha.shutdown()
 
     @patch('time.sleep', Mock())
@@ -874,3 +893,24 @@ class TestHa(unittest.TestCase):
         self.ha.has_lock = false
         # will not say bootstrap from leader as replica can't self elect
         self.assertEquals(self.ha.run_cycle(), "trying to bootstrap from replica 'other'")
+
+    def test_update_cluster_history(self):
+        self.p.get_master_timeline = Mock(return_value=1)
+        self.ha.has_lock = true
+        self.ha.cluster.is_unlocked = false
+        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
+
+    @patch('sys.exit', return_value=1)
+    def test_abort_join(self, exit_mock):
+        self.ha.cluster = get_cluster_not_initialized_without_leader()
+        self.p.is_leader = false
+        self.ha.run_cycle()
+        exit_mock.assert_called_once_with(1)
+
+    def test_after_pause(self):
+        self.ha.has_lock = true
+        self.ha.cluster.is_unlocked = false
+        self.ha.is_paused = true
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: no action.  i am the leader with the lock')
+        self.ha.is_paused = false
+        self.assertEquals(self.ha.run_cycle(), 'no action.  i am the leader with the lock')

tests/test_kubernetes.py

@@ -0,0 +1,107 @@
+import unittest
+
+from mock import Mock, patch
+from patroni.dcs.kubernetes import Kubernetes, KubernetesError, k8s_client, k8s_watch, RetryFailedError
+
+
+def mock_list_namespaced_config_map(self, *args, **kwargs):
+    metadata = {'resource_version': '1', 'labels': {'f': 'b'}, 'name': 'test-config',
+                'annotations': {'initialize': '123', 'config': '{}'}}
+    items = [k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata))]
+    metadata.update({'name': 'test-leader', 'annotations': {'optime': '1234', 'leader': 'p-0', 'ttl': '30s'}})
+    items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
+    metadata.update({'name': 'test-failover', 'annotations': {'leader': 'p-0'}})
+    items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
+    metadata.update({'name': 'test-sync', 'annotations': {'leader': 'p-0'}})
+    items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
+    metadata = k8s_client.V1ObjectMeta(resource_version='1')
+    return k8s_client.V1ConfigMapList(metadata=metadata, items=items)
+
+
+def mock_list_namespaced_pod(self, *args, **kwargs):
+    metadata = k8s_client.V1ObjectMeta(resource_version='1', name='p-0', annotations={'status': '{}'})
+    items = [k8s_client.V1Pod(metadata=metadata)]
+    return k8s_client.V1PodList(items=items)
+
+
+@patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', Mock())
+@patch.object(k8s_client.CoreV1Api, 'create_namespaced_config_map', Mock())
+class TestKubernetes(unittest.TestCase):
+
+    @patch('kubernetes.config.load_kube_config', Mock())
+    @patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map)
+    @patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', mock_list_namespaced_pod)
+    def setUp(self):
+        self.k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10, 'labels': {'f': 'b'}})
+        with patch('time.time', Mock(return_value=1)):
+            self.k.get_cluster()
+
+    @patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map)
+    @patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', mock_list_namespaced_pod)
+    def test_get_cluster(self):
+        self.k.get_cluster()
+        with patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', Mock(side_effect=Exception)):
+            self.assertRaises(KubernetesError, self.k.get_cluster)
+
+    @patch('kubernetes.config.load_kube_config', Mock())
+    @patch.object(k8s_client.CoreV1Api, 'create_namespaced_endpoints', Mock())
+    def test_update_leader(self):
+        k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10,
+                        'labels': {'f': 'b'}, 'use_endpoints': True, 'pod_ip': '10.0.0.0'})
+        self.assertIsNotNone(k.update_leader('123'))
+
+    def test_take_leader(self):
+        self.k.take_leader()
+        self.k._leader_observed_record['leader'] = 'test'
+        self.k.patch_or_create = Mock(return_value=False)
+        self.k.take_leader()
+
+    def test_manual_failover(self):
+        with patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', Mock(side_effect=RetryFailedError(''))):
+            self.k.manual_failover('foo', 'bar')
+
+    def test_set_config_value(self):
+        self.k.set_config_value('{}')
+
+    @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_pod', Mock(return_value=True))
+    def test_touch_member(self):
+        self.k.touch_member({})
+        self.k._name = 'p-1'
+        self.k.touch_member({'state': 'running', 'role': 'replica'})
+        self.k.touch_member({'state': 'stopped', 'role': 'master'})
+
+    def test_initialize(self):
+        self.k.initialize()
+
+    def test_delete_leader(self):
+        self.k.delete_leader()
+
+    def test_cancel_initialization(self):
+        self.k.cancel_initialization()
+
+    @patch.object(k8s_client.CoreV1Api, 'delete_collection_namespaced_config_map',
+                  Mock(side_effect=k8s_client.rest.ApiException(403, '')))
+    def test_delete_cluster(self):
+        self.k.delete_cluster()
+
+    @patch('kubernetes.config.load_kube_config', Mock())
+    @patch.object(k8s_client.CoreV1Api, 'create_namespaced_endpoints',
+                  Mock(side_effect=[k8s_client.rest.ApiException(502, ''), k8s_client.rest.ApiException(500, '')]))
+    def test_delete_sync_state(self):
+        k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10,
+                        'labels': {'f': 'b'}, 'use_endpoints': True, 'pod_ip': '10.0.0.0'})
+        self.assertFalse(k.delete_sync_state())
+
+    def test_watch(self):
+        self.k.set_ttl(10)
+        self.k.watch(None, 0)
+        self.k.watch(None, 0)
+        with patch.object(k8s_watch.Watch, 'stream',
+                          Mock(side_effect=[Exception, [], KeyboardInterrupt,
+                                            [{'raw_object': {'metadata': {'resourceVersion': '2'}}}]])):
+            self.assertFalse(self.k.watch('1', 2))
+            self.assertRaises(KeyboardInterrupt, self.k.watch, '1', 2)
+            self.assertTrue(self.k.watch('1', 2))
+
+    def test_set_history_value(self):
+        self.k.set_history_value('{}')

tests/test_patroni.py

@@ -12,7 +12,7 @@ from patroni.exceptions import DCSError
 from patroni import Patroni, main as _main, patroni_main
 from six.moves import BaseHTTPServer
 from test_etcd import SleepException, etcd_read, etcd_write
-from test_postgresql import Postgresql, psycopg2_connect
+from test_postgresql import Postgresql, psycopg2_connect, MockPostmaster
 
 
 class MockFrozenImporter(object):
@@ -26,7 +26,7 @@ class MockFrozenImporter(object):
 @patch.object(Postgresql, 'write_pg_hba', Mock())
 @patch.object(Postgresql, '_write_postgresql_conf', Mock())
 @patch.object(Postgresql, 'write_recovery_conf', Mock())
-@patch.object(Postgresql, 'is_running', Mock(return_value=True))
+@patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
 @patch.object(Postgresql, 'call_nowait', Mock())
 @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
 @patch.object(AsyncExecutor, 'run', Mock())
@@ -103,6 +103,7 @@ class TestPatroni(unittest.TestCase):
 
     @patch('patroni.config.Config.save_cache', Mock())
     @patch('patroni.config.Config.reload_local_configuration', Mock(return_value=True))
+    @patch('patroni.ha.Ha.is_leader', Mock(return_value=True))
     @patch.object(Postgresql, 'state', PropertyMock(return_value='running'))
     @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
     def test_run(self):
@@ -151,3 +152,7 @@ class TestPatroni(unittest.TestCase):
         self.assertTrue(self.p.nosync)
         self.p.tags['nosync'] = None
         self.assertFalse(self.p.nosync)
+
+    def test_shutdown(self):
+        self.p.api.shutdown = Mock(side_effect=Exception)
+        self.p.shutdown()

tests/test_postgresql.py

@@ -1,8 +1,7 @@
-import errno
+import datetime
 import mock  # for the mock.call method, importing it without a namespace breaks python3
 import os
 import psycopg2
-import psutil
 import shutil
 import subprocess
 import unittest
@@ -10,11 +9,12 @@ import unittest
 from mock import Mock, MagicMock, PropertyMock, patch, mock_open
 from patroni.async_executor import CriticalTask
 from patroni.dcs import Cluster, Leader, Member, SyncState
-from patroni.exceptions import PostgresConnectionException
+from patroni.exceptions import PostgresConnectionException, PostgresException
 from patroni.postgresql import Postgresql, STATE_REJECT, STATE_NO_RESPONSE
+from patroni.postmaster import PostmasterProcess
 from patroni.utils import RetryFailedError
 from six.moves import builtins
-from threading import Thread
+from threading import Thread, current_thread
 
 
 class MockCursor(object):
@@ -35,13 +35,13 @@ class MockCursor(object):
         elif sql.startswith('SELECT slot_name'):
             self.results = [('blabla',), ('foobar',)]
         elif sql.startswith('SELECT CASE WHEN pg_is_in_recovery()'):
-            self.results = [(2,)]
-        elif sql == 'SELECT pg_is_in_recovery()':
-            self.results = [(False, )]
+            self.results = [(1, 2)]
+        elif sql.startswith('SELECT pg_is_in_recovery()'):
+            self.results = [(False, 2)]
         elif sql.startswith('WITH replication_info AS ('):
             replication_info = '[{"application_name":"walreceiver","client_addr":"1.2.3.4",' +\
                                '"state":"streaming","sync_state":"async","sync_priority":0}]'
-            self.results = [('', True, '', '', '', '', False, replication_info)]
+            self.results = [('', 0, '', '', '', '', False, replication_info)]
         elif sql.startswith('SELECT name, setting'):
             self.results = [('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
                             ('search_path', 'public', None, 'string', 'user'),
@@ -51,6 +51,11 @@ class MockCursor(object):
                             ('unix_socket_directories', '/tmp', None, 'string', 'postmaster')]
         elif sql.startswith('IDENTIFY_SYSTEM'):
             self.results = [('1', 2, '0/402EEC0', '')]
+        elif sql.startswith('SELECT isdir, modification'):
+            self.results = [(False, datetime.datetime.now())]
+        elif sql.startswith('SELECT pg_read_file'):
+            self.results = [('1\t0/40159C0\tno recovery target specified\n\n' +
+                             '2\t1/40159C0\tno recovery target specified\n',)]
         elif sql.startswith('TIMELINE_HISTORY '):
             self.results = [('', b'x\t0/40159C0\tno recovery target specified\n\n' +
                                  b'1\t0/40159C0\tno recovery target specified\n\n' +
@@ -96,6 +101,15 @@ class MockConnect(object):
         pass
 
 
+class MockPostmaster(object):
+    def __init__(self, is_running=True, is_single_master=False):
+        self.is_running = Mock(return_value=is_running)
+        self.is_single_master = Mock(return_value=is_single_master)
+        self.wait_for_user_backends_to_close = Mock()
+        self.signal_stop = Mock(return_value=None)
+        self.wait = Mock()
+
+
 def pg_controldata_string(*args, **kwargs):
     return b"""
 pg_control version number:            942
@@ -196,12 +210,11 @@ class TestPostgresql(unittest.TestCase):
     def tearDown(self):
         shutil.rmtree('data')
 
-    def test_get_initdb_options(self):
-        self.assertEquals(self.p.get_initdb_options([{'encoding': 'UTF8'}, 'data-checksums']),
-                          ['--encoding=UTF8', '--data-checksums'])
-        self.assertRaises(Exception, self.p.get_initdb_options, [{'pgdata': 'bar'}])
-        self.assertRaises(Exception, self.p.get_initdb_options, [{'foo': 'bar', 1: 2}])
-        self.assertRaises(Exception, self.p.get_initdb_options, [1])
+    def test__initdb(self):
+        self.assertRaises(Exception, self.p.bootstrap, {'initdb': [{'pgdata': 'bar'}]})
+        self.assertRaises(Exception, self.p.bootstrap, {'initdb': [{'foo': 'bar', 1: 2}]})
+        self.assertRaises(Exception, self.p.bootstrap, {'initdb': [1]})
+        self.assertRaises(Exception, self.p.bootstrap, {'initdb': 1})
 
     @patch('os.path.exists', Mock(return_value=True))
     @patch('os.unlink', Mock())
@@ -213,100 +226,84 @@ class TestPostgresql(unittest.TestCase):
     @patch.object(Postgresql, 'wait_for_port_open')
     @patch.object(Postgresql, 'is_running')
     def test_start(self, mock_is_running, mock_wait_for_port_open, mock_wait_for_startup, mock_popen):
-        mock_is_running.return_value = True
+        mock_is_running.return_value = MockPostmaster()
         mock_wait_for_port_open.return_value = True
         mock_wait_for_startup.return_value = False
         mock_popen.return_value.stdout.readline.return_value = '123'
         self.assertTrue(self.p.start())
-        mock_is_running.return_value = False
-        open(os.path.join(self.data_dir, 'postmaster.pid'), 'w').close()
-        pg_conf = os.path.join(self.data_dir, 'postgresql.conf')
-        open(pg_conf, 'w').close()
-        self.assertFalse(self.p.start(task=CriticalTask()))
-        with open(pg_conf) as f:
-            lines = f.readlines()
-            self.assertTrue("f.oo = 'bar'\n" in lines)
+        mock_is_running.return_value = None
 
-        mock_wait_for_startup.return_value = None
-        self.assertFalse(self.p.start(10))
-        self.assertIsNone(self.p.start())
+        mock_postmaster = MockPostmaster()
+        with patch.object(PostmasterProcess, 'start', return_value=mock_postmaster):
+            pg_conf = os.path.join(self.data_dir, 'postgresql.conf')
+            open(pg_conf, 'w').close()
+            self.assertFalse(self.p.start(task=CriticalTask()))
 
-        mock_wait_for_port_open.return_value = False
+            with open(pg_conf) as f:
+                lines = f.readlines()
+                self.assertTrue("f.oo = 'bar'\n" in lines)
+
+            mock_wait_for_startup.return_value = None
+            self.assertFalse(self.p.start(10))
+            self.assertIsNone(self.p.start())
+
+            mock_wait_for_port_open.return_value = False
+            self.assertFalse(self.p.start())
+            task = CriticalTask()
+            task.cancel()
+            self.assertFalse(self.p.start(task=task))
+
+        self.p.cancel()
         self.assertFalse(self.p.start())
-        task = CriticalTask()
-        task.cancel()
-        self.assertFalse(self.p.start(task=task))
 
     @patch.object(Postgresql, 'pg_isready')
-    @patch.object(Postgresql, 'read_pid_file')
-    @patch.object(Postgresql, '_is_postmaster_pid_running')
     @patch('patroni.postgresql.polling_loop', Mock(return_value=range(1)))
-    def test_wait_for_port_open(self, mock_is_postmaster_pid_running, mock_read_pid_file, mock_pg_isready):
-        mock_is_postmaster_pid_running.return_value = False
+    def test_wait_for_port_open(self, mock_pg_isready):
         mock_pg_isready.return_value = STATE_NO_RESPONSE
+        mock_postmaster = MockPostmaster(is_running=False)
 
         # No pid file and postmaster death
-        mock_read_pid_file.return_value = {}
-        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
+        self.assertFalse(self.p.wait_for_port_open(mock_postmaster, 1))
 
-        mock_is_postmaster_pid_running.return_value = True
+        mock_postmaster.is_running.return_value = True
 
         # timeout
-        mock_read_pid_file.return_value = {'pid', 1}
-        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
-
-        # Garbage pid
-        mock_read_pid_file.return_value = {'pid': 'garbage', 'start_time': '101', 'data_dir': '',
-                                           'socket_dir': '', 'port': '', 'listen_addr': ''}
-        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
-
-        # Not ready
-        mock_read_pid_file.return_value = {'pid': '42', 'start_time': '101', 'data_dir': '',
-                                           'socket_dir': '', 'port': '', 'listen_addr': ''}
-        self.assertFalse(self.p.wait_for_port_open(42, 100., 1))
+        self.assertFalse(self.p.wait_for_port_open(mock_postmaster, 1))
 
         # pg_isready failure
         mock_pg_isready.return_value = 'garbage'
-        self.assertTrue(self.p.wait_for_port_open(42, 100., 1))
+        self.assertTrue(self.p.wait_for_port_open(mock_postmaster, 1))
+
+        # cancelled
+        self.p.cancel()
+        self.assertFalse(self.p.wait_for_port_open(mock_postmaster, 1))
 
     @patch('time.sleep', Mock())
     @patch.object(Postgresql, 'is_running')
-    @patch.object(Postgresql, 'get_pid')
-    def test_stop(self, mock_get_pid, mock_is_running):
+    @patch.object(Postgresql, '_wait_for_connection_close', Mock())
+    def test_stop(self, mock_is_running):
+        # Postmaster is not running
         mock_callback = Mock()
-        mock_is_running.return_value = False
+        mock_is_running.return_value = None
         self.assertTrue(self.p.stop(on_safepoint=mock_callback))
         mock_callback.assert_called()
 
-        with patch.object(Postgresql, '_is_postmaster_pid_running', Mock(return_value=False)), \
-                patch.object(Postgresql, 'data_directory_empty', Mock(return_value=True)):
-            with patch('psutil.Process') as mock_psutil:
-                self.p._postmaster_cached_info = {'pid': 1, 'start_time': 1}
-                mock_psutil.return_value.pid = 1
-                mock_psutil.return_value.create_time.return_value = 1
-                self.assertTrue(self.p.stop())
-                self.p._postmaster_cached_info = {'pid': 1, 'start_time': 1}
-                mock_psutil.return_value.create_time.return_value = 100
-                self.assertTrue(self.p.stop())
-                self.p._postmaster_cached_info = {'pid': 1, 'start_time': 1}
-                mock_psutil.side_effect = psutil.NoSuchProcess('')
-                self.assertTrue(self.p.stop())
-
-        mock_is_running.return_value = True
-        mock_get_pid.return_value = 0
+        # Is running, stopped successfully
+        mock_is_running.return_value = mock_postmaster = MockPostmaster()
         mock_callback.reset_mock()
         self.assertTrue(self.p.stop(on_safepoint=mock_callback))
         mock_callback.assert_called()
-        mock_get_pid.return_value = -1
+        mock_postmaster.signal_stop.assert_called()
+
+        # Stop signal failed
+        mock_postmaster.signal_stop.return_value = False
         self.assertFalse(self.p.stop())
-        mock_get_pid.return_value = 123
-        with patch('os.kill', Mock(side_effect=[OSError(errno.ESRCH, ''), OSError, None])):
-            self.assertTrue(self.p.stop())
-            self.assertFalse(self.p.stop())
-            self.assertTrue(self.p.stop())
-        with patch.object(Postgresql, '_signal_postmaster_stop', Mock(return_value=(123, None))):
-            with patch.object(Postgresql, '_is_postmaster_pid_running', Mock(side_effect=[True, False, False])):
-                self.assertTrue(self.p.stop())
+
+        # Stop signal failed to find process
+        mock_postmaster.signal_stop.return_value = True
+        mock_callback.reset_mock()
+        self.assertTrue(self.p.stop(on_safepoint=mock_callback))
+        mock_callback.assert_called()
 
     def test_restart(self):
         self.p.start = Mock(return_value=False)
@@ -325,12 +322,13 @@ class TestPostgresql(unittest.TestCase):
             self.assertIsNone(self.p.checkpoint())
         self.assertEquals(self.p.checkpoint(), 'not accessible or not healty')
 
-    @patch('subprocess.call', side_effect=OSError)
+    @patch.object(Postgresql, 'cancellable_subprocess_call')
     @patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
-    def test_pg_rewind(self, mock_call):
+    def test_pg_rewind(self, mock_cancellable_subprocess_call):
         r = {'user': '', 'host': '', 'port': '', 'database': '', 'password': ''}
+        mock_cancellable_subprocess_call.return_value = 0
         self.assertTrue(self.p.pg_rewind(r))
-        subprocess.call = mock_call
+        mock_cancellable_subprocess_call.side_effect = OSError
         self.assertFalse(self.p.pg_rewind(r))
 
     def test_check_recovery_conf(self):
@@ -354,7 +352,7 @@ class TestPostgresql(unittest.TestCase):
 
     @patch.object(Postgresql, 'start', Mock())
     @patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
-    @patch.object(Postgresql, '_get_local_timeline_lsn', Mock(return_value=(2, '0/40159C1')))
+    @patch.object(Postgresql, '_get_local_timeline_lsn', Mock(return_value=(2, '40159C1')))
     @patch.object(Postgresql, 'check_leader_is_not_in_recovery')
     def test__check_timeline_and_lsn(self, mock_check_leader_is_not_in_recovery):
         mock_check_leader_is_not_in_recovery.return_value = False
@@ -365,12 +363,8 @@ class TestPostgresql(unittest.TestCase):
         self.p.trigger_check_diverged_lsn()
         with patch('psycopg2.connect', Mock(side_effect=Exception)):
             self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
-        with patch.object(MockCursor, 'fetchone',
-                          Mock(side_effect=[('', 2, '0/0'), ('', b'2\tG/40159C0\tno recovery target specified\n\n')])):
-            self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
         self.p.trigger_check_diverged_lsn()
-        with patch.object(MockCursor, 'fetchone',
-                          Mock(side_effect=[('', 2, '0/0'), ('', b'3\t040159C0\tno recovery target specified\n')])):
+        with patch.object(MockCursor, 'fetchone', Mock(side_effect=[('', 2, '0/0'), ('', b'3\t0/40159C0\tn\n')])):
             self.assertFalse(self.p.rewind_needed_and_possible(self.leader))
         self.p.trigger_check_diverged_lsn()
         with patch.object(MockCursor, 'fetchone', Mock(return_value=('', 1, '0/0'))):
@@ -384,6 +378,7 @@ class TestPostgresql(unittest.TestCase):
         self.p.check_leader_is_not_in_recovery()
         self.p.check_leader_is_not_in_recovery()
 
+    @patch.object(Postgresql, 'cancellable_subprocess_call', Mock(return_value=0))
     @patch.object(Postgresql, 'checkpoint', side_effect=['', '1'])
     @patch.object(Postgresql, 'stop', Mock(return_value=False))
     @patch.object(Postgresql, 'start', Mock())
@@ -420,31 +415,89 @@ class TestPostgresql(unittest.TestCase):
         self.assertFalse(self.p.can_rewind)
 
     @patch('time.sleep', Mock())
+    @patch.object(Postgresql, 'cancellable_subprocess_call')
     @patch.object(Postgresql, 'remove_data_directory', Mock(return_value=True))
-    def test_create_replica(self):
+    def test_create_replica(self, mock_cancellable_subprocess_call):
+        self.p.delete_trigger_file = Mock(side_effect=OSError)
+
+        self.p.config['create_replica_methods'] = ['wale', 'basebackup']
+        self.p.config['wale'] = {'command': 'foo'}
+        mock_cancellable_subprocess_call.return_value = 0
+        self.assertEquals(self.p.create_replica(self.leader), 0)
+        del self.p.config['wale']
+        self.assertEquals(self.p.create_replica(self.leader), 0)
+
+        self.p.config['create_replica_methods'] = ['basebackup']
+        self.p.config['basebackup'] = [{'max_rate': '100M'}, 'no-sync']
+        self.assertEquals(self.p.create_replica(self.leader), 0)
+
+        self.p.config['basebackup'] = [{'max_rate': '100M', 'compress': '9'}]
+        with mock.patch('patroni.postgresql.logger.error', new_callable=Mock()) as mock_logger:
+            self.p.create_replica(self.leader)
+            mock_logger.assert_called_once()
+            self.assertTrue("only one key-value is allowed and value should be a string" in mock_logger.call_args[0][0],
+                            "not matching {0}".format(mock_logger.call_args[0][0]))
+
+        self.p.config['basebackup'] = [42]
+        with mock.patch('patroni.postgresql.logger.error', new_callable=Mock()) as mock_logger:
+            self.p.create_replica(self.leader)
+            mock_logger.assert_called_once()
+            self.assertTrue("value should be string value or a single key-value pair" in mock_logger.call_args[0][0],
+                            "not matching {0}".format(mock_logger.call_args[0][0]))
+
+        self.p.config['basebackup'] = {"foo": "bar"}
+        self.assertEquals(self.p.create_replica(self.leader), 0)
+
+        self.p.config['create_replica_methods'] = ['wale', 'basebackup']
+        del self.p.config['basebackup']
+        mock_cancellable_subprocess_call.return_value = 1
+        self.assertEquals(self.p.create_replica(self.leader), 1)
+
+        mock_cancellable_subprocess_call.side_effect = Exception('foo')
+        self.assertEquals(self.p.create_replica(self.leader), 1)
+
+        mock_cancellable_subprocess_call.side_effect = [1, 0]
+        self.assertEquals(self.p.create_replica(self.leader), 0)
+
+        mock_cancellable_subprocess_call.side_effect = [Exception(), 0]
+        self.assertEquals(self.p.create_replica(self.leader), 0)
+
+        self.p.cancel()
+        self.assertEquals(self.p.create_replica(self.leader), 1)
+
+    @patch('time.sleep', Mock())
+    @patch.object(Postgresql, 'cancellable_subprocess_call')
+    @patch.object(Postgresql, 'remove_data_directory', Mock(return_value=True))
+    def test_create_replica_old_format(self, mock_cancellable_subprocess_call):
+        """ The same test as before but with old 'create_replica_method'
+            to test backward compatibility
+        """
         self.p.delete_trigger_file = Mock(side_effect=OSError)
-        with patch('subprocess.call', Mock(side_effect=[1, 0])):
-            self.assertEquals(self.p.create_replica(self.leader), 0)
-        with patch('subprocess.call', Mock(side_effect=[Exception(), 0])):
-            self.assertEquals(self.p.create_replica(self.leader), 0)
 
         self.p.config['create_replica_method'] = ['wale', 'basebackup']
         self.p.config['wale'] = {'command': 'foo'}
-        with patch('subprocess.call', Mock(return_value=0)):
-            self.assertEquals(self.p.create_replica(self.leader), 0)
-            del self.p.config['wale']
-            self.assertEquals(self.p.create_replica(self.leader), 0)
+        mock_cancellable_subprocess_call.return_value = 0
+        self.assertEquals(self.p.create_replica(self.leader), 0)
+        del self.p.config['wale']
+        self.assertEquals(self.p.create_replica(self.leader), 0)
 
-        with patch('subprocess.call', Mock(side_effect=Exception("foo"))):
-            self.assertEquals(self.p.create_replica(self.leader), 1)
+        self.p.config['create_replica_method'] = ['basebackup']
+        self.p.config['basebackup'] = [{'max_rate': '100M'}, 'no-sync']
+        self.assertEquals(self.p.create_replica(self.leader), 0)
 
-        with patch('subprocess.call', Mock(return_value=1)):
-            self.assertEquals(self.p.create_replica(self.leader), 1)
+        self.p.config['create_replica_method'] = ['wale', 'basebackup']
+        del self.p.config['basebackup']
+        mock_cancellable_subprocess_call.return_value = 1
+        self.assertEquals(self.p.create_replica(self.leader), 1)
+
+    def test_basebackup(self):
+        self.p.cancel()
+        self.p.basebackup(None, None, {'foo': 'bar'})
 
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def test_sync_replication_slots(self):
         self.p.start()
-        cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None, None)
+        cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None, None, None)
         with mock.patch('patroni.postgresql.Postgresql._query', Mock(side_effect=psycopg2.OperationalError)):
             self.p.sync_replication_slots(cluster)
         self.p.sync_replication_slots(cluster)
@@ -457,8 +510,10 @@ class TestPostgresql(unittest.TestCase):
             cluster.members.extend([alias1, alias2])
             self.p.sync_replication_slots(cluster)
             errorlog_mock.assert_called_once()
-            assert "test-3" in errorlog_mock.call_args[0][1]
-            assert "test.3" in errorlog_mock.call_args[0][1]
+            self.assertTrue("test-3" in errorlog_mock.call_args[0][1],
+                            "non matching {0}".format(errorlog_mock.call_args[0][1]))
+            self.assertTrue("test.3" in errorlog_mock.call_args[0][1],
+                            "non matching {0}".format(errorlog_mock.call_args[0][1]))
 
     @patch.object(MockCursor, 'execute', Mock(side_effect=psycopg2.OperationalError))
     def test__query(self):
@@ -474,6 +529,7 @@ class TestPostgresql(unittest.TestCase):
     @patch.object(Postgresql, 'pg_isready', Mock(return_value=STATE_REJECT))
     def test_is_leader(self):
         self.assertTrue(self.p.is_leader())
+        self.p.reset_cluster_info_state()
         with patch.object(Postgresql, '_query', Mock(side_effect=RetryFailedError(''))):
             self.assertRaises(PostgresConnectionException, self.p.is_leader)
 
@@ -489,21 +545,29 @@ class TestPostgresql(unittest.TestCase):
 
     def test_promote(self):
         self.p.set_role('replica')
-        self.assertTrue(self.p.promote())
-        self.assertTrue(self.p.promote())
+        self.assertIsNone(self.p.promote(0))
+        self.assertTrue(self.p.promote(0))
 
-    def test_last_operation(self):
-        self.assertEquals(self.p.last_operation(), '2')
-        Thread(target=self.p.last_operation).start()
+    def test_timeline_wal_position(self):
+        self.assertEquals(self.p.timeline_wal_position(), (1, 2))
+        Thread(target=self.p.timeline_wal_position).start()
 
-    @patch('os.path.isfile', Mock(return_value=True))
-    @patch('os.kill', Mock(side_effect=Exception))
-    @patch('os.getpid', Mock(return_value=2))
-    @patch('os.getppid', Mock(return_value=2))
-    @patch.object(builtins, 'open', mock_open(read_data='-1'))
-    @patch.object(Postgresql, '_version_file_exists', Mock(return_value=True))
-    def test_is_running(self):
-        self.assertFalse(self.p.is_running())
+    @patch.object(PostmasterProcess, 'from_pidfile')
+    def test_is_running(self, mock_frompidfile):
+        # Cached postmaster running
+        mock_postmaster = self.p._postmaster_proc = MockPostmaster()
+        self.assertEquals(self.p.is_running(), mock_postmaster)
+
+        # Cached postmaster not running, no postmaster running
+        mock_postmaster.is_running.return_value = False
+        mock_frompidfile.return_value = None
+        self.assertEquals(self.p.is_running(), None)
+        self.assertEquals(self.p._postmaster_proc, None)
+
+        # No cached postmaster, postmaster running
+        mock_frompidfile.return_value = mock_postmaster2 = MockPostmaster()
+        self.assertEquals(self.p.is_running(), mock_postmaster2)
+        self.assertEquals(self.p._postmaster_proc, mock_postmaster2)
 
     @patch('shlex.split', Mock(side_effect=OSError))
     def test_call_nowait(self):
@@ -515,7 +579,7 @@ class TestPostgresql(unittest.TestCase):
     def test_non_existing_callback(self):
         self.assertFalse(self.p.call_nowait('foobar'))
 
-    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
+    @patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
     def test_is_leader_exception(self):
         self.p.start()
         self.p.query = Mock(side_effect=psycopg2.OperationalError("not supported"))
@@ -550,14 +614,16 @@ class TestPostgresql(unittest.TestCase):
             lines = f.readlines()
             self.assertTrue('host replication replicator 127.0.0.1/32 md5\n' in lines)
 
-    def test_custom_bootstrap(self):
+    @patch.object(Postgresql, 'cancellable_subprocess_call')
+    def test_custom_bootstrap(self, mock_cancellable_subprocess_call):
+        self.p.config.pop('pg_hba')
         config = {'method': 'foo', 'foo': {'command': 'bar'}}
-        with patch('subprocess.call', Mock(return_value=1)):
-            self.assertFalse(self.p.bootstrap(config))
-        with patch('subprocess.call', Mock(side_effect=Exception)):
-            self.assertFalse(self.p.bootstrap(config))
-        with patch('subprocess.call', Mock(return_value=0)),\
-                patch('subprocess.Popen', Mock(side_effect=Exception("42"))),\
+
+        mock_cancellable_subprocess_call.return_value = 1
+        self.assertFalse(self.p.bootstrap(config))
+
+        mock_cancellable_subprocess_call.return_value = 0
+        with patch('subprocess.Popen', Mock(side_effect=Exception("42"))),\
                 patch('os.path.isfile', Mock(return_value=True)),\
                 patch('os.unlink', Mock()),\
                 patch.object(Postgresql, 'save_configuration_files', Mock()),\
@@ -573,6 +639,9 @@ class TestPostgresql(unittest.TestCase):
                 self.p.bootstrap(config)
             self.assertEqual(str(e.exception), '42')
 
+        mock_cancellable_subprocess_call.side_effect = Exception
+        self.assertFalse(self.p.bootstrap(config))
+
     @patch('time.sleep', Mock())
     @patch('os.unlink', Mock())
     @patch.object(Postgresql, 'run_bootstrap_post_init', Mock(return_value=True))
@@ -591,6 +660,12 @@ class TestPostgresql(unittest.TestCase):
         self.p.post_bootstrap({}, task)
         self.assertTrue(task.result)
 
+        self.p.bootstrap(config)
+        with patch.object(Postgresql, 'pending_restart', PropertyMock(return_value=True)), \
+                patch.object(Postgresql, 'restart', Mock()) as mock_restart:
+            self.p.post_bootstrap({}, task)
+            mock_restart.assert_called_once()
+
         self.p.bootstrap(config)
         self.p.set_state('stopped')
         self.p.reload_config({'authentication': {'superuser': {'username': 'p', 'password': 'p'},
@@ -600,26 +675,27 @@ class TestPostgresql(unittest.TestCase):
             self.p.post_bootstrap({}, task)
             mock_restart.assert_called_once()
 
-    def test_run_bootstrap_post_init(self):
-        with patch('subprocess.call', Mock(return_value=1)):
-            self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+    @patch.object(Postgresql, 'cancellable_subprocess_call')
+    def test_run_bootstrap_post_init(self, mock_cancellable_subprocess_call):
+        mock_cancellable_subprocess_call.return_value = 1
+        self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
 
-        with patch('subprocess.call', Mock(side_effect=OSError)):
-            self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+        mock_cancellable_subprocess_call.return_value = 0
+        self.p._superuser.pop('username')
+        self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+        mock_cancellable_subprocess_call.assert_called()
+        args, kwargs = mock_cancellable_subprocess_call.call_args
+        self.assertTrue('PGPASSFILE' in kwargs['env'])
+        self.assertEquals(args[0], ['/bin/false', 'postgres://127.0.0.2:5432/postgres'])
 
-        with patch('subprocess.call', Mock(return_value=0)) as mock_method:
-            self.p._superuser.pop('username')
-            self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
-            mock_method.assert_called()
-            args, kwargs = mock_method.call_args
-            self.assertTrue('PGPASSFILE' in kwargs['env'])
-            self.assertEquals(args[0], ['/bin/false', 'postgres://127.0.0.2:5432/postgres'])
+        mock_cancellable_subprocess_call.reset_mock()
+        self.p._local_address.pop('host')
+        self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
+        mock_cancellable_subprocess_call.assert_called()
+        self.assertEquals(mock_cancellable_subprocess_call.call_args[0][0], ['/bin/false', 'postgres://:5432/postgres'])
 
-            mock_method.reset_mock()
-            self.p._local_address.pop('host')
-            self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
-            mock_method.assert_called()
-            self.assertEquals(mock_method.call_args[0][0], ['/bin/false', 'postgres://:5432/postgres'])
+        mock_cancellable_subprocess_call.side_effect = OSError
+        self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
 
     @patch('patroni.postgresql.Postgresql.create_replica', Mock(return_value=0))
     def test_clone(self):
@@ -745,10 +821,12 @@ class TestPostgresql(unittest.TestCase):
 
     def test_wait_for_startup(self):
         state = {'sleeps': 0, 'num_rejects': 0, 'final_return': 0}
+        self.__thread_ident = current_thread().ident
 
         def increment_sleeps(*args):
-            print("Sleep")
-            state['sleeps'] += 1
+            if current_thread().ident == self.__thread_ident:
+                print("Sleep")
+                state['sleeps'] += 1
 
         def isready_return(*args):
             ret = 1 if state['sleeps'] < state['num_rejects'] else state['final_return']
@@ -782,29 +860,14 @@ class TestPostgresql(unittest.TestCase):
                 self.assertFalse(self.p.wait_for_startup(timeout=2))
                 self.assertEquals(state['sleeps'], 3)
 
-    def test_read_pid_file(self):
-        pidfile = os.path.join(self.data_dir, 'postmaster.pid')
-        if os.path.exists(pidfile):
-            os.remove(pidfile)
-        self.assertEquals(self.p.read_pid_file(), {})
-
-    @patch.object(Postgresql, '_version_file_exists', Mock(return_value=True))
-    @patch('os.path.isfile', Mock(return_value=True))
-    @patch.object(Postgresql, 'read_pid_file')
-    @patch('psutil.Process')
-    def test_is_postmaster_pid_running(self, mock_psutil, mock_read_pid_file):
-        mock_psutil.return_value.create_time.return_value = 1
-        mock_read_pid_file.return_value = {'pid': -100, 'start_time': 1}
-        self.assertTrue(self.p.is_running())
-        with patch('os.getpid', Mock(return_value=100)):
-            mock_read_pid_file.return_value = {'pid': 100, 'start_time': 1}
-            self.assertFalse(self.p.is_running())
-        mock_read_pid_file.return_value = {'pid': 100, 'start_time': 100}
-        self.assertFalse(self.p.is_running())
+        with patch.object(Postgresql, 'check_startup_state_changed', Mock(return_value=False)):
+            self.p.cancel()
+            self.p._state = 'starting'
+            self.assertIsNone(self.p.wait_for_startup())
 
     def test_pick_sync_standby(self):
         cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None,
-                          SyncState(0, self.me.name, self.leadermem.name))
+                          SyncState(0, self.me.name, self.leadermem.name), None)
 
         with patch.object(Postgresql, "query", return_value=[
                     (self.leadermem.name, 'streaming', 'sync'),
@@ -870,43 +933,23 @@ class TestPostgresql(unittest.TestCase):
         self.p.set_synchronous_standby('foo')
         self.p.get_server_parameters(config)
 
-    @patch.object(Postgresql, 'read_pid_file', Mock(return_value={'pid': 'z'}))
-    def test_get_pid(self):
-        self.p.get_pid()
-
-    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
-    @patch.object(Postgresql, '_signal_postmaster_stop', Mock(return_value=(123, None)))
-    @patch.object(Postgresql, 'get_pid', Mock(return_value=123))
     @patch('time.sleep', Mock())
-    @patch.object(Postgresql, '_is_postmaster_pid_running')
-    def test__wait_for_connection_close(self, mock_is_postmaster_pid_running):
-        mock_is_postmaster_pid_running.side_effect = [True, False, False]
-        mock_callback = Mock()
-        self.p.stop(on_safepoint=mock_callback)
-
-        mock_is_postmaster_pid_running.side_effect = [True, False, False]
-        with patch.object(MockCursor, "execute", Mock(side_effect=psycopg2.Error)):
+    def test__wait_for_connection_close(self):
+        mock_postmaster = MockPostmaster()
+        with patch.object(Postgresql, 'is_running', Mock(return_value=mock_postmaster)):
+            mock_postmaster.is_running.side_effect = [True, False, False]
+            mock_callback = Mock()
             self.p.stop(on_safepoint=mock_callback)
 
-    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
-    @patch.object(Postgresql, '_signal_postmaster_stop', Mock(return_value=(123, None)))
-    @patch.object(Postgresql, 'get_pid', Mock(return_value=123))
-    @patch.object(Postgresql, '_is_postmaster_pid_running', Mock(return_value=False))
-    @patch('psutil.Process')
-    def test__wait_for_user_backends_to_close(self, mock_psutil):
-        child = Mock()
-        child.cmdline.return_value = ['foo']
-        mock_psutil.return_value.children.return_value = [child]
-        mock_callback = Mock()
-        self.p.stop(on_safepoint=mock_callback)
+            mock_postmaster.is_running.side_effect = [True, False, False]
+            with patch.object(MockCursor, "execute", Mock(side_effect=psycopg2.Error)):
+                self.p.stop(on_safepoint=mock_callback)
 
-    @patch('os.kill', Mock(side_effect=[OSError(errno.ESRCH, ''), OSError]))
-    @patch('psutil.Process', Mock(side_effect=[psutil.NoSuchProcess]))
-    @patch('time.sleep', Mock())
-    @patch.object(Postgresql, '_is_postmaster_pid_running', Mock(side_effect=[True, False]))
     def test_terminate_starting_postmaster(self):
-        self.p.terminate_starting_postmaster(123)
-        self.p.terminate_starting_postmaster(123)
+        mock_postmaster = MockPostmaster()
+        self.p.terminate_starting_postmaster(mock_postmaster)
+        mock_postmaster.signal_stop.assert_called()
+        mock_postmaster.wait.assert_called()
 
     def test_read_postmaster_opts(self):
         m = mock_open(read_data='/usr/lib/postgres/9.6/bin/postgres "-D" "data/postgresql0" \
@@ -923,13 +966,9 @@ class TestPostgresql(unittest.TestCase):
             self.assertEqual(data, dict())
 
     @patch('subprocess.Popen')
-    @patch.object(builtins, 'open', Mock(return_value=42))
     def test_single_user_mode(self, subprocess_popen_mock):
         subprocess_popen_mock.return_value.wait.return_value = 0
-        self.assertEquals(self.p.single_user_mode(command="CHECKPOINT"), 0)
-        subprocess_popen_mock.return_value = None
-        self.assertEquals(self.p.single_user_mode(), 1)
-        self.assertEquals(self.p.single_user_mode(options={'archive_mode': 'on'}), 1)
+        self.assertEquals(self.p.single_user_mode('CHECKPOINT', {'archive_mode': 'on'}), 0)
 
     @patch('os.listdir', Mock(side_effect=[OSError, ['a', 'b']]))
     @patch('os.unlink', Mock(side_effect=OSError))
@@ -946,10 +985,31 @@ class TestPostgresql(unittest.TestCase):
     def test_fix_cluster_state(self):
         self.assertTrue(self.p.fix_cluster_state())
 
-    def test__update_postmaster_cached_info(self):
-        with open(os.path.join(self.data_dir, 'postmaster.pid'), 'w') as f:
-            f.write('1\n\n1\n')
-        self.p.read_pid_file()
-        with open(os.path.join(self.data_dir, 'postmaster.pid'), 'w') as f:
-            f.write('a\n\n1\n')
-        self.p.read_pid_file()
+    def test_replica_cached_timeline(self):
+        self.assertEquals(self.p.replica_cached_timeline(1), 2)
+
+    def test_get_master_timeline(self):
+        self.assertEquals(self.p.get_master_timeline(), 1)
+
+    def test_cancellable_subprocess_call(self):
+        self.p.cancel()
+        self.assertRaises(PostgresException, self.p.cancellable_subprocess_call, communicate_input=None)
+
+    @patch('patroni.postgresql.polling_loop', Mock(return_value=[0, 0]))
+    def test_cancel(self):
+        self.p._cancellable = Mock()
+        self.p._cancellable.returncode = None
+        self.p.cancel()
+        type(self.p._cancellable).returncode = PropertyMock(side_effect=[None, -15])
+        self.p.cancel()
+
+    @patch.object(Postgresql, 'get_postgres_role_from_data_directory', Mock(return_value='replica'))
+    def test__build_effective_configuration(self):
+        with patch.object(Postgresql, 'controldata',
+                          Mock(return_value={'max_connections setting': '200',
+                                             'max_worker_processes setting': '20',
+                                             'max_prepared_xacts setting': '100',
+                                             'max_locks_per_xact setting': '100'})):
+            self.p.cancel()
+            self.assertFalse(self.p.start())
+            self.assertTrue(self.p.pending_restart)

tests/test_postmaster.py

@@ -0,0 +1,103 @@
+import psutil
+import unittest
+
+from mock import Mock, patch, mock_open
+from patroni.postmaster import PostmasterProcess
+from six.moves import builtins
+
+
+class TestPostmasterProcess(unittest.TestCase):
+    @patch('psutil.Process.__init__', Mock())
+    def test_init(self):
+        proc = PostmasterProcess(-123)
+        self.assertTrue(proc.is_single_user)
+
+    @patch('psutil.Process.create_time')
+    @patch('psutil.Process.__init__')
+    @patch('patroni.postmaster.PostmasterProcess._read_postmaster_pidfile')
+    def test_from_pidfile(self, mock_read, mock_init, mock_create_time):
+        mock_init.side_effect = psutil.NoSuchProcess(123)
+        mock_read.return_value = {}
+        self.assertIsNone(PostmasterProcess.from_pidfile(''))
+        mock_read.return_value = {"pid": "foo"}
+        self.assertIsNone(PostmasterProcess.from_pidfile(''))
+        mock_read.return_value = {"pid": "123"}
+        self.assertIsNone(PostmasterProcess.from_pidfile(''))
+
+        mock_init.side_effect = None
+        with patch.object(psutil.Process, 'pid', 123), \
+                patch.object(psutil.Process, 'ppid', return_value=124), \
+                patch('os.getpid', return_value=125) as mock_ospid, \
+                patch('os.getppid', return_value=126):
+
+            self.assertIsNotNone(PostmasterProcess.from_pidfile(''))
+
+            mock_create_time.return_value = 100000
+            mock_read.return_value = {"pid": "123", "start_time": "200000"}
+            self.assertIsNone(PostmasterProcess.from_pidfile(''))
+
+            mock_read.return_value = {"pid": "123", "start_time": "foobar"}
+            self.assertIsNotNone(PostmasterProcess.from_pidfile(''))
+
+            mock_ospid.return_value = 123
+            mock_read.return_value = {"pid": "123", "start_time": "100000"}
+            self.assertIsNone(PostmasterProcess.from_pidfile(''))
+
+    @patch('psutil.Process.__init__')
+    def test_from_pid(self, mock_init):
+        mock_init.side_effect = psutil.NoSuchProcess(123)
+        self.assertEquals(PostmasterProcess.from_pid(123), None)
+        mock_init.side_effect = None
+        self.assertNotEquals(PostmasterProcess.from_pid(123), None)
+
+    @patch('psutil.Process.__init__', Mock())
+    @patch('psutil.Process.send_signal')
+    @patch('psutil.Process.pid', Mock(return_value=123))
+    def test_signal_stop(self, mock_send_signal):
+        proc = PostmasterProcess(-123)
+        self.assertEquals(proc.signal_stop('immediate'), False)
+
+        mock_send_signal.side_effect = [None, psutil.NoSuchProcess(123), psutil.AccessDenied()]
+        proc = PostmasterProcess(123)
+        self.assertEquals(proc.signal_stop('immediate'), None)
+        self.assertEquals(proc.signal_stop('immediate'), True)
+        self.assertEquals(proc.signal_stop('immediate'), False)
+
+    @patch('psutil.Process.__init__', Mock())
+    @patch('psutil.wait_procs')
+    def test_wait_for_user_backends_to_close(self, mock_wait):
+        c1 = Mock()
+        c1.cmdline = Mock(return_value=["postgres: startup process"])
+        c2 = Mock()
+        c2.cmdline = Mock(return_value=["postgres: postgres postgres [local] idle"])
+        c3 = Mock()
+        c3.cmdline = Mock(side_effect=psutil.NoSuchProcess(123))
+        with patch('psutil.Process.children', Mock(return_value=[c1, c2, c3])):
+            proc = PostmasterProcess(123)
+            self.assertIsNone(proc.wait_for_user_backends_to_close())
+            mock_wait.assert_called_with([c2])
+
+        c3.cmdline = Mock(side_effect=psutil.AccessDenied(123))
+        with patch('psutil.Process.children', Mock(return_value=[c3])):
+            proc = PostmasterProcess(123)
+            self.assertIsNone(proc.wait_for_user_backends_to_close())
+
+    @patch('subprocess.Popen')
+    @patch.object(PostmasterProcess, 'from_pid')
+    @patch.object(PostmasterProcess, '_from_pidfile')
+    def test_start(self, mock_frompidfile, mock_frompid, mock_popen):
+        mock_frompidfile.return_value._is_postmaster_process.return_value = False
+        mock_frompid.return_value = "proc 123"
+        mock_popen.return_value.stdout.readline.return_value = '123'
+        self.assertEquals(PostmasterProcess.start('true', '/tmp', '/tmp/test.conf', []), "proc 123")
+        mock_frompid.assert_called_with(123)
+
+        mock_frompidfile.side_effect = psutil.NoSuchProcess(123)
+        self.assertEquals(PostmasterProcess.start('true', '/tmp', '/tmp/test.conf', []), "proc 123")
+
+    @patch('psutil.Process.__init__', Mock(side_effect=psutil.NoSuchProcess(123)))
+    def test_read_postmaster_pidfile(self):
+        with patch.object(builtins, 'open', Mock(side_effect=IOError)):
+            self.assertIsNone(PostmasterProcess.from_pidfile(''))
+        with patch.object(builtins, 'open', mock_open(read_data='123\n')):
+            self.assertIsNone(PostmasterProcess.from_pidfile(''))

tests/test_wale_restore.py

@@ -7,6 +7,7 @@ from patroni.scripts import wale_restore
 from patroni.scripts.wale_restore import WALERestore, main as _main, get_major_version
 from six.moves import builtins
 from test_postgresql import MockConnect, psycopg2_connect
+from threading import current_thread
 
 
 wale_output_header = (
@@ -42,6 +43,13 @@ class TestWALERestore(unittest.TestCase):
                                         '/etc', 100, 100, 1, 0, WALE_TEST_RETRIES)
 
     def test_should_use_s3_to_create_replica(self):
+        self.__thread_ident = current_thread().ident
+        sleeps = [0]
+
+        def mock_sleep(*args):
+            if current_thread().ident == self.__thread_ident:
+                sleeps[0] += 1
+
         self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
         with patch.object(MockConnect, 'server_version', PropertyMock(return_value=100000)):
             self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
@@ -55,13 +63,11 @@ class TestWALERestore(unittest.TestCase):
 
             self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
 
-            with patch('time.sleep', Mock(return_value=None)) as mock_sleep:
+            with patch('time.sleep', mock_sleep):
                 self.wale_restore.no_master = 1
-                assert self.wale_restore.should_use_s3_to_create_replica()
+                self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
                 # verify retries
-                mock_sleep.assert_has_calls(
-                    [((wale_restore.RETRY_SLEEP_INTERVAL,),)] * WALE_TEST_RETRIES
-                )
+                self.assertEqual(sleeps[0], WALE_TEST_RETRIES)
 
             self.wale_restore.master_connection = ''
             self.assertTrue(self.wale_restore.should_use_s3_to_create_replica())
@@ -104,13 +110,20 @@ class TestWALERestore(unittest.TestCase):
 
     @patch('sys.exit', Mock())
     def test_main(self):
+        self.__thread_ident = current_thread().ident
+        sleeps = [0]
+
+        def mock_sleep(*args):
+            if current_thread().ident == self.__thread_ident:
+                sleeps[0] += 1
+
         with patch.object(WALERestore, 'run', Mock(return_value=0)):
             self.assertEqual(_main(), 0)
 
         with patch.object(WALERestore, 'run', Mock(return_value=1)), \
-                patch('time.sleep', Mock(return_value=None)) as mock_sleep:
+                patch('time.sleep', mock_sleep):
             self.assertEqual(_main(), 1)
-            assert mock_sleep.call_count == WALE_TEST_RETRIES
+            self.assertTrue(sleeps[0], WALE_TEST_RETRIES)
 
     @patch('os.path.isfile', Mock(return_value=True))
     def test_get_major_version(self):

tests/test_zookeeper.py

@@ -58,11 +58,11 @@ class MockKazooClient(Mock):
             raise TypeError("Invalid type for 'path' (string expected)")
         if not isinstance(value, (six.binary_type,)):
             raise TypeError("Invalid type for 'value' (must be a byte string)")
-        if value == b'Exception':
+        if b'Exception' in value:
             raise Exception
         if path.endswith('/initialize') or path == '/service/test/optime/leader':
             raise Exception
-        elif value == b'retry' or (value == b'exists' and self.exists):
+        elif b'retry' in value or (b'exists' in value and self.exists):
             raise NodeExistsError
 
     def create_async(self, path, value=b"", acl=None, ephemeral=False, sequence=False, makepath=False):
@@ -76,10 +76,10 @@ class MockKazooClient(Mock):
             raise TypeError("Invalid type for 'value' (must be a byte string)")
         if path == '/service/bla/optime/leader':
             raise Exception
-        if path == '/service/test/members/bar' and value == b'retry':
+        if path == '/service/test/members/bar' and b'retry' in value:
             return
         if path in ('/service/test/failover', '/service/test/config', '/service/test/sync'):
-            if value == b'Exception':
+            if b'Exception' in value:
                 raise Exception
             elif value == b'ok':
                 return
@@ -145,7 +145,7 @@ class TestZooKeeper(unittest.TestCase):
         self.assertRaises(ZooKeeperError, self.zk.get_cluster)
         cluster = self.zk.get_cluster()
         self.assertIsInstance(cluster.leader, Leader)
-        self.zk.touch_member('foo')
+        self.zk.touch_member({'foo': 'foo'})
 
     def test_delete_leader(self):
         self.assertTrue(self.zk.delete_leader())
@@ -169,17 +169,17 @@ class TestZooKeeper(unittest.TestCase):
     def test_touch_member(self):
         self.zk._name = 'buzz'
         self.zk.get_cluster()
-        self.zk.touch_member('new')
+        self.zk.touch_member({'new': 'new'})
         self.zk._name = 'bar'
-        self.zk.touch_member('new')
+        self.zk.touch_member({'new': 'new'})
         self.zk._name = 'na'
         self.zk._client.exists = 1
-        self.zk.touch_member('Exception')
+        self.zk.touch_member({'Exception': 'Exception'})
         self.zk._name = 'bar'
-        self.zk.touch_member('retry')
+        self.zk.touch_member({'retry': 'retry'})
         self.zk._fetch_cluster = True
         self.zk.get_cluster()
-        self.zk.touch_member('retry')
+        self.zk.touch_member({'retry': 'retry'})
 
     def test_take_leader(self):
         self.zk.take_leader()
@@ -187,7 +187,7 @@ class TestZooKeeper(unittest.TestCase):
             self.zk.take_leader()
 
     def test_update_leader(self):
-        self.assertTrue(self.zk.update_leader())
+        self.assertTrue(self.zk.update_leader(None))
 
     def test_write_leader_optime(self):
         self.zk.last_leader_operation = '0'
@@ -217,3 +217,6 @@ class TestZooKeeper(unittest.TestCase):
         self.zk.set_sync_state_value('ok')
         self.zk.set_sync_state_value('Exception')
         self.zk.delete_sync_state()
+
+    def test_set_history_value(self):
+        self.zk.set_history_value('{}')