Release 1.4.4 (#683)

bump version and update release notes

.gitignore

@@ -1,3 +1,53 @@
-data/*
-*.pyc
-helpers/*.pyc
+*.py[cod]
+
+# vi(m) swap files:
+*.sw?
+
+# C extensions
+*.so
+
+# Packages
+.cache/
+*.egg
+*.eggs
+*.egg-info
+dist
+build
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+lib
+lib64
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+.coverage
+.tox
+nosetests.xml
+coverage.xml
+htmlcov
+junit.xml
+features/output
+dummy
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+pgpass
+scm-source.json
+
+# Sphinx-generated documentation
+docs/build/
+docs/source/_static/
+docs/source/_templates/

.travis.yml

@@ -1,14 +1,153 @@
+sudo: true
+dist: trusty
 language: python
-python:
-  - "2.7"
-  - "3.3"
-  - "3.4"
+env:
+  global:
+  - ETCDVERSION=3.0.17 ZKVERSION=3.4.11 CONSULVERSION=0.7.4
+  - PYVERSIONS="2.7 3.5 3.6"
+  - EXCLUDE_BEHAVE="3.5"
+  - BOTO_CONFIG=/doesnotexist
+matrix:
+  include:
+  - python: "3.5"
+    env: TEST_SUITE="python setup.py"
+  - python: "3.6"
+    env: DCS="etcd" TEST_SUITE="behave"
+  - python: "3.6"
+    env: DCS="exhibitor" TEST_SUITE="behave"
+  - python: "3.6"
+    env: DCS="consul" TEST_SUITE="behave"
+  - python: "3.6"
+    env: DCS="kubernetes" TEST_SUITE="behave"
+branches:
+  only:
+  - master
+  - /^v\d+\.\d+(\.\d+)?$/
+cache:
+  directories:
+  - $HOME/mycache
+before_cache:
+- |
+  rm -fr $HOME/mycache/python*
+  for pv in $PYVERSIONS; do
+      if [[ $TEST_SUITE != "behave" || $pv != $EXCLUDE_BEHAVE ]]; then
+          fpv=$(basename $(readlink $HOME/virtualenv/python${pv}))
+          mv $HOME/virtualenv/${fpv} $HOME/mycache/${fpv}
+      fi
+  done
 install:
-  - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then pip install -r requirements-py2.txt --use-mirrors; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install -r requirements-py3.txt --use-mirrors; fi
-  - pip install coveralls
+  - |
+    set -e
+
+    if [[ $TEST_SUITE == "behave" ]]; then
+        function get_consul() {
+            CC=~/mycache/consul_${CONSULVERSION}
+            if [[ ! -x $CC ]]; then
+                curl -L https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip \
+                    | gunzip > $CC
+                [[ ${PIPESTATUS[0]} == 0 ]] || return 1
+                chmod +x $CC
+            fi
+            ln -s $CC consul
+        }
+
+        function get_etcd() {
+            EC=~/mycache/etcd_${ETCDVERSION}
+            if [[ ! -x $EC ]]; then
+                curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
+                    | tar xz -C . --strip=1 --wildcards --no-anchored etcd
+                [[ ${PIPESTATUS[0]} == 0 ]] || return 1
+                mv etcd $EC
+            fi
+            ln -s $EC etcd
+        }
+
+        function get_kubernetes() {
+            wget -O localkube "https://storage.googleapis.com/minikube/k8sReleases/v1.7.0/localkube-linux-amd64"
+            chmod +x localkube
+            sudo nohup ./localkube --logtostderr=true --enable-dns=false > localkube.log 2>&1 &
+
+            echo "Waiting for localkube to start..."
+            if ! timeout 120 sh -c "while ! curl -ks http://127.0.0.1:8080/ >/dev/null; do sleep 1; done"; then
+                sudo cat localkube.log
+                echo "localkube did not start"
+                exit 1
+            fi
+            echo "Check certificate permissions"
+            sudo chmod 644 /var/lib/localkube/certs/*
+            sudo ls -altr /var/lib/localkube/certs/
+
+            echo "Set up .kube/config"
+            mkdir ~/.kube
+            echo -e "apiVersion: v1\nclusters:\n- cluster:\n    certificate-authority: /var/lib/localkube/certs/ca.crt\n    server: https://127.0.0.1:8443\n  name: local\ncontexts:\n- context:\n    cluster: local\n    user: myself\n  name: local\ncurrent-context: local\nkind: Config\npreferences: {}\nusers:\n- name: myself\n  user:\n    client-certificate: /var/lib/localkube/certs/apiserver.crt\n    client-key: /var/lib/localkube/certs/apiserver.key\n" > ~/.kube/config
+        }
+
+        function get_exhibitor() {
+            ZC=~/mycache/zookeeper-${ZKVERSION}
+            if [[ ! -d $ZC ]]; then
+                curl -L http://www.apache.org/dist/zookeeper/zookeeper-${ZKVERSION}/zookeeper-${ZKVERSION}.tar.gz | tar xz
+                [[ ${PIPESTATUS[0]} == 0 ]] || return 1
+                mv zookeeper-${ZKVERSION}/conf/zoo_sample.cfg zookeeper-${ZKVERSION}/conf/zoo.cfg
+                mv zookeeper-${ZKVERSION} $ZC
+            fi
+            $ZC/bin/zkServer.sh start
+            # following lines are 'emulating' exhibitor REST API
+            while true; do
+                echo -e 'HTTP/1.0 200 OK\nContent-Type: application/json\n\n{"servers":["127.0.0.1"],"port":2181}' \
+                    | nc -l 8181 &> /dev/null
+            done&
+        }
+
+        attempt_num=1
+        until get_${DCS}; do
+            [[ $attempt_num -ge 3 ]] && exit 1
+            echo "Attempt $attempt_num failed! Trying again in $attempt_num seconds..."
+            sleep $(( attempt_num++ ))
+        done
+    fi
+
+    for pv in $PYVERSIONS; do
+        if [[ $TEST_SUITE != "behave" || $pv != $EXCLUDE_BEHAVE ]]; then
+            fpv=$(basename $(readlink $HOME/virtualenv/python$pv))
+            if [[ -d ~/mycache/${fpv} ]]; then
+                mv ~/virtualenv/${fpv} ~/virtualenv/${fpv}.bckp
+                mv ~/mycache/${fpv} ~/virtualenv/${fpv}
+            fi
+            source ~/virtualenv/python${pv}/bin/activate
+            # explicitly install all needed python modules to cache them
+            for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8 mock pytest-cov pytest setuptools'; do
+                pip install $p --upgrade
+            done
+        fi
+    done
 script:
-  - python setup.py test
-  - python setup.py flake8
+  - |
+    for pv in $PYVERSIONS; do
+        if [[ $TEST_SUITE == "behave" && $pv == $EXCLUDE_BEHAVE ]]; then
+            continue
+        fi
+
+        source ~/virtualenv/python${pv}/bin/activate
+
+        if [[ $TEST_SUITE != "behave" ]]; then
+            echo Running unit tests using python${pv}
+            $TEST_SUITE test
+            $TEST_SUITE flake8
+        elif [[ $pv != $EXCLUDE_BEHAVE ]]; then
+            echo Running acceptance tests using python${pv}
+            if ! PATH=.:/usr/lib/postgresql/9.6/bin:$PATH $TEST_SUITE; then
+                # output all log files when tests are failing
+                grep . features/output/*/*postgres?.*
+                exit 1
+            fi
+        fi
+    done
+
+    set +e
 after_success:
+  # before_cache is executed earlier than after_success, so we need to restore one of virtualenv directories
+  - fpv=$(basename $(readlink $HOME/virtualenv/python3.6)) && mv $HOME/mycache/${fpv} $HOME/virtualenv/${fpv}
   - coveralls
+  - if [[ $TEST_SUITE != "behave" ]]; then python-codacy-coverage -r coverage.xml; fi
+  - if [[ $DCS == "exhibitor" ]]; then ~/mycache/zookeeper-${ZKVERSION}/bin/zkServer.sh stop; fi
+  - sudo kill $(jobs -p)

.zappr.yaml

@@ -0,0 +1,13 @@
+# for github.com
+approvals:
+  groups:
+    zalando:
+      minimum: 2
+      from:
+        orgs:
+          - "zalando"
+# team should be valid team id in team service https://teams.auth.zalando.com/api/teams/:id
+X-Zalando-Team: "acid"
+# type should be one of [code, doc, config, tools, secrets]
+# code will be the default value, if X-Zalando-Type is not found in .zappr.yml
+X-Zalando-Type: code

Dockerfile

@@ -1,35 +1,48 @@
 ## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine
 ## It has all the necessary components to play/debug with a single node appliance, running etcd
-FROM ubuntu:14.04
-MAINTAINER Feike Steenbergen <feike.steenberge[email protected]>
+FROM postgres:9.6
+MAINTAINER Alexander Kukushkin <alexander.kukushki[email protected]>
 
-# We need curl
-RUN apt-get update -y && apt-get install curl -y
+RUN export DEBIAN_FRONTEND=noninteractive \
+    && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
+    && apt-get update -y \
+    && apt-get upgrade -y \
+    && apt-get install -y curl jq haproxy python-psycopg2 python-yaml python-requests python-six python-pysocks \
+        python-dateutil python-pip python-setuptools python-prettytable python-wheel python-psutil python locales \
 
-# Add PGDG repositories
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
-RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-RUN apt-get update -y
-RUN apt-get upgrade -y
+    ## Make sure we have a en_US.UTF-8 locale available
+    && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
 
-ENV PGVERSION 9.4
-RUN apt-get install python python-psycopg2 python-yaml python-requests python-boto postgresql-${PGVERSION} python-dnspython python-kazoo python-pip -y
-RUN pip install python-etcd
+    && pip install 'python-etcd>=0.4.3,<0.5' click tzlocal cdiff \
 
-ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:$PATH
+    && mkdir -p /home/postgres \
+    && chown postgres:postgres /home/postgres \
 
-ADD patroni.py /patroni.py
-ADD patroni/ /patroni
+    # Clean up
+    && apt-get remove -y python-pip python-setuptools \
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/* /root/.cache
 
-ENV ETCDVERSION 2.0.13
-RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C /bin --strip=1 --wildcards --no-anchored etcd etcdctl
+ENV ETCDVERSION 3.2.3
+RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
+    | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl
+
+ENV CONFDVERSION 0.11.0
+RUN curl -L https://github.com/kelseyhightower/confd/releases/download/v${CONFDVERSION}/confd-${CONFDVERSION}-linux-amd64 > /usr/local/bin/confd \
+    && chmod +x /usr/local/bin/confd
+
+ADD patronictl.py patroni.py docker/entrypoint.sh /
+ADD patroni /patroni/
+ADD extras/confd /etc/confd
+RUN ln -s /patronictl.py /usr/local/bin/patronictl
 
 ### Setting up a simple script that will serve as an entrypoint
-RUN mkdir /data/ && touch /var/log/etcd.log /var/log/etcd.err /pgpass /patroni/postgres.yml
-RUN chown postgres:postgres -R /patroni/ /data/ /pgpass /var/log/etcd.* /patroni/postgres.yml
-ADD docker/entrypoint.sh /entrypoint.sh
+RUN mkdir /data/ && touch /pgpass /patroni.yml \
+    && chown postgres:postgres -R /patroni/ /data/ /pgpass /patroni.yml /etc/haproxy /var/run/ /var/lib/ /var/log/
 
-EXPOSE 4001 5432 2380
+EXPOSE 2379 5432 8008
 
+ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
 ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
 USER postgres

MAINTAINERS

@@ -0,0 +1,3 @@
+Alexander Kukushkin <[email protected]>
+Feike Steenbergen <[email protected]>
+Oleksii Kliukin <[email protected]>

README.rst

@@ -1,34 +1,111 @@
 |Build Status| |Coverage Status|
 
-Patroni: A Template for PostgreSQL HA with ZooKeeper or etcd
-------------------------------------------------------------
+Patroni: A Template for PostgreSQL HA with ZooKeeper, etcd or Consul
+--------------------------------------------------------------------
 
-Patroni was previously known as Governor.
+You can find a version of this documentation that is searchable and also easier to navigate at `patroni.readthedocs.io <https://patroni.readthedocs.io>`__.
 
-*There are many ways to run high availability with PostgreSQL; here we
-present a template for you to create your own custom fit high
-availability solution using python and distributed configuration store
-(like ZooKeeper or etcd) for maximum accessibility.*
 
-Getting Started
----------------
+There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
+
+Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__, `Consul <https://github.com/hashicorp/consul>`__ or `Kubernetes <https://kubernetes.io>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
+
+We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely.
+
+**Note to Kubernetes users**: Patroni can run natively on top of Kubernetes. Take a look at the `Kubernetes <https://github.com/zalando/patroni/blob/master/docs/kubernetes.rst>`__ chapter of the Patroni documentation.
+
+.. contents::
+    :local:
+    :depth: 1
+    :backlinks: none
+
+=================
+How Patroni Works
+=================
+
+Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features.
+
+For an example of a Docker-based deployment with Patroni, see `Spilo <https://github.com/zalando/spilo>`__, currently in use at Zalando.
+
+For additional background info, see:
+
+* `Elephants on Automatic: HA Clustered PostgreSQL with Helm <https://www.youtube.com/watch?v=CftcVhFMGSY>`_, talk by Josh Berkus and Oleksii Kliukin at KubeCon Berlin 2017
+* `PostgreSQL HA with Kubernetes and Patroni <https://www.youtube.com/watch?v=iruaCgeG7qs>`__, talk by Josh Berkus at KubeCon 2016 (video)
+* `Feb. 2016 Zalando Tech blog post <https://tech.zalando.de/blog/zalandos-patroni-a-template-for-high-availability-postgresql/>`__
+
+==================
+Development Status
+==================
+
+Patroni is in active development and accepts contributions. See our `Contributing <https://github.com/zalando/patroni/blob/master/docs/CONTRIBUTING.rst>`__ section below for more details.
+
+We report new releases information `here <https://github.com/zalando/patroni/releases>`__.
+
+=========
+Community
+=========
+
+There are two places to connect with the Patroni community: `on github <https://github.com/zalando/patroni>`__, via Issues and PRs, and on channel #patroni in the `PostgreSQL Slack <https://postgres-slack.herokuapp.com/>`__.  If you're using Patroni, or just interested, please join us.
+
+===================================
+Technical Requirements/Installation
+===================================
+
+**Pre-requirements for Mac OS**
+
+To install requirements on a Mac, run the following:
+
+::
+
+    brew install postgresql etcd haproxy libyaml python
+
+**General installation for pip**
+
+Patroni can be installed with pip:
+
+::
+
+    pip install patroni[dependencies]
+
+where dependencies can be either empty, or consist of one or more of the following:
+
+etcd
+    `python-etcd` module in order to use Etcd as DCS
+consul
+    `python-consul` module in order to use Consul as DCS
+zookeeper
+    `kazoo` module in order to use Zookeeper as DCS
+exhibitor
+    `kazoo` module in order to use Exhibitor as DCS (same dependencies as for Zookeeper)
+kubernetes
+    `kubernetes` module in order to use Kubernetes as DCS in Patroni
+aws
+    `boto` in order to use AWS callbacks
+
+For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is:
+
+::
+
+    pip install patroni[etcd,aws]
+
+Note that external tools to call in the replica creation or custom bootstap scripts (i.e. WAL-E) should be installed independently of Patroni.
+
+=======================
+Running and Configuring
+=======================
 
 To get started, do the following from different terminals:
-
 ::
 
     > etcd --data-dir=data/etcd
     > ./patroni.py postgres0.yml
     > ./patroni.py postgres1.yml
 
-From there, you will see a high-availability cluster start up. Test
-different settings in the YAML files to see how behavior changes. Kill
-some of the different components to see how the system behaves.
+You will then see a high-availability cluster start up. Test different settings in the YAML files to see how the cluster's behavior changes. Kill some of the components to see how the system behaves.
 
 Add more ``postgres*.yml`` files to create an even larger cluster.
 
-We provide a haproxy configuration, which will give your application a
-single endpoint for connecting to the cluster's leader. To configure,
+Patroni provides an `HAProxy <http://www.haproxy.org/>`__ configuration, which will give your application a single endpoint for connecting to the cluster's leader. To configure,
 run:
 
 ::
@@ -39,150 +116,29 @@ run:
 
     > psql --host 127.0.0.1 --port 5000 postgres
 
-How Patroni works
------------------
-
-For a diagram of the high availability decision loop, see the included a
-PDF:
-`postgres-ha.pdf <https://github.com/zalando/patroni/blob/master/postgres-ha.pdf>`__
-
+==================
 YAML Configuration
-------------------
+==================
 
-For an example file, see ``postgres0.yml``. Below is an explanation of
-settings:
+Go `here <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
 
--  *ttl*: the TTL to acquire the leader lock. Think of it as the length of time before automatic failover process is initiated.
--  *loop\_wait*: the number of seconds the loop will sleep
+=========================
+Environment Configuration
+=========================
 
--  *restapi*:
-    -  *listen*: ip address + port that Patroni will listen to provide health-check information for haproxy.
-    -  *connect\_address*: ip address + port through which restapi is accessible.
-    -  *auth*: (optional) 'username:password' to protect some dangerous REST API endpoints.
-    -  *certfile*: (optional) Specifies a file with the certificate in the PEM format. If certfile is not specified or empty API server will work without SSL.
-    -  *keyfile*: (optional) Specifies a file with the secret key in the PEM format.
+Go `here <https://github.com/zalando/patroni/blob/master/docs/ENVIRONMENT.rst>`__ for comprehensive information about configuring(overriding) settings via environment variables.
 
--  *etcd*:
-    -  *scope*: the relative path used on etcd's http api for this deployment, thus you can run multiple HA deployments from a single etcd
-    -  *ttl*: the TTL to acquire the leader lock. Think of it as the length of time before automatic failover process is initiated.
-    -  *host*: the host:port for the etcd endpoint
+===================
+Replication Choices
+===================
 
--  *zookeeper*:
-    -  *scope*: the relative path used on etcd's http api for this deployment, thus you can run multiple HA deployments from a single etcd
-    -  *session\_timeout*: the TTL to acquire the leader lock. Think of it as the length of time before automatic failover process is initiated.
-    -  *reconnect\_timeout*: how long we should try to reconnect to ZooKeeper after connection loss. After this timeout we assume that we don't have lock anymore and will restart in read-only mode.
-    -  *hosts*: list of ZooKeeper cluster members in format: ['host1:port1', 'host2:port2', 'etc...']
-    -  *exhibitor*: if you are running ZooKeeper cluster under Exhibitor supervisory the following section could be interesting for you
-        -  *poll\_interval*: how often list of ZooKeeper and Exhibitor nodes should be updated from Exhibitor
-        -  *port*: Exhibitor port
-        -  *hosts*: initial list of Exhibitor (ZooKeeper) nodes in format: ['host1', 'host2', 'etc...' ]. This list would be updated automatically when Exhibitor (ZooKeeper) cluster topology changes.
+Patroni uses Postgres' streaming replication, which is asynchronous by default. Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the leader. This setting should be increased or decreased based on business requirements. It's also possible to use synchronous replication for better durability guarantees. See `replication modes documentation <https://github.com/zalando/patroni/blob/master/docs/replication_modes.rst>`__ for details.
 
--  *postgresql*:
-    -  *name*: the name of the Postgres host, must be unique for the cluster
-    -  *listen*: ip address + port that Postgres listening. Must be accessible from other nodes in the cluster if using streaming replication.
-    -  *connect\_address*: ip address + port through which Postgres is accessible from other nodes and applications.
-    -  *data\_dir*: file path to initialize and store Postgres data files
-    -  *maximum\_lag\_on\_failover*: the maximum bytes a follower may lag
-    -  *use\_slots*: whether or not to use replication_slots.  Must be False for PostgreSQL 9.3, and you should comment out max_replication_slots. before it is not eligible become leader
-    -  *pg\_hba*: list of lines which should be added to pg\_hba.conf
-        -  *- host all all 0.0.0.0/0 md5*
+======================================
+Applications Should Not Use Superusers
+======================================
 
-    -  *replication*:
-        -  *username*: replication username, user will be created during initialization
-        -  *password*: replication password, user will be created during initialization
-        -  *network*: network setting for replication in pg\_hba.conf
-
-    -  *callbacks* callback scripts to run on certain actions. Patroni will pass current action, role and cluster name. See scripts/aws.py as an example on how to write them.
-        -  *on\_start*: a script to run when the cluster starts
-        -  *on\_stop*: a script to run when the cluster stops
-        -  *on\_restart*: a script to run when the cluster restarts
-        -  *on\_reload*: a script to run when configuration reload is triggered
-        -  *on\_role\_change*: a script to run when the cluster is being promoted or demoted
-
-    -  *superuser*:
-        -  *password*: password for postgres user. It would be set during initialization
-
-    -  *admin*:
-        -  *username*: admin username, user will be created during initialization. It would have CREATEDB and CREATEROLE privileges
-        -  *password*: admin password, user will be created during initialization.
-
-    -  *recovery\_conf*: additional configuration settings written to recovery.conf when configuring follower
-        -  *parameters*: list of configuration settings for Postgres.  Many of these are required for replication to work.
-
-Replication choices
--------------------
-
-Patroni uses Postgres' streaming replication. By default, this
-replication is asynchronous. For more information, see the `Postgres
-documentation on streaming
-replication <http://www.postgresql.org/docs/current/static/warm-standby.html#STREAMING-REPLICATION>`__.
-
-Patroni's asynchronous replication configuration allows for
-``maximum_lag_on_failover`` settings. This setting ensures failover will
-not occur if a follower is more than a certain number of bytes behind
-the follower. This setting should be increased or decreased based on
-business requirements.
-
-When asynchronous replication is not best for your use-case, investigate
-how Postgres's `synchronous
-replication <http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION>`__
-works. Synchronous replication ensures consistency across a cluster by
-confirming that writes are written to a secondary before returning to
-the connecting client with a success. The cost of synchronous
-replication will be reduced throughput on writes. This throughput will
-be entirely based on network performance. In hosted datacenter
-environments (like AWS, Rackspace, or any network you do not control),
-synchrous replication increases the variability of write performance
-significantly. If followers become inaccessible from the leader, the
-leader will becomes effectively readonly.
-
-To enable a simple synchronous replication test, add the follow lines to
-the ``parameters`` section of your YAML configuration files.
-
-.. code:: YAML
-
-        synchronous_commit: "on"
-        synchronous_standby_names: "*"
-
-When using synchronous replication, use at least a 3-Postgres data nodes
-to ensure write availability if one host fails.
-
-Choosing your replication schema is dependent on the many business
-decisions. Investigate both async and sync replication, as well as other
-HA solutions, to determine which solution is best for you.
-
-Applications should not use superusers
---------------------------------------
-
-When connecting from an application, always use a non-superuser. Patroni
-requires access to the database to function properly. By using a
-superuser from application, you can potentially use the entire
-connection pool, including the connections reserved for superusers with
-the ``superuser_reserved_connections`` setting. If Patroni cannot access
-the Primary, because the connection pool is full, behavior will be
-undesireable.
-
-Requirements on a Mac
----------------------
-
-Run the following on a Mac to install requirements:
-
-::
-
-    brew install postgresql etcd haproxy libyaml python
-    pip install psycopg2 pyyaml
-
-Notice
-------
-
-There are many different ways to do HA with PostgreSQL, see `the
-PostgreSQL
-documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__
-for a complete list.
-
-We call this project a "template" because it is far from a one-size fits
-all, or a plug-and-play replication system. It will have it's own
-caveats. Use wisely.
+When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
 
 .. |Build Status| image:: https://travis-ci.org/zalando/patroni.svg?branch=master
    :target: https://travis-ci.org/zalando/patroni

TODO.md

@@ -1,4 +1,12 @@
 Failover
 ========
-- When determining who should become master, include the minor version of PostgreSQL in the decision
-- Create a way to disable governance of a cluster, something like the existence of a "nogover" or "admin" file in PGDATA will stop governor from changing the cluster state
+- When determining who should become master, include the minor version of PostgreSQL in the decision.
+
+Configuration
+==============
+- Provide a way to change pg_hba.conf of a running cluster on the Patroni level, without changing individual nodes.
+- Provide hooks to store and retrieve cluster-wide passwords without exposing them in a plain-text form to unauthorized users.
+
+Documentation
+==============
+- Document how to run cascading replication and possibly initialize the cluster without an access to the master node.

docker-compose.yml

@@ -0,0 +1,58 @@
+# docker compose file for running a 3-node PostgreSQL cluster
+# with etcd as the SIS
+
+patroni_etcd:
+    container_name: patroni_etcd
+    image: patroni
+    command: --etcd
+
+dbnode1:
+    image: patroni
+    hostname: dbnode1
+    links:
+        - patroni_etcd:patroni_etcd
+    volumes:
+        - ./patroni:/patroni
+    env_file: docker/patroni-secrets.env
+    environment:
+        PATRONI_ETCD_HOST: patroni_etcd:2379
+        PATRONI_NAME: dbnode1
+        PATRONI_SCOPE: testcluster
+
+dbnode2:
+    image: patroni
+    hostname: dbnode2
+    links:
+        - patroni_etcd:patroni_etcd
+    volumes:
+        - ./patroni:/patroni
+    env_file: docker/patroni-secrets.env
+    environment:
+        PATRONI_ETCD_HOST: patroni_etcd:2379
+        PATRONI_NAME: dbnode2
+        PATRONI_SCOPE: testcluster
+
+dbnode3:
+    image: patroni
+    hostname: dbnode3
+    links:
+        - patroni_etcd:patroni_etcd
+    volumes:
+        - ./patroni:/patroni
+    env_file: docker/patroni-secrets.env
+    environment:
+        PATRONI_ETCD_HOST: patroni_etcd:2379
+        PATRONI_NAME: dbnode3
+        PATRONI_SCOPE: testcluster
+
+haproxy:
+    image: patroni
+    links:
+        - patroni_etcd:patroni_etcd
+    ports:
+        - "5000:5000"
+        - "5001:5001"
+    environment:
+        PATRONI_ETCD_HOST: patroni_etcd:2379
+        PATRONI_SCOPE: testcluster
+    command: --confd

docker/dev_patroni_cluster.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-DOCKER_IMAGE="os-registry.stups.zalan.do/acid/patroni:1.0-SNAPSHOT"
+DOCKER_IMAGE="registry.opensource.zalan.do/acid/patroni:1.0-SNAPSHOT"
 MEMBERS=3
 
 
@@ -67,24 +67,38 @@ while getopts "$optspec" optchar; do
     esac
 done
 
-function random_name()
-{
-    cat /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9' | head -c 8
-}
-
-if [ -z ${PATRONI_SCOPE} ]
-then
-    PATRONI_SCOPE=$(random_name)
+if [ -z ${PATRONI_SCOPE} ]; then
+    PATRONI_SCOPE=$(cat /dev/urandom | LC_ALL=C tr -dc 'a-z0-9' | head -c 8)
 fi
 
-etcd_container=$(docker run -P -d --name="${PATRONI_SCOPE}_etcd" "${DOCKER_IMAGE}" --etcd-only)
-etcd_container_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${etcd_container})
-echo "The etcd container is ${etcd_container}, ip=${etcd_container_ip}"
+function docker_run()
+{
+    local name=$1
+    shift
+    container=$(docker run -d --name=$name $*)
+    container_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${container})
+    echo "Started container ${name}, ip=${container_ip}"
+}
 
-for i in $(seq 1 "${MEMBERS}")
-do
-    container_name=$(random_name)
-    patroni_container=$(docker run -P -d --name="${PATRONI_SCOPE}_${container_name}" "${DOCKER_IMAGE}" --etcd="${etcd_container_ip}:4001" --name="${PATRONI_SCOPE}")
-    patroni_container_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${patroni_container})
-    echo "Started Patroni container ${patroni_container}, ip=${patroni_container_ip}"
+
+ETCD_CONTAINER="${PATRONI_SCOPE}_etcd"
+docker_run ${ETCD_CONTAINER} ${DOCKER_IMAGE} --etcd
+
+DOCKER_ARGS="--link=${ETCD_CONTAINER}:${ETCD_CONTAINER} -e PATRONI_SCOPE=${PATRONI_SCOPE} -e PATRONI_ETCD_HOST=${ETCD_CONTAINER}:2379"
+PATRONI_ENV=$(sed 's/#.*//g' docker/patroni-secrets.env | sed -n 's/^PATRONI_.*$/-e &/p' | tr '\n' ' ')
+PATRONI_VOLUME="-v $(dirname $(dirname $(realpath $0)))/patroni:/patroni"
+
+for i in $(seq 1 "${MEMBERS}"); do
+    container_name=postgres${i}
+    docker_run "${PATRONI_SCOPE}_${container_name}" \
+        $PATRONI_VOLUME \
+        $DOCKER_ARGS \
+        $PATRONI_ENV \
+        -e PATRONI_NAME=${container_name} \
+        ${DOCKER_IMAGE}
 done
+
+docker_run "${PATRONI_SCOPE}_haproxy" \
+    -p=5000 -p=5001 \
+    $DOCKER_ARGS \
+    ${DOCKER_IMAGE} --confd

docker/entrypoint.sh

@@ -3,53 +3,56 @@
 function usage()
 {
     cat <<__EOF__
-Usage: $0 
+Usage: $0
 
 Options:
 
-    --etcd      ETCD    Provide an external etcd to connect to
-    --name      NAME    Give the cluster a specific name 
-    --etcd-only         Do not run Patroni, run a standalone etcd
+    --etcd      Do not run Patroni, run a standalone etcd
+    --confd     Do not run Patroni, run a standalone confd
+    --zookeeper Do not run Patroni, run a standalone zookeeper
 
 Examples:
 
-    $0 --etcd=127.17.0.84:4001
-    $0 --etcd-only
-    $0 
-    $0 --name=true_scotsman
+    $0 --etcd
+    $0 --confd
+    $0 --zookeeper
+    $0
 __EOF__
 }
 
 DOCKER_IP=$(hostname --ip-address)
-PATRONI_SCOPE=batman
+PATRONI_SCOPE=${PATRONI_SCOPE:-batman}
+ETCD_ARGS="--data-dir /tmp/etcd.data -advertise-client-urls=http://${DOCKER_IP}:2379 -listen-client-urls=http://0.0.0.0:2379"
 
 optspec=":vh-:"
 while getopts "$optspec" optchar; do
     case "${optchar}" in
         -)
             case "${OPTARG}" in
-                etcd-only)
-                    exec etcd --data-dir /tmp/etcd.data \
-                        -advertise-client-urls=http://${DOCKER_IP}:4001 \
-                        -listen-client-urls=http://0.0.0.0:4001 \
-                        -listen-peer-urls=http://0.0.0.0:2380 
-                    exit 0
+                confd)
+                    haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D
+                    CONFD="confd -prefix=${PATRONI_NAMESPACE:-/service}/$PATRONI_SCOPE -interval=10 -backend"
+                    if [ ! -z ${PATRONI_ZOOKEEPER_HOSTS} ]; then
+                        while ! /usr/share/zookeeper/bin/zkCli.sh -server ${PATRONI_ZOOKEEPER_HOSTS} ls /; do
+                            sleep 1
+                        done
+                        exec $CONFD zookeeper -node ${PATRONI_ZOOKEEPER_HOSTS}
+                    else
+                        while ! curl -s ${PATRONI_ETCD_HOST}/v2/members | jq -r '.members[0].clientURLs[0]' | grep -q http; do
+                            sleep 1
+                        done
+                        exec $CONFD etcd -node $PATRONI_ETCD_HOST
+                    fi
+                    ;;
+                etcd)
+                    exec etcd $ETCD_ARGS
+                    ;;
+                zookeeper)
+                    exec /usr/share/zookeeper/bin/zkServer.sh start-foreground
                     ;;
                 cheat)
                     CHEAT=1
                     ;;
-                name)
-                    PATRONI_SCOPE="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
-                    ;;
-                name=*)
-                    PATRONI_SCOPE=${OPTARG#*=}
-                    ;;
-                etcd)
-                    ETCD_CLUSTER="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
-                    ;;
-                etcd=*)
-                    ETCD_CLUSTER=${OPTARG#*=}
-                    ;;
                 help)
                     usage
                     exit 0
@@ -70,68 +73,43 @@ while getopts "$optspec" optchar; do
     esac
 done
 
-if [ -z ${ETCD_CLUSTER} ]
-then
-    etcd --data-dir /tmp/etcd.data \
-        -advertise-client-urls=http://${DOCKER_IP}:4001 \
-        -listen-client-urls=http://0.0.0.0:4001 \
-        -listen-peer-urls=http://0.0.0.0:2380 > /var/log/etcd.log 2> /var/log/etcd.err &
-    ETCD_CLUSTER="127.0.0.1:4001"
+## We start an etcd
+if [[ -z ${PATRONI_ETCD_HOST} && -z ${PATRONI_ZOOKEEPER_HOSTS} ]]; then
+    etcd $ETCD_ARGS > /var/log/etcd.log 2> /var/log/etcd.err &
+    export PATRONI_ETCD_HOST="127.0.0.1:2379"
 fi
 
-cat > /patroni/postgres.yml <<__EOF__
+export PATRONI_SCOPE
+export PATRONI_NAME="${PATRONI_NAME:-${HOSTNAME}}"
+export PATRONI_RESTAPI_CONNECT_ADDRESS="${DOCKER_IP}:8008"
+export PATRONI_RESTAPI_LISTEN="0.0.0.0:8008"
+export PATRONI_admin_PASSWORD="${PATRONI_admin_PASSWORD:=admin}"
+export PATRONI_admin_OPTIONS="${PATRONI_admin_OPTIONS:-createdb, createrole}"
+export PATRONI_POSTGRESQL_CONNECT_ADDRESS="${DOCKER_IP}:5432"
+export PATRONI_POSTGRESQL_LISTEN="0.0.0.0:5432"
+export PATRONI_POSTGRESQL_DATA_DIR="data/${PATRONI_SCOPE}"
+export PATRONI_REPLICATION_USERNAME="${PATRONI_REPLICATION_USERNAME:-replicator}"
+export PATRONI_REPLICATION_PASSWORD="${PATRONI_REPLICATION_PASSWORD:-abcd}"
+export PATRONI_SUPERUSER_USERNAME="${PATRONI_SUPERUSER_USERNAME:-postgres}"
+export PATRONI_SUPERUSER_PASSWORD="${PATRONI_SUPERUSER_PASSWORD:-postgres}"
+export PATRONI_POSTGRESQL_PGPASS="$HOME/.pgpass"
+
+cat > /patroni.yml <<__EOF__
+bootstrap:
+  dcs:
+    postgresql:
+      use_pg_rewind: true
 
-ttl: &ttl 30
-loop_wait: &loop_wait 10
-scope: &scope ${PATRONI_SCOPE}
-restapi:
-  listen: 127.0.0.1:8008
-  connect_address: 127.0.0.1:8008
-etcd:
-  scope: *scope
-  ttl: *ttl
-  host: ${ETCD_CLUSTER}
-postgresql:
-  name: postgresql_${DOCKER_IP//./_} ## Replication slots do not allow dots in their name
-  scope: *scope
-  listen: 0.0.0.0:5432
-  connect_address: ${DOCKER_IP}:5432
-  data_dir: data/postgresql0
-  maximum_lag_on_failover: 1048576 # 1 megabyte in bytes
   pg_hba:
   - host all all 0.0.0.0/0 md5
-  - hostssl all all 0.0.0.0/0 md5
   - host replication replicator ${DOCKER_IP}/16    md5
-  replication:
-    username: replicator
-    password: rep-pass
-    network:  127.0.0.1/32
-  superuser:
-    password: zalando
-  restore: patroni/scripts/restore.py
-  admin:
-    username: admin
-    password: admin
-  parameters:
-    archive_mode: "on"
-    wal_level: hot_standby
-    archive_command: mkdir -p ../wal_archive && cp %p ../wal_archive/%f
-    max_wal_senders: 20
-    listen_addresses: 0.0.0.0
-    wal_keep_segments: 8
-    archive_timeout: 1800s
-    max_replication_slots: 20
-    hot_standby: "on"
 __EOF__
 
-cat /patroni/postgres.yml
+mkdir -p "$HOME/.config/patroni"
+[ -h "$HOME/.config/patroni/patronictl.yaml" ] || ln -s /patroni.yml "$HOME/.config/patroni/patronictl.yaml"
 
-if [ ! -z $CHEAT ]
-then
-    while :
-    do
-        sleep 60
-    done
-else
-    exec python /patroni.py /patroni/postgres.yml
-fi
+[ -z $CHEAT ] && exec python /patroni.py /patroni.yml
+
+while true; do
+    sleep 60
+done

docker/patroni-secrets.env

@@ -0,0 +1,8 @@
+PATRONI_RESTAPI_USERNAME=admin
+PATRONI_RESTAPI_PASSWORD=admin
+PATRONI_SUPERUSER_USERNAME=postgres
+PATRONI_SUPERUSER_PASSWORD=postgres
+PATRONI_REPLICATION_USERNAME=replicator
+PATRONI_REPLICATION_PASSWORD=replicate
+PATRONI_admin_PASSWORD=admin
+PATRONI_admin_OPTIONS=createdb,createrole

docs/CONTRIBUTING.rst

@@ -0,0 +1,29 @@
+.. _contributing:
+
+Contributing guidelines
+=======================
+
+Wanna contribute to Patroni? Yay - here is how!
+
+Chatting
+--------
+
+Just want to chat with other Patroni users?  Looking for interactive troubleshooting help? Join us on channel #patroni in the `PostgreSQL Slack <https://postgres-slack.herokuapp.com/>`__.
+
+Reporting issues
+----------------
+
+If you have a question about patroni or have a problem using it, please read the :ref:`README <readme>` before filing an issue.
+Also double check with the current issues on our `Issues Tracker <https://github.com/zalando/patroni/issues>`__.
+
+Contributing a pull request
+---------------------------
+
+1) Submit a comment to the relevant issue or create a new issue describing your proposed change.
+2) Do a fork, develop and test your code changes.
+3) Include documentation
+4) Submit a pull request.
+
+You'll get feedback about your pull request as soon as possible.
+
+Happy Patroni hacking ;-)

docs/ENVIRONMENT.rst

@@ -0,0 +1,92 @@
+.. _environment:
+
+Environment Configuration Settings
+==================================
+
+It is possible to override some of the configuration parameters defined in the Patroni configuration file using the system environment variables. This document lists all environment variables handled by Patroni. The values set via those variables always take precedence over the ones set in the Patroni configuration file.
+
+Global/Universal
+----------------
+-  **PATRONI\_CONFIGURATION**: it is possible to set the entire configuration for the Patroni via ``PATRONI_CONFIGURATION`` environment variable. In this case any other environment variables will not be considered!
+-  **PATRONI\_NAME**: name of the node where the current instance of Patroni is running. Must be unique for the cluster.
+-  **PATRONI\_NAMESPACE**: path within the configuration store where Patroni will keep information about the cluster. Default value: "/service"
+-  **PATRONI\_SCOPE**: cluster name
+-  **PATRONI\_LOGLEVEL**: sets the general logging level (see `the docs for Python logging <https://docs.python.org/3.6/library/logging.html#levels>`_)
+-  **PATRONI\_REQUESTS_LOGLEVEL**: sets the logging level for all HTTP requests e.g. Kubernetes API calls (see `the docs for Python logging <https://docs.python.org/3.6/library/logging.html#levels>`_)
+
+Bootstrap configuration
+-----------------------
+It is possible to create new database users right after the successful initialization of a new cluster. This process is defined by the following variables:
+
+-  **PATRONI\_<username>\_PASSWORD='<password>'**
+-  **PATRONI\_<username>\_OPTIONS='list,of,options'**
+
+Example: defining ``PATRONI_admin_PASSWORD=strongpasswd`` and ``PATRONI_admin_OPTIONS='createrole,createdb'`` will cause creation of the user **admin** with the password **strongpasswd** that is allowed to create other users and databases.
+
+Consul
+------
+-  **PATRONI\_CONSUL\_HOST**: the host:port for the Consul endpoint.
+-  **PATRONI\_CONSUL\_URL**: url for the Consul, in format: http(s)://host:port
+-  **PATRONI\_CONSUL\_PORT**: (optional) Consul port
+-  **PATRONI\_CONSUL\_SCHEME**: (optional) **http** or **https**, defaults to **http**
+-  **PATRONI\_CONSUL\_TOKEN**: (optional) ACL token
+-  **PATRONI\_CONSUL\_VERIFY**: (optional) whether to verify the SSL certificate for HTTPS requests
+-  **PATRONI\_CONSUL\_CACERT**: (optional) The ca certificate. If present it will enable validation.
+-  **PATRONI\_CONSUL\_CERT**: (optional) File with the client certificate
+-  **PATRONI\_CONSUL\_KEY**: (optional) File with the client key. Can be empty if the key is part of certificate.
+-  **PATRONI\_CONSUL\_DC**: (optional) Datacenter to communicate with. By default the datacenter of the host is used.
+-  **PATRONI\_CONSUL\_CHECKS**: (optional) list of Consul health checks used for the session. If not specified Consul will use "serfHealth" in additional to the TTL based check created by Patroni. Additional checks, in particular the "serfHealth", may cause the leader lock to expire faster than in `ttl` seconds when the leader instance becomes unavailable.
+
+Etcd
+----
+-  **PATRONI\_ETCD\_HOST**: the host:port for the etcd endpoint.
+-  **PATRONI\_ETCD\_HOSTS**: list of etcd endpoints in format host1:port1,host2:port2,etc...
+-  **PATRONI\_ETCD\_URL**: url for the etcd, in format: http(s)://(username:password@)host:port
+-  **PATRONI\_ETCD\_PROXY**: proxy url for the etcd. If you are connecting to the etcd using proxy, use this parameter instead of **PATRONI\_ETCD\_URL**
+-  **PATRONI\_ETCD\_SRV**: Domain to search the SRV record(s) for cluster autodiscovery.
+-  **PATRONI\_ETCD\_CACERT**: The ca certificate. If present it will enable validation.
+-  **PATRONI\_ETCD\_CERT**: File with the client certificate
+-  **PATRONI\_ETCD\_KEY**: File with the client key. Can be empty if the key is part of certificate.
+
+Exhibitor
+---------
+-  **PATRONI\_EXHIBITOR\_HOSTS**: initial list of Exhibitor (ZooKeeper) nodes in format: 'host1,host2,etc...'. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes.
+-  **PATRONI\_EXHIBITOR\_PORT**: Exhibitor port.
+
+.. _kubernetes_environment:
+
+Kubernetes
+----------
+-  **PATRONI\_KUBERNETES\_NAMESPACE**: (optional) Kubernetes namespace where the Patroni pod is running. Default value is `default`.
+-  **PATRONI\_KUBERNETES\_LABELS**: Labels in format ``{label1: value1, label2: value2}``. These labels will be used to find existing objects (Pods and either Endpoints or ConfigMaps) associated with the current cluster. Also Patroni will set them on every object (Endpoint or ConfigMap) it creates.
+-  **PATRONI\_KUBERNETES\_SCOPE\_LABEL**: (optional) name of the label containing cluster name. Default value is `cluster-name`.
+-  **PATRONI\_KUBERNETES\_ROLE\_LABEL**: (optional) name of the label containing Postgres role (`master` or `replica`). Patroni will set this label on the pod it is running in. Default value is `role`.
+-  **PATRONI\_KUBERNETES\_USE\_ENDPOINTS**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
+-  **PATRONI\_KUBERNETES\_POD\_IP**: (optional) IP address of the pod Patroni is running in. This value is required when `PATRONI_KUBERNETES_USE_ENDPOINTS` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
+-  **PATRONI\_KUBERNETES\_PORTS**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``PATRONI_KUBERNETES_PORTS='{[{"name": "postgresql", "port": 5432}]}'`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `PATRONI_KUBERNETES_USE_ENDPOINTS` is set.
+
+PostgreSQL
+----------
+-  **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
+-  **PATRONI\_POSTGRESQL\_CONNECT\_ADDRESS**: IP address + port through which Postgres is accessible from other nodes and applications.
+-  **PATRONI\_POSTGRESQL\_DATA\_DIR**: The location of the Postgres data directory, either existing or to be initialized by Patroni.
+-  **PATRONI\_POSTGRESQL\_CONFIG\_DIR**: The location of the Postgres configuration directory, defaults to the data directory. Must be writable by Patroni.
+-  **PATRONI\_POSTGRESQL\_BIN_DIR**: Path to PostgreSQL binaries. (pg_ctl, pg_rewind, pg_basebackup, postgres) The  default value is an empty string meaning that PATH environment variable will be used to find the executables.
+-  **PATRONI\_POSTGRESQL\_PGPASS**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni.
+-  **PATRONI\_REPLICATION\_USERNAME**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication
+-  **PATRONI\_REPLICATION\_PASSWORD**: replication password; the user will be created during initialization.
+-  **PATRONI\_SUPERUSER\_USERNAME**: name for the superuser, set during initialization (initdb) and later used by Patroni to connect to the postgres. Also this user is used by pg_rewind.
+-  **PATRONI\_SUPERUSER\_PASSWORD**: password for the superuser, set during initialization (initdb).
+
+REST API
+-------- 
+-  **PATRONI\_RESTAPI\_CONNECT\_ADDRESS**: IP address and port to access the REST API.
+-  **PATRONI\_RESTAPI\_LISTEN**: IP address and port that Patroni will listen to, to provide health-check information for HAProxy.
+-  **PATRONI\_RESTAPI\_USERNAME**: Basic-auth username to protect unsafe REST API endpoints.
+-  **PATRONI\_RESTAPI\_PASSWORD**: Basic-auth password to protect unsafe REST API endpoints.
+-  **PATRONI\_RESTAPI\_CERTFILE**: Specifies the file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL.
+-  **PATRONI\_RESTAPI\_KEYFILE**: Specifies the file with the secret key in the PEM format.
+
+ZooKeeper
+---------
+-  **PATRONI\_ZOOKEEPER\_HOSTS**: comma separated list of ZooKeeper cluster members: "'host1:port1','host2:port2','etc...'". It is important to quote every single entity!

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = Patroni
+SOURCEDIR     = .
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/README.rst

@@ -0,0 +1,125 @@
+.. _readme:
+
+============
+Introduction
+============
+
+Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features.
+
+For an example of a Docker-based deployment with Patroni, see `Spilo <https://github.com/zalando/spilo>`__, currently in use at Zalando.
+
+For additional background info, see:
+
+* `PostgreSQL HA with Kubernetes and Patroni <https://www.youtube.com/watch?v=iruaCgeG7qs>`__, talk by Josh Berkus at KubeCon 2016 (video)
+* `Feb. 2016 Zalando Tech blog post <https://tech.zalando.de/blog/zalandos-patroni-a-template-for-high-availability-postgresql/>`__
+
+
+Development Status
+------------------
+
+Patroni is in active development and accepts contributions. See our :ref:`Contributing <contributing>` section below for more details.
+
+We report new releases information :ref:`here <releases>`.
+
+
+Technical Requirements/Installation
+-----------------------------------
+
+**Pre-requirements for Mac OS**
+
+To install requirements on a Mac, run the following:
+
+::
+
+    brew install postgresql etcd haproxy libyaml python
+
+**General installation for pip**
+
+Patroni can be installed with pip:
+
+::
+
+    pip install patroni[dependencies]
+
+where dependencies can be either empty, or consist of one or more of the following:
+
+etcd
+    `python-etcd` module in order to use Etcd as DCS
+consul
+    `python-consul` module in order to use Consul as DCS
+zookeeper
+    `kazoo` module in order to use Zookeeper as DCS
+exhibitor
+    `kazoo` module in order to use Exhibitor as DCS (same dependencies as for Zookeeper)
+kubernetes
+    `kubernetes` module in order to use Kubernetes as DCS in Patroni
+aws
+    `boto` in order to use AWS callbacks
+
+For example, the command in order to install Patroni together with dependencies for Etcd as a DCS and AWS callbacks is:
+
+::
+
+    pip install patroni[etcd,aws]
+
+Note that external tools to call in the replica creation or custom bootstrap scripts (i.e. WAL-E) should be installed
+independently of Patroni.
+
+
+Running and Configuring
+-----------------------
+
+The following section assumes Patroni repository as being cloned from https://github.com/zalando/patroni. Namely, you
+will need example configuration files `postgres0.yml` and `postgres1.yml`. If you installed Patroni with pip, you can
+obtain those files from the git repository and replace `./patroni.py` below with `patroni` command.
+
+To get started, do the following from different terminals:
+::
+
+    > etcd --data-dir=data/etcd
+    > ./patroni.py postgres0.yml
+    > ./patroni.py postgres1.yml
+
+You will then see a high-availability cluster start up. Test different settings in the YAML files to see how the cluster's behavior changes. Kill some of the components to see how the system behaves.
+
+Add more ``postgres*.yml`` files to create an even larger cluster.
+
+Patroni provides an `HAProxy <http://www.haproxy.org/>`__ configuration, which will give your application a single endpoint for connecting to the cluster's leader. To configure,
+run:
+
+::
+
+    > haproxy -f haproxy.cfg
+
+::
+
+    > psql --host 127.0.0.1 --port 5000 postgres
+
+
+YAML Configuration
+------------------
+
+Go :ref:`here <settings>` for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
+
+
+Environment Configuration
+-------------------------
+
+Go :ref:`here <environment>` for comprehensive information about configuring(overriding) settings via environment variables.
+
+
+Replication Choices
+-------------------
+
+Patroni uses Postgres' streaming replication, which is asynchronous by default. Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the leader. This setting should be increased or decreased based on business requirements. It's also possible to use synchronous replication for better durability guarantees. See :ref:`replication modes documentation <replication_modes>` for details.
+
+
+Applications Should Not Use Superusers
+--------------------------------------
+
+When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
+
+.. |Build Status| image:: https://travis-ci.org/zalando/patroni.svg?branch=master
+   :target: https://travis-ci.org/zalando/patroni
+.. |Coverage Status| image:: https://coveralls.io/repos/zalando/patroni/badge.svg?branch=master
+   :target: https://coveralls.io/r/zalando/patroni?branch=master

docs/SETTINGS.rst

@@ -0,0 +1,156 @@
+.. _settings:
+
+===========================
+YAML Configuration Settings
+===========================
+
+Global/Universal
+----------------
+-  **name**: the name of the host. Must be unique for the cluster.
+-  **namespace**: path within the configuration store where Patroni will keep information about the cluster. Default value: "/service"
+-  **scope**: cluster name
+
+Bootstrap configuration
+-----------------------
+-  **dcs**: This section will be written into `/<namespace>/<scope>/config` of a given configuration store after initializing of new cluster. This is the global configuration for the cluster. If you want to change some parameters for all cluster nodes - just do it in DCS (or via Patroni API) and all nodes will apply this configuration.
+    -  **loop\_wait**: the number of seconds the loop will sleep. Default value: 10
+    -  **ttl**: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process. Default value: 30
+    -  **retry\_timeout**: timeout for DCS and PostgreSQL operation retries. DCS or network issues shorter than this will not cause Patroni to demote the leader. Default value: 10
+    -  **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election.
+    -  **master\_start\_timeout**: the amount of time a master is allowed to recover from failures before failover is triggered. Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Best worst case failover time for master failure is: loop\_wait + master\_start\_timeout + loop\_wait, unless master\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff.
+    -  **synchronous\_mode**: turns on synchronous replication mode. In this mode a replica will be chosen as synchronous and only the latest leader and synchronous replica are able to participate in leader election. Synchronous mode makes sure that successfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See :ref:`replication modes documentation <replication_modes>` for details.
+    -  **postgresql**:
+        -  **use\_pg\_rewind**: whether or not to use pg_rewind
+        -  **use\_slots**: whether or not to use replication_slots. Must be False for PostgreSQL 9.3. You should comment out max_replication_slots before it becomes ineligible for leader status.
+        -  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower. 
+        -  **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work.
+-  **method**: custom script to use for bootstrapping this cluster.
+   See :ref:`custom bootstrap methods documentation <custom_bootstrap>` for details.
+   When ``initdb`` is specified revert to the default ``initdb`` command. ``initdb`` is also triggered when no ``method``
+   parameter is present in the configuration file.
+-  **initdb**: List options to be passed on to initdb.
+        -  **- data-checksums**: Must be enabled when pg_rewind is needed on 9.3.
+        -  **- encoding: UTF8**: default encoding for new databases.
+        -  **- locale: UTF8**: default locale for new databases.
+-  **pg\_hba**: list of lines that you should add to pg\_hba.conf.
+        -  **- host all all 0.0.0.0/0 md5**.
+        -  **- host replication replicator 127.0.0.1/32 md5**: A line like this is required for replication.
+-  **users**: Some additional users which need to be created after initializing new cluster
+    -  **admin**: the name of user
+        -  **password: zalando**:
+        -  **options**: list of options for CREATE USER statement
+            -  **- createrole**
+            -  **- createdb**
+-  **post\_bootstrap** or **post\_init**: An additional script that will be executed after initializing the cluster. The script receives a connection string URL (with the cluster superuser as a user name). The PGPASSFILE variable is set to the location of pgpass file.
+
+.. _consul_settings:
+
+Consul
+------
+Most of the parameters are optional, but you have to specify one of the **host** or **url**
+
+-  **host**: the host:port for the Consul endpoint, in format: http(s)://host:port
+-  **url**: url for the Consul endpoint
+-  **port**: (optional) Consul port
+-  **scheme**: (optional) **http** or **https**, defaults to **http**
+-  **token**: (optional) ACL token
+-  **verify**: (optional) whether to verify the SSL certificate for HTTPS requests
+-  **cacert**: (optional) The ca certificate. If present it will enable validation.
+-  **cert**: (optional) file with the client certificate
+-  **key**: (optional) file with the client key. Can be empty if the key is part of **cert**.
+-  **dc**: (optional) Datacenter to communicate with. By default the datacenter of the host is used.
+-  **checks**: (optional) list of Consul health checks used for the session. If not specified Consul will use "serfHealth" in additional to the TTL based check created by Patroni. Additional checks, in particular the "serfHealth", may cause the leader lock to expire faster than in `ttl` seconds when the leader instance becomes unavailable
+
+Etcd
+----
+Most of the parameters are optional, but you have to specify one of the **host**, **hosts**, **url**, **proxy** or **srv**
+
+-  **host**: the host:port for the etcd endpoint.
+-  **hosts**: list of etcd endpoint in format host1:port1,host2:port2,etc... Could be a comma separated string or an actual yaml list.
+-  **url**: url for the etcd
+-  **proxy**: proxy url for the etcd. If you are connecting to the etcd using proxy, use this parameter instead of **url**
+-  **srv**: Domain to search the SRV record(s) for cluster autodiscovery.
+-  **protocol**: (optional) http or https, if not specified http is used. If the **url** or **proxy** is specified - will take protocol from them.
+-  **username**: (optional) username for etcd authentication
+-  **password**: (optional) password for etcd authentication.
+-  **cacert**: (optional) The ca certificate. If present it will enable validation.
+-  **cert**: (optional) file with the client certificate
+-  **key**: (optional) file with the client key. Can be empty if the key is part of **cert**.
+
+Exhibitor
+---------
+-  **hosts**: initial list of Exhibitor (ZooKeeper) nodes in format: 'host1,host2,etc...'. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes.
+-  **poll\_interval**: how often the list of ZooKeeper and Exhibitor nodes should be updated from Exhibitor
+-  **port**: Exhibitor port.
+
+.. _kubernetes_settings:
+
+Kubernetes
+----------
+-  **namespace**: (optional) Kubernetes namespace where Patroni pod is running. Default value is `default`.
+-  **labels**: Labels in format ``{label1: value1, label2: value2}``. These labels will be used to find existing objects (Pods and either Endpoints or ConfigMaps) associated with the current cluster. Also Patroni will set them on every object (Endpoint or ConfigMap) it creates.
+-  **scope\_label**: (optional) name of the label containing cluster name. Default value is `cluster-name`.
+-  **role\_label**: (optional) name of the label containing role (master or replica). Patroni will set this label on the pod it runs in. Default value is ``role``.
+-  **use\_endpoints**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
+-  **pod\_ip**: (optional) IP address of the pod Patroni is running in. This value is required when `use_endpoints` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
+-  **ports**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``kubernetes.ports: {[{"name": "postgresql", "port": 5432}]}`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `kubernetes.use_endpoints` is set.
+
+.. _postgresql_settings:
+
+PostgreSQL
+----------
+-  **authentication**:
+    -  **superuser**:
+        -  **username**: name for the superuser, set during initialization (initdb) and later used by Patroni to connect to the postgres.
+        -  **password**: password for the superuser, set during initialization (initdb).
+    -  **replication**:
+        -  **username**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication
+        -  **password**: replication password; the user will be created during initialization.
+-  **callbacks**: callback scripts to run on certain actions. Patroni will pass the action, role and cluster name. (See scripts/aws.py as an example of how to write them.)
+        -  **on\_reload**: run this script when configuration reload is triggered.
+        -  **on\_restart**: run this script when the cluster restarts.
+        -  **on\_role\_change**: run this script when the cluster is being promoted or demoted.
+        -  **on\_start**: run this script when the cluster starts.
+        -  **on\_stop**: run this script when the cluster stops.
+-  **connect\_address**: IP address + port through which Postgres is accessible from other nodes and applications.
+-  **create\_replica\_method**: an ordered list of the create methods for turning a Patroni node into a new replica.
+   "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured as its
+   own config item. See :ref:`custom replica creation methods documentation <custom_replica_creation>` for further explanation.
+-  **data\_dir**: The location of the Postgres data directory, either existing or to be initialized by Patroni.
+-  **config\_dir**: The location of the Postgres configuration directory, defaults to the data directory. Must be writable by Patroni.
+-  **bin\_dir**: Path to PostgreSQL binaries (pg_ctl, pg_rewind, pg_basebackup, postgres). The default value is an empty string meaning that PATH environment variable will be used to find the executables.
+-  **listen**: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
+-  **use\_unix\_socket**: specifies that Patroni should prefer to use unix sockets to connect to the cluster. Default value is ``false``. If ``unix_socket_directories`` is definded, Patroni will use first suitable value from it to connect to the cluster and fallback to tcp if nothing is suitable. If ``unix_socket_directories`` is not specified in ``postgresql.parameters``, Patroni will assume that default value should be used and omit ``host`` from connection parameters.
+-  **pgpass**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup, the post_init script and under some other circumstances. The location must be writable by Patroni.
+-  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower.
+-  **custom\_conf** : path to an optional custom ``postgresql.conf`` file, that will be used in place of ``postgresql.base.conf``. The file must exist on all cluster nodes, be readable by PostgreSQL and will be included from its location on the real ``postgresql.conf``. Note that Patroni will not monitor this file for changes, nor backup it. However, its settings can still be overridden by Patroni's own configuration facilities - see :ref:`dynamic configuration <dynamic_configuration>` for details.
+-  **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work.
+-  **pg\_hba**: list of lines that Patroni will use to generate ``pg_hba.conf``. This parameter has higher priority than ``bootstrap.pg_hba``. Together with :ref:`dynamic configuration <dynamic_configuration>` it simplifies management of ``pg_hba.conf``.
+        -  **- host all all 0.0.0.0/0 md5**.
+        -  **- host replication replicator 127.0.0.1/32 md5**: A line like this is required for replication.
+-  **pg\_ctl\_timeout**: How long should pg_ctl wait when doing ``start``, ``stop`` or ``restart``. Default value is 60 seconds.
+-  **use\_pg\_rewind**: try to use pg\_rewind on the former leader when it joins cluster as a replica.
+-  **remove\_data\_directory\_on\_rewind\_failure**: If this option is enabled, Patroni will remove postgres data directory and recreate replica. Otherwise it will try to follow the new leader. Default value is **false**.
+-  **replica\_method**: for each create_replica_method other than basebackup, you would add a configuration section of the same name. At a minimum, this should include "command" with a full path to the actual script to be executed. Other configuration parameters will be passed along to the script in the form "parameter=value".
+
+REST API
+-------- 
+-  **connect\_address**: IP address and port to access the REST API.
+-  **listen**: IP address and port that Patroni will listen to, to provide health-check information for HAProxy.
+-  **Optional**:
+        -  **authentication**:
+            -  **username**: Basic-auth username to protect unsafe REST API endpoints.
+            -  **password**: Basic-auth password to protect unsafe REST API endpoints.
+
+        -  **certfile**: Specifies the file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL.
+        -  **keyfile**: Specifies the file with the secret key in the PEM format.
+
+ZooKeeper
+----------
+-  **hosts**: list of ZooKeeper cluster members in format: ['host1:port1', 'host2:port2', 'etc...'].
+
+Watchdog
+--------
+- **mode**: ``off``, ``automatic`` or ``required``. When ``off`` watchdog is disabled. When ``automatic`` watchdog will be used if available, but ignored if it is not. When ``required`` the node will not become a leader unless watchdog can be successfully enabled.
+- **device**: Path to watchdog device. Defaults to ``/dev/watchdog``.
+- **safety_margin**: Number of seconds of safety margin between watchdog triggering and leader key expiration.

docs/_static/custom.css

@@ -0,0 +1,3 @@
+li {
+   margin-bottom: 0.5em
+}

docs/conf.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Patroni documentation build configuration file, created by
+# sphinx-quickstart on Mon Dec 19 16:54:09 2016.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+
+import sys
+sys.path.insert(0, os.path.abspath('..'))
+
+from patroni.version import __version__
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
+    'sphinx.ext.mathjax',
+    'sphinx.ext.ifconfig',
+    'sphinx.ext.viewcode']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'Patroni'
+copyright = '2015 Compose, Zalando SE'
+author = 'Zalando SE'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = __version__[:__version__.rfind('.')]
+# The full version, including alpha/beta/rc tags.
+release = __version__
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+if not on_rtd:  # only import and set the theme if we're building docs locally
+    import sphinx_rtd_theme
+    html_theme = 'sphinx_rtd_theme'
+    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Patronidoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'Patroni.tex', 'Patroni Documentation',
+     'Zalando SE', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'patroni', 'Patroni Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'Patroni', 'Patroni Documentation',
+     author, 'Patroni', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+
+# -- Options for Epub output ----------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+epub_author = author
+epub_publisher = author
+epub_copyright = copyright
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {'https://docs.python.org/': None}
+
+# A possibility to have an own stylesheet, to add new rules or override existing ones
+# For the latter case, the CSS specificity of the rules should be higher than the default ones
+def setup(app):
+   app.add_stylesheet("custom.css")

docs/dynamic_configuration.rst

@@ -0,0 +1,228 @@
+.. _dynamic_configuration:
+
+Patroni configuration
+=====================
+
+Patroni configuration is stored in the DCS (Distributed Configuration Store). There are 3 types of configuration:
+
+- Dynamic configuration.
+	These options can be set in DCS at any time. If the options changed are not part of the startup configuration,
+	they are applied asynchronously (upon the next wake up cycle) to every node, which gets subsequently reloaded.
+	If the node requires a restart to apply the configuration (for options with context postmaster, if their values
+	have changed), a special flag, ``pending_restart`` indicating this, is set in the members.data JSON.
+	Additionally, the node status also indicates this, by showing ``"restart_pending": true``.
+
+- Local :ref:`configuration <settings>` (patroni.yml).
+	These options are defined in the configuration file and take precedence over dynamic configuration.
+	patroni.yml could be changed and reload in runtime (without restart of Patroni) by sending SIGHUP to the Patroni process or by performing ``POST /reload`` REST-API request.
+
+- Environment :ref:`configuration <environment>`.
+	It is possible to set/override some of the "Local" configuration parameters with environment variables.
+	Environment configuration is very useful when you are running in a dynamic environment and you don't know some of the parameters in advance (for example it's not possible to know your external IP address when you are running inside ``docker``).
+
+Some of the PostgreSQL parameters must hold the same values on the master and the replicas. For those, values set either in the local patroni configuration files or via the environment variables take no effect. To alter or set their values one must change the shared configuration in the DCS. Below is the actual list of such parameters together with the default values:
+
+- max_connections: 100
+- max_locks_per_transaction: 64
+- max_worker_processes: 8
+- max_prepared_transactions: 0
+- wal_level: hot_standby
+- wal_log_hints: on
+- track_commit_timestamp: off
+
+For the parameters below, PostgreSQL does not require equal values among the master and all the replicas. However, considering the possibility of a replica to become the master at any time, it doesn't really make sense to set them differently; therefore, Patroni restricts setting their values to the Dynamic configuration
+
+- max_wal_senders: 5
+- max_replication_slots: 5
+- wal_keep_segments: 8
+
+These parameters are validated to ensure they are sane, or meet a minimum value.
+
+There are some other Postgres parameters controlled by Patroni:
+
+- listen_addresses - is set either from ``postgresql.listen`` or from ``PATRONI_POSTGRESQL_LISTEN`` environment variable
+- port - is set either from ``postgresql.listen`` or from ``PATRONI_POSTGRESQL_LISTEN`` environment variable
+- cluster_name - is set either from ``scope`` or from ``PATRONI_SCOPE`` environment variable
+- hot_standby: on
+
+To be on the safe side parameters from the above lists are not written into ``postgresql.conf``, but passed as a list of arguments to the ``pg_ctl start`` which gives them the highest precedence, even above `ALTER SYSTEM <https://www.postgresql.org/docs/current/static/sql-altersystem.html>`__
+
+
+When applying the local or dynamic configuration options, the following actions are taken:
+
+- The node first checks if there is a postgresql.base.conf or if the ``custom_conf`` parameter is set.
+- If the `custom_conf` parameter is set, it will take the file specified on it as a base configuration, ignoring `postgresql.base.conf` and `postgresql.conf`.
+- If the `custom_conf` parameter is not set and `postgresql.base.conf` exists, it contains the renamed "original" configuration and it will be used as a base configuration.
+- If there is no `custom_conf` nor `postgresql.base.conf`, the original postgresql.conf is taken and renamed to postgresql.base.conf.
+- The dynamic options (with the exceptions above) are dumped into the postgresql.conf and an include is set in
+  postgresql.conf to the used base configuration (either postgresql.base.conf or what is on ``custom_conf``). Therefore, we would be able to apply new options without re-reading the configuration file to check if the include is present not.
+- Some parameters that are essential for Patroni to manage the cluster are overridden using the command line.
+- If some of the options that require restart are changed (we should look at the context in pg_settings and at the actual
+  values of those options), a pending_restart flag of a given node is set. This flag is reset on any restart.
+
+The parameters would be applied in the following order (run-time are given the highest priority):
+
+1. load parameters from file `postgresql.base.conf` (or from a `custom_conf` file, if set)
+2. load parameters from file `postgresql.conf`
+3. load parameters from file `postgresql.auto.conf`
+4. run-time parameter using `-o --name=value`
+
+This allows configuration for all the nodes (2), configuration for a specific node using `ALTER SYSTEM` (3) and ensures that parameters essential to the running of Patroni are enforced (4), as well as leaves room for configuration tools that manage `postgresql.conf` directly without involving Patroni (1).
+
+
+Also, the following Patroni configuration options can be changed only dynamically:
+
+- ttl: 30
+- loop_wait: 10
+- retry_timeouts: 10
+- maximum_lag_on_failover: 1048576
+- postgresql.use_slots: true
+
+Upon changing these options, Patroni will read the relevant section of the configuration stored in DCS and change its
+run-time values.
+
+Patroni nodes are dumping the state of the DCS options to disk upon for every change of the configuration into the file ``patroni.dynamic.json`` located in the Postgres data directory. Only the master is allowed to restore these options from the on-disk dump if these are completely absent from the DCS or if they are invalid.
+
+REST API
+========
+
+We provide a REST API endpoint for working with dynamic configuration.
+
+GET /config
+-----------
+Get current version of dynamic configuration.
+
+.. code-block:: bash
+
+	$ curl -s localhost:8008/config | jq .
+	{
+	  "ttl": 30,
+	  "loop_wait": 10,
+	  "retry_timeout": 10,
+	  "maximum_lag_on_failover": 1048576,
+	  "postgresql": {
+	    "use_slots": true,
+	    "use_pg_rewind": true,
+	    "parameters": {
+	      "hot_standby": "on",
+	      "wal_log_hints": "on",
+	      "wal_keep_segments": 8,
+	      "wal_level": "hot_standby",
+	      "max_wal_senders": 5,
+	      "max_replication_slots": 5,
+	      "max_connections": "100"
+	    }
+	  }
+	}
+
+PATCH /config
+-------------
+Change existing configuration.
+
+.. code-block:: bash
+
+	$ curl -s -XPATCH -d \
+		'{"loop_wait":5,"ttl":20,"postgresql":{"parameters":{"max_connections":"101"}}}' \
+		http://localhost:8008/config | jq .
+	{
+	  "ttl": 20,
+	  "loop_wait": 5,
+	  "maximum_lag_on_failover": 1048576,
+	  "retry_timeout": 10,
+	  "postgresql": {
+	    "use_slots": true,
+	    "use_pg_rewind": true,
+	    "parameters": {
+	      "hot_standby": "on",
+	      "wal_log_hints": "on",
+	      "wal_keep_segments": 8,
+	      "wal_level": "hot_standby",
+	      "max_wal_senders": 5,
+	      "max_replication_slots": 5,
+	      "max_connections": "101"
+	    }
+	  }
+	}
+
+The above REST API call patches the existing configuration and returns the new configuration.
+
+Let's check that the node processed this configuration. First of all it should start printing log lines every 5 seconds (loop_wait=5). The change of "max_connections" requires a restart, so the "restart_pending" flag should be exposed:
+
+.. code-block:: bash
+
+	$ curl -s http://localhost:8008/patroni | jq .
+	{
+	  "pending_restart": true,
+	  "database_system_identifier": "6287881213849985952",
+	  "postmaster_start_time": "2016-06-13 13:13:05.211 CEST",
+	  "xlog": {
+	    "location": 2197818976
+	  },
+	  "patroni": {
+	    "scope": "batman",
+	    "version": "1.0"
+	  },
+	  "state": "running",
+	  "role": "master",
+	  "server_version": 90503
+	}
+
+Removing parameters:
+
+If you want to remove (reset) some setting just patch it with ``null``:
+
+.. code-block:: bash
+
+	$ curl -s -XPATCH -d \
+		'{"postgresql":{"parameters":{"max_connections":null}}}' \
+		http://localhost:8008/config | jq .
+	{
+	  "ttl": 20,
+	  "loop_wait": 5,
+	  "retry_timeout": 10,
+	  "maximum_lag_on_failover": 1048576,
+	  "postgresql": {
+	    "use_slots": true,
+	    "use_pg_rewind": true,
+	    "parameters": {
+	      "hot_standby": "on",
+	      "unix_socket_directories": ".",
+	      "wal_keep_segments": 8,
+	      "wal_level": "hot_standby",
+	      "wal_log_hints": "on",
+	      "max_wal_senders": 5,
+	      "max_replication_slots": 5
+	    }
+	  }
+	}
+
+Above call removes ``postgresql.parameters.max_connections`` from the dynamic configuration.
+
+PUT /config
+-----------
+
+It's also possible to perform the full rewrite of an existing dynamic configuration unconditionally:
+
+.. code-block:: bash
+
+	$ curl -s -XPUT -d \
+		'{"maximum_lag_on_failover":1048576,"retry_timeout":10,"postgresql":{"use_slots":true,"use_pg_rewind":true,"parameters":{"hot_standby":"on","wal_log_hints":"on","wal_keep_segments":8,"wal_level":"hot_standby","unix_socket_directories":".","max_wal_senders":5}},"loop_wait":3,"ttl":20}' \
+		http://localhost:8008/config | jq .
+	{
+	  "ttl": 20,
+	  "maximum_lag_on_failover": 1048576,
+	  "retry_timeout": 10,
+	  "postgresql": {
+	    "use_slots": true,
+	    "parameters": {
+	      "hot_standby": "on",
+	      "unix_socket_directories": ".",
+	      "wal_keep_segments": 8,
+	      "wal_level": "hot_standby",
+	      "wal_log_hints": "on",
+	      "max_wal_senders": 5
+	    },
+	    "use_pg_rewind": true
+	  },
+	  "loop_wait": 3
+	}

docs/ha_loop_diagram.dot

@@ -0,0 +1,148 @@
+// Graphviz source for ha_loop_diagram.png
+// recompile with:
+// dot -Tpng ha_loop_diagram.dot -o ha_loop_diagram.png
+
+digraph G {
+  rankdir=TB;
+  fontname="sans-serif";
+  penwidth="0.3";
+  layout="dot";
+  newrank=true;
+  edge [fontname="sans-serif", 
+        fontsize=12, 
+        color=black, 
+        fontcolor=black];
+  node [fontname=serif, 
+        fontsize=12, 
+        fillcolor=white, 
+        color=black, 
+        fontcolor=black,
+        style=filled];
+  "start" [label=Start, shape="rectangle", fillcolor="green"]
+  "start" -> "load_cluster_from_dcs";
+  "update_member" [label="Persist node state in DCS"]
+  "update_member" -> "start"
+  
+  subgraph cluster_run_cycle {
+    label="run_cycle"
+    "load_cluster_from_dcs" [label="Load cluster from DCS"];
+    "touch_member" [label="Persist node in DCS"];
+    "cluster.has_member" [shape="diamond", label="Is node registered on DCS?"]
+    "cluster.has_member" -> "touch_member"  [label="no" color="red"]
+    "long_action_in_progress?" [shape="diamond" label="Is the PostgreSQL currently being\nstopping/starting/restarting/reinitializing?"]
+    "load_cluster_from_dcs" -> "cluster.has_member";
+    "touch_member" -> "long_action_in_progress?";
+    "cluster.has_member" -> "long_action_in_progress?"  [label="yes" color="green"];
+    "long_action_in_progress?" -> "recovering?" [label="no" color="red"]
+    "recovering?" [label="Was cluster recovering and failed?", shape="diamond"];
+    "recovering?" -> "post_recover" [label="yes" color="green"];
+    "recovering?" -> "data_directory_empty" [label="no" color="red"];
+    "post_recover" [label="Remove leader key (if I was the leader)"];
+    "data_directory_empty" [label="Is data folder empty?", shape="diamond"];
+    "data_directory_empty" -> "cluster_initialize" [label="no" color="red"];
+    "data_belongs_to_cluster" [label="Does data dir belong to cluster?", shape="diamond"];
+    "data_belongs_to_cluster" -> "exit" [label="no" color="red"];
+    "data_belongs_to_cluster" -> "is_healthy" [label="yes" color="green"]
+    "exit" [label="Fail and exit", fillcolor=red];
+    "cluster_initialize" [label="Is cluster initialized on DCS?" shape="diamond"]
+    "cluster_initialize" -> "cluster.has_leader" [label="no" color="red"]
+    "cluster.has_leader" [label="Does the cluster has leader?", shape="diamond"]
+    "cluster.has_leader" -> "dcs.initialize" [label="no", color="red"]
+    "cluster.has_leader" -> "is_healthy" [label="yes", color="green"]
+    "cluster_initialize" -> "data_belongs_to_cluster" [label="yes" color="green"]
+    "dcs.initialize" [label="Initialize new cluster"];
+    "dcs.initialize" -> "is_healthy"
+    "is_healthy" [label="Is node healthy?\n(running Postgres)", shape="diamond"];
+    "recover" [label="Start as read-only\nand set Recover flag"]
+    "is_healthy" -> "recover" [label="no" color="red"]; 
+    "is_healthy" -> "cluster.is_unlocked" [label="yes" color="green"]; 
+    "cluster.is_unlocked" [label="Does the cluster has a leader?", shape="diamond"]
+  }
+
+  "post_recover" -> "update_member"
+  "recover" -> "update_member"
+  "long_action_in_progress?" -> "async_has_lock?" [label="yes" color="green"];
+  "cluster.is_unlocked" -> "unhealthy_is_healthiest" [label="no" color="red"]
+  "cluster.is_unlocked" -> "healthy_has_lock" [label="yes" color="green"]
+  "data_directory_empty" -> "bootstrap.is_unlocked" [label="yes" color="green"]
+
+  subgraph cluster_async {
+    label = "Long action in progress\n(Start/Stop/Restart/Reinitialize)"
+    "async_has_lock?" [label="Do I have the leader lock?", shape="diamond"]
+    "async_update_lock" [label="Renew leader lock"]
+    "async_has_lock?" -> "async_update_lock" [label="yes" color="green"]
+  }
+  "async_update_lock" -> "update_member"
+  "async_has_lock?" -> "update_member" [label="no" color="red"]
+
+  subgraph cluster_bootstrap {
+    label = "Node bootstrap";
+    "bootstrap.is_unlocked" [label="Does the cluster has a leader?", shape="diamond"]
+    "bootstrap.is_initialized" [label="Does the cluster has an initialize key?", shape="diamond"]
+    "bootstrap.is_unlocked" -> "bootstrap.is_initialized" [label="no" color="red"]
+    "bootstrap.is_unlocked" -> "bootstrap.select_node" [label="yes" color="green"]
+    "bootstrap.select_node" [label="Select a node to take a backup from"]
+    "bootstrap.do_bootstrap" [label="Run pg_basebackup\n(async)"]
+    "bootstrap.select_node" -> "bootstrap.do_bootstrap"
+    "bootstrap.is_initialized" -> "bootstrap.initialization_race" [label="no" color="red"]
+    "bootstrap.is_initialized" -> "bootstrap.wait_for_leader" [label="yes" color="green"]
+    "bootstrap.initialization_race" [label="Race for initialize key"]
+    "bootstrap.initialization_race" -> "bootstrap.won_initialize_race?"
+    "bootstrap.won_initialize_race?" [label="Do I won initialize race?", shape="diamond"]
+    "bootstrap.won_initialize_race?" -> "bootstrap.initdb_and_start" [label="yes" color="green"]
+    "bootstrap.won_initialize_race?" -> "bootstrap.wait_for_leader" [label="no" color="red"]
+    "bootstrap.wait_for_leader" [label="Need to wait for leader key"]
+    "bootstrap.initdb_and_start" [label="Run initdb, start postgres and create roles"]
+    "bootstrap.initdb_and_start" -> "bootstrap.success?"
+    "bootstrap.success?" [label="Success", shape="diamond"]
+    "bootstrap.success?" -> "bootstrap.take_leader_key" [label="yes" color="green"]
+    "bootstrap.success?" -> "bootstrap.clean" [label="no" color="red"]
+    "bootstrap.clean" [label="Remove initialize key from DCS\nand data directory from filesystem"]
+    "bootstrap.take_leader_key" [label="Take a leader key in DCS"]
+  }
+
+  "bootstrap.do_bootstrap" -> "update_member"
+  "bootstrap.wait_for_leader" -> "update_member"
+  "bootstrap.clean" -> "update_member"
+  "bootstrap.take_leader_key" -> "update_member"
+
+  subgraph cluster_process_healthy_cluster {
+    label = "process_healthy_cluster"
+    "healthy_has_lock" [label="Am I the owner of the leader lock?", shape=diamond]
+    "healthy_is_leader" [label="Is Postgres running as master?", shape=diamond]
+    "healthy_no_lock" [label="Follow the leader (async,\ncreate/update recovery.conf and restart if necessary)"]
+    "healthy_has_lock" -> "healthy_no_lock" [label="no" color="red"]
+    "healthy_has_lock" -> "healthy_update_leader_lock" [label="yes" color="green"]
+    "healthy_update_leader_lock" [label="Try to update leader lock"]
+    "healthy_update_leader_lock" -> "healthy_update_success"
+    "healthy_update_success" [label="Success?", shape=diamond]
+    "healthy_update_success" -> "healthy_is_leader" [label="yes" color="green"]
+    "healthy_update_success" -> "healthy_demote" [label="no" color="red"]
+    "healthy_demote" [label="Demote (async,\nrestart in read-only)"]
+    "healthy_failover" [label="Promote Postgres to master"]
+    "healthy_is_leader" -> "healthy_failover" [label="no" color="red"]
+  }
+  "healthy_demote" -> "update_member"
+  "healthy_is_leader" -> "update_member" [label="yes" color="green"]
+  "healthy_failover" -> "update_member"
+  "healthy_no_lock" -> "update_member"
+
+  subgraph cluster_process_unhealthy_cluster {
+    label = "process_unhealthy_cluster"
+    "unhealthy_is_healthiest" [label="Am I the healthiest node?", shape="diamond"]
+    "unhealthy_is_healthiest" -> "unhealthy_leader_race" [label="yes", color="green"]
+    "unhealthy_leader_race" [label="Try to create leader key"]
+    "unhealthy_leader_race" -> "unhealthy_acquire_lock"
+    "unhealthy_acquire_lock" [label="Was I able to get the lock?", shape="diamond"]
+    "unhealthy_is_leader" [label="Is Postgres running as master?", shape=diamond]
+    "unhealthy_acquire_lock" -> "unhealthy_is_leader" [label="yes" color="green"]
+    "unhealthy_is_leader" -> "unhealthy_promote" [label="no" color="red"]
+    "unhealthy_promote" [label="Promote to master"]
+    "unhealthy_is_healthiest" -> "unhealthy_follow" [label="no" color="red"]
+    "unhealthy_follow" [label="try to follow somebody else()"]
+    "unhealthy_acquire_lock" -> "unhealthy_follow" [label="no" color="red"]
+  }
+  "unhealthy_follow" -> "update_member"
+  "unhealthy_promote" -> "update_member"
+  "unhealthy_is_leader" -> "update_member" [label="yes" color="green"]
+}

docs/index.rst

@@ -0,0 +1,37 @@
+.. Patroni documentation master file, created by
+   sphinx-quickstart on Mon Dec 19 16:54:09 2016.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Introduction
+============
+
+Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__, `Consul <https://github.com/hashicorp/consul>`__ or `Kubernetes <https://kubernetes.io>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
+
+We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely. There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
+
+**Note to Kubernetes users**: Patroni can run natively on top of Kubernetes. Take a look at the :ref:`Kubernetes <kubernetes>` chapter of the Patroni documentation.
+
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   README
+   dynamic_configuration
+   ENVIRONMENT
+   SETTINGS
+   replica_bootstrap
+   replication_modes
+   pause
+   kubernetes
+   watchdog
+   releases
+   CONTRIBUTING
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/kubernetes.rst

@@ -0,0 +1,53 @@
+.. _kubernetes:
+
+Using Patroni with Kubernetes
+=============================
+
+Patroni can use Kubernetes objects in order to store the state of the cluster and manage the leader key. That makes it
+capable of operating Postgres in Kubernetes environment without any consistency store, namely, one doesn't
+need to run an extra Etcd deployment. There are two different type of Kubernetes objects Patroni can use to store the
+leader and the configuration keys, they are configured with the `kubernetes.use_endpoints` or `PATRONI_KUBERNETES_USE_ENDPOINTS`
+environment variable.
+
+Use Endpoints
+-------------
+
+Despite the fact that this is the recommended mode, it is turned off by default for compatibility reasons. When it is on, Patroni stores
+the cluster configuration and the leader key in the `metadata: annotations` fields of the respective `Endpoints` it creates.
+Changing the leader is safer than when using `ConfigMaps`, since both the annotations, containing the leader information, and the actual addresses
+pointing to the running leader pod are updated simultaneously in one go.
+
+Use ConfigMaps
+--------------
+
+In this mode, Patroni will create ConfigMaps instead of Endpoints and store keys inside meta-data of those ConfigMaps.
+Changing the leader takes at least two updates, one to the leader ConfigMap and another to the respective Endpoint.
+
+There are two ways to direct the traffic to the Postgres master:
+
+- use the `callback script <https://github.com/zalando/patroni/blob/master/kubernetes/callback.py>`_ provided by Patroni
+- configure the Kubernetes Postgres service to use the label selector with the `role_label` (configured in patroni configuration).
+
+Note that in some cases, for instance, when running on OpenShift, there is no alternative to using ConfigMaps.
+
+Configuration
+-------------
+
+Patroni Kubernetes :ref:`settings <kubernetes_settings>` and :ref:`environment variables <kubernetes_environment>` are described in the general chapters of the documentation.
+
+Examples
+--------
+
+- The `kubernetes <https://github.com/zalando/patroni/tree/master/kubernetes>`__ folder of the Patroni repository contains
+  examples of the Docker image, the Kubernetes manifest and the callback script in order to test Patroni Kubernetes setup.
+  Note that in the current state it will not be able to use PersistentVolumes because of permission issues.
+
+- You can find the full-featured Docker image that can use Persistent Volumes in the
+  `Spilo Project <https://github.com/zalando/spilo>`_.
+
+- There is also a `Helm chart <https://github.com/kubernetes/charts/tree/master/incubator/patroni>`_
+  to deploy the Spilo image configured with Patroni running using Kubernetes.
+
+- In order to run your database clusters at scale using Patroni and Spilo, take a look at the
+  `postgres-operator <https://github.com/zalando-incubator/postgres-operator>`_ project. It implements the operator pattern
+  to manage Spilo clusters.

docs/pause.rst

@@ -0,0 +1,35 @@
+.. _pause:
+
+Pause/Resume mode for the cluster
+=================================
+
+The goal
+--------
+
+Under certain circumstances Patroni needs to temporary step down from managing the cluster, while still retaining the cluster state in DCS. Possible use cases are uncommon activities on the cluster, such as major version upgrades or corruption recovery. During those activities nodes are often started and stopped for the reason unknown to Patroni, some nodes can be even temporary promoted, violating the assumption of running only one master. Therefore, Patroni needs to be able to "detach" from the running cluster, implementing an equivalent of the maintenance mode in Pacemaker.
+
+
+
+The implementation
+------------------
+
+When Patroni runs in a paused mode, it does not change the state of PostgreSQL, except for the following cases:
+
+- For each node, the member key in DCS is updated with the current information about the cluster. This causes Patroni to run read-only queries on a member node if the member is running.
+
+- For the Postgres master with the leader lock Patroni updates the lock. If the node with the leader lock stops being the master (i.e. is demoted manually), Patroni will release the lock instead of promoting the node back.
+
+- Manual unscheduled restart, reinitialize and manual failover are allowed. Manual failover is only allowed if the node to failover to is specified. In the paused mode, manual failover does not require a running master node.
+
+- If 'parallel' masters are detected by Patroni, it emits a warning, but does not demote the masters without the leader lock.
+
+- If there is no leader lock in the cluster, the running master acquires the lock. If there is more than one master node, then the first master to acquire the lock wins. If there are no masters altogether, Patroni does not try to promote any replicas. There is an exception in this rule: if there is no leader lock because the old master has demoted itself due to the manual promotion, then only the candidate node mentioned in the promotion request may take the leader lock. When the new leader lock is granted (i.e. after promoting a replica manually), Patroni makes sure the replicas that were streaming from the previous leader will switch to the new one.
+
+- When Postgres is stopped, Patroni does not try to start it. When Patroni is stopped, it does not try to stop the Postgres instance it is managing.
+
+User guide
+----------
+
+``patronictl`` supports ``pause`` and ``resume`` commands.
+
+One can also issue a ``PATCH`` request to the ``{namespace}/{cluster}/config`` key with ``{"pause": true/false/null}``

docs/releases.rst

@@ -0,0 +1,885 @@
+.. _releases:
+
+Release notes
+=============
+
+Version 1.4.4
+-------------
+
+**Stability improvements**
+
+- Fix race condition in poll_failover_result (Alexander Kukushkin)
+
+  It didn't affect directly neither failover nor switchover, but in some rare cases it was reporting success too early, when the former leader released the lock, producing a 'Failed over to "None"' instead of 'Failed over to "desired-node"' message.
+
+- Treat Postgres parameter names as case insensitive (Alexander)
+
+  Most of the Postgres parameters have snake_case names, but there are three exceptions from this rule: DateStyle, IntervalStyle and TimeZone. Postgres accepts those parameters when written in a different case (e.g. timezone = 'some/tzn'); however, Patroni was unable to find case-insensitive matches of those parameter names in pg_settings and ignored such parameters as a result.
+
+- Abort start if attaching to running postgres and cluster not initialized (Alexander)
+
+  Patroni can attach itself to an already running Postgres instance. It is imperative to start running Patroni on the master node before getting to the replicas.
+
+- Fix behavior of patronictl scaffold (Alexander)
+
+  Pass dict object to touch_member instead of json encoded string, DCS implementation will take care of encoding it.
+
+- Don't demote master if failed to update leader key in pause (Alexander)
+
+  During maintenance a DCS may start failing write requests while continuing to responds to read ones. In that case, Patroni used to put the Postgres master node to a read-only mode after failing to update the leader lock in DCS.
+
+- Sync replication slots when Patroni notices a new postmaster process (Alexander)
+
+  If Postgres has been restarted, Patroni has to make sure that list of replication slots matches its expectations.
+
+- Verify sysid and sync replication slots after coming out of pause (Alexander)
+
+  During the `maintenance` mode it may happen that data directory was completely rewritten and therefore we have to make sure that `Database system identifier` still belongs to our cluster and replication slots are in sync with Patroni expectations.
+
+- Fix a possible failure to start not running Postgres on a data directory with postmaster lock file present (Alexander)
+
+  Detect reuse of PID from the postmaster lock file. More likely to hit such problem if you run Patroni and Postgres in the docker container.
+
+- Improve protection of DCS being accidentally wiped (Alexander)
+
+  Patroni has a lot of logic in place to prevent failover in such case; it can also restore all keys back; however, until this change an accidental removal of /config key was switching off pause mode for 1 cycle of HA loop.
+
+- Do not exit when encountering invalid system ID (Oleksii Kliukin)
+
+  Do not exit when the cluster system ID is empty or the one that doesn't pass the validation check. In that case, the cluster most likely needs a reinit; mention it in the result message. Avoid terminating Patroni, as otherwise reinit cannot happen.
+
+**Compatibility with Kubernetes 1.10+**
+
+- Added check for empty subsets (Cody Coons)
+
+  Kubernetes 1.10.0+ started returning `Endpoints.subsets` set to `None` instead of `[]`.
+
+**Bootstrap improvements**
+
+- Make deleting recovery.conf optional (Brad Nicholson)
+
+  If `bootstrap.<custom_bootstrap_method_name>.keep_existing_recovery_conf` is defined and set to ``True``, Patroni will not remove the existing ``recovery.conf`` file. This is useful when bootstrapping from a backup with tools like pgBackRest that generate the appropriate `recovery.conf` for you.
+
+- Allow options to the basebackup built-in method (Oleksii)
+
+  It is now possible to supply options to the built-in basebackup method by defining the `basebackup` section in the configuration, similar to how those are defined for custom replica creation methods. The difference is in the format accepted by the `basebackup` section: since pg_basebackup accepts both `--key=value` and `--key` options, the contents of the section could be either a dictionary of key-value pairs, or a list of either one-element dictionaries or just keys (for the options that don't accept values). See :ref:`replica creation method <custom_replica_creation>` section for additional examples.
+
+
+Version 1.4.3
+-------------
+
+**Improvements in logging**
+
+- Make log level configurable from environment variables (Andy Newton, Keyvan Hedayati)
+
+  `PATRONI_LOGLEVEL` - sets the general logging level
+  `PATRONI_REQUESTS_LOGLEVEL` - sets the logging level for all HTTP requests e.g. Kubernetes API calls
+  See `the docs for Python logging <https://docs.python.org/3.6/library/logging.html#levels>` to get the names of possible log levels
+
+**Stability improvements and bug fixes**
+
+- Don't rediscover etcd cluster topology when watch timed out (Alexander Kukushkin)
+
+  If we have only one host in etcd configuration and exactly this host is not accessible, Patroni was starting discovery of cluster topology and never succeeding. Instead it should just switch to the next available node.
+
+- Write content of bootstrap.pg_hba into a pg_hba.conf after custom bootstrap (Alexander)
+
+  Now it behaves similarly to the usual bootstrap with `initdb`
+
+- Single user mode was waiting for user input and never finish (Alexander)
+
+  Regression was introduced in https://github.com/zalando/patroni/pull/576
+
+
+Version 1.4.2
+-------------
+
+**Improvements in patronictl**
+
+- Rename scheduled failover to scheduled switchover (Alexander Kukushkin)
+
+  Failover and switchover functions were separated in version 1.4, but `patronictl list` was still reporting `Scheduled failover` instead of `Scheduled switchover`.
+
+- Show information about pending restarts (Alexander)
+
+  In order to apply some configuration changes sometimes it is necessary to restart postgres. Patroni was already giving a hint about that in the REST API and when writing node status into DCS, but there were no easy way to display it.
+
+- Make show-config to work with cluster_name from config file (Alexander)
+
+  It works similar to the `patronictl edit-config`
+
+**Stability improvements**
+
+- Avoid calling pg_controldata during bootstrap (Alexander)
+
+  During initdb or custom bootstrap there is a time window when pgdata is not empty but pg_controldata has not been written yet. In such case pg_controldata call was failing with error messages.
+
+- Handle exceptions raised from psutil (Alexander)
+
+  cmdline is read and parsed every time when `cmdline()` method is called. It could happen that the process being examined
+  has already disappeared, in that case `NoSuchProcess` is raised.
+
+**Kubernetes support improvements**
+
+- Don't swallow errors from k8s API (Alexander)
+
+  A call to Kubernetes API could fail for a different number of reasons. In some cases such call should be retried, in some other cases we should log the error message and the exception stack trace. The change here will help debug Kubernetes permission issues.
+
+- Update Kubernetes example Dockerfile to install Patroni from the master branch (Maciej Szulik)
+
+  Before that it was using `feature/k8s`, which became outdated.
+
+- Add proper RBAC to run patroni on k8s (Maciej)
+
+  Add the Service account that is assigned to the pods of the cluster, the role that holds only the necessary permissions, and the rolebinding that connects the Service account and the Role.
+
+
+Version 1.4.1
+-------------
+
+**Fixes in patronictl**
+
+- Don't show current leader in suggested list of members to failover to. (Alexander Kukushkin)
+
+  patronictl failover could still work when there is leader in the cluster and it should be excluded from the list of member where it is possible to failover to.
+
+- Make patronictl switchover compatible with the old Patroni api (Alexander)
+
+  In case if POST /switchover REST API call has failed with status code 501 it will do it once again, but to /failover endpoint.
+
+
+Version 1.4
+-----------
+
+This version adds support for using Kubernetes as a DCS, allowing to run Patroni as a cloud-native agent in Kubernetes without any additional deployments of Etcd, Zookeeper or Consul.
+
+**Upgrade notice**
+
+Installing Patroni via pip will no longer bring in dependencies for (such as libraries for Etcd, Zookeper, Consul or Kubernetes, or support for AWS). In order to enable them one need to list them in pip install command explicitely, for instance `pip install patroni[etcd,kubernetes]`.
+
+**Kubernetes support**
+
+Implement Kubernetes-based DCS. The endpoints meta-data is used in order to store the configuration and the leader key. The meta-data field inside the pods definition is used to store the member-related data.
+In addition to using Endpoints, Patroni supports ConfigMaps. You can find more information about this feature in the :ref:`Kubernetes chapter of the documentation <kubernetes>`
+
+**Stability improvements**
+
+- Factor out postmaster process into a separate object (Ants Aasma)
+
+  This object identifies a running postmaster process via pid and start time and simplifies detection (and resolution) of situations when the postmaster was restarted behind our back or when postgres directory disappeared from the file system.
+
+- Minimize the amount of SELECT's issued by Patroni on every loop of HA cylce (Alexander Kukushkin)
+
+  On every iteration of HA loop Patroni needs to know recovery status and absolute wal position. From now on Patroni will run only single SELECT to get this information instead of two on the replica and three on the master. 
+  
+- Remove leader key on shutdown only when we have the lock (Ants)
+
+  Unconditional removal was generating unnecessary and missleading exceptions.
+
+**Improvements in patronictl**
+
+- Add version command to patronictl (Ants)
+
+  It will show the version of installed Patroni and versions of running Patroni instances (if the cluster name is specified).
+  
+- Make optional specifying cluster_name argument for some of patronictl commands (Alexander, Ants)
+
+  It will work if patronictl is using usual Patroni configuration file with the ``scope`` defined.
+
+- Show information about scheduled switchover and maintenance mode (Alexander)
+
+  Before that it was possible to get this information only from Patroni logs or directly from DCS.
+  
+- Improve ``patronictl reinit`` (Alexander)
+
+  Sometimes ``patronictl reinit`` refused to proceed when Patroni was busy with other actions, namely trying to start postgres. `patronictl` didn't provide any commands to cancel such long running actions and the only (dangerous) workarond was removing a data directory manually. The new implementation of `reinit` forcefully cancells other long-running actions before proceeding with reinit.
+  
+- Implement ``--wait`` flag in ``patronictl pause`` and ``patronictl resume`` (Alexander)
+
+  It will make ``patronictl`` wait until the requested action is acknowledged by all nodes in the cluster.
+  Such behaviour is achieved by exposing the ``pause`` flag for every node in DCS and via the REST API.
+
+- Rename ``patronictl failover`` into ``patronictl switchover`` (Alexander)
+
+  The previous ``failover`` was actually only capable of doing a switchover; it refused to proceed in a cluster without the leader.
+
+- Alter the behavior of ``patronictl failover`` (Alexander)
+
+  It will work even if there is no leader, but in that case you will have to explicitely specify a node which should become the new leader.
+
+**Expose information about timeline and history**
+
+- Expose current timeline in DCS and via API (Alexander)
+
+  Store information about the current timeline for each member of the cluster. This information is accessible via the API and is stored in the DCS
+
+- Store promotion history in the /history key in DCS (Alexander)
+
+  In addition, store the timeline history enriched with the timestamp of the corresponding promotion in the /history key in DCS and update it with each promote.
+
+**Add endpoints for getting synchronous and asynchronous replicas**
+
+- Add new /sync and /async endpoints (Alexander, Oleksii Kliukin)
+
+ Those endpoints (also accessible as /synchronous and /asynchronous) return 200 only for synchronous and asynchornous replicas correspondingly (exclusing those marked as `noloadbalance`).
+
+**Allow multiple hosts for Etcd**
+
+- Add a new `hosts` parameter to Etcd configuration (Alexander)
+
+  This parameter should contain the initial list of hosts that will be used to discover and populate the list of the running etcd cluster members. If for some reason during work this list of discovered hosts is exhausted (no available hosts from that list), Patroni will return to the initial list from the `hosts` parameter.
+
+
+Version 1.3.6
+-------------
+
+**Stability improvements**
+
+- Verify process start time when checking if postgres is running. (Ants Aasma)
+
+  After a crash that doesn't clean up postmaster.pid there could be a new process with the same pid, resulting in a false positive for is_running(), which will lead to all kinds of bad behavior.
+
+- Shutdown postgresql before bootstrap when we lost data directory (ainlolcat)
+  
+  When data directory on the master is forcefully removed, postgres process can still stay alive for some time and prevent the replica created in place of that former master from starting or replicating.
+  The fix makes Patroni cache the postmaster pid and its start time and let it terminate the old postmaster in case it is still running after the corresponding data directory has been removed.
+
+- Perform crash recovery in a single user mode if postgres master dies (Alexander Kukushkin)
+
+  It is unsafe to start immediately as a standby and not possible to run ``pg_rewind`` if postgres hasn't been shut down cleanly.
+  The single user crash recovery only kicks in if ``pg_rewind`` is enabled or there is no master at the moment.
+
+**Consul improvements**
+
+- Make it possible to provide datacenter configuration for Consul (Vilius Okockis, Alexander)
+
+  Before that Patroni was always communicating with datacenter of the host it runs on.
+
+- Always send a token in X-Consul-Token http header (Alexander)
+
+  If ``consul.token`` is defined in Patroni configuration, we will always send it in the 'X-Consul-Token' http header.
+  python-consul module tries to be "consistent" with Consul REST API, which doesn't accept token as a query parameter for `session API <https://www.consul.io/api/session.html>`__, but it still works with 'X-Consul-Token' header.
+
+- Adjust session TTL if supplied value is smaller than the minimum possible (Stas Fomin, Alexander)
+
+  It could happen that the TTL provided in the Patroni configuration is smaller than the minimum one supported by Consul. In that case, Consul agent fails to create a new session.
+  Without a session Patroni cannot create member and leader keys in the Consul KV store, resulting in an unhealthy cluster.
+
+**Other improvements**
+
+- Define custom log format via environment variable ``PATRONI_LOGFORMAT`` (Stas)
+
+  Allow disabling timestamps and other similar fields in Patroni logs if they are already added by the system logger (usually when Patroni runs as a service).
+
+Version 1.3.5
+-------------
+
+**Bugfix**
+
+- Set role to 'uninitialized' if data directory was removed (Alexander Kukushkin)
+
+  If the node was running as a master it was preventing from failover.
+
+**Stability improvement**
+
+- Try to run postmaster in a single-user mode if we tried and failed to start postgres (Alexander)
+
+  Usually such problem happens when node running as a master was terminated and timelines were diverged.
+  If ``recovery.conf`` has ``restore_command`` defined, there are really high chances that postgres will abort startup and leave controldata unchanged.
+  It makes impossible to use ``pg_rewind``, which requires a clean shutdown.
+
+**Consul improvements**
+
+- Make it possible to specify health checks when creating session (Alexander)
+
+  If not specified, Consul will use "serfHealth". From one side it allows fast detection of isolated master, but from another side it makes it impossible for Patroni to tolerate short network lags.
+
+**Bugfix**
+
+- Fix watchdog on Python 3 (Ants Aasma)
+
+  A misunderstanding of the ioctl() call interface. If mutable=False then fcntl.ioctl() actually returns the arg buffer back.
+  This accidentally worked on Python2 because int and str comparison did not return an error.
+  Error reporting is actually done by raising IOError on Python2 and OSError on Python3.
+
+Version 1.3.4
+-------------
+
+**Different Consul improvements**
+
+- Pass the consul token as a header (Andrew Colin Kissa)
+
+  Headers are now the prefered way to pass the token to the consul `API <https://www.consul.io/api/index.html#authentication>`__.
+  
+
+- Advanced configuration for Consul (Alexander Kukushkin)
+
+  possibility to specify ``scheme``, ``token``, client and ca certificates :ref:`details <consul_settings>`.
+
+- compatibility with python-consul-0.7.1 and above (Alexander)
+
+  new python-consul module has changed signature of some methods
+
+- "Could not take out TTL lock" message was never logged (Alexander)
+
+  Not a critical bug, but lack of proper logging complicates investigation in case of problems.
+
+
+**Quote synchronous_standby_names using quote_ident**
+
+- When writing ``synchronous_standby_names`` into the ``postgresql.conf`` its value must be quoted (Alexander)
+
+  If it is not quoted properly, PostgreSQL will effectively disable synchronous replication and continue to work.
+
+
+**Different bugfixes around pause state, mostly related to watchdog** (Alexander)
+
+- Do not send keepalives if watchdog is not active
+- Avoid activating watchdog in a pause mode
+- Set correct postgres state in pause mode
+- Do not try to run queries from API if postgres is stopped
+
+
+Version 1.3.3
+-------------
+
+**Bugfixes**
+
+- synchronous replication was disabled shortly after promotion even when synchronous_mode_strict was turned on (Alexander Kukushkin)
+- create empty ``pg_ident.conf`` file if it is missing after restoring from the backup (Alexander)
+- open access in ``pg_hba.conf`` to all databases, not only postgres (Franco Bellagamba)
+
+
+Version 1.3.2
+-------------
+
+**Bugfix**
+
+- patronictl edit-config didn't work with ZooKeeper (Alexander Kukushkin)
+
+
+Version 1.3.1
+-------------
+
+**Bugfix**
+
+- failover via API was broken due to change in ``_MemberStatus`` (Alexander Kukushkin)
+
+
+Version 1.3
+-----------
+
+Version 1.3 adds custom bootstrap possibility, significantly improves support for pg_rewind, enhances the
+synchronous mode support, adds configuration editing to patronictl and implements watchdog support on Linux.
+In addition, this is the first version to work correctly with PostgreSQL 10.
+
+**Upgrade notice**
+
+There are no known compatibility issues with the new version of Patroni. Configuration from version 1.2 should work
+without any changes. It is possible to upgrade by installing new packages and either  restarting Patroni (will cause
+PostgreSQL restart), or by putting Patroni into a :ref:`pause mode <pause>` first and then restarting Patroni on all
+nodes in the cluster (Patroni in a pause mode will not attempt to stop/start PostgreSQL), resuming from the pause mode
+at the end.
+
+**Custom bootstrap**
+
+- Make the process of bootstrapping the cluster configurable (Alexander Kukushkin)
+
+  Allow custom bootstrap scripts instead of ``initdb`` when initializing the very first node in the cluster.
+  The bootstrap command receives the name of the cluster and the path to the data directory. The resulting cluster can
+  be configured to perform recovery, making it possible to bootstrap from a backup and do point in time recovery. Refer
+  to the :ref:`documentaton page <custom_bootstrap>` for more detailed description of this feature.
+
+**Smarter pg_rewind support**
+
+-  Decide on whether to run pg_rewind by looking at the timeline differences from the current master (Alexander)
+
+   Previously, Patroni had a fixed set of conditions to trigger pg_rewind, namely when starting a former master, when
+   doing a switchover to the designated node for every other node in the cluster or when there is a replica with the
+   nofailover tag. All those cases have in common a chance that some replica may be ahead of the new master. In some cases,
+   pg_rewind did nothing, in some other ones it was not running when necessary. Instead of relying on this limited list
+   of rules make Patroni compare the master and the replica WAL positions (using the streaming replication protocol)
+   in order to reliably decide if rewind is necessary for the replica.
+
+**Synchronous replication mode strict**
+
+-  Enhance synchronous replication support by adding the strict mode (James Sewell, Alexander)
+
+   Normally, when ``synchronous_mode`` is enabled and there are no replicas attached to the master, Patroni will disable
+   synchronous replication in order to keep the master available for writes. The ``synchronous_mode_strict`` option
+   changes that, when it is set Patroni will not disable the synchronous replication in a lack of replicas, effectively
+   blocking all clients writing data to the master. In addition to the synchronous mode guarantee of preventing any data
+   loss due to automatic failover, the strict mode ensures that each write is either durably stored on two nodes or not
+   happening altogether if there is only one node in the cluster.
+
+**Configuration editing with patronictl**
+
+- Add configuration editing to patronictl (Ants Aasma, Alexander)
+
+  Add the ability to patronictl of editing dynamic cluster configuration stored in DCS. Support either specifying the
+  parameter/values from the command-line, invoking the $EDITOR, or applying configuration from the yaml file.
+
+**Linux watchdog support**
+
+- Implement watchdog support for Linux (Ants)
+
+  Support Linux software watchdog in order to reboot the node where Patroni is not running or not responding (e.g because
+  of the high load) The Linux software watchdog reboots the non-responsive node. It is possible to configure the watchdog
+  device to use (`/dev/watchdog` by default) and the mode (on, automatic, off) from the watchdog section of the Patroni
+  configuration. You can get more information from the :ref:`watchdog documentation <watchdog>`.
+
+**Add support for PostgreSQL 10**
+
+- Patroni is compatible with all beta versions of PostgreSQL 10 released so far and we expect it to be compatible with
+  the PostgreSQL 10 when it will be released.
+
+**PostgreSQL-related minor improvements**
+
+- Define pg_hba.conf via the Patroni configuration file or the dynamic configuration in DCS (Alexander)
+
+  Allow to define the contents of ``pg_hba.conf`` in the ``pg_hba`` sub-section of the ``postgresql`` section of the
+  configuration. This simplifies managing ``pg_hba.conf`` on multiple nodes, as one needs to define it only ones in DCS
+  instead of logging to every node, changing it manually and reload the configuration.
+
+  When defined, the contents of this section will replace the current ``pg_hba.conf`` completely. Patroni ignores it
+  if ``hba_file`` PostgreSQL parameter is set.
+
+- Support connecting via a UNIX socket to the local PostgreSQL cluster (Alexander)
+
+  Add the ``use_unix_socket`` option to the ``postgresql`` section of Patroni configuration. When set to true and the
+  PostgreSQL ``unix_socket_directories`` option is not empty, enables Patroni to use the first value from it to connect
+  to the local PostgreSQL cluster. If ``unix_socket_directories`` is not defined, Patroni will assume its default value
+  and omit the ``host`` parameter in the PostgreSQL connection string altogether.
+
+- Support change of superuser and replication credentials on reload (Alexander)
+
+- Support storing of configuration files outside of PostgreSQL data directory (@jouir)
+
+  Add the new configuration ``postgresql`` configuration directive ``config_dir``.
+  It defaults to the data directory and must be writable by Patroni.
+
+**Bug fixes and stability improvements**
+
+- Handle EtcdEventIndexCleared and EtcdWatcherCleared exceptions (Alexander)
+
+  Faster recovery when the watch operation is ended by Etcd by avoiding useless retries.
+
+- Remove error spinning on Etcd failure and reduce log spam (Ants)
+
+  Avoid immediate retrying and emitting stack traces in the log on the second and subsequent Etcd connection failures.
+
+- Export locale variables when forking PostgreSQL processes (Oleksii Kliukin)
+
+  Avoid the `postmaster became multithreaded during startup` fatal error on non-English locales for PostgreSQL built with NLS.
+
+- Extra checks when dropping the replication slot (Alexander)
+
+  In some cases Patroni is prevented from dropping the replication slot by the WAL sender.
+
+- Truncate the replication slot name to 63  (NAMEDATALEN - 1) characters to comply with PostgreSQL naming rules (Nick Scott)
+
+- Fix a race condition resulting in extra connections being opened to the PostgreSQL cluster from Patroni (Alexander)
+
+- Release the leader key when the node restarts with an empty data directory (Alex Kerney)
+
+- Set asynchronous executor busy when running bootstrap without a leader (Alexander)
+
+  Failure to do so could have resulted in errors stating the node belonged to a different cluster, as Patroni proceeded with
+  the normal business while being bootstrapped by a bootstrap method that doesn't require a leader to be present in the
+  cluster.
+
+- Improve WAL-E replica creation method (Joar Wandborg, Alexander).
+
+  - Use csv.DictReader when parsing WAL-E base backup, accepting ISO dates with space-delimited date and time.
+  - Support fetching current WAL position from the replica to estimate the amount of WAL to restore. Previously, the code used to call system information functions that were available only on the master node.
+
+
+Version 1.2
+-----------
+
+This version introduces significant improvements over the handling of synchronous replication, makes the startup process and failover more reliable, adds PostgreSQL 9.6 support and fixes plenty of bugs.
+In addition, the documentation, including these release notes, has been moved to https://patroni.readthedocs.io.
+
+**Synchronous replication**
+
+- Add synchronous replication support. (Ants Aasma)
+
+  Adds a new configuration variable ``synchronous_mode``. When enabled, Patroni will manage ``synchronous_standby_names`` to enable synchronous replication whenever there are healthy standbys available. When synchronous mode is enabled, Patroni will automatically fail over only to a standby that was synchronously replicating at the time of the master failure. This effectively means that no user visible transaction gets lost in such a case. See the
+  :ref:`feature documentation <synchronous_mode>` for the detailed description and implementation details.
+
+**Reliability improvements**
+
+- Do not try to update the leader position stored in the ``leader optime`` key when PostgreSQL is not 100% healthy. Demote immediately when the update of the leader key failed. (Alexander Kukushkin)
+
+- Exclude unhealthy nodes from the list of targets to clone the new replica from. (Alexander)
+
+- Implement retry and timeout strategy for Consul similar to how it is done for Etcd. (Alexander)
+
+- Make ``--dcs`` and ``--config-file`` apply to all options in ``patronictl``. (Alexander)
+
+- Write all postgres parameters into postgresql.conf. (Alexander)
+
+  It allows starting PostgreSQL configured by Patroni with just ``pg_ctl``.
+
+- Avoid exceptions when there are no users in the config. (Kirill Pushkin)
+
+- Allow pausing an unhealthy cluster. Before this fix, ``patronictl`` would bail out if the node it tries to execute pause on is unhealthy. (Alexander)
+
+- Improve the leader watch functionality. (Alexander)
+
+  Previously the replicas were always watching the leader key (sleeping until the timeout or the leader key changes). With this change, they only watch
+  when the replica's PostgreSQL is in the ``running`` state and not when it is stopped/starting or restarting PostgreSQL.
+
+- Avoid running into race conditions when handling SIGCHILD as a PID 1. (Alexander)
+
+  Previously a race condition could occur when running inside the Docker containers, since the same process inside Patroni both spawned new processes
+  and handled SIGCHILD from them. This change uses fork/execs for Patroni and leaves the original PID 1 process responsible for handling signals from children.
+
+- Fix WAL-E restore. (Oleksii Kliukin)
+
+  Previously WAL-E restore used the ``no_master`` flag to avoid consulting with the master altogether, making Patroni always choose restoring
+  from WAL over the ``pg_basebackup``. This change reverts it to the original meaning of ``no_master``, namely Patroni WAL-E restore may be selected as a replication method if the master is not running.
+  The latter is checked by examining the connection string passed to the method. In addition, it makes the retry mechanism more robust and handles other minutia.
+
+- Implement asynchronous DNS resolver cache. (Alexander)
+
+  Avoid failing when DNS is temporary unavailable (for instance, due to an excessive traffic received by the node).
+
+- Implement starting state and master start timeout. (Ants, Alexander)
+
+  Previously ``pg_ctl`` waited for a timeout and then happily trodded on considering PostgreSQL to be running. This caused PostgreSQL to show up in listings as running when it was actually not and caused a race condition that   resulted in either a failover, or a crash recovery, or a crash recovery interrupted by failover and a missed rewind.
+  This change adds a ``master_start_timeout`` parameter and introduces a new state for the main HA loop: ``starting``. When ``master_start_timeout`` is 0 we will failover immediately when the master crashes as soon as there is a failover candidate. Otherwise, Patroni will wait after attempting to start PostgreSQL on the master for the duration of the timeout; when it expires, it will failover if possible. Manual failover requests will be honored during the crash of the master even before the timeout expiration.
+
+  Introduce the ``timeout`` parameter to the ``restart`` API endpoint and ``patronictl``. When it is set and restart takes longer than the timeout, PostgreSQL is considered unhealthy and the other nodes becomes eligible to take the leader lock.
+
+- Fix ``pg_rewind`` behavior in a pause mode. (Ants)
+
+  Avoid unnecessary restart in a pause mode when Patroni thinks it needs to rewind but rewind is not possible (i.e. ``pg_rewind`` is not present). Fallback to default ``libpq`` values for the ``superuser`` (default OS user) if ``superuser`` authentication is missing from the ``pg_rewind`` related Patroni configuration section.
+
+- Serialize callback execution. Kill the previous callback of the same type when the new one is about to run. Fix the issue of spawning zombie processes when running callbacks. (Alexander)
+
+- Avoid promoting a former master when the leader key is set in DCS but update to this leader key fails. (Alexander)
+
+  This avoids the issue of a current master continuing to keep its role when it is partitioned together with the minority of nodes in Etcd and other DCSs that allow "inconsistent reads".
+
+**Miscellaneous**
+
+- Add ``post_init`` configuration option on bootstrap. (Alejandro Martínez)
+
+  Patroni will call the script argument of this option right after running ``initdb`` and starting up PostgreSQL for a new cluster. The script receives a connection URL with ``superuser``
+  and sets ``PGPASSFILE`` to point to the ``.pgpass`` file containing the password. If the script fails, Patroni initialization fails as well. It is useful for adding
+  new users or creating extensions in the new cluster.
+
+- Implement PostgreSQL 9.6 support. (Alexander)
+
+  Use ``wal_level = replica`` as a synonym for ``hot_standby``, avoiding pending_restart flag when it changes from one to another. (Alexander)
+
+**Documentation improvements**
+
+- Add a Patroni main `loop workflow diagram <https://raw.githubusercontent.com/zalando/patroni/master/docs/ha_loop_diagram.png>`__. (Alejandro, Alexander)
+
+- Improve README, adding the Helm chart and links to release notes. (Lauri Apple)
+
+- Move Patroni documentation to ``Read the Docs``. The up-to-date documentation is available at https://patroni.readthedocs.io. (Oleksii)
+
+  Makes the documentation easily viewable from different devices (including smartphones) and searchable.
+
+- Move the package to the semantic versioning. (Oleksii)
+
+  Patroni will follow the major.minor.patch version schema to avoid releasing the new minor version on small but critical bugfixes. We will only publish the release notes for the minor version, which will include all patches.
+
+
+Version 1.1
+-----------
+
+This release improves management of Patroni cluster by bring in pause mode, improves maintenance with scheduled and conditional restarts, makes Patroni interaction with Etcd or Zookeeper more resilient and greatly enhances patronictl.
+
+**Upgrade notice**
+
+When upgrading from releases below 1.0 read about changing of credentials and configuration format at 1.0 release notes.
+
+**Pause mode**
+
+- Introduce pause mode to temporary detach Patroni from managing PostgreSQL instance (Murat Kabilov, Alexander Kukushkin, Oleksii Kliukin).
+
+  Previously, one had to send SIGKILL signal to Patroni to stop it without terminating PostgreSQL. The new pause mode detaches Patroni from PostgreSQL cluster-wide without terminating Patroni. It is similar to the maintenance mode in Pacemaker. Patroni is still responsible for updating member and leader keys in DCS, but it will not start, stop or restart PostgreSQL server in the process. There are a few exceptions, for instance, manual failovers, reinitializes and restarts are still allowed. You can read :ref:`a detailed description of this feature <pause>`.
+
+In addition, patronictl supports new ``pause`` and ``resume`` commands to toggle the pause mode.
+
+**Scheduled and conditional restarts**
+
+- Add conditions to the restart API command (Oleksii)
+
+  This change enhances Patroni restarts by adding a couple of conditions that can be verified in order to do the restart. Among the conditions are restarting when PostgreSQL role is either a master or a replica, checking the PostgreSQL version number or restarting only when restart is necessary in order to apply configuration changes.
+
+- Add scheduled restarts (Oleksii)
+
+  It is now possible to schedule a restart in the future. Only one scheduled restart per node is supported. It is possible to clear the scheduled restart if it is not needed anymore. A combination of scheduled and conditional restarts is supported, making it possible, for instance, to scheduled minor PostgreSQL upgrades in the night, restarting only the instances that are running the outdated minor version without adding postgres-specific logic to administration scripts.
+
+- Add support for conditional and scheduled restarts to patronictl (Murat).
+
+  patronictl restart supports several new options. There is also patronictl flush command to clean the scheduled actions.
+
+**Robust DCS interaction**
+
+- Set Kazoo timeouts depending on the loop_wait (Alexander)
+
+  Originally, ping_timeout and connect_timeout values were calculated from the negotiated session timeout. Patroni loop_wait was not taken into account. As
+  a result, a single retry could take more time than the session timeout, forcing Patroni to release the lock and demote.
+
+  This change set ping and connect timeout to half of the value of loop_wait, speeding up detection of connection issues and  leaving enough time to retry the connection attempt before loosing the lock.
+
+- Update Etcd topology only after original request succeed (Alexander)
+
+  Postpone updating the Etcd topology known to the client until after the original request. When retrieving the cluster topology, implement the retry timeouts depending on the known number of nodes in the Etcd cluster. This makes our client prefer to get the results of the request to having the up-to-date list of nodes.
+
+  Both changes make Patroni connections to DCS more robust in the face of network issues.
+
+**Patronictl, monitoring and configuration**
+
+- Return information about streaming replicas via the API (Feike Steenbergen)
+
+Previously, there was no reliable way to query Patroni about PostgreSQL instances that fail to stream changes (for instance, due to connection issues). This change exposes the contents of pg_stat_replication via the /patroni endpoint.
+
+- Add patronictl scaffold command (Oleksii)
+
+  Add a command to create cluster structure in Etcd. The cluster is created with user-specified sysid and leader, and both leader and member keys are made persistent. This command is useful to create so-called master-less configurations, where Patroni cluster consisting of only replicas replicate  from the external master node that is unaware of Patroni. Subsequently, one
+  may remove the leader key, promoting one of the Patroni nodes and replacing
+  the original master with the Patroni-based HA cluster.
+
+- Add configuration option ``bin_dir`` to locate PostgreSQL binaries (Ants Aasma)
+
+  It is useful to be able to specify the location of PostgreSQL binaries explicitly when Linux distros that support installing multiple PostgreSQL versions at the same time.
+
+- Allow configuration file path to be overridden using ``custom_conf`` of (Alejandro Martínez)
+
+  Allows for custom configuration file paths, which will be unmanaged by Patroni, :ref:`details <postgresql_settings>`.
+
+**Bug fixes and code improvements**
+
+- Make Patroni compatible with new version schema in PostgreSQL 10 and above (Feike)
+
+  Make sure that Patroni understand 2-digits version numbers when doing conditional restarts based on the PostgreSQL version.
+
+- Use pkgutil to find DCS modules (Alexander)
+
+  Use the dedicated python module instead of traversing directories manually in order to find DCS modules.
+
+- Always call on_start callback when starting Patroni (Alexander)
+
+  Previously, Patroni did not call any callbacks when attaching to the already running node with the correct role. Since callbacks are often used to route
+  client connections that could result in the failure to register the running
+  node in the connection routing scheme. With this fix, Patroni calls on_start
+  callback even when attaching to the already running node.
+
+- Do not drop active replication slots (Murat, Oleksii)
+
+  Avoid dropping active physical replication slots on master. PostgreSQL cannot
+  drop such slots anyway. This change makes possible to run non-Patroni managed
+  replicas/consumers on the master.
+
+- Close Patroni connections during start of the PostgreSQL instance (Alexander)
+
+  Forces Patroni to close all former connections when PostgreSQL node is started. Avoids the trap of reusing former connections if postmaster was killed with SIGKILL.
+
+- Replace invalid characters when constructing slot names from member names (Ants)
+
+  Make sure that standby names that do not comply with the slot naming rules don't cause the slot creation and standby startup to fail. Replace the dashes in the slot names with underscores and all other characters not allowed in slot names with their unicode codepoints.
+
+Version 1.0
+-----------
+
+This release introduces the global dynamic configuration that allows dynamic changes of the PostgreSQL and Patroni configuration parameters for the entire HA cluster. It also delivers numerous bugfixes.
+
+**Upgrade notice**
+
+When upgrading from v0.90 or below, always upgrade all replicas before the master. Since we don't store replication credentials in DCS anymore, an old replica won't be able to connect to the new master.
+
+**Dynamic Configuration**
+
+- Implement the dynamic global configuration (Alexander Kukushkin)
+
+  Introduce new REST API endpoint /config to provide PostgreSQL and Patroni configuration parameters that should be set globally for the entire HA cluster (master and all the replicas). Those parameters are set in DCS and in many cases can be applied without disrupting PostgreSQL or Patroni. Patroni sets a special flag called "pending restart" visible via the API when some of the values require the PostgreSQL restart. In that case, restart should be issued manually via the API.
+
+  Patroni SIGHUP or POST to /reload will make it re-read the configuration file.
+
+  See the :ref:`dynamic configuration <dynamic_configuration>`  for the details on which parameters can be changed and the order of processing difference configuration sources.
+
+  The configuration file format *has changed* since the v0.90. Patroni is still compatible with the old configuration files, but in order to take advantage of the bootstrap parameters one needs to change it. Users are encourage to update them by referring to the :ref:`dynamic configuraton documentation page <dynamic_configuration>`.
+
+**More flexible configuration***
+
+- Make postgresql configuration and database name Patroni connects to configurable (Misja Hoebe)
+
+  Introduce `database` and `config_base_name` configuration parameters. Among others, it makes possible to run Patroni with PipelineDB and other PostgreSQL forks.
+
+- Implement possibility to configure some Patroni configuration parameters via environment (Alexander)
+
+  Those include the scope, the node name and the namespace, as well as the secrets and makes it easier to run Patroni in a dynamic environment, i.e. Kubernetes  Please, refer to the :ref:`supported environment variables <environment>` for further details.
+
+- Update the built-in Patroni docker container  to take advantage of environment-based configuration (Feike Steenbergen).
+
+- Add Zookeeper support to Patroni docker image (Alexander)
+
+- Split the Zookeeper and Exhibitor configuration options (Alexander)
+
+- Make patronictl reuse the code from Patroni to read configuration (Alexander)
+
+  This allows patronictl to take advantage of environment-based configuration.
+
+- Set application name to node name in primary_conninfo (Alexander)
+
+  This simplifies identification and configuration of synchronous replication for a given node.
+
+**Stability, security and usability improvements**
+
+- Reset sysid and do not call pg_controldata when restore of backup in progress (Alexander)
+
+  This change reduces the amount of noise generated by Patroni API health checks during the lengthy initialization of this node from the backup.
+
+- Fix a bunch of pg_rewind corner-cases (Alexander)
+
+  Avoid running pg_rewind if the source cluster is not the master.
+
+  In addition, avoid removing the data directory on an unsuccessful rewind, unless the new parameter *remove_data_directory_on_rewind_failure* is set to true. By default it is false.
+
+- Remove passwords from the replication connection string in DCS (Alexander)
+
+  Previously, Patroni always used the replication credentials from the Postgres URL in DCS. That is now changed to take the credentials from the patroni configuration. The secrets (replication username and password) and no longer exposed in DCS.
+
+- Fix the asynchronous machinery around the demote call (Alexander)
+
+  Demote now runs totally asynchronously without blocking the DCS interactions.
+
+- Make patronictl always send the authorization header if it is configured (Alexander)
+
+  This allows patronictl to issue "protected" requests, i.e. restart or reinitialize, when Patroni is configured to require authorization on those.
+
+- Handle the SystemExit exception correctly (Alexander)
+
+  Avoids the issues of Patroni not stopping properly when receiving the SIGTERM
+
+- Sample haproxy templates for confd (Alexander)
+
+  Generates and dynamically changes haproxy configuration from the patroni state in the DCS using confide
+
+- Improve and restructure the documentation to make it more friendly to the new users (Lauri Apple)
+
+- API must report role=master during pg_ctl stop (Alexander)
+
+  Makes the callback calls more reliable, particularly in the cluster stop case. In addition, introduce the `pg_ctl_timeout` option to set the timeout for the start, stop and restart calls via the `pg_ctl`.
+
+- Fix the retry logic in etcd (Alexander)
+
+  Make retries more predictable and robust.
+
+- Make Zookeeper code more resilient against short network hiccups (Alexander)
+
+  Reduce the connection timeouts to make Zookeeper connection attempts more frequent.
+
+Version 0.90
+------------
+
+This releases adds support for Consul, includes a new *noloadbalance* tag, changes the behavior of the *clonefrom* tag, improves *pg_rewind* handling and improves *patronictl* control program.
+
+**Consul support**
+
+- Implement Consul support (Alexander Kukushkin)
+
+  Patroni runs against Consul, in addition to Etcd and Zookeeper. the connection parameters can be configured in the YAML file.
+
+**New and improved tags**
+
+- Implement *noloadbalance* tag (Alexander)
+
+  This tag makes Patroni always return that the replica is not available to the load balancer.
+
+- Change the implementation of the *clonefrom* tag (Alexander)
+
+  Previously, a node name had to be supplied to the *clonefrom*, forcing a tagged replica to clone from the specific node. The new implementation makes *clonefrom* a boolean tag: if it is set to true, the replica becomes a candidate for other replicas to clone from it. When multiple candidates are present, the replicas picks one randomly.
+
+**Stability and security improvements**
+
+- Numerous reliability improvements (Alexander)
+
+  Removes some spurious error messages, improves the stability of the failover, addresses some corner cases with reading data from DCS, shutdown, demote and reattaching of the former leader.
+
+- Improve systems script to avoid killing Patroni children on stop (Jan Keirse, Alexander Kukushkin)
+
+  Previously, when stopping Patroni, *systemd* also sent a signal to PostgreSQL. Since Patroni also tried to stop PostgreSQL by itself, it resulted in sending to different shutdown requests (the smart shutdown, followed by the fast shutdown). That resulted in replicas disconnecting too early and a former master not being able to rejoin after demote. Fix by Jan with prior research by Alexander.
+
+- Eliminate some cases where the former master was unable to call pg_rewind before rejoining as a replica (Oleksii Kliukin)
+
+  Previously, we only called *pg_rewind* if the former master had crashed. Change this to always run pg_rewind for the former master as long as pg_rewind is present in the system. This fixes the case when the master is shut down before the replicas managed to get the latest changes (i.e. during the "smart" shutdown).
+
+- Numerous improvements to unit- and acceptance- tests, in particular, enable support for Zookeeper and Consul (Alexander).
+
+- Make Travis CI faster and implement support for running tests against Zookeeper (Exhibitor) and Consul (Alexander)
+
+  Both unit and acceptance tests run automatically against Etcd, Zookeeper and Consul on each commit or pull-request.
+
+- Clear environment variables before calling PostgreSQL commands from Patroni (Feike Steenbergen)
+
+  This prevents  a possibility of reading system environment variables by connecting to the PostgreSQL cluster managed by Patroni.
+
+**Configuration and control changes**
+
+- Unify patronictl and Patroni configuration (Feike)
+
+  patronictl can use the same configuration file as Patroni itself.
+
+- Enable Patroni to read the configuration from the environment variables (Oleksii)
+
+  This simplifies generating configuration for Patroni automatically, or merging a single configuration from different sources.
+
+- Include database system identifier in the information returned by the API (Feike)
+
+- Implement *delete_cluster* for all available DCSs (Alexander)
+
+  Enables support for DCSs other than Etcd in patronictl.
+
+
+Version 0.80
+------------
+
+This release adds support for *cascading replication* and simplifies Patroni management by providing *scheduled failovers*. One may use older versions of Patroni (in particular, 0.78) combined with this one in order to migrate to the new release. Note that the scheduled failover and cascading replication related features will only work with Patroni 0.80 and above.
+
+**Cascading replication**
+
+ - Add support for the *replicatefrom* and *clonefrom* tags for the patroni node (Oleksii Kliukin).
+
+ The tag *replicatefrom*  allows a replica to use an arbitrary node a source, not necessary the master. The *clonefrom* does the same for the initial backup. Together, they enable Patroni to fully support cascading replication.
+
+- Add support for running replication methods to initialize the replica even without a running replication connection (Oleksii).
+
+ This is useful in order to create replicas from the snapshots stored on S3 or FTP.  A replication method that does not require a running replication connection should supply *no_master: true* in the yaml configuration. Those scripts will still be called in order if the replication connection is present.
+
+**Patronictl, API and DCS improvements**
+
+- Implement scheduled failovers (Feike Steenbergen).
+
+  Failovers can be scheduled to happen at a certain time in the future, using either patronictl, or API calls.
+
+- Add support for *dbuser* and *password* parameters in patronictl (Feike).
+
+- Add PostgreSQL version to the health check output (Feike).
+
+- Improve Zookeeper support in patronictl (Oleksandr Shulgin)
+
+- Migrate to python-etcd 0.43 (Alexander Kukushkin)
+
+**Configuration**
+
+- Add a sample systems configuration script for Patroni (Jan Keirse).
+
+- Fix the problem of Patroni ignoring the superuser name specified in the configuration file for DB connections  (Alexander).
+
+- Fix the handling of CTRL-C by creating a separate session ID and process group for the postmaster launched by Patroni (Alexander).
+
+**Tests**
+
+- Add acceptance tests with *behave* in order to check real-world scenarios of running Patroni (Alexander, Oleksii).
+
+  The tests can be launched manually using the *behave* command. They are also launched automatically for pull requests and after commits.
+
+  Release notes for some older versions can be found on `project's github page <https://github.com/zalando/patroni/releases>`__.

docs/replica_bootstrap.rst

@@ -0,0 +1,130 @@
+Replica imaging and bootstrap
+=============================
+
+Patroni allows customizing creation of a new replica. It also supports defining what happens when the new empty cluster
+is being bootstrapped. The distinction between two is well defined: Patroni creates replicas only if the ``initialize``
+key is present in DCS for the cluster. If there is no ``initialize`` key - Patroni calls bootstrap exclusively on the
+first node that takes the initialize key lock.
+
+.. _custom_bootstrap:
+
+Bootstrap
+---------
+
+PostgreSQL provides ``initdb`` command to initialize a new cluster and Patroni calls it by default. In certain cases,
+particularly when creating a new cluster as a copy of an existing one, it is necessary to replace a built-in method with
+custom actions. Patroni supports executing user-defined scripts to bootstrap new clusters, supplying some required
+arguments to them, i.e. the name of the cluster and the path to the data directory. This is configured in the
+``bootstrap`` section of the Patroni configuration. For example:
+
+.. code:: YAML
+
+    bootstrap:
+        method: <custom_bootstrap_method_name>
+        <custom_bootstrap_method_name>:
+            command: <path_to_custom_bootstrap_script> [param1 [, ...]]
+            keep_existing_recovery_conf: False
+            recovery_conf:
+                recovery_target_action: promote
+                recovery_target_timeline: latest
+                restore_command: <method_specific_restore_command>
+
+
+Each bootstrap method must define at least a ``name`` and a ``command``. A special ``initdb`` method is available to trigger
+the default behavior, in which case ``method`` parameter can be omitted altogether. The ``command`` can be specified using either
+an absolute path, or the one relative to the ``patroni`` command location. In addition to the fixed parameters defined
+in the configuration files, Patroni supplies two cluster-specific ones:
+
+--scope
+    Name of the cluster to be bootstrapped
+--datadir
+    Path to the data directory of the cluster instance to be bootstrapped
+
+If the bootstrap script returns 0, Patroni tries to configure and start the PostgreSQL instance produced by it. If any
+of the intermediate steps fail, or the script returns a non-zero value, Patroni assumes that the bootstrap has failed,
+cleans up after itself and releases the initialize lock to give another node the opportunity to bootstrap.
+
+If a ``recovery_conf`` block is defined in the same section as the custom bootstrap method, Patroni will generate a
+``recovery.conf`` before starting the newly bootstrapped instance. Typically, such recovery.conf should contain at least
+one of the ``recovery_target_*`` parameters, together with the ``recovery_target_timeline`` set to ``promote``.
+
+If ``keep_existing_recovery_conf`` is defined and set to ``True``, Patroni will not remove the existing ``recovery.conf`` file if it exists.
+This is useful when bootstrapping from a backup with tools like pgBackRest that generate the appropriate ``recovery.conf`` for you.
+
+ .. note:: Bootstrap methods are neither chained, nor fallen-back to the default one in case the primary one fails
+
+
+.. _custom_replica_creation:
+
+Building replicas
+-----------------
+
+Patroni uses tried and proven ``pg_basebackup`` in order to create new replicas. One downside of it is that it requires
+a running master node. Another one is the lack of 'on-the-fly' compression for the backup data and no built-in cleanup
+for outdated backup files. Some people prefer other backup solutions, such as ``WAL-E``, ``pgBackRest``, ``Barman`` and
+others, or simply roll their own scripts. In order to accommodate all those use-cases Patroni supports running custom
+scripts to clone a new replica. Those are configured in the ``postgresql`` configuration block:
+
+.. code:: YAML
+
+    postgresql:
+        create_replica_method:
+            - wal_e
+            - basebackup
+        wal_e:
+            command: patroni_wale_restore
+            no_master: 1
+            envdir: {{WALE_ENV_DIR}}
+            use_iam: 1
+        basebackup:
+            max-rate: '100M'
+
+
+The ``create_replica_method`` defines available replica creation methods and the order of executing them. Patroni will
+stop on the first one that returns 0. Each method should define a separate section in the configuration file, listing the command
+to execute and any custom parameters that should be passed to that command. All parameters will be passed in a
+``--name=value`` format. Besides user-defined parameters, Patroni supplies a couple of cluster-specific ones:
+
+--scope
+    Which cluster this replica belongs to
+--datadir
+    Path to the data directory of the replica
+--role
+    Always 'replica'
+--connstring
+    Connection string to connect to the cluster member to clone from (master or other replica). The user in the
+    connection string can execute SQL and replication protocol commands.
+
+A special ``no_master`` parameter, if defined, allows Patroni to call the replica creation method even if there is no
+running master or replicas. In that case, an empty string will be passed in a connection string. This is useful for
+restoring the formerly running cluster from the binary backup.
+
+A ``basebackup`` method is a special case: it will be used if ``create_replica_method`` is empty, although it is possible
+to list it explicitly among the ``create_replica_method`` methods. This method initializes a new replica with the
+``pg_basebackup``, the base backup is taken from the master unless there are replicas with ``clonefrom`` tag, in which case one
+of such replicas will be used as the origin for pg_basebackup. It works without any configuration; however, it is
+possible to specify a ``basebackup`` configuration section. Same rules as with the other method configuration apply,
+namely, only long (with --) options should be specified there. Not all parameters make sense, if you override a connection
+string or provide an option to created tar-ed or compressed base backups, patroni won't be able to make a replica out
+of it. There is no validation performed on the names or values of the parameters passed to the ``basebackup`` section.
+You can specify basebackup parameters as either a map (key-value pairs) or a list of elements, where each element
+could be either a key-value pair or a single key (for options that does not receive any values, for instance, ``--verbose``).
+Consider those 2 examples:
+
+.. code:: YAML
+
+    postgresql:
+        basebackup:
+            max-rate: '100M'
+            checkpoint: 'fast'
+
+and
+
+.. code:: YAML
+
+    postgresql:
+        basebackup:
+            - verbose
+            - max-rate: '100M'
+
+If all replica creation methods fail, Patroni will try again all methods in order during the next event loop cycle.

docs/replication_modes.rst

@@ -0,0 +1,68 @@
+.. _replication_modes:
+
+=================
+Replication modes
+=================
+
+Patroni uses PostgreSQL streaming replication. For more information about streaming replication, see the `Postgres documentation <http://www.postgresql.org/docs/current/static/warm-standby.html#STREAMING-REPLICATION>`__. By default Patroni configures PostgreSQL for asynchronous replication. Choosing your replication schema is dependent on your business considerations. Investigate both async and sync replication, as well as other HA solutions, to determine which solution is best for you.
+
+Asynchronous mode durability
+----------------------------
+
+In asynchronous mode the cluster is allowed to lose some committed transactions to ensure availability. When the primary server fails or becomes unavailable for any other reason Patroni will automatically promote a sufficiently healthy standby to primary. Any transactions that have not been replicated to that standby remain in a "forked timeline" on the primary, and are effectively unrecoverable [1]_.
+
+The amount of transactions that can be lost is controlled via ``maximum_lag_on_failover`` parameter. Because the primary transaction log position is not sampled in real time, in reality the amount of lost data on failover is worst case bounded by  ``maximum_lag_on_failover`` bytes of transaction log plus the amount that is written in the last ``ttl`` seconds (``loop_wait``/2 seconds in the average case). However typical steady state replication delay is well under a second.
+
+PostgreSQL synchronous replication
+----------------------------------
+
+You can use Postgres's `synchronous replication <http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION>`__ with Patroni. Synchronous replication ensures consistency across a cluster by confirming that writes are written to a secondary before returning to the connecting client with a success. The cost of synchronous replication: reduced throughput on writes. This throughput will be entirely based on network performance.
+
+In hosted datacenter environments (like AWS, Rackspace, or any network you do not control), synchronous replication significantly increases the variability of write performance. If followers become inaccessible from the leader, the leader effectively becomes read-only.
+
+To enable a simple synchronous replication test, add the following lines to the ``parameters`` section of your YAML configuration files:
+
+.. code:: YAML
+
+        synchronous_commit: "on"
+        synchronous_standby_names: "*"
+
+When using PostgreSQL synchronous replication, use at least three Postgres data nodes to ensure write availability if one host fails.
+
+Using PostgreSQL synchronous replication does not guarantee zero lost transactions under all circumstances. When the primary and the secondary that is currently acting as a synchronous replica fail simultaneously a third node that might not contain all transactions will be promoted.
+
+.. _synchronous_mode:
+
+Synchronous mode
+----------------
+
+For use cases where losing committed transactions is not permissible you can turn on Patroni's ``synchronous_mode``. When ``synchronous_mode`` is turned on Patroni will not promote a standby unless it is certain that the standby contains all transactions that may have returned a successful commit status to client [2]_. This means that the system may be unavailable for writes even though some servers are available. System administrators can still use manual failover commands to promote a standby even if it results in transaction loss.
+
+Turning on ``synchronous_mode`` does not guarantee multi node durability of commits under all circumstances. When no suitable standby is available, primary server will still accept writes, but does not guarantee their replication. When the primary fails in this mode no standby will be promoted. When the host that used to be the primary comes back it will get promoted automatically, unless system administrator performed a manual failover. This behavior makes synchronous mode usable with 2 node clusters.
+
+When ``synchronous_mode`` is on and a standby crashes, commits will block until next iteration of Patroni runs and switches the primary to standalone mode (worst case delay for writes ``ttl`` seconds, average case ``loop_wait``/2 seconds). Manually shutting down or restarting a standby will not cause a commit service interruption. Standby will signal the primary to release itself from synchronous standby duties before PostgreSQL shutdown is initiated.
+
+You can ensure that a standby never becomes the synchronous standby by setting ``nosync`` tag to true. This is recommended to set for standbys that are behind slow network connections and would cause performance degradation when becoming a synchronous standby.
+
+Synchronous mode can be switched on and off via Patroni REST interface. See :ref:`dynamic configuration <dynamic_configuration>` for instructions.
+
+
+Synchronous mode implementation
+-------------------------------
+
+When in synchronous mode Patroni maintains synchronization state in the DCS, containing the latest primary and current synchronous standby. This state is updated with strict ordering constraints to ensure the following invariants:
+
+- A node must be marked as the latest leader whenever it can accept write transactions. Patroni crashing or PostgreSQL not shutting down can cause violations of this invariant.
+
+- A node must be set as the synchronous standby in PostgreSQL as long as it is published as the synchronous standby.
+
+- A node that is not the leader or current synchronous standby is not allowed to promote itself automatically.
+
+Patroni will only ever assign one standby to ``synchronous_standby_names`` because with multiple candidates it is not possible to know which node was acting as synchronous during the failure.
+
+On each HA loop iteration Patroni re-evaluates synchronous standby choice. If the current synchronous standby is connected and has not requested its synchronous status to be removed it remains picked. Otherwise the cluster member available for sync that is furthest ahead in replication is picked.
+
+
+.. [1] The data is still there, but recovering it requires a manual recovery effort by data recovery specialists. When Patroni is allowed to rewind with ``use_pg_rewind`` the forked timeline will be automatically erased to rejoin the failed primary with the cluster.
+
+.. [2] Clients can change the behavior per transaction using PostgreSQL's ``synchronous_commit`` setting. Transactions with ``synchronous_commit`` values of ``off`` and ``local`` may be lost on fail over, but will not be blocked by replication delays.

docs/watchdog.rst

@@ -0,0 +1,39 @@
+.. _watchdog:
+
+Watchdog support
+================
+
+Having multiple PostgreSQL servers running as master can result in transactions lost due to diverging timelines. This situation is also called a split-brain problem. To avoid split-brain Patroni needs to ensure PostgreSQL will not accept any transaction commits after leader key expires in the DCS. Under normal circumstances Patroni will try to achieve this by stopping PostgreSQL when leader lock update fails for any reason. However, this may fail to happen due to various reasons:
+
+- Patroni has crashed due to a bug, out-of-memory condition or by being accidentally killed by a system administrator.
+
+- Shutting down PostgreSQL is too slow.
+
+- Patroni does not get to run due to high load on the system, the VM being paused by the hypervisor, or other infrastructure issues.
+
+To guarantee correct behavior under these conditions Patroni supports watchdog devices. Watchdog devices are software or hardware mechanisms that will reset the whole system when they do not get a keepalive heartbeat within a specified timeframe. This adds an additional layer of fail safe in case usual Patroni split-brain protection mechanisms fail.
+
+Patroni will try to activate the watchdog before promoting PostgreSQL to master. If watchdog activation fails and watchdog mode is ``required`` then the node will refuse to become master. When deciding to participate in leader election Patroni will also check that watchdog configuration will allow it to become leader at all. After demoting PostgreSQL (for example due to a manual failover) Patroni will disable the watchdog again. Watchdog will also be disabled while Patroni is in paused state.
+
+By default Patroni will set up the watchdog to expire 5 seconds before TTL expires. With the default setup of ``loop_wait=10`` and ``ttl=30`` this gives HA loop at least 15 seconds (``ttl`` - ``safety_margin`` - ``loop_wait``) to complete before the system gets forcefully reset. By default accessing DCS is configured to time out after 10 seconds. This means that when DCS is unavailable, for example due to network issues, Patroni and PostgreSQL will have at least 5 seconds (``ttl`` - ``safety_margin`` - ``loop_wait`` - ``retry_timeout``) to come to a state where all client connections are terminated.
+
+Safety margin is the amount of time that Patroni reserves for time between leader key update and watchdog keepalive. Patroni will try to send a keepalive immediately after confirmation of leader key update. If Patroni process is suspended for extended amount of time at exactly the right moment the keepalive may be delayed for more than the safety margin without triggering the watchdog. This results in a window of time where watchdog will not trigger before leader key expiration, invalidating the guarantee. To be absolutely sure that watchdog will trigger under all circumstances set up the watchdog to expire after half of TTL by setting ``safety_margin`` to -1 to set watchdog timeout to ``ttl // 2``. If you need this guarantee you probably should increase ``ttl`` and/or reduce ``loop_wait`` and ``retry_timeout``.
+
+Currently watchdogs are only supported using Linux watchdog device interface.
+
+Setting up software watchdog on Linux
+-------------------------------------
+
+Default Patroni configuration will try to use ``/dev/watchdog`` on Linux if it is accessible to Patroni. For most use cases using software watchdog built into the Linux kernel is secure enough.
+
+To enable software watchdog issue the following commands as root before starting Patroni:
+
+.. code-block:: bash
+
+    modprobe softdog
+    # Replace postgres with the user you will be running patroni under
+    chown postgres /dev/watchdog
+
+For testing it may be helpful to disable rebooting by adding ``soft_noboot=1`` to the modprobe command line. In this case the watchdog will just log a line in kernel ring buffer, visible via `dmesg`.
+
+Patroni will log information about the watchdog when it is successfully enabled.

extras/README.md

@@ -0,0 +1,13 @@
+### confd
+
+`confd` directory contains haproxy template files for the [confd](https://github.com/kelseyhightower/confd) -- lightweight configuration management tool
+You need to copy content of `confd` directory into /etcd/confd and run confd service:
+```bash
+$ confd -prefix=/service/$PATRONI_SCOPE -backend etcd -node $PATRONI_ETCD_HOST -interval=10
+```
+It will periodically update haproxy.cfg with the actual list of Patroni nodes from `etcd` and "reload" haproxy when it is necessary.
+
+
+### startup-scripts
+
+`startup-scripts` directory contains startup scripts for various OSes and management tools for Patroni.

extras/confd/conf.d/haproxy.toml

@@ -0,0 +1,13 @@
+[template]
+#prefix = "/service/batman"
+#owner = "haproxy"
+#mode = "0644"
+src = "haproxy.tmpl"
+dest = "/etc/haproxy/haproxy.cfg"
+
+check_cmd = "/usr/sbin/haproxy -c -f {{ .src }}"
+reload_cmd = "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D -sf $(cat /var/run/haproxy.pid)"
+
+keys = [
+    "/members/",
+]

extras/confd/templates/haproxy.tmpl

@@ -0,0 +1,32 @@
+global
+	maxconn 100
+
+defaults
+	log	global
+	mode	tcp
+	retries 2
+	timeout client 30m
+	timeout connect 4s
+	timeout server 30m
+	timeout check 5s
+
+listen stats
+	mode http
+	bind *:7000
+	stats enable
+	stats uri /
+
+listen master
+	bind *:5000
+	option httpchk OPTIONS /master
+	http-check expect status 200
+	default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
+{{range gets "/members/*"}}	server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}}
+{{end}}
+listen replicas
+	bind *:5001
+	option httpchk OPTIONS /replica
+	http-check expect status 200
+	default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
+{{range gets "/members/*"}}	server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}}
+{{end}}

extras/startup-scripts/README.md

@@ -8,3 +8,15 @@ Scripts supplied:
 ### patroni.upstart.conf
 
 Upstart job for Ubuntu 12.04 or 14.04.  Requires Upstart > 1.4. Intended for systems where Patroni has been installed on a base system, rather than in Docker.
+
+### patroni.service
+Systemd service file, to be copied to /etc/systemd/system/patroni.service, tested on Centos 7.1 with Patroni installed from pip. 
+
+### patroni
+Init.d service file for Debian-like distributions. Copy it to /etc/init.d/, make executable:
+```chmod 755 /etc/init.d/patroni``` and run with ```service patroni start```, or make it starting on boot with ```update-rc.d patroni defaults```. Also you might edit some configuration variables in it:
+PATRONI for patroni.py location
+CONF for configuration file
+LOGFILE for log (script creates it if does not exist)
+
+Note. If you have several versions of Postgres installed, please add to POSTGRES_VERSION the release number which you wish to run. Script uses this value to append PATH environment with correct path to Postgres bin.

extras/startup-scripts/patroni

@@ -0,0 +1,144 @@
+#!/bin/sh
+#
+### BEGIN INIT INFO
+# Provides:          patroni
+# Required-Start:    $remote_fs $syslog
+# Required-Stop:     $remote_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Patroni init script
+# Description:       Runners to orchestrate a high-availability PostgreSQL
+### END INIT INFO
+
+### BEGIN USER CONFIGURATION
+
+CONF="/etc/patroni/postgres.yml"
+LOGFILE="/var/log/patroni.log"
+USER="postgres"
+GROUP="postgres"
+
+NAME=patroni
+PATRONI="/opt/patroni/$NAME.py"
+PIDFILE="/var/run/$NAME.pid"
+
+# Set this parameter, if you have several Postgres versions installed
+# POSTGRES_VERSION="9.4"
+POSTGRES_VERSION=""
+
+### END USER CONFIGURATION
+
+. /lib/lsb/init-functions
+
+# Loading this library for get_versions() function
+if test ! -e /usr/share/postgresql-common/init.d-functions; then
+  log_failure_msg "Probably postgresql-common does not installed."
+  exit 1
+else
+  . /usr/share/postgresql-common/init.d-functions
+fi
+
+# Is there Patroni executable?
+if test ! -e $PATRONI; then
+  log_failure_msg "Patroni executable $PATRONI does not exist."
+  exit 1
+fi
+
+# Is there Patroni configuration file?
+if test ! -e $CONF; then
+  log_failure_msg "Patroni configuration file $CONF does not exist."
+  exit 1
+fi
+
+# Create logfile if doesn't exist
+if test ! -e $LOGFILE; then
+  log_action_msg "Creating logfile for Patroni..."
+  touch $LOGFILE
+  chown $USER:$GROUP $LOGFILE
+fi
+
+prepare_pgpath() {
+  if [ "$POSTGRES_VERSION" != "" ]; then
+    if [ -x /usr/lib/postgresql/$POSTGRES_VERSION/bin/pg_ctl ]; then
+      PGPATH="/usr/lib/postgresql/$POSTGRES_VERSION/bin"
+    else
+      log_failure_msg "Postgres version incorrect, check POSTGRES_VERSION variable."
+      exit 0
+    fi
+  else
+    get_versions
+    if echo $versions | grep -q -e "\s"; then
+      log_warning_msg "You have several Postgres versions installed. Please, use POSTGRES_VERSION to define correct environment."
+    else
+      versions=`echo $versions | sed -e 's/^[ \t]*//'`
+      PGPATH="/usr/lib/postgresql/$versions/bin"
+    fi
+  fi
+}
+
+get_pid() {
+  if test -e $PIDFILE; then
+    PID=`cat $PIDFILE`
+    CHILDPID=`ps --ppid $PID -o %p --no-headers`
+  else
+    log_failure_msg "Could not find PID file. Patroni probably down."
+    exit 1
+  fi
+}
+
+
+case "$1" in
+  start)
+    prepare_pgpath
+    PGPATH=$PATH:$PGPATH
+    log_success_msg "Starting Patroni\n"
+    exec start-stop-daemon --start --quiet \
+        --background \
+        --pidfile $PIDFILE --make-pidfile \
+        --chuid $USER:$GROUP \
+        --chdir `eval echo ~$USER` \
+        --exec $PATRONI \
+        --startas /bin/sh -- \
+        -c "/usr/bin/env PATH=$PGPATH /usr/bin/python $PATRONI $CONF >> $LOGFILE 2>&1"
+    ;;
+
+  stop)
+    log_success_msg "Stopping Patroni"
+    get_pid
+    start-stop-daemon --stop --pid $CHILDPID
+    start-stop-daemon --stop --pidfile $PIDFILE --remove-pidfile --quiet
+    ;;
+
+  reload)
+    log_success_msg "Reloading Patroni configuration"
+    get_pid
+    kill -HUP $CHILDPID
+    ;;
+
+  status)
+    get_pid
+    if start-stop-daemon -T --pid $CHILDPID; then
+      log_success_msg "Patroni is running\n"
+      exit 0
+    else
+      log_warning_msg "Patroni in not running\n"
+    fi
+    ;;
+
+  restart)
+    $0 stop
+    $0 start
+    ;;
+
+  *)
+    echo "Usage: /etc/init.d/$NAME {start|stop|restart|reload|status}"
+    exit 1
+    ;;
+esac
+
+if [ $? -eq 0 ]; then
+        echo .
+        exit 0
+else
+        echo " failed"
+        exit 1
+fi

extras/startup-scripts/patroni.service

@@ -0,0 +1,31 @@
+# This is an example systemd config file for Patroni
+# You can copy it to "/etc/systemd/system/patroni.service",
+
+[Unit]
+Description=Runners to orchestrate a high-availability PostgreSQL
+After=syslog.target network.target
+
+[Service]
+Type=simple
+
+User=postgres
+Group=postgres
+
+# Where to send early-startup messages from the server
+# This is normally controlled by the global default set by systemd
+# StandardOutput=syslog
+
+ExecStart=/bin/patroni /etc/patroni.yml
+
+# only kill the patroni process, not it's children, so it will gracefully stop postgres
+KillMode=process
+
+# Give a reasonable amount of time for the server to start up/shut down
+TimeoutSec=30
+
+# Do not restart the service if it crashes, we want to manually inspect database on failure
+Restart=no
+
+[Install]
+WantedBy=multi-user.target
+

features/backup_create.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+while getopts ":-:" optchar; do
+    [[ "${optchar}" == "-" ]] || continue
+    case "${OPTARG}" in
+        datadir=* )
+            PGDATA=${OPTARG#*=}
+            ;;
+        dbname=* )
+            DBNAME=${OPTARG#*=}
+            ;;
+        walmethod=* )
+            WALMETHOD=${OPTARG#*=}
+            ;;
+    esac
+done
+
+[[ -z $PGDATA || -z $DBNAME || -z $WALMETHOD ]] && exit 1
+
+[[ $WALMETHOD != "none" ]] && WALMETHOD="-X $WALMETHOD" || WALMETHOD=""
+
+exec pg_basebackup -D $PGDATA $WALMETHOD -c fast -d $DBNAME

features/backup_restore.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -x
+
+while getopts ":-:" optchar; do
+    [[ "${optchar}" == "-" ]] || continue
+    case "${OPTARG}" in
+        datadir=* )
+            PGDATA=${OPTARG#*=}
+            ;;
+        sourcedir=* )
+            SOURCE=${OPTARG#*=}
+            ;;
+    esac
+done
+
+[[ -z $PGDATA || -z $SOURCE ]] && exit 1
+
+mkdir -p $(dirname $PGDATA)
+
+exec cp -af $SOURCE $PGDATA

features/basic_replication.feature

@@ -0,0 +1,51 @@
+Feature: basic replication
+  We should check that the basic bootstrapping, replication and failover works.
+
+  Scenario: check replication of a single table
+    Given I start postgres0
+    Then postgres0 is a leader after 10 seconds
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "loop_wait": 2, "synchronous_mode": true}
+    Then I receive a response code 200
+    When I start postgres1
+    And I configure and start postgres2 with a tag replicatefrom postgres0
+    And "sync" key in DCS has leader=postgres0 after 20 seconds
+    And I add the table foo to postgres0
+    Then table foo is present on postgres1 after 20 seconds
+    Then table foo is present on postgres2 after 20 seconds
+
+  Scenario: check restart of sync replica
+    Given I run patronictl.py restart batman postgres2 --force
+    And "sync" key in DCS has sync_standby=postgres1 after 2 seconds
+    And I run patronictl.py restart batman postgres1 --force
+    Then I receive a response returncode 0
+    And "sync" key in DCS has sync_standby=postgres2 after 10 seconds
+    And I sleep for 2 seconds
+    When I issue a GET request to http://127.0.0.1:8010/sync
+    Then I receive a response code 200
+    When I issue a GET request to http://127.0.0.1:8009/async
+    Then I receive a response code 200
+
+  Scenario: check the basic failover in synchronous mode
+    Given I run patronictl.py pause batman
+    Then I receive a response returncode 0
+    When I sleep for 2 seconds
+    And I shut down postgres0
+    And I run patronictl.py resume batman 
+    Then I receive a response returncode 0
+    And postgres2 role is the primary after 24 seconds
+    When I issue a PATCH request to http://127.0.0.1:8010/config with {"synchronous_mode": null, "master_start_timeout": 0}
+    Then I receive a response code 200
+    When I add the table bar to postgres2
+    Then table bar is present on postgres1 after 20 seconds
+
+  Scenario: check immediate failover when master_start_timeout=0
+    Given I kill postmaster on postgres2
+    Then postgres1 is a leader after 10 seconds
+    And postgres1 role is the primary after 10 seconds
+
+  Scenario: check rejoin of the former master with pg_rewind
+    Given I add the table splitbrain to postgres0
+    And I start postgres0
+    Then postgres0 role is the secondary after 20 seconds
+    When I add the table buz to postgres1
+    Then table buz is present on postgres0 after 20 seconds

features/cascading_replication.feature

@@ -0,0 +1,14 @@
+Feature: cascading replication
+	We should check that patroni can do base backup and streaming from the replica
+
+Scenario: check a base backup and streaming replication from a replica
+	Given I start postgres0
+	And postgres0 is a leader after 10 seconds
+	And I configure and start postgres1 with a tag clonefrom true
+	And replication works from postgres0 to postgres1 after 20 seconds
+	And I create label with "postgres0" in postgres0 data directory
+	And I create label with "postgres1" in postgres1 data directory
+	And "members/postgres1" key in DCS has state=running after 12 seconds
+	And I configure and start postgres2 with a tag replicatefrom postgres1
+	Then replication works from postgres0 to postgres2 after 30 seconds
+	And there is a label with "postgres1" in postgres2 data directory

features/custom_bootstrap.feature

@@ -0,0 +1,17 @@
+Feature: custom bootstrap
+    We should check that patroni can bootstrap a new cluster from a backup
+
+Scenario: clone existing cluster using pg_basebackup
+    Given I start postgres0
+    Then postgres0 is a leader after 10 seconds
+    When I add the table foo to postgres0
+    And I start postgres1 in a cluster batman1 as a clone of postgres0
+    Then postgres1 is a leader of batman1 after 10 seconds
+    Then table foo is present on postgres1 after 10 seconds
+
+Scenario: make a backup and do a restore into a new cluster
+    Given I add the table bar to postgres1
+    And I do a backup of postgres1
+    When I start postgres2 in a cluster batman2 from backup
+    Then postgres2 is a leader of batman2 after 10 seconds
+    And table bar is present on postgres2 after 10 seconds

features/environment.py

@@ -0,0 +1,808 @@
+import abc
+import consul
+import datetime
+import etcd
+import kazoo.client
+import kazoo.exceptions
+import os
+import psutil
+import psycopg2
+import json
+import shutil
+import signal
+import six
+import subprocess
+import tempfile
+import threading
+import time
+import yaml
+
+
+@six.add_metaclass(abc.ABCMeta)
+class AbstractController(object):
+
+    def __init__(self, context, name, work_directory, output_dir):
+        self._context = context
+        self._name = name
+        self._work_directory = work_directory
+        self._output_dir = output_dir
+        self._handle = None
+        self._log = None
+
+    def _has_started(self):
+        return self._handle and self._handle.pid and self._handle.poll() is None
+
+    def _is_running(self):
+        return self._has_started()
+
+    @abc.abstractmethod
+    def _is_accessible(self):
+        """process is accessible for queries"""
+
+    @abc.abstractmethod
+    def _start(self):
+        """start process"""
+
+    def start(self, max_wait_limit=5):
+        if self._is_running():
+            return True
+
+        self._log = open(os.path.join(self._output_dir, self._name + '.log'), 'a')
+        self._handle = self._start()
+
+        assert self._has_started(), "Process {0} is not running after being started".format(self._name)
+
+        max_wait_limit *= self._context.timeout_multiplier
+        for _ in range(max_wait_limit):
+            if self._is_accessible():
+                break
+            time.sleep(1)
+        else:
+            assert False,\
+                "{0} instance is not available for queries after {1} seconds".format(self._name, max_wait_limit)
+
+    def stop(self, kill=False, timeout=15, _=False):
+        term = False
+        start_time = time.time()
+
+        timeout *= self._context.timeout_multiplier
+        while self._handle and self._is_running():
+            if kill:
+                self._handle.kill()
+            elif not term:
+                self._handle.terminate()
+                term = True
+            time.sleep(1)
+            if not kill and time.time() - start_time > timeout:
+                kill = True
+
+        if self._log:
+            self._log.close()
+
+    def cancel_background(self):
+        pass
+
+
+class PatroniController(AbstractController):
+    __PORT = 5440
+    PATRONI_CONFIG = '{}.yml'
+    """ starts and stops individual patronis"""
+
+    def __init__(self, context, name, work_directory, output_dir, custom_config=None):
+        super(PatroniController, self).__init__(context, 'patroni_' + name, work_directory, output_dir)
+        PatroniController.__PORT += 1
+        self._data_dir = os.path.join(work_directory, 'data', name)
+        self._connstring = None
+        if custom_config and 'watchdog' in custom_config:
+            self.watchdog = WatchdogMonitor(name, work_directory, output_dir)
+            custom_config['watchdog'] = {'driver': 'testing', 'device': self.watchdog.fifo_path, 'mode': 'required'}
+        else:
+            self.watchdog = None
+
+        self._scope = (custom_config or {}).get('scope', 'batman')
+        self._config = self._make_patroni_test_config(name, custom_config)
+        self._closables = []
+
+        self._conn = None
+        self._curs = None
+
+    def write_label(self, content):
+        with open(os.path.join(self._data_dir, 'label'), 'w') as f:
+            f.write(content)
+
+    def read_label(self):
+        try:
+            with open(os.path.join(self._data_dir, 'label'), 'r') as f:
+                return f.read().strip()
+        except IOError:
+            return None
+
+    def add_tag_to_config(self, tag, value):
+        with open(self._config) as r:
+            config = yaml.safe_load(r)
+            config['tags']['tag'] = value
+            with open(self._config, 'w') as w:
+                yaml.safe_dump(config, w, default_flow_style=False)
+
+    def _start(self):
+        if self.watchdog:
+            self.watchdog.start()
+        if isinstance(self._context.dcs_ctl, KubernetesController):
+            self._context.dcs_ctl.create_pod(self._name[8:], self._scope)
+            os.environ['PATRONI_KUBERNETES_POD_IP'] = '10.0.0.' + self._name[-1]
+        return subprocess.Popen(['coverage', 'run', '--source=patroni', '-p', 'patroni.py', self._config],
+                                stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory)
+
+    def stop(self, kill=False, timeout=15, postgres=False):
+        if postgres:
+            return subprocess.call(['pg_ctl', '-D', self._data_dir, 'stop', '-mi', '-w'])
+        super(PatroniController, self).stop(kill, timeout)
+        if isinstance(self._context.dcs_ctl, KubernetesController):
+            self._context.dcs_ctl.delete_pod(self._name[8:])
+        if self.watchdog:
+            self.watchdog.stop()
+
+    def _is_accessible(self):
+        cursor = self.query("SELECT 1", fail_ok=True)
+        if cursor is not None:
+            cursor.execute("SET synchronous_commit TO 'local'")
+            return True
+
+    def _make_patroni_test_config(self, name, custom_config):
+        patroni_config_name = self.PATRONI_CONFIG.format(name)
+        patroni_config_path = os.path.join(self._output_dir, patroni_config_name)
+
+        with open(patroni_config_name) as f:
+            config = yaml.safe_load(f)
+            config.pop('etcd', None)
+
+        host = config['postgresql']['listen'].split(':')[0]
+
+        config['postgresql']['listen'] = config['postgresql']['connect_address'] = '{0}:{1}'.format(host, self.__PORT)
+
+        config['name'] = name
+        config['postgresql']['data_dir'] = self._data_dir
+        config['postgresql']['use_unix_socket'] = True
+        config['postgresql']['parameters'].update({
+            'logging_collector': 'on', 'log_destination': 'csvlog', 'log_directory': self._output_dir,
+            'log_filename': name + '.log', 'log_statement': 'all', 'log_min_messages': 'debug1',
+            'unix_socket_directories': self._data_dir})
+
+        if 'bootstrap' in config:
+            config['bootstrap']['post_bootstrap'] = 'psql -w -c "SELECT 1"'
+            if 'initdb' in config['bootstrap']:
+                config['bootstrap']['initdb'].extend([{'auth': 'md5'}, {'auth-host': 'md5'}])
+
+        if custom_config is not None:
+            def recursive_update(dst, src):
+                for k, v in src.items():
+                    if k in dst and isinstance(dst[k], dict):
+                        recursive_update(dst[k], v)
+                    else:
+                        dst[k] = v
+            recursive_update(config, custom_config)
+
+        with open(patroni_config_path, 'w') as f:
+            yaml.safe_dump(config, f, default_flow_style=False)
+
+        user = config['postgresql'].get('authentication', config['postgresql']).get('superuser', {})
+        self._connkwargs = {k: user[n] for n, k in [('username', 'user'), ('password', 'password')] if n in user}
+        self._connkwargs.update({'host': host, 'port': self.__PORT, 'database': 'postgres'})
+
+        self._replication = config['postgresql'].get('authentication', config['postgresql']).get('replication', {})
+        self._replication.update({'host': host, 'port': self.__PORT, 'database': 'postgres'})
+
+        return patroni_config_path
+
+    def _connection(self):
+        if not self._conn or self._conn.closed != 0:
+            self._conn = psycopg2.connect(**self._connkwargs)
+            self._conn.autocommit = True
+        return self._conn
+
+    def _cursor(self):
+        if not self._curs or self._curs.closed or self._curs.connection.closed != 0:
+            self._curs = self._connection().cursor()
+        return self._curs
+
+    def query(self, query, fail_ok=False):
+        try:
+            cursor = self._cursor()
+            cursor.execute(query)
+            return cursor
+        except psycopg2.Error:
+            if not fail_ok:
+                raise
+
+    def check_role_has_changed_to(self, new_role, timeout=10):
+        bound_time = time.time() + timeout
+        recovery_status = new_role != 'primary'
+        while time.time() < bound_time:
+            cur = self.query("SELECT pg_is_in_recovery()", fail_ok=True)
+            if cur:
+                row = cur.fetchone()
+                if row and row[0] == recovery_status:
+                    return True
+            time.sleep(1)
+        return False
+
+    def get_watchdog(self):
+        return self.watchdog
+
+    def _get_pid(self):
+        try:
+            pidfile = os.path.join(self._data_dir, 'postmaster.pid')
+            if not os.path.exists(pidfile):
+                return None
+            return int(open(pidfile).readline().strip())
+        except Exception:
+            return None
+
+    def database_is_running(self):
+        pid = self._get_pid()
+        if not pid:
+            return False
+        try:
+            os.kill(pid, 0)
+        except OSError:
+            return False
+        return True
+
+    def patroni_hang(self, timeout):
+        hang = ProcessHang(self._handle.pid, timeout)
+        self._closables.append(hang)
+        hang.start()
+
+    def checkpoint_hang(self, timeout):
+        pid = self._get_pid()
+        if not pid:
+            return False
+        proc = psutil.Process(pid)
+        for child in proc.children():
+            if 'checkpoint' in child.cmdline()[0]:
+                checkpointer = child
+                break
+        else:
+            return False
+        hang = ProcessHang(checkpointer.pid, timeout)
+        self._closables.append(hang)
+        hang.start()
+        return True
+
+    def cancel_background(self):
+        for obj in self._closables:
+            obj.close()
+        self._closables = []
+
+    def terminate_backends(self):
+        pid = self._get_pid()
+        if not pid:
+            return False
+        proc = psutil.Process(pid)
+        for p in proc.children():
+            if 'process' not in p.cmdline()[0]:
+                p.terminate()
+
+    @property
+    def backup_source(self):
+        return 'postgres://{username}:{password}@{host}:{port}/{database}'.format(**self._replication)
+
+    def backup(self, dest='basebackup'):
+        subprocess.call([PatroniPoolController.BACKUP_SCRIPT, '--walmethod=none',
+                         '--datadir=' + os.path.join(self._output_dir, dest),
+                         '--dbname=' + self.backup_source])
+
+
+class ProcessHang(object):
+
+    """A background thread implementing a cancelable process hang via SIGSTOP."""
+
+    def __init__(self, pid, timeout):
+        self._cancelled = threading.Event()
+        self._thread = threading.Thread(target=self.run)
+        self.pid = pid
+        self.timeout = timeout
+
+    def start(self):
+        self._thread.start()
+
+    def run(self):
+        os.kill(self.pid, signal.SIGSTOP)
+        try:
+            self._cancelled.wait(self.timeout)
+        finally:
+            os.kill(self.pid, signal.SIGCONT)
+
+    def close(self):
+        self._cancelled.set()
+        self._thread.join()
+
+
+class AbstractDcsController(AbstractController):
+
+    _CLUSTER_NODE = '/service/{0}'
+
+    def __init__(self, context, mktemp=True):
+        work_directory = mktemp and tempfile.mkdtemp() or None
+        super(AbstractDcsController, self).__init__(context, self.name(), work_directory, context.pctl.output_dir)
+
+    def _is_accessible(self):
+        return self._is_running()
+
+    def stop(self, kill=False, timeout=15):
+        """ terminate process and wipe out the temp work directory, but only if we actually started it"""
+        super(AbstractDcsController, self).stop(kill=kill, timeout=timeout)
+        if self._work_directory:
+            shutil.rmtree(self._work_directory)
+
+    def path(self, key=None, scope='batman'):
+        return self._CLUSTER_NODE.format(scope) + (key and '/' + key or '')
+
+    @abc.abstractmethod
+    def query(self, key, scope='batman'):
+        """ query for a value of a given key """
+
+    @abc.abstractmethod
+    def cleanup_service_tree(self):
+        """ clean all contents stored in the tree used for the tests """
+
+    @classmethod
+    def get_subclasses(cls):
+        for subclass in cls.__subclasses__():
+            for subsubclass in subclass.get_subclasses():
+                yield subsubclass
+            yield subclass
+
+    @classmethod
+    def name(cls):
+        return cls.__name__[:-10].lower()
+
+
+class ConsulController(AbstractDcsController):
+
+    def __init__(self, context):
+        super(ConsulController, self).__init__(context)
+        os.environ['PATRONI_CONSUL_HOST'] = 'localhost:8500'
+        self._client = consul.Consul()
+        self._config_file = None
+
+    def _start(self):
+        self._config_file = self._work_directory + '.json'
+        with open(self._config_file, 'wb') as f:
+            f.write(b'{"session_ttl_min":"5s","server":true,"bootstrap":true,"advertise_addr":"127.0.0.1"}')
+        return subprocess.Popen(['consul', 'agent', '-config-file', self._config_file, '-data-dir',
+                                 self._work_directory], stdout=self._log, stderr=subprocess.STDOUT)
+
+    def stop(self, kill=False, timeout=15):
+        super(ConsulController, self).stop(kill=kill, timeout=timeout)
+        if self._config_file:
+            os.unlink(self._config_file)
+
+    def _is_running(self):
+        try:
+            return bool(self._client.status.leader())
+        except Exception:
+            return False
+
+    def path(self, key=None, scope='batman'):
+        return super(ConsulController, self).path(key, scope)[1:]
+
+    def query(self, key, scope='batman'):
+        _, value = self._client.kv.get(self.path(key, scope))
+        return value and value['Value'].decode('utf-8')
+
+    def cleanup_service_tree(self):
+        self._client.kv.delete(self.path(scope=''), recurse=True)
+
+    def start(self, max_wait_limit=15):
+        super(ConsulController, self).start(max_wait_limit)
+
+
+class EtcdController(AbstractDcsController):
+
+    """ handles all etcd related tasks, used for the tests setup and cleanup """
+
+    def __init__(self, context):
+        super(EtcdController, self).__init__(context)
+        os.environ['PATRONI_ETCD_HOST'] = 'localhost:2379'
+        self._client = etcd.Client(port=2379)
+
+    def _start(self):
+        return subprocess.Popen(["etcd", "--debug", "--data-dir", self._work_directory],
+                                stdout=self._log, stderr=subprocess.STDOUT)
+
+    def query(self, key, scope='batman'):
+        try:
+            return self._client.get(self.path(key, scope)).value
+        except etcd.EtcdKeyNotFound:
+            return None
+
+    def cleanup_service_tree(self):
+        try:
+            self._client.delete(self.path(scope=''), recursive=True)
+        except (etcd.EtcdKeyNotFound, etcd.EtcdConnectionFailed):
+            return
+        except Exception as e:
+            assert False, "exception when cleaning up etcd contents: {0}".format(e)
+
+    def _is_running(self):
+        # if etcd is running, but we didn't start it
+        try:
+            return bool(self._client.machines)
+        except Exception:
+            return False
+
+
+class KubernetesController(AbstractDcsController):
+
+    def __init__(self, context):
+        super(KubernetesController, self).__init__(context)
+        self._namespace = 'default'
+        self._labels = {"application": "patroni"}
+        self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
+        os.environ['PATRONI_KUBERNETES_LABELS'] = json.dumps(self._labels)
+        os.environ['PATRONI_KUBERNETES_USE_ENDPOINTS'] = 'true'
+
+        from kubernetes import client as k8s_client, config as k8s_config
+        k8s_config.load_kube_config(context='local')
+        self._client = k8s_client
+        self._api = self._client.CoreV1Api()
+
+    def _start(self):
+        pass
+
+    def create_pod(self, name, scope):
+        labels = self._labels.copy()
+        labels['cluster-name'] = scope
+        metadata = self._client.V1ObjectMeta(namespace=self._namespace, name=name, labels=labels)
+        spec = self._client.V1PodSpec(containers=[self._client.V1Container(name=name, image='empty')])
+        body = self._client.V1Pod(metadata=metadata, spec=spec)
+        self._api.create_namespaced_pod(self._namespace, body)
+
+    def delete_pod(self, name):
+        try:
+            self._api.delete_namespaced_pod(name, self._namespace, self._client.V1DeleteOptions())
+        except:
+            pass
+        while True:
+            try:
+                self._api.read_namespaced_pod(name, self._namespace)
+            except:
+                break
+
+    def query(self, key, scope='batman'):
+        if key.startswith('members/'):
+            pod = self._api.read_namespaced_pod(key[8:], self._namespace)
+            return (pod.metadata.annotations or {}).get('status', '')
+        else:
+            try:
+                e = self._api.read_namespaced_endpoints(scope + ('' if key == 'leader' else '-' + key), self._namespace)
+                if key == 'leader':
+                    return e.metadata.annotations[key]
+                else:
+                    return json.dumps(e.metadata.annotations)
+            except:
+                return None
+
+    def cleanup_service_tree(self):
+        try:
+            self._api.delete_collection_namespaced_pod(self._namespace, label_selector=self._label_selector)
+        except:
+            pass
+        try:
+            self._api.delete_collection_namespaced_endpoints(self._namespace, label_selector=self._label_selector)
+        except:
+            pass
+
+        while True:
+            result = self._api.list_namespaced_pod(self._namespace, label_selector=self._label_selector)
+            if len(result.items) < 1:
+                break
+
+    def _is_running(self):
+        return True
+
+
+class ZooKeeperController(AbstractDcsController):
+
+    """ handles all zookeeper related tasks, used for the tests setup and cleanup """
+
+    def __init__(self, context, export_env=True):
+        super(ZooKeeperController, self).__init__(context, False)
+        if export_env:
+            os.environ['PATRONI_ZOOKEEPER_HOSTS'] = "'localhost:2181'"
+        self._client = kazoo.client.KazooClient()
+
+    def _start(self):
+        pass  # TODO: implement later
+
+    def query(self, key, scope='batman'):
+        try:
+            return self._client.get(self.path(key, scope))[0].decode('utf-8')
+        except kazoo.exceptions.NoNodeError:
+            return None
+
+    def cleanup_service_tree(self):
+        try:
+            self._client.delete(self.path(scope=''), recursive=True)
+        except (kazoo.exceptions.NoNodeError):
+            return
+        except Exception as e:
+            assert False, "exception when cleaning up zookeeper contents: {0}".format(e)
+
+    def _is_running(self):
+        # if zookeeper is running, but we didn't start it
+        if self._client.connected:
+            return True
+        try:
+            return self._client.start(1) or True
+        except Exception:
+            return False
+
+
+class ExhibitorController(ZooKeeperController):
+
+    def __init__(self, context):
+        super(ExhibitorController, self).__init__(context, False)
+        os.environ.update({'PATRONI_EXHIBITOR_HOSTS': 'localhost', 'PATRONI_EXHIBITOR_PORT': '8181'})
+
+
+class PatroniPoolController(object):
+
+    BACKUP_SCRIPT = 'features/backup_create.sh'
+
+    def __init__(self, context):
+        self._context = context
+        self._dcs = None
+        self._output_dir = None
+        self._patroni_path = None
+        self._processes = {}
+        self.create_and_set_output_directory('')
+        self.known_dcs = {subclass.name(): subclass for subclass in AbstractDcsController.get_subclasses()}
+
+    @property
+    def patroni_path(self):
+        if self._patroni_path is None:
+            cwd = os.path.realpath(__file__)
+            while True:
+                cwd, entry = os.path.split(cwd)
+                if entry == 'features' or cwd == '/':
+                    break
+            self._patroni_path = cwd
+        return self._patroni_path
+
+    @property
+    def output_dir(self):
+        return self._output_dir
+
+    def start(self, name, max_wait_limit=20, custom_config=None):
+        if name not in self._processes:
+            self._processes[name] = PatroniController(self._context, name, self.patroni_path,
+                                                      self._output_dir, custom_config)
+        self._processes[name].start(max_wait_limit)
+
+    def __getattr__(self, func):
+        if func not in ['stop', 'query', 'write_label', 'read_label', 'check_role_has_changed_to', 'add_tag_to_config',
+                        'get_watchdog', 'database_is_running', 'checkpoint_hang', 'patroni_hang',
+                        'terminate_backends', 'backup']:
+            raise AttributeError("PatroniPoolController instance has no attribute '{0}'".format(func))
+
+        def wrapper(name, *args, **kwargs):
+            return getattr(self._processes[name], func)(*args, **kwargs)
+        return wrapper
+
+    def stop_all(self):
+        for ctl in self._processes.values():
+            ctl.cancel_background()
+            ctl.stop()
+        self._processes.clear()
+
+    def create_and_set_output_directory(self, feature_name):
+        feature_dir = os.path.join(self.patroni_path, 'features/output', feature_name.replace(' ', '_'))
+        if os.path.exists(feature_dir):
+            shutil.rmtree(feature_dir)
+        os.makedirs(feature_dir)
+        self._output_dir = feature_dir
+
+    def clone(self, from_name, cluster_name, to_name):
+        f = self._processes[from_name]
+        custom_config = {
+            'scope': cluster_name,
+            'bootstrap': {
+                'method': 'pg_basebackup',
+                'pg_basebackup': {
+                    'command': self.BACKUP_SCRIPT + ' --walmethod=stream --dbname=' + f.backup_source
+                }
+            },
+            'postgresql': {
+                'parameters': {
+                    'archive_mode': 'on',
+                    'archive_command': 'mkdir -p {0} && test ! -f {0}/%f && cp %p {0}/%f'.format(
+                            os.path.join(self._output_dir, 'wal_archive'))
+                },
+                'authentication': {
+                    'superuser': {'password': 'zalando1'},
+                    'replication': {'password': 'rep-pass1'}
+                }
+            }
+        }
+        self.start(to_name, custom_config=custom_config)
+
+    def bootstrap_from_backup(self, name, cluster_name):
+        custom_config = {
+            'scope': cluster_name,
+            'bootstrap': {
+                'method': 'backup_restore',
+                'backup_restore': {
+                    'command': 'features/backup_restore.sh --sourcedir=' + os.path.join(self._output_dir, 'basebackup'),
+                    'recovery_conf': {
+                        'recovery_target_action': 'promote',
+                        'recovery_target_timeline': 'latest',
+                        'restore_command': 'cp {0}/wal_archive/%f %p'.format(self._output_dir)
+                    }
+                }
+            },
+            'postgresql': {
+                'authentication': {
+                    'superuser': {'password': 'zalando2'},
+                    'replication': {'password': 'rep-pass2'}
+                }
+            }
+        }
+        self.start(name, custom_config=custom_config)
+
+    @property
+    def dcs(self):
+        if self._dcs is None:
+            self._dcs = os.environ.pop('DCS', 'etcd')
+            assert self._dcs in self.known_dcs, 'Unsupported dcs: ' + self._dcs
+        return self._dcs
+
+
+class WatchdogMonitor(object):
+    """Testing harness for emulating a watchdog device as a named pipe. Because we can't easily emulate ioctl's we
+    require a custom driver on Patroni side. The device takes no action, only notes if it was pinged and/or triggered.
+    """
+    def __init__(self, name, work_directory, output_dir):
+        self.fifo_path = os.path.join(work_directory, 'data', 'watchdog.{0}.fifo'.format(name))
+        self.fifo_file = None
+        self._stop_requested = False  # Relying on bool setting being atomic
+        self._thread = None
+        self.last_ping = None
+        self.was_pinged = False
+        self.was_closed = False
+        self._was_triggered = False
+        self.timeout = 60
+        self._log_file = open(os.path.join(output_dir, 'watchdog.{0}.log'.format(name)), 'w')
+        self._log("watchdog {0} initialized".format(name))
+
+    def _log(self, msg):
+        tstamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")
+        self._log_file.write("{0}: {1}\n".format(tstamp, msg))
+
+    def start(self):
+        assert self._thread is None
+        self._stop_requested = False
+        self._log("starting fifo {0}".format(self.fifo_path))
+        fifo_dir = os.path.dirname(self.fifo_path)
+        if os.path.exists(self.fifo_path):
+            os.unlink(self.fifo_path)
+        elif not os.path.exists(fifo_dir):
+            os.mkdir(fifo_dir)
+        os.mkfifo(self.fifo_path)
+        self.last_ping = time.time()
+
+        self._thread = threading.Thread(target=self.run)
+        self._thread.start()
+
+    def run(self):
+        try:
+            while not self._stop_requested:
+                self._log("opening")
+                self.fifo_file = os.open(self.fifo_path, os.O_RDONLY)
+                try:
+                    self._log("Fifo {0} connected".format(self.fifo_path))
+                    self.was_closed = False
+                    while not self._stop_requested:
+                        c = os.read(self.fifo_file, 1)
+
+                        if c == b'X':
+                            self._log("Stop requested")
+                            return
+                        elif c == b'':
+                            self._log("Pipe closed")
+                            break
+                        elif c == b'C':
+                            command = b''
+                            c = os.read(self.fifo_file, 1)
+                            while c != b'\n' and c != b'':
+                                command += c
+                                c = os.read(self.fifo_file, 1)
+                            command = command.decode('utf8')
+
+                            if command.startswith('timeout='):
+                                self.timeout = int(command.split('=')[1])
+                                self._log("timeout={0}".format(self.timeout))
+                        elif c in [b'V', b'1']:
+                            cur_time = time.time()
+                            if cur_time - self.last_ping > self.timeout:
+                                self._log("Triggered")
+                                self._was_triggered = True
+                            if c == b'V':
+                                self._log("magic close")
+                                self.was_closed = True
+                            elif c == b'1':
+                                self.was_pinged = True
+                                self._log("ping after {0} seconds".format(cur_time - (self.last_ping or cur_time)))
+                                self.last_ping = cur_time
+                        else:
+                            self._log('Unknown command {0} received from fifo'.format(c))
+                finally:
+                    self.was_closed = True
+                    self._log("closing")
+                    os.close(self.fifo_file)
+        except Exception as e:
+            self._log("Error {0}".format(e))
+        finally:
+            self._log("stopping")
+            self._log_file.flush()
+            if os.path.exists(self.fifo_path):
+                os.unlink(self.fifo_path)
+
+    def stop(self):
+        self._log("Monitor stop")
+        self._stop_requested = True
+        try:
+            if os.path.exists(self.fifo_path):
+                fd = os.open(self.fifo_path, os.O_WRONLY)
+                os.write(fd, b'X')
+                os.close(fd)
+        except Exception as e:
+            self._log("err while closing: {0}".format(str(e)))
+        if self._thread:
+            self._thread.join()
+            self._thread = None
+
+    def reset(self):
+        self._log("reset")
+        self.was_pinged = self.was_closed = self._was_triggered = False
+
+    @property
+    def was_triggered(self):
+        delta = time.time() - self.last_ping
+        triggered = self._was_triggered or not self.was_closed and delta > self.timeout
+        self._log("triggered={0}, {1}s left".format(triggered, self.timeout - delta))
+        return triggered
+
+
+# actions to execute on start/stop of the tests and before running invidual features
+def before_all(context):
+    os.environ.update({'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password'})
+    context.ci = 'TRAVIS_BUILD_NUMBER' in os.environ or 'BUILD_NUMBER' in os.environ
+    context.timeout_multiplier = 2 if context.ci else 1
+    context.pctl = PatroniPoolController(context)
+    context.dcs_ctl = context.pctl.known_dcs[context.pctl.dcs](context)
+    context.dcs_ctl.start()
+    try:
+        context.dcs_ctl.cleanup_service_tree()
+    except AssertionError:  # after_all handlers won't be executed in before_all
+        context.dcs_ctl.stop()
+        raise
+
+
+def after_all(context):
+    context.dcs_ctl.stop()
+    subprocess.call(['coverage', 'combine'])
+    subprocess.call(['coverage', 'report'])
+
+
+def before_feature(context, feature):
+    """ create per-feature output directory to collect Patroni and PostgreSQL logs """
+    context.pctl.create_and_set_output_directory(feature.name)
+
+
+def after_feature(context, feature):
+    """ stop all Patronis, remove their data directory and cleanup the keys in etcd """
+    context.pctl.stop_all()
+    shutil.rmtree(os.path.join(context.pctl.patroni_path, 'data'))
+    context.dcs_ctl.cleanup_service_tree()

features/patroni_api.feature

@@ -0,0 +1,99 @@
+Feature: patroni api
+	We should check that patroni correctly responds to valid and not-valid API requests.
+
+Scenario: check API requests on a stand-alone server
+	Given I start postgres0
+	And postgres0 is a leader after 10 seconds
+	When I issue a GET request to http://127.0.0.1:8008/
+	Then I receive a response code 200
+	And I receive a response state running
+	And I receive a response role master
+	When I issue a GET request to http://127.0.0.1:8008/replica
+	Then I receive a response code 503
+	When I run patronictl.py reinit batman postgres0 --force
+	Then I receive a response returncode 0
+	And I receive a response output "Failed: reinitialize for member postgres0, status code=503, (I am the leader, can not reinitialize)"
+	When I run patronictl.py switchover batman --master postgres0 --force
+	Then I receive a response returncode 1
+	And I receive a response output "Error: No candidates found to switchover to"
+	When I issue a POST request to http://127.0.0.1:8008/switchover with {"leader": "postgres0"}
+	Then I receive a response code 412
+	And I receive a response text switchover is not possible: cluster does not have members except leader
+	When I issue an empty POST request to http://127.0.0.1:8008/failover
+	Then I receive a response code 400
+	When I issue a POST request to http://127.0.0.1:8008/failover with {"foo": "bar"}
+	Then I receive a response code 400
+	And I receive a response text "Failover could be performed only to a specific candidate"
+
+Scenario: check local configuration reload
+	Given I issue an empty POST request to http://127.0.0.1:8008/reload
+	Then I receive a response code 200
+	And I receive a response text nothing changed
+	When I add tag new_tag new_value to postgres0 config
+	And I issue an empty POST request to http://127.0.0.1:8008/reload
+	Then I receive a response code 202
+
+Scenario: check dynamic configuration change via DCS
+	Given I run patronictl.py edit-config -s 'ttl=10' -s 'loop_wait=2' -p 'max_connections=101' --force batman
+	Then I receive a response returncode 0
+	And I receive a response output "+loop_wait: 2"
+	And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 11 seconds
+	When I issue a GET request to http://127.0.0.1:8008/config
+	Then I receive a response code 200
+	And I receive a response loop_wait 2
+	When I issue a GET request to http://127.0.0.1:8008/patroni
+	Then I receive a response code 200
+	And I receive a response tags {'tag': 'new_value'}
+
+Scenario: check API requests for the primary-replica pair in the pause mode
+	Given I run patronictl.py pause batman
+	Then I receive a response returncode 0
+	When I start postgres1
+	Then replication works from postgres0 to postgres1 after 20 seconds
+	When I issue a GET request to http://127.0.0.1:8009/replica
+	Then I receive a response code 200
+	And I receive a response state running
+	And I receive a response role replica
+	When I run patronictl.py reinit batman postgres1 --force
+	Then I receive a response returncode 0
+	And I receive a response output "Success: reinitialize for member postgres1"
+	When I run patronictl.py restart batman postgres0 --force
+	Then I receive a response returncode 0
+	And I receive a response output "Success: restart on member postgres0"
+	And postgres0 role is the primary after 5 seconds
+	When I sleep for 10 seconds
+	Then postgres1 role is the secondary after 15 seconds
+
+Scenario: check the switchover via the API in the pause mode
+	Given I issue a POST request to http://127.0.0.1:8008/switchover with {"leader": "postgres0", "candidate": "postgres1"}
+	Then I receive a response code 200
+	And postgres1 is a leader after 5 seconds
+	And postgres1 role is the primary after 10 seconds
+	And postgres0 role is the secondary after 10 seconds
+	And replication works from postgres1 to postgres0 after 20 seconds
+
+Scenario: check the scheduled switchover
+	Given I issue a scheduled switchover from postgres1 to postgres0 in 3 seconds
+	Then I receive a response returncode 1
+	And I receive a response output "Can't schedule switchover in the paused state"
+	When I run patronictl.py resume batman
+	Then I receive a response returncode 0
+	Given I issue a scheduled switchover from postgres1 to postgres0 in 3 seconds
+	Then I receive a response returncode 0
+	And postgres0 is a leader after 20 seconds
+	And postgres0 role is the primary after 10 seconds
+	And postgres1 role is the secondary after 10 seconds
+	And replication works from postgres0 to postgres1 after 25 seconds
+
+Scenario: check the scheduled restart
+	Given I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"superuser_reserved_connections": "6"}}}
+	Then I receive a response code 200
+		And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 5 seconds
+	Given I issue a scheduled restart at http://127.0.0.1:8008 in 3 seconds with {"role": "replica"}
+	Then I receive a response code 202
+		And I sleep for 4 seconds
+		And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 10 seconds
+	Given I issue a scheduled restart at http://127.0.0.1:8008 in 3 seconds with {"restart_pending": "True"}
+	Then I receive a response code 202
+		And Response on GET http://127.0.0.1:8008/patroni does not contain pending_restart after 10 seconds
+

features/steps/basic_replication.py

@@ -0,0 +1,61 @@
+import psycopg2 as pg
+
+from behave import step, then
+from time import sleep, time
+
+
+@step('I start {name:w}')
+def start_patroni(context, name):
+    return context.pctl.start(name)
+
+
+@step('I shut down {name:w}')
+def stop_patroni(context, name):
+    return context.pctl.stop(name, timeout=60)
+
+
+@step('I kill {name:w}')
+def kill_patroni(context, name):
+    return context.pctl.stop(name, kill=True)
+
+
+@step('I kill postmaster on {name:w}')
+def stop_postgres(context, name):
+    return context.pctl.stop(name, postgres=True)
+
+
+@step('I add the table {table_name:w} to {pg_name:w}')
+def add_table(context, table_name, pg_name):
+    # parse the configuration file and get the port
+    try:
+        context.pctl.query(pg_name, "CREATE TABLE {0}()".format(table_name))
+    except pg.Error as e:
+        assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e)
+
+
+@then('Table {table_name:w} is present on {pg_name:w} after {max_replication_delay:d} seconds')
+def table_is_present_on(context, table_name, pg_name, max_replication_delay):
+    max_replication_delay *= context.timeout_multiplier
+    for _ in range(int(max_replication_delay)):
+        if context.pctl.query(pg_name, "SELECT 1 FROM {0}".format(table_name), fail_ok=True) is not None:
+            break
+        sleep(1)
+    else:
+        assert False,\
+            "Table {0} is not present on {1} after {2} seconds".format(table_name, pg_name, max_replication_delay)
+
+
+@then('{pg_name:w} role is the {pg_role:w} after {max_promotion_timeout:d} seconds')
+def check_role(context, pg_name, pg_role, max_promotion_timeout):
+    max_promotion_timeout *= context.timeout_multiplier
+    assert context.pctl.check_role_has_changed_to(pg_name, pg_role, timeout=int(max_promotion_timeout)),\
+        "{0} role didn't change to {1} after {2} seconds".format(pg_name, pg_role, max_promotion_timeout)
+
+
+@step('replication works from {master:w} to {replica:w} after {time_limit:d} seconds')
+@then('replication works from {master:w} to {replica:w} after {time_limit:d} seconds')
+def replication_works(context, master, replica, time_limit):
+    context.execute_steps(u"""
+        When I add the table test_{0} to {1}
+        Then table test_{0} is present on {2} after {3} seconds
+    """.format(int(time()), master, replica, time_limit))

features/steps/cascading_replication.py

@@ -0,0 +1,35 @@
+import json
+import time
+
+from behave import step, then
+
+
+@step('I configure and start {name:w} with a tag {tag_name:w} {tag_value:w}')
+def start_patroni_with_a_name_value_tag(context, name, tag_name, tag_value):
+    return context.pctl.start(name, custom_config={'tags': {tag_name: tag_value}})
+
+
+@then('There is a label with "{content:w}" in {name:w} data directory')
+def check_label(context, content, name):
+    label = context.pctl.read_label(name)
+    assert label == content, "{0} is not equal to {1}".format(label, content)
+
+
+@step('I create label with "{content:w}" in {name:w} data directory')
+def write_label(context, content, name):
+    context.pctl.write_label(name, content)
+
+
+@step('"{name}" key in DCS has {key:w}={value:w} after {time_limit:d} seconds')
+def check_member(context, name, key, value, time_limit):
+    time_limit *= context.timeout_multiplier
+    max_time = time.time() + int(time_limit)
+    while time.time() < max_time:
+        try:
+            response = json.loads(context.dcs_ctl.query(name))
+            if response.get(key) == value:
+                return
+        except Exception:
+            pass
+        time.sleep(1)
+    assert False, "{0} does not have {1}={2} in dcs after {3} seconds".format(name, key, value, time_limit)

features/steps/custom_bootstrap.py

@@ -0,0 +1,27 @@
+import time
+
+from behave import step, then
+
+
+@step('I start {name:w} in a cluster {cluster_name:w} as a clone of {name2:w}')
+def start_cluster_clone(context, name, cluster_name, name2):
+    context.pctl.clone(name2, cluster_name, name)
+
+
+@step('I start {name:w} in a cluster {cluster_name:w} from backup')
+def start_cluster_from_backup(context, name, cluster_name):
+    context.pctl.bootstrap_from_backup(name, cluster_name)
+
+
+@then('{name:w} is a leader of {cluster_name:w} after {time_limit:d} seconds')
+def is_a_leader(context, name, cluster_name, time_limit):
+    time_limit *= context.timeout_multiplier
+    max_time = time.time() + int(time_limit)
+    while (context.dcs_ctl.query("leader", scope=cluster_name) != name):
+        time.sleep(1)
+        assert time.time() < max_time, "{0} is not a leader in dcs after {1} seconds".format(name, time_limit)
+
+
+@step('I do a backup of {name:w}')
+def do_backup(context, name):
+    context.pctl.backup(name)

features/steps/patroni_api.py

@@ -0,0 +1,162 @@
+import base64
+import json
+import os
+import parse
+import requests
+import shlex
+import subprocess
+import time
+import yaml
+
+from behave import register_type, step, then
+from dateutil import tz
+from datetime import datetime, timedelta
+
+tzutc = tz.tzutc()
+
+
+@parse.with_pattern(r'https?://(?:\w|\.|:|/)+')
+def parse_url(text):
+    return text
+
+
+register_type(url=parse_url)
+
+
+# there is no way we can find out if the node has already
+# started as a leader without checking the DCS. We cannot
+# just rely on the database availability, since there is
+# a short gap between the time PostgreSQL becomes available
+# and Patroni assuming the leader role.
+@step('{name:w} is a leader after {time_limit:d} seconds')
+@then('{name:w} is a leader after {time_limit:d} seconds')
+def is_a_leader(context, name, time_limit):
+    time_limit *= context.timeout_multiplier
+    max_time = time.time() + int(time_limit)
+    while (context.dcs_ctl.query("leader") != name):
+        time.sleep(1)
+        assert time.time() < max_time, "{0} is not a leader in dcs after {1} seconds".format(name, time_limit)
+
+
+@step('I sleep for {value:d} seconds')
+def sleep_for_n_seconds(context, value):
+    time.sleep(int(value))
+
+
+def _set_response(context, response):
+    context.status_code = response.status_code
+    data = response.content.decode('utf-8')
+    ct = response.headers.get('content-type', '')
+    if ct.startswith('application/json') or\
+            ct.startswith('text/yaml') or\
+            ct.startswith('text/x-yaml') or\
+            ct.startswith('application/yaml') or\
+            ct.startswith('application/x-yaml'):
+        try:
+            context.response = yaml.safe_load(data)
+        except ValueError:
+            context.response = data
+    else:
+        context.response = data
+
+
+@step('I issue a GET request to {url:url}')
+def do_get(context, url):
+    try:
+        r = requests.get(url)
+    except requests.exceptions.RequestException:
+        context.status_code = None
+        context.response = None
+    else:
+        _set_response(context, r)
+
+
+@step('I issue an empty POST request to {url:url}')
+def do_post_empty(context, url):
+    do_request(context, 'POST', url, None)
+
+
+@step('I issue a {request_method:w} request to {url:url} with {data}')
+def do_request(context, request_method, url, data):
+    data = data and json.loads(data) or {}
+    headers = {'Authorization': 'Basic ' + base64.b64encode('username:password'.encode('utf-8')).decode('utf-8'),
+               'Content-Type': 'application/json'}
+    try:
+        if request_method == 'PATCH':
+            r = requests.patch(url, headers=headers, json=data)
+        else:
+            r = requests.post(url, headers=headers, json=data)
+    except requests.exceptions.RequestException:
+        context.status_code = None
+        context.response = None
+    else:
+        _set_response(context, r)
+
+
+@step('I run {cmd}')
+def do_run(context, cmd):
+    cmd = ['coverage', 'run', '--source=patroni', '-p'] + shlex.split(cmd)
+    try:
+        # XXX: Dirty hack! We need to take name/passwd from the config!
+        env = os.environ.copy()
+        env.update({'PATRONI_RESTAPI_USERNAME': 'username', 'PATRONI_RESTAPI_PASSWORD': 'password'})
+        response = subprocess.check_output(cmd, stderr=subprocess.STDOUT, env=env)
+        context.status_code = 0
+    except subprocess.CalledProcessError as e:
+        response = e.output
+        context.status_code = e.returncode
+    context.response = response.decode('utf-8').strip()
+
+
+@then('I receive a response {component:w} {data}')
+def check_response(context, component, data):
+    if component == 'code':
+        assert context.status_code == int(data),\
+            "status code {0} != {1}, response: {2}".format(context.status_code, data, context.response)
+    elif component == 'returncode':
+        assert context.status_code == int(data), "return code {0} != {1}, {2}".format(context.status_code,
+                                                                                      data, context.response)
+    elif component == 'text':
+        assert context.response == data.strip('"'), "response {0} does not contain {1}".format(context.response, data)
+    elif component == 'output':
+        assert data.strip('"') in context.response, "response {0} does not contain {1}".format(context.response, data)
+    else:
+        assert component in context.response, "{0} is not part of the response".format(component)
+        assert str(context.response[component]) == str(data), "{0} does not contain {1}".format(component, data)
+
+
+@step('I issue a scheduled switchover from {from_host:w} to {to_host:w} in {in_seconds:d} seconds')
+def scheduled_switchover(context, from_host, to_host, in_seconds):
+    context.execute_steps(u"""
+        Given I run patronictl.py switchover batman --master {0} --candidate {1} --scheduled "{2}" --force
+    """.format(from_host, to_host, datetime.now(tzutc) + timedelta(seconds=int(in_seconds))))
+
+
+@step('I issue a scheduled restart at {url:url} in {in_seconds:d} seconds with {data}')
+def scheduled_restart(context, url, in_seconds, data):
+    data = data and json.loads(data) or {}
+    data.update(schedule='{0}'.format((datetime.now(tzutc) + timedelta(seconds=int(in_seconds))).isoformat()))
+    context.execute_steps(u"""Given I issue a POST request to {0}/restart with {1}""".format(url, json.dumps(data)))
+
+
+@step('I add tag {tag:w} {value:w} to {pg_name:w} config')
+def add_tag_to_config(context, tag, value, pg_name):
+    context.pctl.add_tag_to_config(pg_name, tag, value)
+
+
+@then('Response on GET {url} contains {value} after {timeout:d} seconds')
+def check_http_response(context, url, value, timeout, negate=False):
+    timeout *= context.timeout_multiplier
+    for _ in range(int(timeout)):
+        r = requests.get(url)
+        if (value in r.content.decode('utf-8')) != negate:
+            break
+        time.sleep(1)
+    else:
+        assert False,\
+            "Value {0} is {1} present in response after {2} seconds".format(value, "not" if not negate else "", timeout)
+
+
+@then('Response on GET {url} does not contain {value} after {timeout:d} seconds')
+def check_not_in_http_response(context, url, value, timeout):
+    check_http_response(context, url, value, timeout, negate=True)

features/steps/watchdog.py

@@ -0,0 +1,74 @@
+from behave import step, then
+import time
+
+
+def polling_loop(timeout, interval=1):
+    """Returns an iterator that returns values until timeout has passed. Timeout is measured from start of iteration."""
+    start_time = time.time()
+    iteration = 0
+    end_time = start_time + timeout
+    while time.time() < end_time:
+        yield iteration
+        iteration += 1
+        time.sleep(interval)
+
+
+@step('I start {name:w} with watchdog')
+def start_patroni_with_watchdog(context, name):
+    return context.pctl.start(name, custom_config={'watchdog': True})
+
+
+@step('{name:w} watchdog has been pinged after {timeout:d} seconds')
+def watchdog_was_pinged(context, name, timeout):
+    for _ in polling_loop(timeout):
+        if context.pctl.get_watchdog(name).was_pinged:
+            return True
+    return False
+
+
+@then('{name:w} watchdog has been closed')
+def watchdog_was_closed(context, name):
+    assert context.pctl.get_watchdog(name).was_closed
+
+
+@step('I reset {name:w} watchdog state')
+def watchdog_reset_pinged(context, name):
+    context.pctl.get_watchdog(name).reset()
+
+
+@then('{name:w} watchdog is triggered after {timeout:d} seconds')
+def watchdog_was_triggered(context, name, timeout):
+    for _ in polling_loop(timeout):
+        if context.pctl.get_watchdog(name).was_triggered:
+            return True
+    assert False
+
+
+@then('{name:w} watchdog was not triggered')
+def watchdog_was_not_triggered(context, name):
+    assert not context.pctl.get_watchdog(name).was_triggered
+
+
+@step('{name:w} checkpoint takes {timeout:d} seconds')
+def checkpoint_hang(context, name, timeout):
+    assert context.pctl.checkpoint_hang(name, timeout)
+
+
+@step('{name:w} hangs for {timeout:d} seconds')
+def patroni_hang(context, name, timeout):
+    return context.pctl.patroni_hang(name, timeout)
+
+
+@step('I terminate {name:w} user processes')
+def terminate_backends(context, name):
+    return context.pctl.terminate_backends(name)
+
+
+@step('Sleep for {timeout:d} seconds')
+def dcs_connection_lost(context, timeout):
+    time.sleep(timeout)
+
+
+@then('{name:w} database is running')
+def database_is_running(context, name):
+    assert context.pctl.database_is_running(name)

features/watchdog.feature

@@ -0,0 +1,31 @@
+Feature: watchdog
+  Verify that watchdog gets pinged and triggered under appropriate circumstances.
+
+  Scenario: watchdog is opened and pinged
+    Given I start postgres0 with watchdog
+    Then postgres0 is a leader after 10 seconds
+    And postgres0 role is the primary after 10 seconds
+    And postgres0 watchdog has been pinged after 10 seconds
+
+  Scenario: watchdog is disabled during pause
+    Given I run patronictl.py pause batman
+    Then I receive a response returncode 0
+    When I sleep for 2 seconds
+    Then postgres0 watchdog has been closed
+
+  Scenario: watchdog is opened and pinged after resume
+    Given I reset postgres0 watchdog state
+    And I run patronictl.py resume batman
+    Then I receive a response returncode 0
+    And postgres0 watchdog has been pinged after 10 seconds
+
+  Scenario: watchdog is disabled when shutting down
+    Given I shut down postgres0
+    Then postgres0 watchdog has been closed
+
+  Scenario: watchdog is triggered if patroni stops responding
+    Given I reset postgres0 watchdog state
+    And I start postgres0 with watchdog
+    Then postgres0 role is the primary after 10 seconds
+    When postgres0 hangs for 30 seconds
+    Then postgres0 watchdog is triggered after 30 seconds

haproxy.cfg

@@ -1,21 +1,25 @@
 global
-	maxconn 100
+    maxconn 100
 
 defaults
-	log	global
-	mode	tcp
-	retries 2
-	timeout client 30m
-	timeout connect 4s
-	timeout server 30m
-	timeout check 5s
+    log global
+    mode tcp
+    retries 2
+    timeout client 30m
+    timeout connect 4s
+    timeout server 30m
+    timeout check 5s
 
-frontend ft_postgresql
-	bind *:5000
-	default_backend bk_db
+listen stats
+    mode http
+    bind *:7000
+    stats enable
+    stats uri /
 
-backend bk_db
-	option httpchk GET
-
-  server postgresql_127.0.0.1_5432 127.0.0.1:5432 maxconn 100 check port 8008
-  server postgresql_127.0.0.1_5433 127.0.0.1:5433 maxconn 100 check port 8009
+listen batman
+    bind *:5000
+    option httpchk
+    http-check expect status 200
+    default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
+    server postgresql_127.0.0.1_5432 127.0.0.1:5432 maxconn 100 check port 8008
+    server postgresql_127.0.0.1_5433 127.0.0.1:5433 maxconn 100 check port 8009

kubernetes/Dockerfile

@@ -0,0 +1,32 @@
+FROM postgres:9.6
+MAINTAINER Alexander Kukushkin <[email protected]>
+
+RUN export DEBIAN_FRONTEND=noninteractive \
+    && echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
+    && apt-get update -y \
+    && apt-get upgrade -y \
+    && apt-get install -y git curl jq python-psycopg2 python-yaml python-requests python-six python-pysocks \
+        python-dateutil python-pip python-prettytable python-wheel python-psutil python locales \
+
+    ## Make sure we have a en_US.UTF-8 locale available
+    && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
+
+    && pip install setuptools pip --upgrade \
+    && pip install 'git+https://github.com/zalando/patroni.git#egg=patroni[kubernetes]' \
+
+    && mkdir -p /home/postgres \
+    && chown postgres:postgres /home/postgres \
+
+    # Clean up
+    && apt-get remove -y git python-pip python-setuptools \
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/* /root/.cache
+
+ADD entrypoint.sh callback.py /
+
+EXPOSE 5432 8008
+ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
+USER postgres
+WORKDIR /home/postgres
+CMD ["/bin/bash", "/entrypoint.sh"]

kubernetes/callback.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+
+import logging
+import os
+import socket
+import sys
+import time
+
+from kubernetes import client as k8s_client, config as k8s_config
+from urllib3.exceptions import HTTPError
+from six.moves.http_client import HTTPException
+
+logger = logging.getLogger(__name__)
+
+
+class CoreV1Api(k8s_client.CoreV1Api):
+
+    def retry(func):
+        def wrapped(*args, **kwargs):
+            count = 0
+            while True:
+                try:
+                    return func(*args, **kwargs)
+                except (HTTPException, HTTPError, socket.error, socket.timeout):
+                    if count >= 10:
+                        raise
+                    logger.info('Throttling API requests...')
+                    time.sleep(2 ** count * 0.5)
+                    count += 1
+        return wrapped
+
+    @retry
+    def patch_namespaced_endpoints(self, *args, **kwargs):
+        return super(CoreV1Api, self).patch_namespaced_endpoints(*args, **kwargs)
+
+
+def patch_master_endpoint(api, namespace, cluster):
+    addresses = [k8s_client.V1EndpointAddress(ip=os.environ['POD_IP'])]
+    ports = [k8s_client.V1EndpointPort(port=5432)]
+    subsets = [k8s_client.V1EndpointSubset(addresses=addresses, ports=ports)]
+    body = k8s_client.V1Endpoints(subsets=subsets)
+    return api.patch_namespaced_endpoints(cluster, namespace, body)
+
+
+def main():
+    logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
+    if len(sys.argv) != 4 or sys.argv[1] not in ('on_start', 'on_stop', 'on_role_change'):
+        sys.exit('Usage: %s <action> <role> <cluster_name>', sys.argv[0])
+
+    action, role, cluster = sys.argv[1:4]
+
+    k8s_config.load_incluster_config()
+    k8s_api = CoreV1Api()
+
+    namespace = os.environ['KUBERNETES_NAMESPACE']
+
+    if role == 'master' and action in ('on_start', 'on_role_change'):
+        patch_master_endpoint(k8s_api, namespace, cluster)
+
+
+if __name__ == '__main__':
+    main()

kubernetes/entrypoint.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+cat > /home/postgres/patroni.yml <<__EOF__
+bootstrap:
+  dcs:
+    postgresql:
+      use_pg_rewind: true
+  initdb:
+  - auth-host: md5
+  - auth-local: trust
+  - encoding: UTF8
+  - locale: en_US.UTF-8
+  - data-checksums
+  pg_hba:
+  - host all all 0.0.0.0/0 md5
+  - host replication ${PATRONI_REPLICATION_USERNAME} ${POD_IP}/16    md5
+restapi:
+  connect_address: '${POD_IP}:8008'
+postgresql:
+  connect_address: '${POD_IP}:5432'
+  authentication:
+    superuser:
+      password: '${PATRONI_SUPERUSER_PASSWORD}'
+    replication:
+      password: '${PATRONI_REPLICATION_PASSWORD}'
+  callbacks:
+    on_start: /callback.py
+    on_stop: /callback.py
+    on_role_change: /callback.py
+__EOF__
+
+unset PATRONI_SUPERUSER_PASSWORD PATRONI_REPLICATION_PASSWORD
+export KUBERNETES_NAMESPACE=$PATRONI_KUBERNETES_NAMESPACE
+export POD_NAME=$PATRONI_NAME
+
+exec /usr/bin/python /usr/local/bin/patroni /home/postgres/patroni.yml

kubernetes/patroni_k8s.yaml

@@ -0,0 +1,182 @@
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  name: &cluster_name patronidemo
+  labels:
+    application: patroni
+    cluster-name: *cluster_name
+spec:
+  replicas: 3
+  serviceName: *cluster_name
+  template:
+    metadata:
+      labels:
+        application: patroni
+        cluster-name: *cluster_name
+    spec:
+      serviceAccountName: patronidemo
+      containers:
+      - name: *cluster_name
+        image: patroni  # docker build -t patroni .
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 8008
+          protocol: TCP
+        - containerPort: 5432
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /home/postgres/pgdata
+          name: pgdata
+        env:
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        - name: PATRONI_KUBERNETES_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: PATRONI_KUBERNETES_LABELS
+          value: '{application: patroni, cluster-name: patronidemo}'
+        - name: PATRONI_SUPERUSER_USERNAME
+          value: postgres
+        - name: PATRONI_SUPERUSER_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: *cluster_name
+              key: superuser-password
+        - name: PATRONI_REPLICATION_USERNAME
+          value: standby
+        - name: PATRONI_REPLICATION_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: *cluster_name
+              key: replication-password
+        - name: PATRONI_SCOPE
+          value: *cluster_name
+        - name: PATRONI_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: PATRONI_POSTGRESQL_DATA_DIR
+          value: /home/postgres/pgdata/pgroot/data
+        - name: PATRONI_POSTGRESQL_PGPASS
+          value: /tmp/pgpass
+        - name: PATRONI_POSTGRESQL_LISTEN
+          value: '0.0.0.0:5432'
+        - name: PATRONI_RESTAPI_LISTEN
+          value: '0.0.0.0:8008'
+      terminationGracePeriodSeconds: 0
+      volumes:
+      - name: pgdata
+        emptyDir: {}
+#  volumeClaimTemplates:
+#  - metadata:
+#      labels:
+#        application: spilo
+#        spilo-cluster: *cluster_name
+#      annotations:
+#        volume.alpha.kubernetes.io/storage-class: anything
+#      name: pgdata
+#    spec:
+#      accessModes:
+#      - ReadWriteOnce
+#      resources:
+#        requests:
+#          storage: 5Gi
+
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: &cluster_name patronidemo
+  labels:
+    application: patroni
+    cluster-name: *cluster_name
+subsets: []
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: &cluster_name patronidemo
+  labels:
+    application: patroni
+    cluster-name: *cluster_name
+spec:
+  type: ClusterIP
+  ports:
+  - port: 5432
+    targetPort: 5432
+
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: &cluster_name patronidemo
+  labels:
+    application: patroni
+    cluster-name: *cluster_name
+type: Opaque
+data:
+  superuser-password: emFsYW5kbw==
+  replication-password: cmVwLXBhc3M=
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: patronidemo
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: patronidemo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - patch
+  - update
+  # the following three privileges are necessary only when using endpoints
+  - create
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: patronidemo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: patronidemo
+subjects:
+- kind: ServiceAccount
+  name: patronidemo

mkbinary.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -e
+
+pip install --ignore-installed setuptools==19.2 pyinstaller
+pyinstaller --clean --onefile patroni.spec

patroni.spec

@@ -0,0 +1,39 @@
+# -*- mode: python -*-
+
+block_cipher = None
+
+
+def hiddenimports():
+    import sys
+    sys.path.insert(0, '.')
+    try:
+        import patroni.dcs
+        return patroni.dcs.dcs_modules()
+    finally:
+        sys.path.pop(0)
+
+
+a = Analysis(['patroni/__main__.py'],
+             pathex=[],
+             binaries=None,
+             datas=None,
+             hiddenimports=hiddenimports(),
+             hookspath=[],
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher)
+
+pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
+
+exe = EXE(pyz,
+    a.scripts,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    name='patroni',
+    debug=False,
+    strip=False,
+    upx=True,
+    console=True)

patroni/__init__.py

@@ -1,75 +1,206 @@
 import logging
 import os
+import signal
 import sys
 import time
-import yaml
-
-from patroni.api import RestApiServer
-from patroni.etcd import Etcd
-from patroni.ha import Ha
-from patroni.postgresql import Postgresql
-from patroni.utils import setup_signal_handlers, reap_children
-from patroni.zookeeper import ZooKeeper
 
 logger = logging.getLogger(__name__)
 
 
-class Patroni:
+class Patroni(object):
 
-    def __init__(self, config):
-        self.nap_time = config['loop_wait']
-        self.postgresql = Postgresql(config['postgresql'])
-        self.dcs = self.get_dcs(self.postgresql.name, config)
-        host, port = config['restapi']['listen'].split(':')
-        self.api = RestApiServer(self, config['restapi'])
+    def __init__(self):
+        from patroni.api import RestApiServer
+        from patroni.config import Config
+        from patroni.dcs import get_dcs
+        from patroni.ha import Ha
+        from patroni.postgresql import Postgresql
+        from patroni.version import __version__
+        from patroni.watchdog import Watchdog
+
+        self.setup_signal_handlers()
+
+        self.version = __version__
+        self.config = Config()
+        self.dcs = get_dcs(self.config)
+        self.watchdog = Watchdog(self.config)
+        self.load_dynamic_configuration()
+
+        self.postgresql = Postgresql(self.config['postgresql'])
+        self.api = RestApiServer(self, self.config['restapi'])
         self.ha = Ha(self)
-        self.next_run = time.time()
 
-    @staticmethod
-    def get_dcs(name, config):
-        if 'etcd' in config:
-            return Etcd(name, config['etcd'])
-        if 'zookeeper' in config:
-            return ZooKeeper(name, config['zookeeper'])
-        raise Exception('Can not find sutable configuration of distributed configuration store')
+        self.tags = self.get_tags()
+        self.next_run = time.time()
+        self.scheduled_restart = {}
+
+    def load_dynamic_configuration(self):
+        from patroni.exceptions import DCSError
+        while True:
+            try:
+                cluster = self.dcs.get_cluster()
+                if cluster and cluster.config and cluster.config.data:
+                    if self.config.set_dynamic_configuration(cluster.config):
+                        self.dcs.reload_config(self.config)
+                        self.watchdog.reload_config(self.config)
+                elif not self.config.dynamic_configuration and 'bootstrap' in self.config:
+                    if self.config.set_dynamic_configuration(self.config['bootstrap']['dcs']):
+                        self.dcs.reload_config(self.config)
+                break
+            except DCSError:
+                logger.warning('Can not get cluster from dcs')
+
+    def get_tags(self):
+        return {tag: value for tag, value in self.config.get('tags', {}).items()
+                if tag not in ('clonefrom', 'nofailover', 'noloadbalance', 'nosync') or value}
+
+    @property
+    def nofailover(self):
+        return bool(self.tags.get('nofailover', False))
+
+    @property
+    def nosync(self):
+        return bool(self.tags.get('nosync', False))
+
+    def reload_config(self):
+        try:
+            self.tags = self.get_tags()
+            self.dcs.reload_config(self.config)
+            self.watchdog.reload_config(self.config)
+            self.api.reload_config(self.config['restapi'])
+            self.postgresql.reload_config(self.config['postgresql'])
+        except Exception:
+            logger.exception('Failed to reload config_file=%s', self.config.config_file)
+
+    @property
+    def replicatefrom(self):
+        return self.tags.get('replicatefrom')
+
+    def sighup_handler(self, *args):
+        self._received_sighup = True
+
+    def sigterm_handler(self, *args):
+        if not self._received_sigterm:
+            self._received_sigterm = True
+            sys.exit()
+
+    @property
+    def noloadbalance(self):
+        return bool(self.tags.get('noloadbalance', False))
 
     def schedule_next_run(self):
-        self.next_run += self.nap_time
+        self.next_run += self.dcs.loop_wait
         current_time = time.time()
         nap_time = self.next_run - current_time
         if nap_time <= 0:
             self.next_run = current_time
-        elif self.dcs.watch(nap_time):
+            # Release the GIL so we don't starve anyone waiting on async_executor lock
+            time.sleep(0.001)
+            # Warn user that Patroni is not keeping up
+            logger.warning("Loop time exceeded, rescheduling immediately.")
+        elif self.ha.watch(nap_time):
             self.next_run = time.time()
 
     def run(self):
         self.api.start()
         self.next_run = time.time()
 
-        while True:
+        while not self._received_sigterm:
+            if self._received_sighup:
+                self._received_sighup = False
+                if self.config.reload_local_configuration():
+                    self.reload_config()
+
             logger.info(self.ha.run_cycle())
-            reap_children()
+
+            if self.dcs.cluster and self.dcs.cluster.config and self.dcs.cluster.config.data \
+                    and self.config.set_dynamic_configuration(self.dcs.cluster.config):
+                self.reload_config()
+
+            if self.postgresql.role != 'uninitialized':
+                self.config.save_cache()
+
             self.schedule_next_run()
 
+    def setup_signal_handlers(self):
+        self._received_sighup = False
+        self._received_sigterm = False
+        signal.signal(signal.SIGHUP, self.sighup_handler)
+        signal.signal(signal.SIGTERM, self.sigterm_handler)
 
-def main():
-    logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
-    logging.getLogger('requests').setLevel(logging.WARNING)
-    setup_signal_handlers()
+    def shutdown(self):
+        self.api.shutdown()
+        self.ha.shutdown()
 
-    if len(sys.argv) < 2 or not os.path.isfile(sys.argv[1]):
-        print('Usage: {} config.yml'.format(sys.argv[0]))
-        return
 
-    with open(sys.argv[1], 'r') as f:
-        config = yaml.load(f)
+def patroni_main():
+    logformat = os.environ.get('PATRONI_LOGFORMAT', '%(asctime)s %(levelname)s: %(message)s')
+    loglevel = os.environ.get('PATRONI_LOGLEVEL', 'INFO')
+    requests_loglevel = os.environ.get('PATRONI_REQUESTS_LOGLEVEL', 'WARNING')
+    logging.basicConfig(format=logformat, level=loglevel)
+    logging.getLogger('requests').setLevel(requests_loglevel)
 
-    patroni = Patroni(config)
+    patroni = Patroni()
     try:
         patroni.run()
     except KeyboardInterrupt:
         pass
     finally:
-        patroni.api.shutdown()
-        patroni.postgresql.stop()
-        patroni.dcs.delete_leader()
+        patroni.shutdown()
+
+
+def pg_ctl_start(args):
+    import subprocess
+    postmaster = subprocess.Popen(args)
+    print(postmaster.pid)
+
+
+def call_self(args, **kwargs):
+    """This function executes Patroni once again with provided arguments.
+
+    :args: list of arguments to call Patroni with.
+    :returns: `Popen` object"""
+
+    exe = [sys.executable]
+    if not getattr(sys, 'frozen', False):  # Binary distribution?
+        exe.append(sys.argv[0])
+
+    import subprocess
+    return subprocess.Popen(exe + args, **kwargs)
+
+
+def main():
+    if os.getpid() != 1:
+        if len(sys.argv) > 5 and sys.argv[1] == 'pg_ctl_start':
+            return pg_ctl_start(sys.argv[2:])
+        return patroni_main()
+
+    pid = 0
+
+    # Looks like we are in a docker, so we will act like init
+    def sigchld_handler(signo, stack_frame):
+        try:
+            while True:
+                ret = os.waitpid(-1, os.WNOHANG)
+                if ret == (0, 0):
+                    break
+                elif ret[0] != pid:
+                    logging.info('Reaped pid=%s, exit status=%s', *ret)
+        except OSError:
+            pass
+
+    def passtochild(signo, stack_frame):
+        if pid:
+            os.kill(pid, signo)
+
+    signal.signal(signal.SIGCHLD, sigchld_handler)
+    signal.signal(signal.SIGHUP, passtochild)
+    signal.signal(signal.SIGINT, passtochild)
+    signal.signal(signal.SIGUSR1, passtochild)
+    signal.signal(signal.SIGUSR2, passtochild)
+    signal.signal(signal.SIGQUIT, passtochild)
+    signal.signal(signal.SIGTERM, passtochild)
+
+    patroni = call_self(sys.argv[1:])
+    pid = patroni.pid
+    patroni.wait()

patroni/api.py

@@ -3,9 +3,12 @@ import fcntl
 import json
 import logging
 import psycopg2
+import time
+import dateutil.parser
+import datetime
 
-from patroni.exceptions import PostgresConnectionException
-from patroni.utils import Retry, RetryFailedError
+from patroni.postgresql import PostgresConnectionException, PostgresException, Postgresql
+from patroni.utils import deep_compare, parse_bool, patch_config, Retry, RetryFailedError, parse_int, tzutc
 from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 from six.moves.socketserver import ThreadingMixIn
 from threading import Thread
@@ -21,27 +24,57 @@ def check_auth(func):
     def do_PUT_foo():
         pass
     """
-    def wrapper(handler):
+    def wrapper(handler, *args, **kwargs):
         if handler.check_auth_header():
-            return func(handler)
+            return func(handler, *args, **kwargs)
     return wrapper
 
 
 class RestApiHandler(BaseHTTPRequestHandler):
 
-    def send_auth_request(self, body):
-        self.send_response(401)
-        self.send_header('WWW-Authenticate', 'Basic realm=\"Patroni\"')
-        self.send_header('Content-type', 'text/html')
+    def _write_response(self, status_code, body, content_type='text/html', headers=None):
+        self.send_response(status_code)
+        headers = headers or {}
+        if content_type:
+            headers['Content-Type'] = content_type
+        for name, value in headers.items():
+            self.send_header(name, value)
         self.end_headers()
         self.wfile.write(body.encode('utf-8'))
 
+    def _write_json_response(self, status_code, response):
+        self._write_response(status_code, json.dumps(response), content_type='application/json')
+
+    def send_auth_request(self, body):
+        headers = {'WWW-Authenticate': 'Basic realm="' + self.server.patroni.__class__.__name__ + '"'}
+        self._write_response(401, body, headers=headers)
+
     def check_auth_header(self):
         auth_header = self.headers.get('Authorization')
         status = self.server.check_auth_header(auth_header)
         return not status or self.send_auth_request(status)
 
-    def do_GET(self):
+    def _write_status_response(self, status_code, response):
+        patroni = self.server.patroni
+        tags = patroni.ha.get_effective_tags()
+        if tags:
+            response['tags'] = tags
+        if patroni.postgresql.sysid:
+            response['database_system_identifier'] = patroni.postgresql.sysid
+        if patroni.postgresql.pending_restart:
+            response['pending_restart'] = True
+        response['patroni'] = {'version': patroni.version, 'scope': patroni.postgresql.scope}
+        if patroni.scheduled_restart and isinstance(patroni.scheduled_restart, dict):
+            response['scheduled_restart'] = patroni.scheduled_restart.copy()
+            del response['scheduled_restart']['postmaster_start_time']
+            response['scheduled_restart']['schedule'] = (response['scheduled_restart']['schedule']).isoformat()
+        if not patroni.ha.watchdog.is_healthy:
+            response['watchdog_failed'] = True
+        if patroni.ha.is_paused():
+            response['pause'] = True
+        self._write_json_response(status_code, response)
+
+    def do_GET(self, write_status_code_only=False):
         """Default method for processing all GET requests which can not be routed to other methods"""
 
         path = '/master' if self.path == '/' else self.path
@@ -49,6 +82,14 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         patroni = self.server.patroni
         cluster = patroni.dcs.cluster
+
+        def is_synchronous():
+            return (cluster.is_synchronous_mode() and cluster.sync
+                    and cluster.sync.sync_standby == patroni.postgresql.name)
+
+        def is_balanceable_replica():
+            return response.get('role') == 'replica' and not patroni.noloadbalance
+
         if cluster:  # dcs available
             if cluster.leader and cluster.leader.name == patroni.postgresql.name:  # is_leader
                 status_code = 200 if 'master' in path else 503
@@ -56,70 +97,296 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 status_code = 503
             elif response['role'] == 'master':  # running as master but without leader lock!!!!
                 status_code = 503
-            elif response['role'] in path:
-                status_code = 200
+            elif path in ('/sync', '/synchronous'):
+                status_code = 200 if is_balanceable_replica() and is_synchronous() else 503
+            elif path in ('/async', '/asynchronous'):
+                status_code = 200 if is_balanceable_replica() and not is_synchronous() else 503
+            elif response['role'] in path:  # response['role'] != 'master'
+                status_code = 503 if patroni.noloadbalance else 200
             else:
                 status_code = 503
         elif 'role' in response and response['role'] in path:
-            status_code = 200
+            status_code = 503 if response['role'] != 'master' and patroni.noloadbalance else 200
         elif patroni.ha.restart_scheduled() and patroni.postgresql.role == 'master' and 'master' in path:
             # exceptional case for master node when the postgres is being restarted via API
             status_code = 200
         else:
             status_code = 503
 
-        self.send_response(status_code)
-        self.send_header('Content-Type', 'application/json')
-        self.end_headers()
-        self.wfile.write(json.dumps(response).encode('utf-8'))
+        if write_status_code_only:  # when haproxy sends OPTIONS request it reads only status code and nothing more
+            message = self.responses[status_code][0]
+            self.wfile.write('{0} {1} {2}\r\n'.format(self.protocol_version, status_code, message).encode('utf-8'))
+        else:
+            self._write_status_response(status_code, response)
+
+    def do_OPTIONS(self):
+        self.do_GET(write_status_code_only=True)
 
     def do_GET_patroni(self):
         response = self.get_postgresql_status(True)
+        self._write_status_response(200, response)
 
-        self.send_response(200)
-        self.send_header('Content-Type', 'application/json')
-        self.end_headers()
-        self.wfile.write(json.dumps(response).encode('utf-8'))
+    def do_GET_config(self):
+        cluster = self.server.patroni.dcs.cluster or self.server.patroni.dcs.get_cluster()
+        if cluster.config:
+            self._write_json_response(200, cluster.config.data)
+        else:
+            self.send_error(502)
+
+    def _read_json_content(self, body_is_optional=False):
+        if 'content-length' not in self.headers:
+            return self.send_error(411) if not body_is_optional else {}
+        try:
+            content_length = int(self.headers.get('content-length'))
+            if content_length == 0 and body_is_optional:
+                return {}
+            request = json.loads(self.rfile.read(content_length).decode('utf-8'))
+            if isinstance(request, dict) and (request or body_is_optional):
+                return request
+        except Exception:
+            logger.exception('Bad request')
+        self.send_error(400)
+
+    @check_auth
+    def do_PATCH_config(self):
+        request = self._read_json_content()
+        if request:
+            cluster = self.server.patroni.dcs.get_cluster()
+            data = cluster.config.data.copy()
+            if patch_config(data, request):
+                value = json.dumps(data, separators=(',', ':'))
+                if not self.server.patroni.dcs.set_config_value(value, cluster.config.index):
+                    return self.send_error(409)
+            self.server.patroni.ha.wakeup()
+            self._write_json_response(200, data)
+
+    @check_auth
+    def do_PUT_config(self):
+        request = self._read_json_content()
+        if request:
+            cluster = self.server.patroni.dcs.get_cluster()
+            if not deep_compare(request, cluster.config.data):
+                value = json.dumps(request, separators=(',', ':'))
+                if not self.server.patroni.dcs.set_config_value(value):
+                    return self.send_error(502)
+            self._write_json_response(200, request)
+
+    @check_auth
+    def do_POST_reload(self):
+        try:
+            if self.server.patroni.config.reload_local_configuration(True):
+                status_code = 202
+                response = 'reload scheduled'
+                self.server.patroni.sighup_handler()
+            else:
+                status_code = 200
+                response = 'nothing changed'
+        except Exception as e:
+            status_code = 500
+            response = str(e)
+        self._write_response(status_code, response)
+
+    @staticmethod
+    def parse_schedule(schedule, action):
+        """ parses the given schedule and validates at """
+        error = None
+        scheduled_at = None
+        try:
+            scheduled_at = dateutil.parser.parse(schedule)
+            if scheduled_at.tzinfo is None:
+                error = 'Timezone information is mandatory for the scheduled {0}'.format(action)
+                status_code = 400
+            elif scheduled_at < datetime.datetime.now(tzutc):
+                error = 'Cannot schedule {0} in the past'.format(action)
+                status_code = 422
+            else:
+                status_code = None
+        except (ValueError, TypeError):
+            logger.exception('Invalid scheduled %s time: %s', action, schedule)
+            error = 'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601'
+            status_code = 422
+        return (status_code, error, scheduled_at)
 
     @check_auth
     def do_POST_restart(self):
-        status_code = 503
-        data = b'restart failed'
-        try:
-            status, msg = self.server.patroni.ha.restart()
-            status_code = 200 if status else 503
-            data = msg.encode('utf-8')
-        except:
-            logger.exception('Exception during restart')
+        status_code = 500
+        data = 'restart failed'
+        request = self._read_json_content(body_is_optional=True)
+        cluster = self.server.patroni.dcs.get_cluster()
+        if request is None:
+            # failed to parse the json
+            return
+        if request:
+            logger.debug("received restart request: {0}".format(request))
 
-        self.send_response(status_code)
-        self.send_header('Content-Type', 'text/html')
-        self.end_headers()
-        self.wfile.write(data)
+        if cluster.is_paused() and 'schedule' in request:
+            self._write_response(status_code, "Can't schedule restart in the paused state")
+            return
+
+        for k in request:
+            if k == 'schedule':
+                (_, data, request[k]) = self.parse_schedule(request[k], "restart")
+                if _:
+                    status_code = _
+                    break
+            elif k == 'role':
+                if request[k] not in ('master', 'replica'):
+                    status_code = 400
+                    data = "PostgreSQL role should be either master or replica"
+                    break
+            elif k == 'postgres_version':
+                try:
+                    Postgresql.postgres_version_to_int(request[k])
+                except PostgresException as e:
+                    status_code = 400
+                    data = e.value
+                    break
+            elif k == 'timeout':
+                request[k] = parse_int(request[k], 's')
+                if request[k] is None or request[k] <= 0:
+                    status_code = 400
+                    data = "Timeout should be a positive number of seconds"
+                    break
+            elif k != 'restart_pending':
+                status_code = 400
+                data = "Unknown filter for the scheduled restart: {0}".format(k)
+                break
+        else:
+            if 'schedule' not in request:
+                try:
+                    status, data = self.server.patroni.ha.restart(request)
+                    status_code = 200 if status else 503
+                except Exception:
+                    logger.exception('Exception during restart')
+                    status_code = 400
+            else:
+                if self.server.patroni.ha.schedule_future_restart(request):
+                    data = "Restart scheduled"
+                    status_code = 202
+                else:
+                    data = "Another restart is already scheduled"
+                    status_code = 409
+        self._write_response(status_code, data)
+
+    @check_auth
+    def do_DELETE_restart(self):
+        if self.server.patroni.ha.delete_future_restart():
+            data = "scheduled restart deleted"
+            code = 200
+        else:
+            data = "no restarts are scheduled"
+            code = 404
+        self._write_response(code, data)
 
     @check_auth
     def do_POST_reinitialize(self):
-        ha = self.server.patroni.ha
-        cluster = ha.dcs.get_cluster()
-        if cluster.is_unlocked():
-            status_code = 503
-            data = b'Cluster has no leader, can not reinitialize'
-        elif cluster.leader.name == ha.state_handler.name:
-            status_code = 503
-            data = b'I am the leader, can not reinitialize'
-        else:
-            action = ha.schedule_reinitialize()
-            if action is not None:
-                status_code = 503
-                data = (action + ' already in progress').encode('utf-8')
-            else:
-                status_code = 200
-                data = b'reinitialize scheduled'
+        request = self._read_json_content(body_is_optional=True)
 
-        self.send_response(status_code)
-        self.send_header('Content-Type', 'text/html')
-        self.end_headers()
-        self.wfile.write(data)
+        if request:
+            logger.debug('received reinitialize request: %s', request)
+
+        force = isinstance(request, dict) and parse_bool(request.get('force')) or False
+
+        data = self.server.patroni.ha.reinitialize(force)
+        if data is None:
+            status_code = 200
+            data = 'reinitialize started'
+        else:
+            status_code = 503
+        self._write_response(status_code, data)
+
+    def poll_failover_result(self, leader, candidate, action):
+        timeout = max(10, self.server.patroni.dcs.loop_wait)
+        for _ in range(0, timeout*2):
+            time.sleep(1)
+            try:
+                cluster = self.server.patroni.dcs.get_cluster()
+                if not cluster.is_unlocked() and cluster.leader.name != leader:
+                    if not candidate or candidate == cluster.leader.name:
+                        return 200, 'Successfully {0}ed over to "{1}"'.format(action[:-4], cluster.leader.name)
+                    else:
+                        return 200, '{0}ed over to "{1}" instead of "{2}"'.format(action[:-4].title(),
+                                                                                  cluster.leader.name, candidate)
+                if not cluster.failover:
+                    return 503, action.title() + ' failed'
+            except Exception as e:
+                logger.debug('Exception occured during polling %s result: %s', action, e)
+        return 503, action.title() + ' status unknown'
+
+    def is_failover_possible(self, cluster, leader, candidate, action):
+        if leader and (not cluster.leader or cluster.leader.name != leader):
+            return 'leader name does not match'
+        if candidate:
+            if action == 'switchover' and cluster.is_synchronous_mode() and cluster.sync.sync_standby != candidate:
+                return 'candidate name does not match with sync_standby'
+            members = [m for m in cluster.members if m.name == candidate]
+            if not members:
+                return 'candidate does not exists'
+        elif cluster.is_synchronous_mode():
+            members = [m for m in cluster.members if m.name == cluster.sync.sync_standby]
+            if not members:
+                return action + ' is not possible: can not find sync_standby'
+        else:
+            members = [m for m in cluster.members if m.name != cluster.leader.name and m.api_url]
+            if not members:
+                return action + ' is not possible: cluster does not have members except leader'
+        for st in self.server.patroni.ha.fetch_nodes_statuses(members):
+            if st.failover_limitation() is None:
+                return None
+        return action + ' is not possible: no good candidates have been found'
+
+    @check_auth
+    def do_POST_failover(self, action='failover'):
+        request = self._read_json_content()
+        (status_code, data) = (400, '')
+        if not request:
+            return
+
+        leader = request.get('leader')
+        candidate = request.get('candidate') or request.get('member')
+        scheduled_at = request.get('scheduled_at')
+        cluster = self.server.patroni.dcs.get_cluster()
+
+        logger.info("received %s request with leader=%s candidate=%s scheduled_at=%s",
+                    action, leader, candidate, scheduled_at)
+
+        if action == 'failover' and not candidate:
+            data = 'Failover could be performed only to a specific candidate'
+        elif action == 'switchover' and not leader:
+            data = 'Switchover could be performed only from a specific leader'
+
+        if not data and scheduled_at:
+            if not leader:
+                data = 'Scheduled {0} is possible only from a specific leader'.format(action)
+            if not data and cluster.is_paused():
+                data = "Can't schedule {0} in the paused state".format(action)
+            if not data:
+                (status_code, data, scheduled_at) = self.parse_schedule(scheduled_at, action)
+
+        if not data and cluster.is_paused() and not candidate:
+            data = action.title() + ' is possible only to a specific candidate in a paused state'
+
+        if not data and not scheduled_at:
+            data = self.is_failover_possible(cluster, leader, candidate, action)
+            if data:
+                status_code = 412
+
+        if not data:
+            if self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
+                self.server.patroni.ha.wakeup()
+                if scheduled_at:
+                    data = action.title() + ' scheduled'
+                    status_code = 202
+                else:
+                    status_code, data = self.poll_failover_result(cluster.leader and cluster.leader.name,
+                                                                  candidate, action)
+            else:
+                data = 'failed to write {0} key into DCS'.format(action)
+                status_code = 503
+        self._write_response(status_code, data)
+
+    def do_POST_switchover(self):
+        self.do_POST_failover(action='switchover')
 
     def parse_request(self):
         """Override parse_request method to enrich basic functionality of `BaseHTTPRequestHandler` class
@@ -147,56 +414,67 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
     def get_postgresql_status(self, retry=False):
         try:
-            row = self.query("""SELECT to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
-                                       pg_is_in_recovery(),
-                                       CASE WHEN pg_is_in_recovery()
-                                            THEN 0
-                                            ELSE pg_xlog_location_diff(pg_current_xlog_location(), '0/0')::bigint
-                                       END,
-                                       pg_xlog_location_diff(pg_last_xlog_receive_location(), '0/0')::bigint,
-                                       pg_xlog_location_diff(pg_last_xlog_replay_location(), '0/0')::bigint,
-                                       pg_is_in_recovery() AND pg_is_xlog_replay_paused()""", retry=retry)[0]
-            return {
+            if self.server.patroni.postgresql.state not in ('running', 'restarting', 'starting'):
+                raise RetryFailedError('')
+            stmt = ("WITH replication_info AS ("
+                    "SELECT usename, application_name, client_addr, state, sync_state, sync_priority"
+                    " FROM pg_stat_replication) SELECT"
+                    " to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),"
+                    " CASE WHEN pg_is_in_recovery() THEN 0"
+                    " ELSE ('x' || SUBSTR(pg_{0}file_name(pg_current_{0}_{1}()), 1, 8))::bit(32)::int END,"
+                    " CASE WHEN pg_is_in_recovery() THEN 0"
+                    " ELSE pg_{0}_{1}_diff(pg_current_{0}_{1}(), '0/0')::bigint END,"
+                    " pg_{0}_{1}_diff(COALESCE(pg_last_{0}_receive_{1}(), pg_last_{0}_replay_{1}()), '0/0')::bigint,"
+                    " pg_{0}_{1}_diff(pg_last_{0}_replay_{1}(), '0/0')::bigint,"
+                    " to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),"
+                    " pg_is_in_recovery() AND pg_is_{0}_replay_paused(),"
+                    " (SELECT array_to_json(array_agg(row_to_json(ri))) FROM replication_info ri)")
+
+            row = self.query(stmt.format(self.server.patroni.postgresql.wal_name,
+                                         self.server.patroni.postgresql.lsn_name), retry=retry)[0]
+
+            result = {
                 'state': self.server.patroni.postgresql.state,
                 'postmaster_start_time': row[0],
-                'role': 'replica' if row[1] else 'master',
+                'role': 'replica' if row[1] == 0 else 'master',
+                'server_version': self.server.patroni.postgresql.server_version,
                 'xlog': ({
                     'received_location': row[3],
                     'replayed_location': row[4],
-                    'paused': row[5]} if row[1] else {
+                    'replayed_timestamp': row[5],
+                    'paused': row[6]} if row[1] == 0 else {
                     'location': row[2]
                 })
             }
+
+            if row[1] > 0:
+                result['timeline'] = row[1]
+            else:
+                cluster = self.server.patroni.dcs.cluster
+                leader_timeline = None if not cluster or cluster.is_unlocked() else cluster.leader.timeline
+                result['timeline'] = self.server.patroni.postgresql.replica_cached_timeline(leader_timeline)
+
+            if row[7]:
+                result['replication'] = row[7]
+
+            return result
         except (psycopg2.Error, RetryFailedError, PostgresConnectionException):
             state = self.server.patroni.postgresql.state
-            if state in ['stopped', 'starting', 'stopping', 'restarting', 'running']:
+            if state == 'running':
                 logger.exception('get_postgresql_status')
-                state = 'unknown' if state == 'running' else state
-            return {'state': state}
+                state = 'unknown'
+            return {'state': state, 'role': self.server.patroni.postgresql.role}
+
+    def log_message(self, fmt, *args):
+        logger.debug("API thread: %s - - [%s] %s", self.client_address[0], self.log_date_time_string(), fmt % args)
 
 
 class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
 
     def __init__(self, patroni, config):
-        self._auth_key = base64.b64encode(config['auth'].encode('utf-8')).decode('utf-8') if 'auth' in config else None
-        host, port = config['listen'].split(':')
-        HTTPServer.__init__(self, (host, int(port)), RestApiHandler)
-        Thread.__init__(self, target=self.serve_forever)
-        self._set_fd_cloexec(self.socket)
-
-        protocol = 'http'
-
-        # wrap socket with ssl if 'certfile' is defined in a config.yaml
-        # Sometime it's also needed to pass reference to a 'keyfile'.
-        options = {option: config[option] for option in ['certfile', 'keyfile'] if option in config}
-        if options.get('certfile', None):
-            import ssl
-            self.socket = ssl.wrap_socket(self.socket, server_side=True, **options)
-            protocol = 'https'
-
-        self.connection_string = '{}://{}/patroni'.format(protocol, config.get('connect_address', config['listen']))
-
         self.patroni = patroni
+        self.__initialize(config)
+        self.__set_config_parameters(config)
         self.daemon = True
 
     def query(self, sql, *params):
@@ -216,11 +494,47 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC)
 
     def check_basic_auth_key(self, key):
-        return self._auth_key == key
+        return self.__auth_key == key
 
     def check_auth_header(self, auth_header):
-        if self._auth_key:
+        if self.__auth_key:
             if auth_header is None:
                 return 'no auth header received'
             if not auth_header.startswith('Basic ') or not self.check_basic_auth_key(auth_header[6:]):
                 return 'not authenticated'
+
+    @staticmethod
+    def __get_ssl_options(config):
+        return {option: config[option] for option in ['certfile', 'keyfile'] if option in config}
+
+    def __set_connection_string(self, connect_address):
+        self.connection_string = '{0}://{1}/patroni'.format(self.__protocol, connect_address or self.__listen)
+
+    def __set_config_parameters(self, config):
+        self.__auth_key = base64.b64encode(config['auth'].encode('utf-8')).decode('utf-8') if 'auth' in config else None
+        self.__set_connection_string(config.get('connect_address'))
+
+    def __initialize(self, config):
+        self.__ssl_options = self.__get_ssl_options(config)
+        self.__listen = config['listen']
+        host, port = config['listen'].rsplit(':', 1)
+        HTTPServer.__init__(self, (host, int(port)), RestApiHandler)
+        Thread.__init__(self, target=self.serve_forever)
+        self._set_fd_cloexec(self.socket)
+
+        self.__protocol = 'http'
+
+        # wrap socket with ssl if 'certfile' is defined in a config.yaml
+        # Sometime it's also needed to pass reference to a 'keyfile'.
+        if self.__ssl_options.get('certfile'):
+            import ssl
+            self.socket = ssl.wrap_socket(self.socket, server_side=True, **self.__ssl_options)
+            self.__protocol = 'https'
+        self.__set_connection_string(config.get('connect_address'))
+
+    def reload_config(self, config):
+        self.__set_config_parameters(config)
+        if self.__listen != config['listen'] or self.__ssl_options != self.__get_ssl_options(config):
+            self.shutdown()
+            self.__initialize(config)
+            self.start()

patroni/async_executor.py

@@ -1,28 +1,78 @@
 import logging
-from threading import Lock, Thread
+from threading import Event, Lock, RLock, Thread
 
 logger = logging.getLogger(__name__)
 
 
-class AsyncExecutor:
+class CriticalTask(object):
+    """Represents a critical task in a background process that we either need to cancel or get the result of.
 
+    Fields of this object may be accessed only when holding a lock on it. To perform the critical task the background
+    thread must, while holding lock on this object, check `is_cancelled` flag, run the task and mark the task as
+    complete using `complete()`.
+
+    The main thread must hold async lock to prevent the task from completing, hold lock on critical task object,
+    call cancel. If the task has completed `cancel()` will return False and `result` field will contain the result of
+    the task. When cancel returns True it is guaranteed that the background task will notice the `is_cancelled` flag.
+    """
     def __init__(self):
-        Lock.__init__(self)
-        self._busy = False
-        self._thread_lock = Lock()
+        self._lock = Lock()
+        self.is_cancelled = False
+        self.result = None
+
+    def reset(self):
+        """Must be called every time the background task is finished.
+
+        Must be called from async thread. Caller must hold lock on async executor when calling."""
+        self.is_cancelled = False
+        self.result = None
+
+    def cancel(self):
+        """Tries to cancel the task, returns True if the task has already run.
+
+        Caller must hold lock on async executor and the task when calling."""
+        if self.result is not None:
+            return False
+        self.is_cancelled = True
+        return True
+
+    def complete(self, result):
+        """Mark task as completed along with a result.
+
+        Must be called from async thread. Caller must hold lock on task when calling."""
+        self.result = result
+
+    def __enter__(self):
+        self._lock.acquire()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._lock.release()
+
+
+class AsyncExecutor(object):
+
+    def __init__(self, state_handler, ha_wakeup):
+        self.state_handler = state_handler
+        self._ha_wakeup = ha_wakeup
+        self._thread_lock = RLock()
         self._scheduled_action = None
-        self._scheduled_action_lock = Lock()
+        self._scheduled_action_lock = RLock()
+        self._is_cancelled = False
+        self._finish_event = Event()
+        self.critical_task = CriticalTask()
 
     @property
     def busy(self):
-        return self._busy
+        return self.scheduled_action is not None
 
-    def schedule(self, action, immediately=False):
+    def schedule(self, action):
         with self._scheduled_action_lock:
             if self._scheduled_action is not None:
                 return self._scheduled_action
             self._scheduled_action = action
-            self._busy = immediately
+            self._is_cancelled = False
+            self._finish_event.set()
         return None
 
     @property
@@ -35,21 +85,47 @@ class AsyncExecutor:
             self._scheduled_action = None
 
     def run(self, func, args=()):
+        wakeup = False
         try:
-            return func(*args) if args else func()
-        except:
+            with self:
+                if self._is_cancelled:
+                    return
+                self._finish_event.clear()
+
+            self.state_handler.reset_is_cancelled()
+            # if the func returned something (not None) - wake up main HA loop
+            wakeup = func(*args) if args else func()
+            return wakeup
+        except Exception:
             logger.exception('Exception during execution of long running task %s', self.scheduled_action)
         finally:
             with self:
-                self._busy = False
                 self.reset_scheduled_action()
+                self._finish_event.set()
+                with self.critical_task:
+                    self.critical_task.reset()
+            if wakeup is not None:
+                self._ha_wakeup()
 
     def run_async(self, func, args=()):
-        self._busy = True
         Thread(target=self.run, args=(func, args)).start()
 
+    def cancel(self):
+        with self:
+            with self._scheduled_action_lock:
+                if self._scheduled_action is None:
+                    return
+                logger.warning('Cancelling long running task %s', self._scheduled_action)
+            self._is_cancelled = True
+
+        self.state_handler.cancel()
+        self._finish_event.wait()
+
+        with self:
+            self.reset_scheduled_action()
+
     def __enter__(self):
         self._thread_lock.acquire()
 
-    def __exit__(self, type, value, traceback):
+    def __exit__(self, *args):
         self._thread_lock.release()

patroni/callback_executor.py

@@ -0,0 +1,37 @@
+import logging
+import subprocess
+from threading import Event, Lock, Thread
+
+logger = logging.getLogger(__name__)
+
+
+class CallbackExecutor(Thread):
+
+    def __init__(self):
+        super(CallbackExecutor, self).__init__()
+        self.daemon = True
+        self._lock = Lock()
+        self._cmd = None
+        self._process = None
+        self._callback_event = Event()
+        self.start()
+
+    def call(self, cmd):
+        with self._lock:
+            if self._process and self._process.poll() is None:
+                self._process.kill()
+                logger.warning('Killed the old callback process because it was still running: %s', self._cmd)
+        self._cmd = cmd
+        self._callback_event.set()
+
+    def run(self):
+        while True:
+            self._callback_event.wait()
+            self._callback_event.clear()
+            with self._lock:
+                try:
+                    self._process = subprocess.Popen(self._cmd, close_fds=True)
+                except Exception:
+                    logger.exception('Failed to execute %s',  self._cmd)
+                    continue
+            self._process.wait()

patroni/config.py

@@ -0,0 +1,333 @@
+import json
+import logging
+import os
+import sys
+import tempfile
+import yaml
+
+from collections import defaultdict
+from copy import deepcopy
+from patroni.dcs import ClusterConfig
+from patroni.postgresql import Postgresql
+from patroni.utils import deep_compare, parse_bool, parse_int, patch_config
+from requests.structures import CaseInsensitiveDict
+
+logger = logging.getLogger(__name__)
+
+
+class Config(object):
+    """
+    This class is responsible for:
+
+      1) Building and giving access to `effective_configuration` from:
+         * `Config.__DEFAULT_CONFIG` -- some sane default values
+         * `dynamic_configuration` -- configuration stored in DCS
+         * `local_configuration` -- configuration from `config.yml` or environment
+
+      2) Saving and loading `dynamic_configuration` into 'patroni.dynamic.json' file
+         located in local_configuration['postgresql']['data_dir'] directory.
+         This is necessary to be able to restore `dynamic_configuration`
+         if DCS was accidentally wiped
+
+      3) Loading of configuration file in the old format and converting it into new format
+
+      4) Mimicking some of the `dict` interfaces to make it possible
+         to work with it as with the old `config` object.
+    """
+
+    PATRONI_ENV_PREFIX = 'PATRONI_'
+    PATRONI_CONFIG_VARIABLE = PATRONI_ENV_PREFIX + 'CONFIGURATION'
+
+    __CACHE_FILENAME = 'patroni.dynamic.json'
+    __DEFAULT_CONFIG = {
+        'ttl': 30, 'loop_wait': 10, 'retry_timeout': 10,
+        'maximum_lag_on_failover': 1048576,
+        'master_start_timeout': 300,
+        'synchronous_mode': False,
+        'synchronous_mode_strict': False,
+        'postgresql': {
+            'bin_dir': '',
+            'use_slots': True,
+            'parameters': CaseInsensitiveDict({p: v[0] for p, v in Postgresql.CMDLINE_OPTIONS.items()})
+        },
+        'watchdog': {
+            'mode': 'automatic',
+        }
+    }
+
+    def __init__(self):
+        self._modify_index = -1
+        self._dynamic_configuration = {}
+
+        self.__environment_configuration = self._build_environment_configuration()
+
+        # Patroni reads the configuration from the command-line argument if it exists, otherwise from the environment
+        self._config_file = len(sys.argv) >= 2 and os.path.isfile(sys.argv[1]) and sys.argv[1]
+        if self._config_file:
+            self._local_configuration = self._load_config_file()
+        else:
+            config_env = os.environ.pop(self.PATRONI_CONFIG_VARIABLE, None)
+            self._local_configuration = config_env and yaml.safe_load(config_env) or self.__environment_configuration
+            if not self._local_configuration:
+                print('Usage: {0} config.yml'.format(sys.argv[0]))
+                print('\tPatroni may also read the configuration from the {0} environment variable'.
+                      format(self.PATRONI_CONFIG_VARIABLE))
+                sys.exit(1)
+
+        self.__effective_configuration = self._build_effective_configuration({}, self._local_configuration)
+        self._data_dir = self.__effective_configuration['postgresql']['data_dir']
+        self._cache_file = os.path.join(self._data_dir, self.__CACHE_FILENAME)
+        self._load_cache()
+        self._cache_needs_saving = False
+
+    @property
+    def config_file(self):
+        return self._config_file
+
+    @property
+    def dynamic_configuration(self):
+        return deepcopy(self._dynamic_configuration)
+
+    def check_mode(self, mode):
+        return bool(parse_bool(self._dynamic_configuration.get(mode)))
+
+    def _load_config_file(self):
+        """Loads config.yaml from filesystem and applies some values which were set via ENV"""
+        with open(self._config_file) as f:
+            config = yaml.safe_load(f)
+            patch_config(config, self.__environment_configuration)
+            return config
+
+    def _load_cache(self):
+        if os.path.isfile(self._cache_file):
+            try:
+                with open(self._cache_file) as f:
+                    self.set_dynamic_configuration(json.load(f))
+            except Exception:
+                logger.exception('Exception when loading file: %s', self._cache_file)
+
+    def save_cache(self):
+        if self._cache_needs_saving:
+            tmpfile = fd = None
+            try:
+                (fd, tmpfile) = tempfile.mkstemp(prefix=self.__CACHE_FILENAME, dir=self._data_dir)
+                with os.fdopen(fd, 'w') as f:
+                    fd = None
+                    json.dump(self.dynamic_configuration, f)
+                tmpfile = os.rename(tmpfile, self._cache_file)
+                self._cache_needs_saving = False
+            except Exception:
+                logger.exception('Exception when saving file: %s', self._cache_file)
+                if fd:
+                    try:
+                        os.close(fd)
+                    except Exception:
+                        logger.error('Can not close temporary file %s', tmpfile)
+                if tmpfile and os.path.exists(tmpfile):
+                    try:
+                        os.remove(tmpfile)
+                    except Exception:
+                        logger.error('Can not remove temporary file %s', tmpfile)
+
+    # configuration could be either ClusterConfig or dict
+    def set_dynamic_configuration(self, configuration):
+        if isinstance(configuration, ClusterConfig):
+            if self._modify_index == configuration.modify_index:
+                return False  # If the index didn't changed there is nothing to do
+            self._modify_index = configuration.modify_index
+            configuration = configuration.data
+
+        if not deep_compare(self._dynamic_configuration, configuration):
+            try:
+                self.__effective_configuration = self._build_effective_configuration(configuration,
+                                                                                     self._local_configuration)
+                self._dynamic_configuration = configuration
+                self._cache_needs_saving = True
+                return True
+            except Exception:
+                logger.exception('Exception when setting dynamic_configuration')
+
+    def reload_local_configuration(self, dry_run=False):
+        if self.config_file:
+            try:
+                configuration = self._load_config_file()
+                if not deep_compare(self._local_configuration, configuration):
+                    new_configuration = self._build_effective_configuration(self._dynamic_configuration, configuration)
+                    if dry_run:
+                        return not deep_compare(new_configuration, self.__effective_configuration)
+                    self._local_configuration = configuration
+                    self.__effective_configuration = new_configuration
+                    return True
+            except Exception:
+                logger.exception('Exception when reloading local configuration from %s', self.config_file)
+                if dry_run:
+                    raise
+
+    @staticmethod
+    def _process_postgresql_parameters(parameters, is_local=False):
+        ret = {}
+        for name, value in (parameters or {}).items():
+            if name not in Postgresql.CMDLINE_OPTIONS or not is_local and Postgresql.CMDLINE_OPTIONS[name][1](value):
+                ret[name] = value
+        return ret
+
+    def _safe_copy_dynamic_configuration(self, dynamic_configuration):
+        config = deepcopy(self.__DEFAULT_CONFIG)
+
+        for name, value in dynamic_configuration.items():
+            if name == 'postgresql':
+                for name, value in (value or {}).items():
+                    if name == 'parameters':
+                        config['postgresql'][name].update(self._process_postgresql_parameters(value))
+                    elif name not in ('connect_address', 'listen', 'data_dir', 'pgpass', 'authentication'):
+                        config['postgresql'][name] = deepcopy(value)
+            elif name in config:  # only variables present in __DEFAULT_CONFIG allowed to be overriden from DCS
+                if name in ('synchronous_mode', 'synchronous_mode_strict'):
+                    config[name] = value
+                else:
+                    config[name] = int(value)
+        return config
+
+    @staticmethod
+    def _build_environment_configuration():
+        ret = defaultdict(dict)
+
+        def _popenv(name):
+            return os.environ.pop(Config.PATRONI_ENV_PREFIX + name.upper(), None)
+
+        for param in ('name', 'namespace', 'scope'):
+            value = _popenv(param)
+            if value:
+                ret[param] = value
+
+        def _set_section_values(section, params):
+            for param in params:
+                value = _popenv(section + '_' + param)
+                if value:
+                    ret[section][param] = value
+
+        _set_section_values('restapi', ['listen', 'connect_address', 'certfile', 'keyfile'])
+        _set_section_values('postgresql', ['listen', 'connect_address', 'data_dir', 'pgpass', 'bin_dir'])
+
+        def _get_auth(name):
+            ret = {}
+            for param in ('username', 'password'):
+                value = _popenv(name + '_' + param)
+                if value:
+                    ret[param] = value
+            return ret
+
+        restapi_auth = _get_auth('restapi')
+        if restapi_auth:
+            ret['restapi']['authentication'] = restapi_auth
+
+        authentication = {}
+        for user_type in ('replication', 'superuser'):
+            entry = _get_auth(user_type)
+            if entry:
+                authentication[user_type] = entry
+
+        if authentication:
+            ret['postgresql']['authentication'] = authentication
+
+        users = {}
+
+        def _parse_list(value):
+            if not (value.strip().startswith('-') or '[' in value):
+                value = '[{0}]'.format(value)
+            try:
+                return yaml.safe_load(value)
+            except Exception:
+                logger.exception('Exception when parsing list %s', value)
+                return None
+
+        for param in list(os.environ.keys()):
+            if param.startswith(Config.PATRONI_ENV_PREFIX):
+                name, suffix = (param[8:].split('_', 1) + [''])[:2]
+                if name and suffix:
+                    # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..)
+                    if suffix in ('HOST', 'HOSTS', 'PORT', 'SRV', 'URL', 'PROXY', 'CACERT', 'CERT',
+                                  'KEY', 'VERIFY', 'TOKEN', 'CHECKS', 'DC', 'NAMESPACE', 'CONTEXT',
+                                  'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL', 'POD_IP', 'PORTS', 'LABELS'):
+                        value = os.environ.pop(param)
+                        if suffix == 'PORT':
+                            value = value and parse_int(value)
+                        elif suffix in ('HOSTS', 'PORTS', 'CHECKS'):
+                            value = value and _parse_list(value)
+                        elif suffix == 'LABELS':
+                            if not value.strip().startswith('{'):
+                                value = '{{{0}}}'.format(value)
+                            try:
+                                value = yaml.safe_load(value)
+                            except Exception:
+                                logger.exception('Exception when parsing dict %s', value)
+                                value = None
+                        if value:
+                            ret[name.lower()][suffix.lower()] = value
+                    # PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>
+                    # CREATE USER "<username>" WITH <OPTIONS> PASSWORD '<password>'
+                    elif suffix == 'PASSWORD':
+                        password = os.environ.pop(param)
+                        if password:
+                            users[name] = {'password': password}
+                            options = os.environ.pop(param[:-9] + '_OPTIONS', None)
+                            options = options and _parse_list(options)
+                            if options:
+                                users[name]['options'] = options
+        if users:
+            ret['bootstrap']['users'] = users
+
+        return ret
+
+    def _build_effective_configuration(self, dynamic_configuration, local_configuration):
+        config = self._safe_copy_dynamic_configuration(dynamic_configuration)
+        for name, value in local_configuration.items():
+            if name == 'postgresql':
+                for name, value in (value or {}).items():
+                    if name == 'parameters':
+                        config['postgresql'][name].update(self._process_postgresql_parameters(value, True))
+                    elif name != 'use_slots':  # replication slots must be enabled/disabled globally
+                        config['postgresql'][name] = deepcopy(value)
+            elif name not in config or name in ['watchdog']:
+                config[name] = deepcopy(value) if value else {}
+
+        # restapi server expects to get restapi.auth = 'username:password'
+        if 'authentication' in config['restapi']:
+            config['restapi']['auth'] = '{username}:{password}'.format(**config['restapi']['authentication'])
+
+        # special treatment for old config
+
+        # 'exhibitor' inside 'zookeeper':
+        if 'zookeeper' in config and 'exhibitor' in config['zookeeper']:
+            config['exhibitor'] = config['zookeeper'].pop('exhibitor')
+            config.pop('zookeeper')
+
+        pg_config = config['postgresql']
+        # no 'authentication' in 'postgresql', but 'replication' and 'superuser'
+        if 'authentication' not in pg_config:
+            pg_config['use_pg_rewind'] = 'pg_rewind' in pg_config
+            pg_config['authentication'] = {u: pg_config[u] for u in ('replication', 'superuser') if u in pg_config}
+        # no 'superuser' in 'postgresql'.'authentication'
+        if 'superuser' not in pg_config['authentication'] and 'pg_rewind' in pg_config:
+            pg_config['authentication']['superuser'] = pg_config['pg_rewind']
+
+        # no 'name' in config
+        if 'name' not in config and 'name' in pg_config:
+            config['name'] = pg_config['name']
+
+        pg_config.update({p: config[p] for p in ('name', 'scope', 'retry_timeout',
+                          'synchronous_mode', 'maximum_lag_on_failover') if p in config})
+
+        return config
+
+    def get(self, key, default=None):
+        return self.__effective_configuration.get(key, default)
+
+    def __contains__(self, key):
+        return key in self.__effective_configuration
+
+    def __getitem__(self, key):
+        return self.__effective_configuration[key]
+
+    def copy(self):
+        return deepcopy(self.__effective_configuration)

patroni/dcs.py

@@ -1,267 +0,0 @@
-import abc
-import json
-
-from collections import namedtuple
-from patroni.exceptions import DCSError
-from six.moves.urllib_parse import urlparse, urlunparse, parse_qsl
-from threading import Event, Lock
-
-
-def parse_connection_string(value):
-    """Original Governor stores connection strings for each cluster members if a following format:
-        postgres://{username}:{password}@{connect_address}/postgres
-    Since each of our patroni instances provides own REST API endpoint it's good to store this information
-    in DCS among with postgresql connection string. In order to not introduce new keys and be compatible with
-    original Governor we decided to extend original connection string in a following way:
-        postgres://{username}:{password}@{connect_address}/postgres?application_name={api_url}
-    This way original Governor could use such connection string as it is, because of feature of `libpq` library.
-
-    This method is able to split connection string stored in DCS into two parts, `conn_url` and `api_url`"""
-
-    scheme, netloc, path, params, query, fragment = urlparse(value)
-    conn_url = urlunparse((scheme, netloc, path, params, '', fragment))
-    api_url = ([v for n, v in parse_qsl(query) if n == 'application_name'] or [None])[0]
-    return conn_url, api_url
-
-
-class Member(namedtuple('Member', 'index,name,session,data')):
-
-    """Immutable object (namedtuple) which represents single member of PostgreSQL cluster.
-    Consists of the following fields:
-    :param index: modification index of a given member key in a Configuration Store
-    :param name: name of PostgreSQL cluster member
-    :param session: either session id or just ttl in seconds
-    :param data: arbitrary data i.e. conn_url, api_url, xlog location, state, role, tags, etc...
-
-    There are two mandatory keys in a data:
-    conn_url: connection string containing host, user and password which could be used to access this member.
-    api_url: REST API url of patroni instance"""
-
-    @staticmethod
-    def from_node(index, name, session, data):
-        """
-        >>> Member.from_node(-1, '', '', '{"conn_url": "postgres://foo@bar/postgres"}') is not None
-        True
-        >>> Member.from_node(-1, '', '', '{')
-        Member(index=-1, name='', session='', data={})
-        """
-        if data.startswith('postgres'):
-            conn_url, api_url = parse_connection_string(data)
-            data = {'conn_url': conn_url, 'api_url': api_url}
-        else:
-            try:
-                data = json.loads(data)
-            except:
-                data = {}
-        return Member(index, name, session, data)
-
-    @property
-    def conn_url(self):
-        return self.data.get('conn_url', None)
-
-    @property
-    def api_url(self):
-        return self.data.get('api_url', None)
-
-
-class Leader(namedtuple('Leader', 'index,session,member')):
-
-    """Immutable object (namedtuple) which represents leader key.
-    Consists of the following fields:
-    :param index: modification index of a leader key in a Configuration Store
-    :param session: either session id or just ttl in seconds
-    :param member: reference to a `Member` object which represents current leader (see `Cluster.members`)"""
-
-    @property
-    def name(self):
-        return self.member.name
-
-    @property
-    def conn_url(self):
-        return self.member.conn_url
-
-
-class Failover(namedtuple('Failover', 'index,leader,member')):
-
-    @staticmethod
-    def from_node(index, value):
-        t = [a.strip() for a in value.split(':')] + ['']
-        return Failover(index, t[0], t[1]) if t[0] or t[1] else None
-
-
-class Cluster(namedtuple('Cluster', 'initialize,leader,last_leader_operation,members,failover')):
-
-    """Immutable object (namedtuple) which represents PostgreSQL cluster.
-    Consists of the following fields:
-    :param initialize: boolean, shows whether this cluster has initialization key stored in DC or not.
-    :param leader: `Leader` object which represents current leader of the cluster
-    :param last_leader_operation: int or long object containing position of last known leader operation.
-        This value is stored in `/optime/leader` key
-    :param members: list of Member object, all PostgreSQL cluster members including leader
-    :param failover: reference to `Failover` object"""
-
-    def is_unlocked(self):
-        return not (self.leader and self.leader.name)
-
-
-class AbstractDCS:
-
-    __metaclass__ = abc.ABCMeta
-
-    _INITIALIZE = 'initialize'
-    _LEADER = 'leader'
-    _FAILOVER = 'failover'
-    _MEMBERS = 'members/'
-    _OPTIME = 'optime'
-    _LEADER_OPTIME = _OPTIME + '/' + _LEADER
-
-    def __init__(self, name, config):
-        """
-        :param name: name of current instance (the same value as `~Postgresql.name`)
-        :param config: dict, reference to config section of selected DCS.
-            i.e.: `zookeeper` for zookeeper, `etcd` for etcd, etc...
-        """
-        self._name = name
-        self._scope = config['scope']
-        self._base_path = '/service/' + self._scope
-
-        self._cluster = None
-        self._cluster_thread_lock = Lock()
-        self.event = Event()
-
-    def client_path(self, path):
-        return '/'.join([self._base_path, path.lstrip('/')])
-
-    @property
-    def initialize_path(self):
-        return self.client_path(self._INITIALIZE)
-
-    @property
-    def members_path(self):
-        return self.client_path(self._MEMBERS)
-
-    @property
-    def member_path(self):
-        return self.client_path(self._MEMBERS + self._name)
-
-    @property
-    def leader_path(self):
-        return self.client_path(self._LEADER)
-
-    @property
-    def failover_path(self):
-        return self.client_path(self._FAILOVER)
-
-    @property
-    def leader_optime_path(self):
-        return self.client_path(self._LEADER_OPTIME)
-
-    @abc.abstractmethod
-    def _load_cluster(self):
-        """Internally this method should build  `Cluster` object which
-           represents current state and topology of the cluster in DCS.
-           this method supposed to be called only by `get_cluster` method.
-
-           raise `~DCSError` in case of communication or other problems with DCS.
-           If the current node was running as a master and exception raised,
-           instance would be demoted."""
-
-    def get_cluster(self):
-        with self._cluster_thread_lock:
-            try:
-                self._load_cluster()
-            except:
-                self._cluster = None
-                raise
-            return self._cluster
-
-    @property
-    def cluster(self):
-        with self._cluster_thread_lock:
-            return self._cluster
-
-    def reset_cluster(self):
-        with self._cluster_thread_lock:
-            self._cluster = None
-
-    @abc.abstractmethod
-    def write_leader_optime(self, last_operation):
-        """write current xlog location into `/optime/leader` key in DCS
-        :param last_operation: absolute xlog location in bytes"""
-
-    @abc.abstractmethod
-    def update_leader(self):
-        """Update leader key (or session) ttl
-
-        :returns: `!True` if leader key (or session) has been updated successfully.
-            If not, `!False` must be returned and current instance would be demoted.
-
-        You have to use CAS (Compare And Swap) operation in order to update leader key,
-        for example for etcd `prevValue` parameter must be used."""
-
-    @abc.abstractmethod
-    def attempt_to_acquire_leader(self):
-        """Attempt to acquire leader lock
-        This method should create `/leader` key with value=`~self._name`
-        :returns: `!True` if key has been created successfully.
-
-        Key must be created atomically. In case if key already exists it should not be
-        overwritten and `!False` must be returned"""
-
-    @abc.abstractmethod
-    def set_failover_value(self, value, index=None):
-        """Create or update `/failover` key"""
-
-    def manual_failover(self, leader, member, index=None):
-        return self.set_failover_value(leader + (':' + member if member else ''), index)
-
-    def current_leader(self):
-        try:
-            cluster = self.get_cluster()
-            return None if cluster.is_unlocked() else cluster.leader
-        except DCSError:
-            return None
-
-    @abc.abstractmethod
-    def touch_member(self, connection_string, ttl=None):
-        """Update member key in DCS.
-        This method should create or update key with the name = '/members/' + `~self._name`
-        and value = connection_string in a given DCS.
-
-        :param connection_string: how this instance can be accessed by other instances
-        :param ttl: ttl for member key, optional parameter. If it is None `~self.member_ttl will be used`
-        :returns: `!True` on success otherwise `!False`
-        """
-
-    @abc.abstractmethod
-    def take_leader(self):
-        """This method should create leader key with value = `~self._name` and ttl=`~self.ttl`
-        Since it could be called only on initial cluster bootstrap it could create this key regardless,
-        overwriting the key if necessary."""
-
-    @abc.abstractmethod
-    def initialize(self):
-        """Race for cluster initialization.
-        :returns: `!True` if key has been created successfully.
-
-        this method should create atomically initialize key and return `!True`
-        otherwise it should return `!False`"""
-
-    @abc.abstractmethod
-    def delete_leader(self):
-        """Voluntarily remove leader key from DCS
-        This method should remove leader key if current instance is the leader"""
-
-    @abc.abstractmethod
-    def cancel_initialization(self):
-        """ Removes the initialize key for a cluster """
-
-    def watch(self, timeout):
-        """If the current node is a master it should just sleep.
-        Any other node should watch for changes of leader key with a given timeout
-
-        :param timeout: timeout in seconds
-        :returns: `!True` if you would like to reschedule the next run of ha cycle"""
-
-        self.event.wait(timeout)
-        return self.event.isSet()

patroni/dcs/__init__.py

@@ -0,0 +1,614 @@
+import abc
+import dateutil
+import importlib
+import inspect
+import json
+import logging
+import os
+import pkgutil
+import six
+import sys
+
+from collections import namedtuple
+from patroni.exceptions import PatroniException
+from patroni.utils import parse_bool
+from random import randint
+from six.moves.urllib_parse import urlparse, urlunparse, parse_qsl
+from threading import Event, Lock
+
+logger = logging.getLogger(__name__)
+
+
+def parse_connection_string(value):
+    """Original Governor stores connection strings for each cluster members if a following format:
+        postgres://{username}:{password}@{connect_address}/postgres
+    Since each of our patroni instances provides own REST API endpoint it's good to store this information
+    in DCS among with postgresql connection string. In order to not introduce new keys and be compatible with
+    original Governor we decided to extend original connection string in a following way:
+        postgres://{username}:{password}@{connect_address}/postgres?application_name={api_url}
+    This way original Governor could use such connection string as it is, because of feature of `libpq` library.
+
+    This method is able to split connection string stored in DCS into two parts, `conn_url` and `api_url`"""
+
+    scheme, netloc, path, params, query, fragment = urlparse(value)
+    conn_url = urlunparse((scheme, netloc, path, params, '', fragment))
+    api_url = ([v for n, v in parse_qsl(query) if n == 'application_name'] or [None])[0]
+    return conn_url, api_url
+
+
+def dcs_modules():
+    """Get names of DCS modules, depending on execution environment. If being packaged with PyInstaller,
+    modules aren't discoverable dynamically by scanning source directory because `FrozenImporter` doesn't
+    implement `iter_modules` method. But it is still possible to find all potential DCS modules by
+    iterating through `toc`, which contains list of all "frozen" resources."""
+
+    dcs_dirname = os.path.dirname(__file__)
+    module_prefix = __package__ + '.'
+
+    if getattr(sys, 'frozen', False):
+        importer = pkgutil.get_importer(dcs_dirname)
+        return [module for module in list(importer.toc) if module.startswith(module_prefix) and module.count('.') == 2]
+    else:
+        return [module_prefix + name for _, name, is_pkg in pkgutil.iter_modules([dcs_dirname]) if not is_pkg]
+
+
+def get_dcs(config):
+    available_implementations = set()
+    for module_name in dcs_modules():
+        try:
+            module = importlib.import_module(module_name)
+            for name in filter(lambda name: not name.startswith('__'), dir(module)):  # iterate through module content
+                item = getattr(module, name)
+                name = name.lower()
+                # try to find implementation of AbstractDCS interface, class name must match with module_name
+                if inspect.isclass(item) and issubclass(item, AbstractDCS) and __package__ + '.' + name == module_name:
+                    available_implementations.add(name)
+                    if name in config:  # which has configuration section in the config file
+                        # propagate some parameters
+                        config[name].update({p: config[p] for p in ('namespace', 'name', 'scope', 'loop_wait',
+                                             'patronictl', 'ttl', 'retry_timeout') if p in config})
+                        return item(config[name])
+        except ImportError:
+            if not config.get('patronictl'):
+                logger.info('Failed to import %s', module_name)
+    raise PatroniException("""Can not find suitable configuration of distributed configuration store
+Available implementations: """ + ', '.join(available_implementations))
+
+
+class Member(namedtuple('Member', 'index,name,session,data')):
+
+    """Immutable object (namedtuple) which represents single member of PostgreSQL cluster.
+    Consists of the following fields:
+    :param index: modification index of a given member key in a Configuration Store
+    :param name: name of PostgreSQL cluster member
+    :param session: either session id or just ttl in seconds
+    :param data: arbitrary data i.e. conn_url, api_url, xlog location, state, role, tags, etc...
+
+    There are two mandatory keys in a data:
+    conn_url: connection string containing host, user and password which could be used to access this member.
+    api_url: REST API url of patroni instance"""
+
+    @staticmethod
+    def from_node(index, name, session, data):
+        """
+        >>> Member.from_node(-1, '', '', '{"conn_url": "postgres://foo@bar/postgres"}') is not None
+        True
+        >>> Member.from_node(-1, '', '', '{')
+        Member(index=-1, name='', session='', data={})
+        """
+        if data.startswith('postgres'):
+            conn_url, api_url = parse_connection_string(data)
+            data = {'conn_url': conn_url, 'api_url': api_url}
+        else:
+            try:
+                data = json.loads(data)
+            except (TypeError, ValueError):
+                data = {}
+        return Member(index, name, session, data)
+
+    @property
+    def conn_url(self):
+        return self.data.get('conn_url')
+
+    def conn_kwargs(self, auth=None):
+        ret = self.data.get('conn_kwargs')
+        if ret:
+            ret = ret.copy()
+        else:
+            r = urlparse(self.conn_url)
+            ret = {
+                'host': r.hostname,
+                'port': r.port or 5432,
+                'database': r.path[1:]
+            }
+            self.data['conn_kwargs'] = ret.copy()
+
+        if auth and isinstance(auth, dict):
+            if 'username' in auth:
+                ret['user'] = auth['username']
+            if 'password' in auth:
+                ret['password'] = auth['password']
+        return ret
+
+    @property
+    def api_url(self):
+        return self.data.get('api_url')
+
+    @property
+    def tags(self):
+        return self.data.get('tags', {})
+
+    @property
+    def nofailover(self):
+        return self.tags.get('nofailover', False)
+
+    @property
+    def replicatefrom(self):
+        return self.tags.get('replicatefrom')
+
+    @property
+    def clonefrom(self):
+        return self.tags.get('clonefrom', False) and bool(self.conn_url)
+
+    @property
+    def state(self):
+        return self.data.get('state', 'unknown')
+
+    @property
+    def is_running(self):
+        return self.state == 'running'
+
+
+class Leader(namedtuple('Leader', 'index,session,member')):
+
+    """Immutable object (namedtuple) which represents leader key.
+    Consists of the following fields:
+    :param index: modification index of a leader key in a Configuration Store
+    :param session: either session id or just ttl in seconds
+    :param member: reference to a `Member` object which represents current leader (see `Cluster.members`)"""
+
+    @property
+    def name(self):
+        return self.member.name
+
+    def conn_kwargs(self, auth=None):
+        return self.member.conn_kwargs(auth)
+
+    @property
+    def conn_url(self):
+        return self.member.conn_url
+
+    @property
+    def timeline(self):
+        return self.member.data.get('timeline')
+
+
+class Failover(namedtuple('Failover', 'index,leader,candidate,scheduled_at')):
+
+    """
+    >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader"}'))
+    True
+    >>> 'Failover' in str(Failover.from_node(1, {"leader": "cluster_leader"}))
+    True
+    >>> 'Failover' in str(Failover.from_node(1, '{"leader": "cluster_leader", "member": "cluster_candidate"}'))
+    True
+    >>> Failover.from_node(1, 'null') is None
+    False
+    >>> n = '{"leader": "cluster_leader", "member": "cluster_candidate", "scheduled_at": "2016-01-14T10:09:57.1394Z"}'
+    >>> 'tzinfo=' in str(Failover.from_node(1, n))
+    True
+    >>> Failover.from_node(1, None) is None
+    False
+    >>> Failover.from_node(1, '{}') is None
+    False
+    >>> 'abc' in Failover.from_node(1, 'abc:def')
+    True
+    """
+    @staticmethod
+    def from_node(index, value):
+        if isinstance(value, dict):
+            data = value
+        elif value:
+            try:
+                data = json.loads(value)
+                if not isinstance(data, dict):
+                    data = {}
+            except ValueError:
+                t = [a.strip() for a in value.split(':')]
+                leader = t[0]
+                candidate = t[1] if len(t) > 1 else None
+                return Failover(index, leader, candidate, None) if leader or candidate else None
+        else:
+            data = {}
+
+        if data.get('scheduled_at'):
+            data['scheduled_at'] = dateutil.parser.parse(data['scheduled_at'])
+
+        return Failover(index, data.get('leader'), data.get('member'), data.get('scheduled_at'))
+
+    def __len__(self):
+        return int(bool(self.leader)) + int(bool(self.candidate))
+
+
+class ClusterConfig(namedtuple('ClusterConfig', 'index,data,modify_index')):
+
+    @staticmethod
+    def from_node(index, data, modify_index=None):
+        """
+        >>> ClusterConfig.from_node(1, '{') is None
+        True
+        """
+
+        try:
+            data = json.loads(data)
+        except (TypeError, ValueError):
+            return None
+        return ClusterConfig(index, data, modify_index or index)
+
+
+class SyncState(namedtuple('SyncState', 'index,leader,sync_standby')):
+    """Immutable object (namedtuple) which represents last observed synhcronous replication state
+
+    :param index: modification index of a synchronization key in a Configuration Store
+    :param leader: reference to member that was leader
+    :param sync_standby: standby that was last synchronized to leader
+    """
+
+    @staticmethod
+    def from_node(index, value):
+        """
+        >>> SyncState.from_node(1, None).leader is None
+        True
+        >>> SyncState.from_node(1, '{}').leader is None
+        True
+        >>> SyncState.from_node(1, '{').leader is None
+        True
+        >>> SyncState.from_node(1, '[]').leader is None
+        True
+        >>> SyncState.from_node(1, '{"leader": "leader"}').leader == "leader"
+        True
+        >>> SyncState.from_node(1, {"leader": "leader"}).leader == "leader"
+        True
+        """
+        if isinstance(value, dict):
+            data = value
+        elif value:
+            try:
+                data = json.loads(value)
+                if not isinstance(data, dict):
+                    data = {}
+            except (TypeError, ValueError):
+                data = {}
+        else:
+            data = {}
+        return SyncState(index, data.get('leader'), data.get('sync_standby'))
+
+    def matches(self, name):
+        """
+        Returns if a node name matches one of the nodes in the sync state
+
+        >>> s = SyncState(1, 'foo', 'bar')
+        >>> s.matches('foo')
+        True
+        >>> s.matches('bar')
+        True
+        >>> s.matches('baz')
+        False
+        >>> s.matches(None)
+        False
+        >>> SyncState(1, None, None).matches('foo')
+        False
+        """
+        return name is not None and name in (self.leader, self.sync_standby)
+
+
+class TimelineHistory(namedtuple('TimelineHistory', 'index,lines')):
+    """Object representing timeline history file"""
+
+    @staticmethod
+    def from_node(index, value):
+        """
+        >>> h = TimelineHistory.from_node(1, 2)
+        >>> h.lines
+        []
+        """
+        try:
+            lines = json.loads(value)
+        except (TypeError, ValueError):
+            lines = None
+        if not isinstance(lines, list):
+            lines = []
+        return TimelineHistory(index, lines)
+
+
+class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operation,members,failover,sync,history')):
+
+    """Immutable object (namedtuple) which represents PostgreSQL cluster.
+    Consists of the following fields:
+    :param initialize: shows whether this cluster has initialization key stored in DC or not.
+    :param config: global dynamic configuration, reference to `ClusterConfig` object
+    :param leader: `Leader` object which represents current leader of the cluster
+    :param last_leader_operation: int or long object containing position of last known leader operation.
+        This value is stored in `/optime/leader` key
+    :param members: list of Member object, all PostgreSQL cluster members including leader
+    :param failover: reference to `Failover` object
+    :param sync: reference to `SyncState` object, last observed synchronous replication state.
+    :param history: reference to `TimelineHistory` object
+    """
+
+    def is_unlocked(self):
+        return not (self.leader and self.leader.name)
+
+    def has_member(self, member_name):
+        return any(m for m in self.members if m.name == member_name)
+
+    def get_member(self, member_name, fallback_to_leader=True):
+        return ([m for m in self.members if m.name == member_name] or [self.leader if fallback_to_leader else None])[0]
+
+    def get_clone_member(self, exclude):
+        exclude = [exclude] + [self.leader.name] if self.leader else []
+        candidates = [m for m in self.members if m.clonefrom and m.is_running and m.name not in exclude]
+        return candidates[randint(0, len(candidates) - 1)] if candidates else self.leader
+
+    def check_mode(self, mode):
+        return bool(self.config and parse_bool(self.config.data.get(mode)))
+
+    def is_paused(self):
+        return self.check_mode('pause')
+
+    def is_synchronous_mode(self):
+        return self.check_mode('synchronous_mode')
+
+
+@six.add_metaclass(abc.ABCMeta)
+class AbstractDCS(object):
+
+    _INITIALIZE = 'initialize'
+    _CONFIG = 'config'
+    _LEADER = 'leader'
+    _FAILOVER = 'failover'
+    _HISTORY = 'history'
+    _MEMBERS = 'members/'
+    _OPTIME = 'optime'
+    _LEADER_OPTIME = _OPTIME + '/' + _LEADER
+    _SYNC = 'sync'
+
+    def __init__(self, config):
+        """
+        :param config: dict, reference to config section of selected DCS.
+            i.e.: `zookeeper` for zookeeper, `etcd` for etcd, etc...
+        """
+        self._name = config['name']
+        self._base_path = os.path.join('/', config.get('namespace', '/service/').strip('/'), config['scope'])
+        self._set_loop_wait(config.get('loop_wait', 10))
+
+        self._ctl = bool(config.get('patronictl', False))
+        self._cluster = None
+        self._cluster_thread_lock = Lock()
+        self._last_leader_operation = ''
+        self.event = Event()
+
+    def client_path(self, path):
+        return '/'.join([self._base_path, path.lstrip('/')])
+
+    @property
+    def initialize_path(self):
+        return self.client_path(self._INITIALIZE)
+
+    @property
+    def config_path(self):
+        return self.client_path(self._CONFIG)
+
+    @property
+    def members_path(self):
+        return self.client_path(self._MEMBERS)
+
+    @property
+    def member_path(self):
+        return self.client_path(self._MEMBERS + self._name)
+
+    @property
+    def leader_path(self):
+        return self.client_path(self._LEADER)
+
+    @property
+    def failover_path(self):
+        return self.client_path(self._FAILOVER)
+
+    @property
+    def history_path(self):
+        return self.client_path(self._HISTORY)
+
+    @property
+    def leader_optime_path(self):
+        return self.client_path(self._LEADER_OPTIME)
+
+    @property
+    def sync_path(self):
+        return self.client_path(self._SYNC)
+
+    @abc.abstractmethod
+    def set_ttl(self, ttl):
+        """Set the new ttl value for leader key"""
+
+    @abc.abstractmethod
+    def set_retry_timeout(self, retry_timeout):
+        """Set the new value for retry_timeout"""
+
+    def _set_loop_wait(self, loop_wait):
+        self._loop_wait = loop_wait
+
+    def reload_config(self, config):
+        self._set_loop_wait(config['loop_wait'])
+        self.set_ttl(config['ttl'])
+        self.set_retry_timeout(config['retry_timeout'])
+
+    @property
+    def loop_wait(self):
+        return self._loop_wait
+
+    @abc.abstractmethod
+    def _load_cluster(self):
+        """Internally this method should build  `Cluster` object which
+           represents current state and topology of the cluster in DCS.
+           this method supposed to be called only by `get_cluster` method.
+
+           raise `~DCSError` in case of communication or other problems with DCS.
+           If the current node was running as a master and exception raised,
+           instance would be demoted."""
+
+    def get_cluster(self):
+        with self._cluster_thread_lock:
+            try:
+                self._load_cluster()
+            except Exception:
+                self._cluster = None
+                raise
+            return self._cluster
+
+    @property
+    def cluster(self):
+        with self._cluster_thread_lock:
+            return self._cluster
+
+    def reset_cluster(self):
+        with self._cluster_thread_lock:
+            self._cluster = None
+
+    @abc.abstractmethod
+    def _write_leader_optime(self, last_operation):
+        """write current xlog location into `/optime/leader` key in DCS
+        :param last_operation: absolute xlog location in bytes
+        :returns: `!True` on success."""
+
+    def write_leader_optime(self, last_operation):
+        if self._last_leader_operation != last_operation and self._write_leader_optime(last_operation):
+            self._last_leader_operation = last_operation
+
+    @abc.abstractmethod
+    def _update_leader(self):
+        """Update leader key (or session) ttl
+
+        :returns: `!True` if leader key (or session) has been updated successfully.
+            If not, `!False` must be returned and current instance would be demoted.
+
+        You have to use CAS (Compare And Swap) operation in order to update leader key,
+        for example for etcd `prevValue` parameter must be used."""
+
+    def update_leader(self, last_operation):
+        """Update leader key (or session) ttl and optime/leader
+
+        :param last_operation: absolute xlog location in bytes
+        :returns: `!True` if leader key (or session) has been updated successfully.
+            If not, `!False` must be returned and current instance would be demoted."""
+
+        ret = self._update_leader()
+        if ret and last_operation:
+            self.write_leader_optime(last_operation)
+        return ret
+
+    @abc.abstractmethod
+    def attempt_to_acquire_leader(self, permanent=False):
+        """Attempt to acquire leader lock
+        This method should create `/leader` key with value=`~self._name`
+        :param permanent: if set to `!True`, the leader key will never expire.
+         Used in patronictl for the external master
+        :returns: `!True` if key has been created successfully.
+
+        Key must be created atomically. In case if key already exists it should not be
+        overwritten and `!False` must be returned"""
+
+    @abc.abstractmethod
+    def set_failover_value(self, value, index=None):
+        """Create or update `/failover` key"""
+
+    def manual_failover(self, leader, candidate, scheduled_at=None, index=None):
+        failover_value = {}
+        if leader:
+            failover_value['leader'] = leader
+
+        if candidate:
+            failover_value['member'] = candidate
+
+        if scheduled_at:
+            failover_value['scheduled_at'] = scheduled_at.isoformat()
+        return self.set_failover_value(json.dumps(failover_value, separators=(',', ':')), index)
+
+    @abc.abstractmethod
+    def set_config_value(self, value, index=None):
+        """Create or update `/config` key"""
+
+    @abc.abstractmethod
+    def touch_member(self, data, ttl=None, permanent=False):
+        """Update member key in DCS.
+        This method should create or update key with the name = '/members/' + `~self._name`
+        and value = data in a given DCS.
+
+        :param data: information about instance (including connection strings)
+        :param ttl: ttl for member key, optional parameter. If it is None `~self.member_ttl will be used`
+        :param permanent: if set to `!True`, the member key will never expire.
+         Used in patronictl for the external master.
+        :returns: `!True` on success otherwise `!False`
+        """
+
+    @abc.abstractmethod
+    def take_leader(self):
+        """This method should create leader key with value = `~self._name` and ttl=`~self.ttl`
+        Since it could be called only on initial cluster bootstrap it could create this key regardless,
+        overwriting the key if necessary."""
+
+    @abc.abstractmethod
+    def initialize(self, create_new=True, sysid=""):
+        """Race for cluster initialization.
+
+        :param create_new: False if the key should already exist (in the case we are setting the system_id)
+        :param sysid: PostgreSQL cluster system identifier, if specified, is written to the key
+        :returns: `!True` if key has been created successfully.
+
+        this method should create atomically initialize key and return `!True`
+        otherwise it should return `!False`"""
+
+    @abc.abstractmethod
+    def delete_leader(self):
+        """Voluntarily remove leader key from DCS
+        This method should remove leader key if current instance is the leader"""
+
+    @abc.abstractmethod
+    def cancel_initialization(self):
+        """ Removes the initialize key for a cluster """
+
+    @abc.abstractmethod
+    def delete_cluster(self):
+        """Delete cluster from DCS"""
+
+    @staticmethod
+    def sync_state(leader, sync_standby):
+        """Build sync_state dict"""
+        return {'leader': leader, 'sync_standby': sync_standby}
+
+    def write_sync_state(self, leader, sync_standby, index=None):
+        sync_value = self.sync_state(leader, sync_standby)
+        return self.set_sync_state_value(json.dumps(sync_value, separators=(',', ':')), index)
+
+    @abc.abstractmethod
+    def set_history_value(self, value):
+        """"""
+
+    @abc.abstractmethod
+    def set_sync_state_value(self, value, index=None):
+        """"""
+
+    @abc.abstractmethod
+    def delete_sync_state(self, index=None):
+        """"""
+
+    def watch(self, leader_index, timeout):
+        """If the current node is a master it should just sleep.
+        Any other node should watch for changes of leader key with a given timeout
+
+        :param leader_index: index of a leader key
+        :param timeout: timeout in seconds
+        :returns: `!True` if you would like to reschedule the next run of ha cycle"""
+
+        self.event.wait(timeout)
+        return self.event.isSet()

patroni/dcs/consul.py

@@ -0,0 +1,416 @@
+from __future__ import absolute_import
+import json
+import logging
+import os
+import socket
+import ssl
+import time
+import urllib3
+
+from consul import ConsulException, NotFound, base
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
+from patroni.exceptions import DCSError
+from patroni.utils import deep_compare, parse_bool, Retry, RetryFailedError, split_host_port
+from urllib3.exceptions import HTTPError
+from six.moves.urllib.parse import urlencode, urlparse
+from six.moves.http_client import HTTPException
+
+logger = logging.getLogger(__name__)
+
+
+class ConsulError(DCSError):
+    pass
+
+
+class ConsulInternalError(ConsulException):
+    """An internal Consul server error occurred"""
+
+
+class InvalidSessionTTL(ConsulInternalError):
+    """Session TTL is too small or too big"""
+
+
+class HTTPClient(object):
+
+    def __init__(self, host='127.0.0.1', port=8500, token=None, scheme='http', verify=True, cert=None, ca_cert=None):
+        self.token = token
+        self._read_timeout = 10
+        self.base_uri = '{0}://{1}:{2}'.format(scheme, host, port)
+        kwargs = {}
+        if cert:
+            if isinstance(cert, tuple):
+                # Key and cert are separate
+                kwargs['cert_file'] = cert[0]
+                kwargs['key_file'] = cert[1]
+            else:
+                # combined certificate
+                kwargs['cert_file'] = cert
+        if ca_cert:
+            kwargs['ca_certs'] = ca_cert
+        if verify or ca_cert:
+            kwargs['cert_reqs'] = ssl.CERT_REQUIRED
+        self.http = urllib3.PoolManager(num_pools=10, **kwargs)
+        self._ttl = None
+
+    def set_read_timeout(self, timeout):
+        self._read_timeout = timeout/3.0
+
+    @property
+    def ttl(self):
+        return self._ttl
+
+    def set_ttl(self, ttl):
+        ret = self._ttl != ttl
+        self._ttl = ttl
+        return ret
+
+    @staticmethod
+    def response(response):
+        data = response.data.decode('utf-8')
+        if response.status == 500:
+            msg = '{0} {1}'.format(response.status, data)
+            if data.startswith('Invalid Session TTL'):
+                raise InvalidSessionTTL(msg)
+            else:
+                raise ConsulInternalError(msg)
+        return base.Response(response.status, response.headers, data)
+
+    def uri(self, path, params=None):
+        return '{0}{1}{2}'.format(self.base_uri, path, params and '?' + urlencode(params) or '')
+
+    def __getattr__(self, method):
+        if method not in ('get', 'post', 'put', 'delete'):
+            raise AttributeError("HTTPClient instance has no attribute '{0}'".format(method))
+
+        def wrapper(callback, path, params=None, data=''):
+            # python-consul doesn't allow to specify ttl smaller then 10 seconds
+            # because session_ttl_min defaults to 10s, so we have to do this ugly dirty hack...
+            if method == 'put' and path == '/v1/session/create':
+                ttl = '"ttl": "{0}s"'.format(self._ttl)
+                if not data or data == '{}':
+                    data = '{' + ttl + '}'
+                else:
+                    data = data[:-1] + ', ' + ttl + '}'
+            kwargs = {'retries': 0, 'preload_content': False, 'body': data}
+            if method == 'get' and isinstance(params, dict) and 'index' in params:
+                kwargs['timeout'] = (float(params['wait'][:-1]) if 'wait' in params else 300) + 1
+            else:
+                kwargs['timeout'] = self._read_timeout
+            token = params.pop('token', self.token) if isinstance(params, dict) else self.token
+            if token:
+                kwargs['headers'] = {'X-Consul-Token': token}
+            return callback(self.response(self.http.request(method.upper(), self.uri(path, params), **kwargs)))
+        return wrapper
+
+
+class ConsulClient(base.Consul):
+
+    def __init__(self, *args, **kwargs):
+        self._cert = kwargs.pop('cert', None)
+        self._ca_cert = kwargs.pop('ca_cert', None)
+        self._token = kwargs.get('token')
+        super(ConsulClient, self).__init__(*args, **kwargs)
+
+    def connect(self, *args, **kwargs):
+        kwargs.update(dict(zip(['host', 'port', 'scheme', 'verify'], args)))
+        if self._cert:
+            kwargs['cert'] = self._cert
+        if self._ca_cert:
+            kwargs['ca_cert'] = self._ca_cert
+        if self._token:
+            kwargs['token'] = self._token
+        return HTTPClient(**kwargs)
+
+
+def catch_consul_errors(func):
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except (RetryFailedError, ConsulException, HTTPException, HTTPError, socket.error, socket.timeout):
+            return False
+    return wrapper
+
+
+class Consul(AbstractDCS):
+
+    def __init__(self, config):
+        super(Consul, self).__init__(config)
+        self._scope = config['scope']
+        self._session = None
+        self.__do_not_watch = False
+        self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
+                            retry_exceptions=(ConsulInternalError, HTTPException,
+                                              HTTPError, socket.error, socket.timeout))
+
+        self._my_member_data = {}
+        kwargs = {}
+        if 'url' in config:
+            r = urlparse(config['url'])
+            config.update({'scheme': r.scheme, 'host': r.hostname, 'port': r.port or 8500})
+        elif 'host' in config:
+            host, port = split_host_port(config.get('host', '127.0.0.1:8500'), 8500)
+            config['host'] = host
+            if 'port' not in config:
+                config['port'] = int(port)
+
+        if config.get('cacert'):
+            config['ca_cert'] = config.pop('cacert')
+
+        if config.get('key') and config.get('cert'):
+            config['cert'] = (config['cert'], config['key'])
+
+        config_keys = ('host', 'port', 'token', 'scheme', 'cert', 'ca_cert', 'dc')
+        kwargs = {p: config.get(p) for p in config_keys if config.get(p)}
+
+        verify = config.get('verify')
+        if not isinstance(verify, bool):
+            verify = parse_bool(verify)
+        if isinstance(verify, bool):
+            kwargs['verify'] = verify
+
+        self._client = ConsulClient(**kwargs)
+        self.set_retry_timeout(config['retry_timeout'])
+        self.set_ttl(config.get('ttl') or 30)
+        self._last_session_refresh = 0
+        self.__session_checks = config.get('checks')
+        if not self._ctl:
+            self.create_session()
+
+    def retry(self, *args, **kwargs):
+        return self._retry.copy()(*args, **kwargs)
+
+    def create_session(self):
+        while not self._session:
+            try:
+                self.refresh_session()
+            except ConsulError:
+                logger.info('waiting on consul')
+                time.sleep(5)
+
+    def set_ttl(self, ttl):
+        if self._client.http.set_ttl(ttl/2.0):  # Consul multiplies the TTL by 2x
+            self._session = None
+            self.__do_not_watch = True
+
+    def set_retry_timeout(self, retry_timeout):
+        self._retry.deadline = retry_timeout
+        self._client.http.set_read_timeout(retry_timeout)
+
+    def adjust_ttl(self):
+        try:
+            settings = self._client.agent.self()
+            min_ttl = (settings['Config']['SessionTTLMin'] or 10000000000)/1000000000.0
+            logger.warning('Changing Session TTL from %s to %s', self._client.http.ttl, min_ttl)
+            self._client.http.set_ttl(min_ttl)
+        except Exception:
+            logger.exception('adjust_ttl')
+
+    def _do_refresh_session(self):
+        """:returns: `!True` if it had to create new session"""
+        if self._session and self._last_session_refresh + self._loop_wait > time.time():
+            return False
+
+        if self._session:
+            try:
+                self._client.session.renew(self._session)
+            except NotFound:
+                self._session = None
+        ret = not self._session
+        if ret:
+            try:
+                self._session = self._client.session.create(name=self._scope + '-' + self._name,
+                                                            checks=self.__session_checks,
+                                                            lock_delay=0.001, behavior='delete')
+            except InvalidSessionTTL:
+                logger.exception('session.create')
+                self.adjust_ttl()
+                raise
+
+        self._last_session_refresh = time.time()
+        return ret
+
+    def refresh_session(self):
+        try:
+            return self.retry(self._do_refresh_session)
+        except (ConsulException, RetryFailedError):
+            logger.exception('refresh_session')
+        raise ConsulError('Failed to renew/create session')
+
+    def client_path(self, path):
+        return super(Consul, self).client_path(path)[1:]
+
+    @staticmethod
+    def member(node):
+        return Member.from_node(node['ModifyIndex'], os.path.basename(node['Key']), node.get('Session'), node['Value'])
+
+    def _load_cluster(self):
+        try:
+            path = self.client_path('/')
+            _, results = self.retry(self._client.kv.get, path, recurse=True)
+
+            if results is None:
+                raise NotFound
+
+            nodes = {}
+            for node in results:
+                node['Value'] = (node['Value'] or b'').decode('utf-8')
+                nodes[os.path.relpath(node['Key'], path)] = node
+
+            # get initialize flag
+            initialize = nodes.get(self._INITIALIZE)
+            initialize = initialize and initialize['Value']
+
+            # get global dynamic configuration
+            config = nodes.get(self._CONFIG)
+            config = config and ClusterConfig.from_node(config['ModifyIndex'], config['Value'])
+
+            # get timeline history
+            history = nodes.get(self._HISTORY)
+            history = history and TimelineHistory.from_node(history['ModifyIndex'], history['Value'])
+
+            # get last leader operation
+            last_leader_operation = nodes.get(self._LEADER_OPTIME)
+            last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation['Value'])
+
+            # get list of members
+            members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
+
+            # get leader
+            leader = nodes.get(self._LEADER)
+            if not self._ctl and leader and leader['Value'] == self._name \
+                    and self._session != leader.get('Session', 'x'):
+                logger.info('I am leader but not owner of the session. Removing leader node')
+                self._client.kv.delete(self.leader_path, cas=leader['ModifyIndex'])
+                leader = None
+
+            if leader:
+                member = Member(-1, leader['Value'], None, {})
+                member = ([m for m in members if m.name == leader['Value']] or [member])[0]
+                leader = Leader(leader['ModifyIndex'], leader.get('Session'), member)
+
+            # failover key
+            failover = nodes.get(self._FAILOVER)
+            if failover:
+                failover = Failover.from_node(failover['ModifyIndex'], failover['Value'])
+
+            # get synchronization state
+            sync = nodes.get(self._SYNC)
+            sync = SyncState.from_node(sync and sync['ModifyIndex'], sync and sync['Value'])
+
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
+        except NotFound:
+            self._cluster = Cluster(None, None, None, None, [], None, None, None)
+        except Exception:
+            logger.exception('get_cluster')
+            raise ConsulError('Consul is not responding properly')
+
+    def touch_member(self, data, ttl=None, permanent=False):
+        cluster = self.cluster
+        member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
+        create_member = not permanent and self.refresh_session()
+
+        if member and (create_member or member.session != self._session):
+            try:
+                self._client.kv.delete(self.member_path)
+                create_member = True
+            except Exception:
+                return False
+
+        if not create_member and member and deep_compare(data, self._my_member_data):
+            return True
+
+        try:
+            args = {} if permanent else {'acquire': self._session}
+            self._client.kv.put(self.member_path, json.dumps(data, separators=(',', ':')), **args)
+            self._my_member_data = data
+            return True
+        except Exception:
+            logger.exception('touch_member')
+        return False
+
+    @catch_consul_errors
+    def _do_attempt_to_acquire_leader(self, kwargs):
+        return self.retry(self._client.kv.put, self.leader_path, self._name, **kwargs)
+
+    def attempt_to_acquire_leader(self, permanent=False):
+        if not self._session and not permanent:
+            self.refresh_session()
+
+        ret = self._do_attempt_to_acquire_leader({} if permanent else {'acquire': self._session})
+        if not ret:
+            logger.info('Could not take out TTL lock')
+        return ret
+
+    def take_leader(self):
+        return self.attempt_to_acquire_leader()
+
+    @catch_consul_errors
+    def set_failover_value(self, value, index=None):
+        return self._client.kv.put(self.failover_path, value, cas=index)
+
+    @catch_consul_errors
+    def set_config_value(self, value, index=None):
+        return self._client.kv.put(self.config_path, value, cas=index)
+
+    @catch_consul_errors
+    def _write_leader_optime(self, last_operation):
+        return self._client.kv.put(self.leader_optime_path, last_operation)
+
+    @catch_consul_errors
+    def _update_leader(self):
+        if self._session:
+            self.retry(self._client.session.renew, self._session)
+            self._last_session_refresh = time.time()
+        return bool(self._session)
+
+    @catch_consul_errors
+    def initialize(self, create_new=True, sysid=''):
+        kwargs = {'cas': 0} if create_new else {}
+        return self.retry(self._client.kv.put, self.initialize_path, sysid, **kwargs)
+
+    @catch_consul_errors
+    def cancel_initialization(self):
+        return self.retry(self._client.kv.delete, self.initialize_path)
+
+    @catch_consul_errors
+    def delete_cluster(self):
+        return self.retry(self._client.kv.delete, self.client_path(''), recurse=True)
+
+    @catch_consul_errors
+    def set_history_value(self, value):
+        return self._client.kv.put(self.history_path, value)
+
+    @catch_consul_errors
+    def delete_leader(self):
+        cluster = self.cluster
+        if cluster and isinstance(cluster.leader, Leader) and cluster.leader.name == self._name:
+            return self._client.kv.delete(self.leader_path, cas=cluster.leader.index)
+
+    @catch_consul_errors
+    def set_sync_state_value(self, value, index=None):
+        return self.retry(self._client.kv.put, self.sync_path, value, cas=index)
+
+    @catch_consul_errors
+    def delete_sync_state(self, index=None):
+        return self.retry(self._client.kv.delete, self.sync_path, cas=index)
+
+    def watch(self, leader_index, timeout):
+        if self.__do_not_watch:
+            self.__do_not_watch = False
+            return True
+
+        if leader_index:
+            end_time = time.time() + timeout
+            while timeout >= 1:
+                try:
+                    idx, _ = self._client.kv.get(self.leader_path, index=leader_index, wait=str(timeout) + 's')
+                    return str(idx) != str(leader_index)
+                except (ConsulException, HTTPException, HTTPError, socket.error, socket.timeout):
+                    logging.exception('watch')
+
+                timeout = end_time - time.time()
+
+        try:
+            return super(Consul, self).watch(None, timeout)
+        finally:
+            self.event.clear()

patroni/dcs/etcd.py

@@ -0,0 +1,592 @@
+from __future__ import absolute_import
+import etcd
+import json
+import logging
+import os
+import urllib3.util.connection
+import random
+import requests
+import six
+import socket
+import time
+
+from dns.exception import DNSException
+from dns import resolver
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
+from patroni.exceptions import DCSError
+from patroni.utils import Retry, RetryFailedError, split_host_port
+from urllib3.exceptions import HTTPError, ReadTimeoutError
+from requests.exceptions import RequestException
+from six.moves.queue import Queue
+from six.moves.http_client import HTTPException
+from six.moves.urllib_parse import urlparse
+from threading import Thread
+
+logger = logging.getLogger(__name__)
+
+
+def uri(protocol, host, port, endpoint=''):
+    return '{0}://{1}:{2}{3}'.format(protocol, host, port, endpoint)
+
+
+class EtcdError(DCSError):
+    pass
+
+
+class DnsCachingResolver(Thread):
+
+    def __init__(self, cache_time=600.0, cache_fail_time=30.0):
+        super(DnsCachingResolver, self).__init__()
+        self._cache = {}
+        self._cache_time = cache_time
+        self._cache_fail_time = cache_fail_time
+        self._resolve_queue = Queue()
+        self.daemon = True
+        self.start()
+
+    def run(self):
+        while True:
+            (host, port), attempt = self._resolve_queue.get()
+            response = self._do_resolve(host, port)
+            if response:
+                self._cache[(host, port)] = (time.time(), response)
+            else:
+                if attempt < 10:
+                    self.resolve_async(host, port, attempt + 1)
+                    time.sleep(1)
+
+    def resolve(self, host, port):
+        current_time = time.time()
+        cached_time, response = self._cache.get((host, port), (0, []))
+        time_passed = current_time - cached_time
+        if time_passed > self._cache_time or (not response and time_passed > self._cache_fail_time):
+            new_response = self._do_resolve(host, port)
+            if new_response:
+                self._cache[(host, port)] = (current_time, new_response)
+                response = new_response
+        return response
+
+    def resolve_async(self, host, port, attempt=0):
+        self._resolve_queue.put(((host, port), attempt))
+
+    @staticmethod
+    def _do_resolve(host, port):
+        try:
+            return socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, socket.IPPROTO_TCP)
+        except socket.gaierror:
+            logger.warning('failed to resolve host %s', host)
+            return []
+
+
+class Client(etcd.Client):
+
+    def __init__(self, config, dns_resolver, cache_ttl=300):
+        self._dns_resolver = dns_resolver
+        self.set_machines_cache_ttl(cache_ttl)
+        self._machines_cache_updated = 0
+        args = {p: config.get(p) for p in ('host', 'port', 'protocol', 'use_proxies', 'username', 'password',
+                                           'cert', 'ca_cert') if config.get(p)}
+        super(Client, self).__init__(read_timeout=config['retry_timeout'], **args)
+        self._config = config
+        self._load_machines_cache()
+        self._allow_reconnect = not self._use_proxies
+
+    def _build_request_parameters(self):
+        kwargs = {'headers': self._get_headers(), 'redirect': self.allow_redirect}
+
+        # calculate the number of retries and timeout *per node*
+        # actual number of retries depends on the number of nodes
+        etcd_nodes = len(self._machines_cache) + 1
+        kwargs['retries'] = 0 if etcd_nodes > 3 else (1 if etcd_nodes > 1 else 2)
+
+        # if etcd_nodes > 3:
+        #     kwargs.update({'retries': 0, 'timeout': float(self.read_timeout)/etcd_nodes})
+        # elif etcd_nodes > 1:
+        #     kwargs.update({'retries': 1, 'timeout': self.read_timeout/2.0/etcd_nodes})
+        # else:
+        #     kwargs.update({'retries': 2, 'timeout': self.read_timeout/3.0})
+        kwargs['timeout'] = self.read_timeout/float(kwargs['retries'] + 1)/etcd_nodes
+        return kwargs
+
+    def set_machines_cache_ttl(self, cache_ttl):
+        self._machines_cache_ttl = cache_ttl
+
+    @property
+    def machines(self):
+        """Original `machines` method(property) of `etcd.Client` class raise exception
+        when it failed to get list of etcd cluster members. This method is being called
+        only when request failed on one of the etcd members during `api_execute` call.
+        For us it's more important to execute original request rather then get new topology
+        of etcd cluster. So we will catch this exception and return empty list of machines.
+        Later, during next `api_execute` call we will forcefully update machines_cache.
+
+        Also this method implements the same timeout-retry logic as `api_execute`, because
+        the original method was retrying 2 times with the `read_timeout` on each node."""
+
+        kwargs = self._build_request_parameters()
+
+        while True:
+            try:
+                response = self.http.request(self._MGET, self._base_uri + self.version_prefix + '/machines', **kwargs)
+                machines = [n.strip() for n in self._handle_server_response(response).data.decode('utf-8').split(',')]
+                logger.debug("Retrieved list of machines: %s", machines)
+                random.shuffle(machines)
+                for url in machines:
+                    r = urlparse(url)
+                    port = r.port or (443 if r.scheme == 'https' else 80)
+                    self._dns_resolver.resolve_async(r.hostname, port)
+                return machines
+            except Exception as e:
+                # We can't get the list of machines, if one server is in the
+                # machines cache, try on it
+                logger.error("Failed to get list of machines from %s%s: %r", self._base_uri, self.version_prefix, e)
+                if self._machines_cache:
+                    self._base_uri = self._machines_cache.pop(0)
+                    logger.info("Retrying on %s", self._base_uri)
+                elif self._update_machines_cache:
+                    raise etcd.EtcdException("Could not get the list of servers, "
+                                             "maybe you provided the wrong "
+                                             "host(s) to connect to?")
+                else:
+                    return []
+
+    def set_read_timeout(self, timeout):
+        self._read_timeout = timeout
+
+    def _do_http_request(self, request_executor, method, url, fields=None, **kwargs):
+        try:
+            response = request_executor(method, url, fields=fields, **kwargs)
+            response.data.decode('utf-8')
+            self._check_cluster_id(response)
+        except (HTTPError, HTTPException, socket.error, socket.timeout) as e:
+            if (isinstance(fields, dict) and fields.get("wait") == "true" and
+                    isinstance(e, ReadTimeoutError)):
+                logger.debug("Watch timed out.")
+                raise etcd.EtcdWatchTimedOut("Watch timed out: {0}".format(e), cause=e)
+            logger.error("Request to server %s failed: %r", self._base_uri, e)
+            logger.info("Reconnection allowed, looking for another server.")
+            self._base_uri = self._next_server(cause=e)
+            response = False
+        return response
+
+    def api_execute(self, path, method, params=None, timeout=None):
+        if not path.startswith('/'):
+            raise ValueError('Path does not start with /')
+
+        kwargs = {'fields': params, 'preload_content': False}
+
+        if method in [self._MGET, self._MDELETE]:
+            request_executor = self.http.request
+        elif method in [self._MPUT, self._MPOST]:
+            request_executor = self.http.request_encode_body
+            kwargs['encode_multipart'] = False
+        else:
+            raise etcd.EtcdException('HTTP method {0} not supported'.format(method))
+
+        # Update machines_cache if previous attempt of update has failed
+        if self._update_machines_cache:
+            self._load_machines_cache()
+        elif time.time() - self._machines_cache_updated > self._machines_cache_ttl:
+            self._machines_cache = self.machines
+            if self._base_uri in self._machines_cache:
+                self._machines_cache.remove(self._base_uri)
+            self._machines_cache_updated = time.time()
+
+        kwargs.update(self._build_request_parameters())
+
+        if timeout is not None:
+            kwargs.update({'retries': 0, 'timeout': timeout})
+
+        response = False
+
+        try:
+            some_request_failed = False
+            while not response:
+                response = self._do_http_request(request_executor, method, self._base_uri + path, **kwargs)
+
+                if response is False:
+                    some_request_failed = True
+            if some_request_failed and not self._use_proxies:
+                self._machines_cache = self.machines
+                if self._base_uri in self._machines_cache:
+                    self._machines_cache.remove(self._base_uri)
+        except etcd.EtcdConnectionFailed as e:
+            if isinstance(e, etcd.EtcdWatchTimedOut) and self._machines_cache:
+                self._base_uri = self._next_server()
+            else:
+                self._update_machines_cache = True
+            if not response:
+                raise
+        return self._handle_server_response(response)
+
+    @staticmethod
+    def get_srv_record(host):
+        try:
+            return [(r.target.to_text(True), r.port) for r in resolver.query(host, 'SRV')]
+        except DNSException:
+            return []
+
+    def _get_machines_cache_from_srv(self, srv):
+        """Fetch list of etcd-cluster member by resolving _etcd-server._tcp. SRV record.
+        This record should contain list of host and peer ports which could be used to run
+        'GET http://{host}:{port}/members' request (peer protocol)"""
+
+        ret = []
+        for r in ['-client-ssl', '-client', '-ssl', '', '-server-ssl', '-server']:
+            protocol = 'https' if '-ssl' in r else 'http'
+            endpoint = '/members' if '-server' in r else ''
+            for host, port in self.get_srv_record('_etcd{0}._tcp.{1}'.format(r, srv)):
+                url = uri(protocol, host, port, endpoint)
+                if endpoint:
+                    try:
+                        response = requests.get(url, timeout=self.read_timeout, verify=False)
+                        if response.ok:
+                            for member in response.json():
+                                ret.extend(member['clientURLs'])
+                            break
+                    except RequestException:
+                        logger.exception('GET %s', url)
+                else:
+                    ret.append(url)
+            if ret:
+                self._protocol = protocol
+                break
+        else:
+            logger.warning('Can not resolve SRV for %s', srv)
+        return list(set(ret))
+
+    def _get_machines_cache_from_dns(self, host, port):
+        """One host might be resolved into multiple ip addresses. We will make list out of it"""
+        if self.protocol == 'http':
+            ret = []
+            for af, _, _, _, sa in self._dns_resolver.resolve(host, port):
+                host, port = sa[:2]
+                if af == socket.AF_INET6:
+                    host = '[{0}]'.format(host)
+                ret.append(uri(self.protocol, host, port))
+            if ret:
+                return list(set(ret))
+        return [uri(self.protocol, host, port)]
+
+    def _load_machines_cache(self):
+        """This method should fill up `_machines_cache` from scratch.
+        It could happen only in two cases:
+        1. During class initialization
+        2. When all etcd members failed"""
+
+        self._update_machines_cache = True
+
+        if 'srv' not in self._config and 'host' not in self._config and 'hosts' not in self._config:
+            raise Exception('Neither srv, hosts, host nor url are defined in etcd section of config')
+
+        if self._use_proxies:
+            self._machines_cache = [uri(self.protocol, self._config['host'], self._config['port'])]
+        else:
+            self._machines_cache = []
+
+            if 'srv' in self._config:
+                self._machines_cache = self._get_machines_cache_from_srv(self._config['srv'])
+
+            if not self._machines_cache and 'hosts' in self._config:
+                self._machines_cache = list(self._config['hosts'])
+
+            if not self._machines_cache and 'host' in self._config:
+                self._machines_cache = self._get_machines_cache_from_dns(self._config['host'], self._config['port'])
+
+        # Can not bootstrap list of etcd-cluster members, giving up
+        if not self._machines_cache:
+            raise etcd.EtcdException
+
+        # After filling up initial list of machines_cache we should ask etcd-cluster about actual list
+        self._base_uri = self._next_server()
+        self._machines_cache = self.machines
+
+        if self._base_uri in self._machines_cache:
+            self._machines_cache.remove(self._base_uri)
+
+        self._update_machines_cache = False
+        self._machines_cache_updated = time.time()
+
+
+class Etcd(AbstractDCS):
+
+    def __init__(self, config):
+        super(Etcd, self).__init__(config)
+        self._ttl = int(config.get('ttl') or 30)
+        self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
+                            retry_exceptions=(etcd.EtcdLeaderElectionInProgress,
+                                              etcd.EtcdWatcherCleared,
+                                              etcd.EtcdEventIndexCleared))
+        self._client = self.get_etcd_client(config)
+        self.__do_not_watch = False
+        self._has_failed = False
+
+    def retry(self, *args, **kwargs):
+        return self._retry.copy()(*args, **kwargs)
+
+    def _handle_exception(self, e, name='', do_sleep=False, raise_ex=None):
+        if not self._has_failed:
+            logger.exception(name)
+        else:
+            logger.error(e)
+            if do_sleep:
+                time.sleep(1)
+        self._has_failed = True
+        if isinstance(raise_ex, Exception):
+            raise raise_ex
+
+    def catch_etcd_errors(func):
+        def wrapper(self, *args, **kwargs):
+            try:
+                retval = func(self, *args, **kwargs) is not None
+                self._has_failed = False
+                return retval
+            except (RetryFailedError, etcd.EtcdException) as e:
+                self._handle_exception(e)
+                return False
+            except Exception as e:
+                self._handle_exception(e, raise_ex=EtcdError('unexpected error'))
+
+        return wrapper
+
+    @staticmethod
+    def get_etcd_client(config):
+        if 'proxy' in config:
+            config['use_proxies'] = True
+            config['url'] = config['proxy']
+
+        if 'url' in config:
+            r = urlparse(config['url'])
+            config.update({'protocol': r.scheme, 'host': r.hostname, 'port': r.port or 2379,
+                           'username': r.username, 'password': r.password})
+        elif 'hosts' in config:
+            hosts = config.pop('hosts')
+            default_port = config.pop('port', 2379)
+            protocol = config.get('protocol', 'http')
+
+            if isinstance(hosts, six.string_types):
+                hosts = hosts.split(',')
+
+            config['hosts'] = []
+            for value in hosts:
+                if isinstance(value, six.string_types):
+                    config['hosts'].append(uri(protocol, *split_host_port(value, default_port)))
+        elif 'host' in config:
+            host, port = split_host_port(config['host'], 2379)
+            config['host'] = host
+            if 'port' not in config:
+                config['port'] = int(port)
+
+        if config.get('cacert'):
+            config['ca_cert'] = config.pop('cacert')
+
+        if config.get('key') and config.get('cert'):
+            config['cert'] = (config['cert'], config['key'])
+
+        for p in ('discovery_srv', 'srv_domain'):
+            if p in config:
+                config['srv'] = config.pop(p)
+
+        dns_resolver = DnsCachingResolver()
+
+        def create_connection_patched(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                                      source_address=None, socket_options=None):
+            host, port = address
+            if host.startswith('['):
+                host = host.strip('[]')
+            err = None
+            for af, socktype, proto, _, sa in dns_resolver.resolve(host, port):
+                sock = None
+                try:
+                    sock = socket.socket(af, socktype, proto)
+                    if socket_options:
+                        for opt in socket_options:
+                            sock.setsockopt(*opt)
+                    if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
+                        sock.settimeout(timeout)
+                    if source_address:
+                        sock.bind(source_address)
+                    sock.connect(sa)
+                    return sock
+
+                except socket.error as e:
+                    err = e
+                    if sock is not None:
+                        sock.close()
+                        sock = None
+
+            if err is not None:
+                raise err
+
+            raise socket.error("getaddrinfo returns an empty list")
+
+        urllib3.util.connection.create_connection = create_connection_patched
+
+        client = None
+        while not client:
+            try:
+                client = Client(config, dns_resolver)
+            except etcd.EtcdException:
+                logger.info('waiting on etcd')
+                time.sleep(5)
+        return client
+
+    def set_ttl(self, ttl):
+        ttl = int(ttl)
+        self.__do_not_watch = self._ttl != ttl
+        self._ttl = ttl
+        self._client.set_machines_cache_ttl(ttl*10)
+
+    def set_retry_timeout(self, retry_timeout):
+        self._retry.deadline = retry_timeout
+        self._client.set_read_timeout(retry_timeout)
+
+    @staticmethod
+    def member(node):
+        return Member.from_node(node.modifiedIndex, os.path.basename(node.key), node.ttl, node.value)
+
+    def _load_cluster(self):
+        try:
+            result = self.retry(self._client.read, self.client_path(''), recursive=True)
+            nodes = {os.path.relpath(node.key, result.key): node for node in result.leaves}
+
+            # get initialize flag
+            initialize = nodes.get(self._INITIALIZE)
+            initialize = initialize and initialize.value
+
+            # get global dynamic configuration
+            config = nodes.get(self._CONFIG)
+            config = config and ClusterConfig.from_node(config.modifiedIndex, config.value)
+
+            # get timeline history
+            history = nodes.get(self._HISTORY)
+            history = history and TimelineHistory.from_node(history.modifiedIndex, history.value)
+
+            # get last leader operation
+            last_leader_operation = nodes.get(self._LEADER_OPTIME)
+            last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation.value)
+
+            # get list of members
+            members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
+
+            # get leader
+            leader = nodes.get(self._LEADER)
+            if leader:
+                member = Member(-1, leader.value, None, {})
+                member = ([m for m in members if m.name == leader.value] or [member])[0]
+                index = result.etcd_index if result.etcd_index > leader.modifiedIndex else leader.modifiedIndex + 1
+                leader = Leader(index, leader.ttl, member)
+
+            # failover key
+            failover = nodes.get(self._FAILOVER)
+            if failover:
+                failover = Failover.from_node(failover.modifiedIndex, failover.value)
+
+            # get synchronization state
+            sync = nodes.get(self._SYNC)
+            sync = SyncState.from_node(sync and sync.modifiedIndex, sync and sync.value)
+
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
+        except etcd.EtcdKeyNotFound:
+            self._cluster = Cluster(None, None, None, None, [], None, None, None)
+        except Exception as e:
+            self._handle_exception(e, 'get_cluster', raise_ex=EtcdError('Etcd is not responding properly'))
+        self._has_failed = False
+
+    @catch_etcd_errors
+    def touch_member(self, data, ttl=None, permanent=False):
+        data = json.dumps(data, separators=(',', ':'))
+        return self.retry(self._client.set, self.member_path, data, None if permanent else ttl or self._ttl)
+
+    @catch_etcd_errors
+    def take_leader(self):
+        return self.retry(self._client.set, self.leader_path, self._name, self._ttl)
+
+    def attempt_to_acquire_leader(self, permanent=False):
+        try:
+            return bool(self.retry(self._client.write,
+                                   self.leader_path,
+                                   self._name,
+                                   ttl=None if permanent else self._ttl,
+                                   prevExist=False))
+        except etcd.EtcdAlreadyExist:
+            logger.info('Could not take out TTL lock')
+        except (RetryFailedError, etcd.EtcdException):
+            pass
+        return False
+
+    @catch_etcd_errors
+    def set_failover_value(self, value, index=None):
+        return self._client.write(self.failover_path, value, prevIndex=index or 0)
+
+    @catch_etcd_errors
+    def set_config_value(self, value, index=None):
+        return self._client.write(self.config_path, value, prevIndex=index or 0)
+
+    @catch_etcd_errors
+    def _write_leader_optime(self, last_operation):
+        return self._client.set(self.leader_optime_path, last_operation)
+
+    @catch_etcd_errors
+    def _update_leader(self):
+        return self.retry(self._client.test_and_set, self.leader_path, self._name, self._name, self._ttl)
+
+    @catch_etcd_errors
+    def initialize(self, create_new=True, sysid=""):
+        return self.retry(self._client.write, self.initialize_path, sysid, prevExist=(not create_new))
+
+    @catch_etcd_errors
+    def delete_leader(self):
+        return self._client.delete(self.leader_path, prevValue=self._name)
+
+    @catch_etcd_errors
+    def cancel_initialization(self):
+        return self.retry(self._client.delete, self.initialize_path)
+
+    @catch_etcd_errors
+    def delete_cluster(self):
+        return self.retry(self._client.delete, self.client_path(''), recursive=True)
+
+    @catch_etcd_errors
+    def set_history_value(self, value):
+        return self._client.write(self.history_path, value)
+
+    @catch_etcd_errors
+    def set_sync_state_value(self, value, index=None):
+        return self.retry(self._client.write, self.sync_path, value, prevIndex=index or 0)
+
+    @catch_etcd_errors
+    def delete_sync_state(self, index=None):
+        return self.retry(self._client.delete, self.sync_path, prevIndex=index or 0)
+
+    def watch(self, leader_index, timeout):
+        if self.__do_not_watch:
+            self.__do_not_watch = False
+            return True
+
+        if leader_index:
+            end_time = time.time() + timeout
+
+            while timeout >= 1:  # when timeout is too small urllib3 doesn't have enough time to connect
+                try:
+                    self._client.watch(self.leader_path, index=leader_index, timeout=timeout + 0.5)
+                    self._has_failed = False
+                    # Synchronous work of all cluster members with etcd is less expensive
+                    # than reestablishing http connection every time from every replica.
+                    return True
+                except etcd.EtcdWatchTimedOut:
+                    self._client.http.clear()
+                    self._has_failed = False
+                    return False
+                except (etcd.EtcdEventIndexCleared, etcd.EtcdWatcherCleared):  # Watch failed
+                    self._has_failed = False
+                    return True  # leave the loop, because watch with the same parameters will fail anyway
+                except etcd.EtcdException as e:
+                    self._handle_exception(e, 'watch', True)
+
+                timeout = end_time - time.time()
+
+        try:
+            return super(Etcd, self).watch(None, timeout)
+        finally:
+            self.event.clear()

patroni/dcs/exhibitor.py

@@ -0,0 +1,74 @@
+import logging
+import random
+import requests
+import time
+
+from patroni.dcs.zookeeper import ZooKeeper
+from requests.exceptions import RequestException
+
+logger = logging.getLogger(__name__)
+
+
+class ExhibitorEnsembleProvider(object):
+
+    TIMEOUT = 3.1
+
+    def __init__(self, hosts, port, uri_path='/exhibitor/v1/cluster/list', poll_interval=300):
+        self._exhibitor_port = port
+        self._uri_path = uri_path
+        self._poll_interval = poll_interval
+        self._exhibitors = hosts
+        self._master_exhibitors = hosts
+        self._zookeeper_hosts = ''
+        self._next_poll = None
+        while not self.poll():
+            logger.info('waiting on exhibitor')
+            time.sleep(5)
+
+    def poll(self):
+        if self._next_poll and self._next_poll > time.time():
+            return False
+
+        json = self._query_exhibitors(self._exhibitors)
+        if not json:
+            json = self._query_exhibitors(self._master_exhibitors)
+
+        if isinstance(json, dict) and 'servers' in json and 'port' in json:
+            self._next_poll = time.time() + self._poll_interval
+            zookeeper_hosts = ','.join([h + ':' + str(json['port']) for h in sorted(json['servers'])])
+            if self._zookeeper_hosts != zookeeper_hosts:
+                logger.info('ZooKeeper connection string has changed: %s => %s', self._zookeeper_hosts, zookeeper_hosts)
+                self._zookeeper_hosts = zookeeper_hosts
+                self._exhibitors = json['servers']
+                return True
+        return False
+
+    def _query_exhibitors(self, exhibitors):
+        random.shuffle(exhibitors)
+        for host in exhibitors:
+            uri = 'http://{0}:{1}{2}'.format(host, self._exhibitor_port, self._uri_path)
+            try:
+                response = requests.get(uri, timeout=self.TIMEOUT)
+                return response.json()
+            except RequestException:
+                pass
+        return None
+
+    @property
+    def zookeeper_hosts(self):
+        return self._zookeeper_hosts
+
+
+class Exhibitor(ZooKeeper):
+
+    def __init__(self, config):
+        interval = config.get('poll_interval', 300)
+        self._ensemble_provider = ExhibitorEnsembleProvider(config['hosts'], config['port'], poll_interval=interval)
+        config = config.copy()
+        config['hosts'] = self._ensemble_provider.zookeeper_hosts
+        super(Exhibitor, self).__init__(config)
+
+    def _load_cluster(self):
+        if self._ensemble_provider.poll():
+            self._client.set_hosts(self._ensemble_provider.zookeeper_hosts)
+        return super(Exhibitor, self)._load_cluster()

patroni/dcs/kubernetes.py

@@ -0,0 +1,409 @@
+from __future__ import absolute_import
+import datetime
+import functools
+import json
+import logging
+import socket
+import sys
+import time
+
+from kubernetes import client as k8s_client, config as k8s_config, watch as k8s_watch
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
+from patroni.exceptions import DCSError
+from patroni.utils import deep_compare, tzutc, Retry, RetryFailedError
+from urllib3.exceptions import HTTPError
+from six.moves.http_client import HTTPException
+
+logger = logging.getLogger(__name__)
+
+
+class KubernetesError(DCSError):
+    pass
+
+
+class KubernetesRetriableException(k8s_client.rest.ApiException):
+
+    def __init__(self, orig):
+        super(KubernetesRetriableException, self).__init__(orig.status, orig.reason)
+        self.body = orig.body
+        self.headers = orig.headers
+
+
+class CoreV1ApiProxy(object):
+
+    def __init__(self, use_endpoints=False):
+        self._api = k8s_client.CoreV1Api()
+        self._request_timeout = None
+        self._use_endpoints = use_endpoints
+
+    def set_timeout(self, timeout):
+        self._request_timeout = (1, timeout / 3.0)
+
+    def __getattr__(self, func):
+        if func.endswith('_kind'):
+            func = func[:-4] + ('endpoints' if self._use_endpoints else 'config_map')
+
+        def wrapper(*args, **kwargs):
+            if '_request_timeout' not in kwargs:
+                kwargs['_request_timeout'] = self._request_timeout
+            try:
+                return getattr(self._api, func)(*args, **kwargs)
+            except k8s_client.rest.ApiException as e:
+                if e.status in (502, 503, 504):  # XXX
+                    raise KubernetesRetriableException(e)
+                raise
+        return wrapper
+
+
+def catch_kubernetes_errors(func):
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except k8s_client.rest.ApiException as e:
+            if e.status == 403:
+                logger.exception('Permission denied')
+            elif e.status != 409:  # Object exists or conflict in resource_version
+                logger.exception('Unexpected error from Kubernetes API')
+            return False
+        except (RetryFailedError, HTTPException, HTTPError, socket.error, socket.timeout):
+            return False
+    return wrapper
+
+
+class Kubernetes(AbstractDCS):
+
+    def __init__(self, config):
+        self._labels = config['labels']
+        self._labels[config.get('scope_label', 'cluster-name')] = config['scope']
+        self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
+        self._namespace = config.get('namespace') or 'default'
+        self._role_label = config.get('role_label', 'role')
+        config['namespace'] = ''
+        super(Kubernetes, self).__init__(config)
+        self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
+                            retry_exceptions=(KubernetesRetriableException, HTTPException,
+                                              HTTPError, socket.error, socket.timeout))
+        self._ttl = None
+        try:
+            k8s_config.load_incluster_config()
+        except k8s_config.ConfigException:
+            k8s_config.load_kube_config(context=config.get('context', 'local'))
+
+        self.__subsets = None
+        use_endpoints = config.get('use_endpoints') and (config.get('patronictl') or 'pod_ip' in config)
+        if use_endpoints:
+            addresses = [k8s_client.V1EndpointAddress(ip=config['pod_ip'])]
+            ports = []
+            for p in config.get('ports', [{}]):
+                port = {'port': int(p.get('port', '5432'))}
+                port.update({n: p[n] for n in ('name', 'protocol') if p.get(n)})
+                ports.append(k8s_client.V1EndpointPort(**port))
+            self.__subsets = [k8s_client.V1EndpointSubset(addresses=addresses, ports=ports)]
+        self._api = CoreV1ApiProxy(use_endpoints)
+        self.set_retry_timeout(config['retry_timeout'])
+        self.set_ttl(config.get('ttl') or 30)
+        self._leader_observed_record = {}
+        self._leader_observed_time = None
+        self._leader_resource_version = None
+        self._leader_observed_subsets = []
+        self.__do_not_watch = False
+
+    def retry(self, *args, **kwargs):
+        return self._retry.copy()(*args, **kwargs)
+
+    def client_path(self, path):
+        return super(Kubernetes, self).client_path(path)[1:].replace('/', '-')
+
+    @property
+    def leader_path(self):
+        return self._base_path[1:] if self.__subsets else super(Kubernetes, self).leader_path
+
+    def set_ttl(self, ttl):
+        ttl = int(ttl)
+        self.__do_not_watch = self._ttl != ttl
+        self._ttl = ttl
+
+    def set_retry_timeout(self, retry_timeout):
+        self._retry.deadline = retry_timeout
+        self._api.set_timeout(retry_timeout)
+
+    @staticmethod
+    def member(pod):
+        annotations = pod.metadata.annotations or {}
+        member = Member.from_node(pod.metadata.resource_version, pod.metadata.name, None, annotations.get('status', ''))
+        member.data['pod_labels'] = pod.metadata.labels
+        return member
+
+    def _load_cluster(self):
+        try:
+            # get list of members
+            response = self.retry(self._api.list_namespaced_pod, self._namespace, label_selector=self._label_selector)
+            members = [self.member(pod) for pod in response.items]
+
+            response = self.retry(self._api.list_namespaced_kind, self._namespace, label_selector=self._label_selector)
+            nodes = {item.metadata.name: item for item in response.items}
+
+            config = nodes.get(self.config_path)
+            metadata = config and config.metadata
+            annotations = metadata and metadata.annotations or {}
+
+            # get initialize flag
+            initialize = annotations.get(self._INITIALIZE)
+
+            # get global dynamic configuration
+            config = ClusterConfig.from_node(metadata and metadata.resource_version,
+                                             annotations.get(self._CONFIG) or '{}')
+
+            # get timeline history
+            history = TimelineHistory.from_node(metadata and metadata.resource_version,
+                                                annotations.get(self._HISTORY) or '[]')
+
+            leader = nodes.get(self.leader_path)
+            metadata = leader and leader.metadata
+            self._leader_resource_version = metadata.resource_version if metadata else None
+            self._leader_observed_subsets = leader.subsets if self.__subsets and leader and leader.subsets else []
+            annotations = metadata and metadata.annotations or {}
+
+            # get last leader operation
+            last_leader_operation = annotations.get(self._OPTIME)
+            last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation)
+
+            # get leader
+            leader_record = {n: annotations.get(n) for n in (self._LEADER, 'acquireTime',
+                             'ttl', 'renewTime', 'transitions') if n in annotations}
+            if (leader_record or self._leader_observed_record) and leader_record != self._leader_observed_record:
+                self._leader_observed_record = leader_record
+                self._leader_observed_time = time.time()
+
+            leader = leader_record.get(self._LEADER)
+            try:
+                ttl = int(leader_record.get('ttl')) or self._ttl
+            except (TypeError, ValueError):
+                ttl = self._ttl
+
+            if not metadata or not self._leader_observed_time or self._leader_observed_time + ttl < time.time():
+                leader = None
+
+            if metadata:
+                member = Member(-1, leader, None, {})
+                member = ([m for m in members if m.name == leader] or [member])[0]
+                leader = Leader(response.metadata.resource_version, None, member)
+
+            # failover key
+            failover = nodes.get(self.failover_path)
+            metadata = failover and failover.metadata
+            failover = Failover.from_node(metadata and metadata.resource_version, metadata and metadata.annotations)
+
+            # get synchronization state
+            sync = nodes.get(self.sync_path)
+            metadata = sync and sync.metadata
+            sync = SyncState.from_node(metadata and metadata.resource_version,  metadata and metadata.annotations)
+
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
+        except Exception:
+            logger.exception('get_cluster')
+            raise KubernetesError('Kubernetes API is not responding properly')
+
+    @staticmethod
+    def compare_ports(p1, p2):
+        return p1.name == p2.name and p1.port == p2.port and (p1.protocol or 'TCP') == (p2.protocol or 'TCP')
+
+    @staticmethod
+    def subsets_changed(last_observed_subsets, subsets):
+        """
+        >>> Kubernetes.subsets_changed([], [])
+        False
+        >>> Kubernetes.subsets_changed([], [k8s_client.V1EndpointSubset()])
+        True
+        >>> s1 = [k8s_client.V1EndpointSubset(addresses=[k8s_client.V1EndpointAddress(ip='1.2.3.4')])]
+        >>> s2 = [k8s_client.V1EndpointSubset(addresses=[k8s_client.V1EndpointAddress(ip='1.2.3.5')])]
+        >>> Kubernetes.subsets_changed(s1, s2)
+        True
+        >>> a = [k8s_client.V1EndpointAddress(ip='1.2.3.4')]
+        >>> s1 = [k8s_client.V1EndpointSubset(addresses=a, ports=[k8s_client.V1EndpointPort(protocol='TCP', port=1)])]
+        >>> s2 = [k8s_client.V1EndpointSubset(addresses=a, ports=[k8s_client.V1EndpointPort(port=5432)])]
+        >>> Kubernetes.subsets_changed(s1, s2)
+        True
+        >>> p1 = k8s_client.V1EndpointPort(name='port1', port=1)
+        >>> p2 = k8s_client.V1EndpointPort(name='port2', port=2)
+        >>> p3 = k8s_client.V1EndpointPort(name='port3', port=3)
+        >>> s1 = [k8s_client.V1EndpointSubset(addresses=a, ports=[p1, p2])]
+        >>> s2 = [k8s_client.V1EndpointSubset(addresses=a, ports=[p2, p3])]
+        >>> Kubernetes.subsets_changed(s1, s2)
+        True
+        >>> s2 = [k8s_client.V1EndpointSubset(addresses=a, ports=[p2, p1])]
+        >>> Kubernetes.subsets_changed(s1, s2)
+        False
+        """
+        if len(last_observed_subsets) != len(subsets):
+            return True
+        if subsets == []:
+            return False
+        if len(last_observed_subsets[0].addresses or []) != 1 or \
+                last_observed_subsets[0].addresses[0].ip != subsets[0].addresses[0].ip or \
+                len(last_observed_subsets[0].ports) != len(subsets[0].ports):
+            return True
+        if len(subsets[0].ports) == 1:
+            return not Kubernetes.compare_ports(last_observed_subsets[0].ports[0], subsets[0].ports[0])
+        observed_ports = {p.name: p for p in last_observed_subsets[0].ports}
+        for p in subsets[0].ports:
+            if p.name not in observed_ports or not Kubernetes.compare_ports(p, observed_ports.pop(p.name)):
+                return True
+        return False
+
+    @catch_kubernetes_errors
+    def patch_or_create(self, name, annotations, resource_version=None, patch=False, retry=True, subsets=None):
+        metadata = {'namespace': self._namespace, 'name': name, 'labels': self._labels, 'annotations': annotations}
+        if patch or resource_version:
+            if resource_version is not None:
+                metadata['resource_version'] = resource_version
+            func = functools.partial(self._api.patch_namespaced_kind, name)
+        else:
+            func = functools.partial(self._api.create_namespaced_kind)
+            # skip annotations with null values
+            metadata['annotations'] = {k: v for k, v in metadata['annotations'].items() if v is not None}
+
+        metadata = k8s_client.V1ObjectMeta(**metadata)
+        if subsets is not None and self.__subsets:
+            endpoints = {'metadata': metadata}
+            if self.subsets_changed(self._leader_observed_subsets, subsets):
+                endpoints['subsets'] = subsets
+            body = k8s_client.V1Endpoints(**endpoints)
+        else:
+            body = k8s_client.V1ConfigMap(metadata=metadata)
+        return self.retry(func, self._namespace, body) if retry else func(self._namespace, body)
+
+    def _write_leader_optime(self, last_operation):
+        """Unused"""
+
+    def _update_leader(self):
+        """Unused"""
+
+    def update_leader(self, last_operation):
+        now = datetime.datetime.now(tzutc).isoformat()
+        annotations = {self._LEADER: self._name, 'ttl': str(self._ttl), 'renewTime': now,
+                       'acquireTime': self._leader_observed_record.get('acquireTime') or now,
+                       'transitions': self._leader_observed_record.get('transitions') or '0'}
+        if last_operation:
+            annotations[self._OPTIME] = last_operation
+
+        ret = self.patch_or_create(self.leader_path, annotations, self._leader_resource_version, subsets=self.__subsets)
+        if ret:
+            self._leader_resource_version = ret.metadata.resource_version
+        return ret
+
+    def attempt_to_acquire_leader(self, permanent=False):
+        now = datetime.datetime.now(tzutc).isoformat()
+        annotations = {self._LEADER: self._name, 'ttl': str(sys.maxsize if permanent else self._ttl),
+                       'renewTime': now, 'acquireTime': now, 'transitions': '0'}
+        if self._leader_observed_record:
+            try:
+                transitions = int(self._leader_observed_record.get('transitions'))
+            except (TypeError, ValueError):
+                transitions = 0
+
+            if self._leader_observed_record.get(self._LEADER) != self._name:
+                transitions += 1
+            else:
+                annotations['acquireTime'] = self._leader_observed_record.get('acquireTime') or now
+            annotations['transitions'] = str(transitions)
+        ret = self.patch_or_create(self.leader_path, annotations, self._leader_resource_version, subsets=self.__subsets)
+        if ret:
+            self._leader_resource_version = ret.metadata.resource_version
+        else:
+            logger.info('Could not take out TTL lock')
+        return ret
+
+    def take_leader(self):
+        return self.attempt_to_acquire_leader()
+
+    def set_failover_value(self, value, index=None):
+        """Unused"""
+
+    def manual_failover(self, leader, candidate, scheduled_at=None, index=None):
+        annotations = {'leader': leader or None, 'member': candidate or None, 'scheduled_at': scheduled_at}
+        patch = bool(self.cluster and isinstance(self.cluster.failover, Failover) and self.cluster.failover.index)
+        return self.patch_or_create(self.failover_path, annotations, index, bool(index or patch), False)
+
+    def set_config_value(self, value, index=None):
+        patch = bool(index or self.cluster and self.cluster.config and self.cluster.config.index)
+        return self.patch_or_create(self.config_path, {self._CONFIG: value}, index, patch, False)
+
+    @catch_kubernetes_errors
+    def touch_member(self, data, ttl=None, permanent=False):
+        cluster = self.cluster
+        if cluster and cluster.leader and cluster.leader.name == self._name:
+            role = 'master'
+        elif data['state'] == 'running' and data['role'] != 'master':
+            role = data['role']
+        else:
+            role = None
+
+        member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
+        pod_labels = member and member.data.pop('pod_labels', None)
+        ret = pod_labels is not None and pod_labels.get(self._role_label) == role and deep_compare(data, member.data)
+
+        if not ret:
+            metadata = {'namespace': self._namespace, 'name': self._name, 'labels': {self._role_label: role},
+                        'annotations': {'status': json.dumps(data, separators=(',', ':'))}}
+            body = k8s_client.V1Pod(metadata=k8s_client.V1ObjectMeta(**metadata))
+            ret = self._api.patch_namespaced_pod(self._name, self._namespace, body)
+        return ret
+
+    def initialize(self, create_new=True, sysid=""):
+        cluster = self.cluster
+        resource_version = cluster.config.index if cluster and cluster.config and cluster.config.index else None
+        return self.patch_or_create(self.config_path, {self._INITIALIZE: sysid}, resource_version)
+
+    def delete_leader(self):
+        if self.cluster and isinstance(self.cluster.leader, Leader) and self.cluster.leader.name == self._name:
+            self.patch_or_create(self.leader_path, {self._LEADER: None}, self._leader_resource_version, True, False, [])
+            self.reset_cluster()
+
+    def cancel_initialization(self):
+        self.patch_or_create(self.config_path, {self._INITIALIZE: None}, self.cluster.config.index, True)
+
+    @catch_kubernetes_errors
+    def delete_cluster(self):
+        self.retry(self._api.delete_collection_namespaced_kind, self._namespace, label_selector=self._label_selector)
+
+    def set_history_value(self, value):
+        patch = bool(self.cluster and self.cluster.config and self.cluster.config.index)
+        return self.patch_or_create(self.config_path, {self._HISTORY: value}, None, patch, False)
+
+    def set_sync_state_value(self, value, index=None):
+        """Unused"""
+
+    def write_sync_state(self, leader, sync_standby, index=None):
+        return self.patch_or_create(self.sync_path, self.sync_state(leader, sync_standby), index, False)
+
+    def delete_sync_state(self, index=None):
+        return self.write_sync_state(None, None, index)
+
+    def watch(self, leader_index, timeout):
+        if self.__do_not_watch:
+            self.__do_not_watch = False
+            return True
+
+        if leader_index:
+            end_time = time.time() + timeout
+            w = k8s_watch.Watch()
+            while timeout >= 1:
+                try:
+                    for event in w.stream(self._api.list_namespaced_kind, self._namespace,
+                                          resource_version=leader_index, timeout_seconds=int(timeout + 0.5),
+                                          field_selector='metadata.name=' + self.leader_path,
+                                          _request_timeout=(1, timeout + 1)):
+                        return event['raw_object'].get('metadata', {}).get('resourceVersion') != leader_index
+                    return False
+                except KeyboardInterrupt:
+                    raise
+                except Exception:
+                    logging.exception('watch')
+
+                timeout = end_time - time.time()
+
+        try:
+            return super(Kubernetes, self).watch(None, timeout)
+        finally:
+            self.event.clear()

patroni/dcs/zookeeper.py

@@ -0,0 +1,344 @@
+import json
+import logging
+import time
+
+from kazoo.client import KazooClient, KazooState, KazooRetry
+from kazoo.exceptions import NoNodeError, NodeExistsError
+from kazoo.handlers.threading import SequentialThreadingHandler
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
+from patroni.exceptions import DCSError
+from patroni.utils import deep_compare
+
+logger = logging.getLogger(__name__)
+
+
+class ZooKeeperError(DCSError):
+    pass
+
+
+class PatroniSequentialThreadingHandler(SequentialThreadingHandler):
+
+    def __init__(self, connect_timeout):
+        super(PatroniSequentialThreadingHandler, self).__init__()
+        self.set_connect_timeout(connect_timeout)
+
+    def set_connect_timeout(self, connect_timeout):
+        self._connect_timeout = max(1.0, connect_timeout/2.0)  # try to connect to zookeeper node during loop_wait/2
+
+    def create_connection(self, *args, **kwargs):
+        """This method is trying to establish connection with one of the zookeeper nodes.
+           Somehow strategy "fail earlier and retry more often" works way better comparing to
+           the original strategy "try to connect with specified timeout".
+           Since we want to try connect to zookeeper more often (with the smaller connect_timeout),
+           he have to override `create_connection` method in the `SequentialThreadingHandler`
+           class (which is used by `kazoo.Client`).
+
+        :param args: always contains `tuple(host, port)` as the first element and could contain
+                     `connect_timeout` (negotiated session timeout) as the second element."""
+
+        args = list(args)
+        if len(args) == 1:
+            args.append(self._connect_timeout)
+        else:
+            args[1] = max(self._connect_timeout, args[1]/10.0)
+        return super(PatroniSequentialThreadingHandler, self).create_connection(*args, **kwargs)
+
+
+class ZooKeeper(AbstractDCS):
+
+    def __init__(self, config):
+        super(ZooKeeper, self).__init__(config)
+
+        hosts = config.get('hosts', [])
+        if isinstance(hosts, list):
+            hosts = ','.join(hosts)
+
+        self._client = KazooClient(hosts, handler=PatroniSequentialThreadingHandler(config['retry_timeout']),
+                                   timeout=config['ttl'], connection_retry=KazooRetry(max_delay=1, max_tries=-1,
+                                   sleep_func=time.sleep), command_retry=KazooRetry(deadline=config['retry_timeout'],
+                                   max_delay=1, max_tries=-1, sleep_func=time.sleep))
+        self._client.add_listener(self.session_listener)
+
+        self._my_member_data = {}
+        self._fetch_cluster = True
+
+        self._orig_kazoo_connect = self._client._connection._connect
+        self._client._connection._connect = self._kazoo_connect
+
+        self._client.start()
+
+    def _kazoo_connect(self, host, port):
+        """Kazoo is using Ping's to determine health of connection to zookeeper. If there is no
+        response on Ping after Ping interval (1/2 from read_timeout) it will consider current
+        connection dead and try to connect to another node. Without this "magic" it was taking
+        up to 2/3 from session timeout (ttl) to figure out that connection was dead and we had
+        only small time for reconnect and retry.
+
+        This method is needed to return different value of read_timeout, which is not calculated
+        from negotiated session timeout but from value of `loop_wait`. And it is 2 sec smaller
+        than loop_wait, because we can spend up to 2 seconds when calling `touch_member()` and
+        `write_leader_optime()` methods, which also may hang..."""
+
+        ret = self._orig_kazoo_connect(host, port)
+        return max(self.loop_wait - 2, 2)*1000, ret[1]
+
+    def session_listener(self, state):
+        if state in [KazooState.SUSPENDED, KazooState.LOST]:
+            self.cluster_watcher(None)
+
+    def cluster_watcher(self, event):
+        self._fetch_cluster = True
+        self.event.set()
+
+    def reload_config(self, config):
+        self.set_retry_timeout(config['retry_timeout'])
+
+        loop_wait = config['loop_wait']
+
+        loop_wait_changed = self._loop_wait != loop_wait
+        self._loop_wait = loop_wait
+        self._client.handler.set_connect_timeout(loop_wait)
+
+        # We need to reestablish connection to zookeeper if we want to change
+        # read_timeout (and Ping interval respectively), because read_timeout
+        # is calculated in `_kazoo_connect` method. If we are changing ttl at
+        # the same time, set_ttl method will reestablish connection and return
+        # `!True`, otherwise we will close existing connection and let kazoo
+        # open the new one.
+        if not self.set_ttl(int(config['ttl'] * 1000)) and loop_wait_changed:
+            self._client._connection._socket.close()
+
+    def set_ttl(self, ttl):
+        """It is not possible to change ttl (session_timeout) in zookeeper without
+        destroying old session and creating the new one. This method returns `!True`
+        if session_timeout has been changed (`restart()` has been called)."""
+        if self._client._session_timeout != ttl:
+            self._client._session_timeout = ttl
+            self._client.restart()
+            return True
+
+    def set_retry_timeout(self, retry_timeout):
+        retry = self._client.retry if isinstance(self._client.retry, KazooRetry) else self._client._retry
+        retry.deadline = retry_timeout
+
+    def get_node(self, key, watch=None):
+        try:
+            ret = self._client.get(key, watch)
+            return (ret[0].decode('utf-8'), ret[1])
+        except NoNodeError:
+            return None
+
+    @staticmethod
+    def member(name, value, znode):
+        return Member.from_node(znode.version, name, znode.ephemeralOwner, value)
+
+    def get_children(self, key, watch=None):
+        try:
+            return self._client.get_children(key, watch)
+        except NoNodeError:
+            return []
+
+    def load_members(self, sync_standby):
+        members = []
+        for member in self.get_children(self.members_path, self.cluster_watcher):
+            watch = member == sync_standby and self.cluster_watcher or None
+            data = self.get_node(self.members_path + member, watch)
+            if data is not None:
+                members.append(self.member(member, *data))
+        return members
+
+    def _inner_load_cluster(self):
+        self._fetch_cluster = False
+        self.event.clear()
+        nodes = set(self.get_children(self.client_path(''), self.cluster_watcher))
+        if not nodes:
+            self._fetch_cluster = True
+
+        # get initialize flag
+        initialize = (self.get_node(self.initialize_path) or [None])[0] if self._INITIALIZE in nodes else None
+
+        # get global dynamic configuration
+        config = self.get_node(self.config_path, watch=self.cluster_watcher) if self._CONFIG in nodes else None
+        config = config and ClusterConfig.from_node(config[1].version, config[0], config[1].mzxid)
+
+        # get timeline history
+        history = self.get_node(self.history_path, watch=self.cluster_watcher) if self._HISTORY in nodes else None
+        history = history and TimelineHistory.from_node(history[1].mzxid, history[0])
+
+        # get last leader operation
+        last_leader_operation = self._OPTIME in nodes and self._fetch_cluster and self.get_node(self.leader_optime_path)
+        last_leader_operation = last_leader_operation and int(last_leader_operation[0]) or 0
+
+        # get synchronization state
+        sync = self.get_node(self.sync_path, watch=self.cluster_watcher) if self._SYNC in nodes else None
+        sync = SyncState.from_node(sync and sync[1].version, sync and sync[0])
+
+        # get list of members
+        sync_standby = sync.leader == self._name and sync.sync_standby or None
+        members = self.load_members(sync_standby) if self._MEMBERS[:-1] in nodes else []
+
+        # get leader
+        leader = self.get_node(self.leader_path) if self._LEADER in nodes else None
+        if leader:
+            client_id = self._client.client_id
+            if not self._ctl and leader[0] == self._name and client_id is not None \
+                    and client_id[0] != leader[1].ephemeralOwner:
+                logger.info('I am leader but not owner of the session. Removing leader node')
+                self._client.delete(self.leader_path)
+                leader = None
+
+            if leader:
+                member = Member(-1, leader[0], None, {})
+                member = ([m for m in members if m.name == leader[0]] or [member])[0]
+                leader = Leader(leader[1].version, leader[1].ephemeralOwner, member)
+                self._fetch_cluster = member.index == -1
+
+        # failover key
+        failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
+        failover = failover and Failover.from_node(failover[1].version, failover[0])
+
+        self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover, sync, history)
+
+    def _load_cluster(self):
+        if self._fetch_cluster or self._cluster is None:
+            try:
+                self._client.retry(self._inner_load_cluster)
+            except Exception:
+                logger.exception('get_cluster')
+                self.cluster_watcher(None)
+                raise ZooKeeperError('ZooKeeper in not responding properly')
+
+    def _create(self, path, value, **kwargs):
+        try:
+            self._client.retry(self._client.create, path, value.encode('utf-8'), **kwargs)
+            return True
+        except Exception:
+            return False
+
+    def attempt_to_acquire_leader(self, permanent=False):
+        ret = self._create(self.leader_path, self._name, makepath=True, ephemeral=not permanent)
+        if not ret:
+            logger.info('Could not take out TTL lock')
+        return ret
+
+    def __set_failover_or_sync_state_value(self, key, value, index=None):
+        try:
+            self._client.retry(self._client.set, key, value.encode('utf-8'), version=index or -1)
+            return True
+        except NoNodeError:
+            return value == '' or (index is None and self._create(key, value))
+        except Exception:
+            logging.exception('set_failover_value')
+            return False
+
+    def set_failover_value(self, value, index=None):
+        return self.__set_failover_or_sync_state_value(self.failover_path, value, index)
+
+    def set_config_value(self, value, index=None):
+        try:
+            self._client.retry(self._client.set, self.config_path, value.encode('utf-8'), version=index or -1)
+            return True
+        except NoNodeError:
+            return index is None and self._create(self.config_path, value)
+        except Exception:
+            logging.exception('set_config_value')
+            return False
+
+    def initialize(self, create_new=True, sysid=""):
+        return self._create(self.initialize_path, sysid, makepath=True) if create_new \
+            else self._client.retry(self._client.set, self.initialize_path, sysid.encode("utf-8"))
+
+    def touch_member(self, data, ttl=None, permanent=False):
+        cluster = self.cluster
+        member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
+        encoded_data = json.dumps(data, separators=(',', ':')).encode('utf-8')
+        if member and self._client.client_id is not None and member.session != self._client.client_id[0]:
+            try:
+                self._client.delete_async(self.member_path).get(timeout=1)
+            except NoNodeError:
+                pass
+            except Exception:
+                return False
+            member = None
+
+        if member:
+            if deep_compare(data, self._my_member_data):
+                return True
+        else:
+            try:
+                self._client.create_async(self.member_path, encoded_data, makepath=True,
+                                          ephemeral=not permanent).get(timeout=1)
+                self._my_member_data = data
+                return True
+            except Exception as e:
+                if not isinstance(e, NodeExistsError):
+                    logger.exception('touch_member')
+                    return False
+        try:
+            self._client.set_async(self.member_path, encoded_data).get(timeout=1)
+            self._my_member_data = data
+            return True
+        except Exception:
+            logger.exception('touch_member')
+
+        return False
+
+    def take_leader(self):
+        return self.attempt_to_acquire_leader()
+
+    def __write_leader_optime_or_history_value(self, key, value):
+        value = value.encode('utf-8')
+        try:
+            self._client.set_async(key, value).get(timeout=1)
+            return True
+        except NoNodeError:
+            try:
+                self._client.create_async(key, value, makepath=True).get(timeout=1)
+                return True
+            except Exception:
+                logger.exception('Failed to create %s', key)
+        except Exception:
+            logger.exception('Failed to update %s', key)
+        return False
+
+    def _write_leader_optime(self, last_operation):
+        return self.__write_leader_optime_or_history_value(self.leader_optime_path, last_operation)
+
+    def _update_leader(self):
+        return True
+
+    def delete_leader(self):
+        self._client.restart()
+        self._my_member_data = None
+        return True
+
+    def _cancel_initialization(self):
+        node = self.get_node(self.initialize_path)
+        if node:
+            self._client.delete(self.initialize_path, version=node[1].version)
+
+    def cancel_initialization(self):
+        try:
+            self._client.retry(self._cancel_initialization)
+        except Exception:
+            logger.exception("Unable to delete initialize key")
+
+    def delete_cluster(self):
+        try:
+            return self._client.retry(self._client.delete, self.client_path(''), recursive=True)
+        except NoNodeError:
+            return True
+
+    def set_history_value(self, value):
+        return self.__write_leader_optime_or_history_value(self.history_path, value)
+
+    def set_sync_state_value(self, value, index=None):
+        return self.__set_failover_or_sync_state_value(self.sync_path, value, index)
+
+    def delete_sync_state(self, index=None):
+        return self.set_sync_state_value("{}", index)
+
+    def watch(self, leader_index, timeout):
+        if super(ZooKeeper, self).watch(leader_index, timeout):
+            self._fetch_cluster = True
+        return self._fetch_cluster

patroni/etcd.py

@@ -1,273 +0,0 @@
-from __future__ import absolute_import
-import etcd
-import logging
-import os
-import random
-import requests
-import socket
-import time
-import urllib3
-
-from dns.exception import DNSException
-from dns import resolver
-from patroni.dcs import AbstractDCS, Cluster, Failover, Leader, Member
-from patroni.exceptions import DCSError
-from patroni.utils import Retry, RetryFailedError, sleep
-from requests.exceptions import RequestException
-
-logger = logging.getLogger(__name__)
-
-
-class EtcdError(DCSError):
-    pass
-
-
-class Client(etcd.Client):
-
-    def __init__(self, config):
-        super(Client, self).__init__(read_timeout=5)
-        self._config = config
-        self._load_machines_cache()
-        self._allow_reconnect = True
-
-    @property
-    def machines(self):
-        """Original `machines` method(property) of `etcd.Client` class raise exception
-        when it failed to get list of etcd cluster members. This method is being called
-        only when request failed on one of the etcd members during `api_execute` call.
-        For us it's more important to execute original request rather then get new
-        topology of etcd cluster. So we will catch this exception and return valid list
-        of machines with setting flag `self._update_machines_cache` to `!True`.
-        Later, during next `api_execute` call we will forcefully update machines_cache"""
-        try:
-            ret = super(Client, self).machines
-            random.shuffle(ret)
-            return ret
-        except etcd.EtcdException:
-            if self._update_machines_cache:  # We are updating machines_cache
-                raise  # This exception is fatal, we should re-raise it.
-            self._update_machines_cache = True
-            return [self._base_uri]
-
-    def api_execute(self, path, method, **kwargs):
-        # Update machines_cache if previous attempt of update has failed
-        self._update_machines_cache and self._load_machines_cache()
-        return super(Client, self).api_execute(path, method, **kwargs)
-
-    @staticmethod
-    def get_srv_record(host):
-        try:
-            return [(str(r.target).rstrip('.'), r.port) for r in resolver.query('_etcd-server._tcp.' + host, 'SRV')]
-        except DNSException:
-            logger.exception('Can not resolve SRV for %s', host)
-        return []
-
-    # try to workarond bug in python-etcd: https://github.com/jplana/python-etcd/issues/81
-    def _result_from_response(self, response):
-        try:
-            response.data.decode('utf-8')
-        except urllib3.exceptions.TimeoutError:
-            raise
-        except Exception as e:
-            raise etcd.EtcdException('Unable to decode server response: %s' % e)
-        return super(Client, self)._result_from_response(response)
-
-    def _get_machines_cache_from_srv(self, discovery_srv):
-        """Fetch list of etcd-cluster member by resolving _etcd-server._tcp. SRV record.
-        This record should contain list of host and peer ports which could be used to run
-        'GET http://{host}:{port}/members' request (peer protocol)"""
-
-        ret = []
-        for host, port in self.get_srv_record(discovery_srv):
-            url = '{}://{}:{}/members'.format(self._protocol, host, port)
-            try:
-                response = requests.get(url, timeout=5)
-                if response.ok:
-                    for member in response.json():
-                        ret.extend(member['clientURLs'])
-                    break
-            except RequestException:
-                logger.exception('GET %s', url)
-        return list(set(ret))
-
-    def _get_machines_cache_from_dns(self, addr):
-        """One host might be resolved into multiple ip addresses. We will make list out of it"""
-
-        ret = []
-        host, port = addr.split(':')
-        try:
-            for r in set(socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)):
-                ret.append('{}://{}:{}'.format(self._protocol, r[4][0], r[4][1]))
-        except socket.error:
-            logger.exception('Can not resolve %s', host)
-        return list(set(ret)) if ret else ['{}://{}:{}'.format(self._protocol, host, port)]
-
-    def _load_machines_cache(self):
-        """This method should fill up `_machines_cache` from scratch.
-        It could happen only in two cases:
-        1. During class initialization
-        2. When all etcd members failed"""
-
-        self._update_machines_cache = True
-
-        if 'discovery_srv' not in self._config and 'host' not in self._config:
-            raise Exception('Neither discovery_srv nor host are defined in etcd section of config')
-
-        self._machines_cache = []
-
-        if 'discovery_srv' in self._config:
-            self._machines_cache = self._get_machines_cache_from_srv(self._config['discovery_srv'])
-
-        if not self._machines_cache and 'host' in self._config:
-            self._machines_cache = self._get_machines_cache_from_dns(self._config['host'])
-
-        # Can not bootstrap list of etcd-cluster members, giving up
-        if not self._machines_cache:
-            raise etcd.EtcdException
-
-        # After filling up initial list of machines_cache we should ask etcd-cluster about actual list
-        self._base_uri = self._machines_cache.pop(0)
-        self._machines_cache = self.machines
-        self._base_uri in self._machines_cache and self._machines_cache.remove(self._base_uri)
-
-        self._update_machines_cache = False
-
-
-def catch_etcd_errors(func):
-    def wrapper(*args, **kwargs):
-        try:
-            return not func(*args, **kwargs) is None
-        except (RetryFailedError, etcd.EtcdException):
-            return False
-    return wrapper
-
-
-class Etcd(AbstractDCS):
-
-    def __init__(self, name, config):
-        super(Etcd, self).__init__(name, config)
-        self.ttl = config['ttl']
-        self._retry = Retry(deadline=10, max_delay=1, max_tries=-1,
-                            retry_exceptions=(etcd.EtcdConnectionFailed,
-                                              etcd.EtcdLeaderElectionInProgress,
-                                              etcd.EtcdWatcherCleared,
-                                              etcd.EtcdEventIndexCleared))
-        self.client = self.get_etcd_client(config)
-
-    def retry(self, *args, **kwargs):
-        return self._retry.copy()(*args, **kwargs)
-
-    def get_etcd_client(self, config):
-        client = None
-        while not client:
-            try:
-                client = Client(config)
-            except etcd.EtcdException:
-                logger.info('waiting on etcd')
-                sleep(5)
-        return client
-
-    @staticmethod
-    def member(node):
-        return Member.from_node(node.modifiedIndex, os.path.basename(node.key), node.ttl, node.value)
-
-    def _load_cluster(self):
-        try:
-            result = self.retry(self.client.read, self.client_path(''), recursive=True)
-            nodes = {os.path.relpath(node.key, result.key): node for node in result.leaves}
-
-            # get initialize flag
-            initialize = bool(nodes.get(self._INITIALIZE, False))
-
-            # get last leader operation
-            last_leader_operation = nodes.get(self._LEADER_OPTIME, None)
-            last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation.value)
-
-            # get list of members
-            members = [self.member(n) for k, n in nodes.items() if k.startswith(self._MEMBERS) and k.count('/') == 1]
-
-            # get leader
-            leader = nodes.get(self._LEADER, None)
-            if leader:
-                member = Member(-1, leader.value, None, {})
-                member = ([m for m in members if m.name == leader.value] or [member])[0]
-                leader = Leader(leader.modifiedIndex, leader.ttl, member)
-
-            # failover key
-            failover = nodes.get(self._FAILOVER, None)
-            if failover:
-                failover = Failover.from_node(failover.modifiedIndex, failover.value)
-
-            self._cluster = Cluster(initialize, leader, last_leader_operation, members, failover)
-        except etcd.EtcdKeyNotFound:
-            self._cluster = Cluster(False, None, None, [], None)
-        except:
-            logger.exception('get_cluster')
-            raise EtcdError('Etcd is not responding properly')
-
-    @catch_etcd_errors
-    def touch_member(self, connection_string, ttl=None):
-        return self.retry(self.client.set, self.member_path, connection_string, ttl or self.ttl)
-
-    @catch_etcd_errors
-    def take_leader(self):
-        return self.retry(self.client.set, self.leader_path, self._name, self.ttl)
-
-    def attempt_to_acquire_leader(self):
-        try:
-            return bool(self.retry(self.client.write, self.leader_path, self._name, ttl=self.ttl, prevExist=False))
-        except etcd.EtcdAlreadyExist:
-            logger.info('Could not take out TTL lock')
-        except (RetryFailedError, etcd.EtcdException):
-            pass
-        return False
-
-    @catch_etcd_errors
-    def set_failover_value(self, value, index=None):
-        return self.client.write(self.failover_path, value, prevIndex=index or 0)
-
-    @catch_etcd_errors
-    def write_leader_optime(self, last_operation):
-        return self.client.set(self.leader_optime_path, last_operation)
-
-    @catch_etcd_errors
-    def update_leader(self):
-        return self.retry(self.client.test_and_set, self.leader_path, self._name, self._name, self.ttl)
-
-    @catch_etcd_errors
-    def initialize(self):
-        return self.retry(self.client.write, self.initialize_path, self._name, prevExist=False)
-
-    @catch_etcd_errors
-    def delete_leader(self):
-        return self.client.delete(self.leader_path, prevValue=self._name)
-
-    @catch_etcd_errors
-    def cancel_initialization(self):
-        return self.retry(self.client.delete, self.initialize_path, prevValue=self._name)
-
-    def watch(self, timeout):
-        cluster = self.cluster
-        # watch on leader key changes if it is defined and current node is not lock owner
-        if cluster and cluster.leader and cluster.leader.name != self._name:
-            end_time = time.time() + timeout
-            index = cluster.leader.index
-
-            while index and timeout >= 1:  # when timeout is too small urllib3 doesn't have enough time to connect
-                try:
-                    self.client.watch(self.leader_path, index=index + 1, timeout=timeout + 0.5)
-                    # Synchronous work of all cluster members with etcd is less expensive
-                    # than reestablishing http connection every time from every replica.
-                    return True
-                except urllib3.exceptions.TimeoutError:
-                    self.client.http.clear()
-                    return False
-                except etcd.EtcdException:
-                    logging.exception('watch')
-
-                timeout = end_time - time.time()
-
-        try:
-            return super(Etcd, self).watch(timeout)
-        finally:
-            self.event.clear()

patroni/exceptions.py

@@ -23,3 +23,7 @@ class DCSError(PatroniException):
 
 class PostgresConnectionException(PostgresException):
     pass
+
+
+class WatchdogError(PatroniException):
+    pass

patroni/postmaster.py

@@ -0,0 +1,165 @@
+import logging
+import os
+import psutil
+import re
+import signal
+import subprocess
+
+from patroni import call_self
+
+logger = logging.getLogger(__name__)
+
+STOP_SIGNALS = {
+    'smart': signal.SIGTERM,
+    'fast': signal.SIGINT,
+    'immediate': signal.SIGQUIT,
+}
+
+
+class PostmasterProcess(psutil.Process):
+
+    def __init__(self, pid):
+        self.is_single_user = False
+        if pid < 0:
+            pid = -pid
+            self.is_single_user = True
+        super(PostmasterProcess, self).__init__(pid)
+
+    @staticmethod
+    def _read_postmaster_pidfile(data_dir):
+        """Reads and parses postmaster.pid from the data directory
+
+        :returns dictionary of values if successful, empty dictionary otherwise
+        """
+        pid_line_names = ['pid', 'data_dir', 'start_time', 'port', 'socket_dir', 'listen_addr', 'shmem_key']
+        try:
+            with open(os.path.join(data_dir, 'postmaster.pid')) as f:
+                return {name: line.rstrip('\n') for name, line in zip(pid_line_names, f)}
+        except IOError:
+            return {}
+
+    def _is_postmaster_process(self):
+        try:
+            start_time = int(self._postmaster_pid.get('start_time', 0))
+            if start_time and abs(self.create_time() - start_time) > 3:
+                logger.info('Too much difference between %s and %s', self.create_time(), start_time)
+                return False
+        except ValueError:
+            logger.warning('Garbage start time value in pid file: %r', self._postmaster_pid.get('start_time'))
+
+        # Extra safety check. The process can't be ourselves, our parent or our direct child.
+        if self.pid == os.getpid() or self.pid == os.getppid() or self.ppid() == os.getpid():
+            logger.info('Patroni (pid=%s, ppid=%s), "fake postmaster" (pid=%s, ppid=%s)',
+                        os.getpid(), os.getppid(), self.pid, self.ppid())
+            return False
+
+        return True
+
+    @classmethod
+    def _from_pidfile(cls, data_dir):
+        postmaster_pid = PostmasterProcess._read_postmaster_pidfile(data_dir)
+        try:
+            pid = int(postmaster_pid.get('pid', 0))
+            if pid:
+                proc = cls(pid)
+                proc._postmaster_pid = postmaster_pid
+                return proc
+        except ValueError:
+            pass
+
+    @staticmethod
+    def from_pidfile(data_dir):
+        try:
+            proc = PostmasterProcess._from_pidfile(data_dir)
+            return proc if proc and proc._is_postmaster_process() else None
+        except psutil.NoSuchProcess:
+            return None
+
+    @classmethod
+    def from_pid(cls, pid):
+        try:
+            return cls(pid)
+        except psutil.NoSuchProcess:
+            return None
+
+    def signal_stop(self, mode):
+        """Signal postmaster process to stop
+
+        :returns None if signaled, True if process is already gone, False if error
+        """
+        if self.is_single_user:
+            logger.warning("Cannot stop server; single-user server is running (PID: {0})".format(self.pid))
+            return False
+        try:
+            self.send_signal(STOP_SIGNALS[mode])
+        except psutil.NoSuchProcess:
+            return True
+        except psutil.AccessDenied as e:
+            logger.warning("Could not send stop signal to PostgreSQL (error: {0})".format(e))
+            return False
+
+        return None
+
+    def wait_for_user_backends_to_close(self):
+        # These regexps are cross checked against versions PostgreSQL 9.1 .. 9.6
+        aux_proc_re = re.compile("(?:postgres:)( .*:)? (?:""(?:startup|logger|checkpointer|writer|wal writer|"
+                                 "autovacuum launcher|autovacuum worker|stats collector|wal receiver|archiver|"
+                                 "wal sender) process|bgworker: )")
+
+        try:
+            user_backends = []
+            user_backends_cmdlines = []
+            for child in self.children():
+                try:
+                    cmdline = child.cmdline()[0]
+                    if not aux_proc_re.match(cmdline):
+                        user_backends.append(child)
+                        user_backends_cmdlines.append(cmdline)
+                except psutil.NoSuchProcess:
+                    pass
+            if user_backends:
+                logger.debug('Waiting for user backends %s to close', ', '.join(user_backends_cmdlines))
+                psutil.wait_procs(user_backends)
+            logger.debug("Backends closed")
+        except psutil.Error:
+            logger.exception('wait_for_user_backends_to_close')
+
+    @staticmethod
+    def start(pgcommand, data_dir, conf, options):
+        # Unfortunately `pg_ctl start` does not return postmaster pid to us. Without this information
+        # it is hard to know the current state of postgres startup, so we had to reimplement pg_ctl start
+        # in python. It will start postgres, wait for port to be open and wait until postgres will start
+        # accepting connections.
+        # Important!!! We can't just start postgres using subprocess.Popen, because in this case it
+        # will be our child for the rest of our live and we will have to take care of it (`waitpid`).
+        # So we will use the same approach as pg_ctl uses: start a new process, which will start postgres.
+        # This process will write postmaster pid to stdout and exit immediately. Now it's responsibility
+        # of init process to take care about postmaster.
+        # In order to make everything portable we can't use fork&exec approach here, so  we will call
+        # ourselves and pass list of arguments which must be used to start postgres.
+        env = {p: os.environ[p] for p in ('PATH', 'LC_ALL', 'LANG') if p in os.environ}
+        try:
+            proc = PostmasterProcess._from_pidfile(data_dir)
+            if proc and not proc._is_postmaster_process():
+                # Upon start postmaster process performs various safety checks if there is a postmaster.pid
+                # file in the data directory. Although Patroni already detected that the running process
+                # corresponding to the postmaster.pid is not a postmaster, the new postmaster might fail
+                # to start, because it thinks that postmaster.pid is already locked.
+                # Important!!! Unlink of postmaster.pid isn't an option, because it has a lot of nasty race conditions.
+                # Luckily there is a workaround to this problem, we can pass the pid from postmaster.pid
+                # in the `PG_GRANDPARENT_PID` environment variable and postmaster will ignore it.
+                env['PG_GRANDPARENT_PID'] = str(proc.pid)
+        except psutil.NoSuchProcess:
+            pass
+
+        proc = call_self(['pg_ctl_start', pgcommand, '-D', data_dir,
+                          '--config-file={}'.format(conf)] + options, close_fds=True,
+                         preexec_fn=os.setsid, stdout=subprocess.PIPE, env=env)
+        pid = int(proc.stdout.readline().strip())
+        proc.wait()
+        logger.info('postmaster pid=%s', pid)
+
+        # TODO: In an extremely unlikely case, the process could have exited and the pid reassigned. The start
+        # initiation time is not accurate enough to compare to create time as start time would also likely
+        # be relatively close. We need the subprocess extract pid+start_time in a race free manner.
+        return PostmasterProcess.from_pid(pid)

patroni/scripts/aws.py

@@ -6,67 +6,73 @@ from requests.exceptions import RequestException
 import sys
 import boto.ec2
 
+from patroni.utils import Retry, RetryFailedError
+
 logger = logging.getLogger(__name__)
 
 
-class AWSConnection:
+class AWSConnection(object):
+
     def __init__(self, cluster_name):
         self.available = False
         self.cluster_name = cluster_name if cluster_name is not None else 'unknown'
+        self._retry = Retry(deadline=300, max_delay=30, max_tries=-1, retry_exceptions=(boto.exception.StandardError,))
         try:
             # get the instance id
-            r = requests.get('http://169.254.169.254/latest/dynamic/instance-identity/document', timeout=0.1)
+            r = requests.get('http://169.254.169.254/latest/dynamic/instance-identity/document', timeout=2.1)
         except RequestException:
-            logger.info("cannot query AWS meta-data")
+            logger.error('cannot query AWS meta-data')
             return
+
         if r.ok:
             try:
                 content = r.json()
                 self.instance_id = content['instanceId']
                 self.region = content['region']
-            except Exception as e:
-                logger.info('unable to fetch instance id and region from AWS meta-data: {}'.format(e))
+            except Exception:
+                logger.exception('unable to fetch instance id and region from AWS meta-data')
                 return
             self.available = True
 
+    def retry(self, *args, **kwargs):
+        return self._retry.copy()(*args, **kwargs)
+
     def aws_available(self):
         return self.available
 
-    def _tag_ebs(self, role):
+    def _tag_ebs(self, conn, role):
         """ set tags, carrying the cluster name, instance role and instance id for the EBS storage """
-        if not self.available:
-            return False
-
         tags = {'Name': 'spilo_' + self.cluster_name, 'Role': role, 'Instance': self.instance_id}
-        try:
-            conn = boto.ec2.connect_to_region(self.region)
-            volumes = conn.get_all_volumes(filters={'attachment.instance-id': self.instance_id})
-            conn.create_tags([v.id for v in volumes], tags)
-        except Exception as e:
-            logger.info('could not set tags for EBS storage devices attached: {}'.format(e))
-            return False
-        return True
+        volumes = conn.get_all_volumes(filters={'attachment.instance-id': self.instance_id})
+        conn.create_tags([v.id for v in volumes], tags)
 
-    def _tag_ec2(self, role):
+    def _tag_ec2(self, conn, role):
         """ tag the current EC2 instance with a cluster role """
-        if not self.available:
-            return False
         tags = {'Role': role}
-        try:
-            conn = boto.ec2.connect_to_region(self.region)
-            conn.create_tags([self.instance_id], tags)
-        except Exception as e:
-            logger.info("could not set tags for EC2 instance {}: {}".format(self.instance_id, e))
-            return False
-        return True
+        conn.create_tags([self.instance_id], tags)
 
     def on_role_change(self, new_role):
-        ret = self._tag_ec2(new_role)
-        return self._tag_ebs(new_role) and ret
+        if not self.available:
+            return False
+        try:
+            conn = self.retry(boto.ec2.connect_to_region, self.region)
+            self.retry(self._tag_ec2, conn, new_role)
+            self.retry(self._tag_ebs, conn, new_role)
+        except RetryFailedError:
+            logger.warning("Unable to communicate to AWS "
+                           "when setting tags for the EC2 instance {0} "
+                           "and attached EBS volumes".format(self.instance_id))
+            return False
+        return True
 
 
-if __name__ == '__main__':
+def main():
+    logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
     if len(sys.argv) == 4 and sys.argv[1] in ('on_start', 'on_stop', 'on_role_change'):
         AWSConnection(cluster_name=sys.argv[3]).on_role_change(sys.argv[2])
     else:
         sys.exit("Usage: {0} action role name".format(sys.argv[0]))
+
+
+if __name__ == '__main__':
+    main()

patroni/scripts/restore.py

@@ -1,216 +0,0 @@
-#!/usr/bin/env python
-# arguments are:
-#   - cluster scope
-#   - cluster role
-#   - master connection string
-
-# for the AWS, the folliowing environment variables should be defined:
-# - WALE_ENV_DIR: directory where WAL-E environment is kept
-# - WAL_S3_BUCKET: a name of the S3 bucket for WAL-E
-# - WALE_BACKUP_THRESHOLD_MEGABYTES if WAL amount is above that - use pg_basebackup
-# - WALE_BACKUP_THRESHOLD_PERCENTAGE if WAL size exceeds a certain percentage of the
-#   latest backup size
-from collections import namedtuple
-import logging
-import os
-import psycopg2
-import subprocess
-import sys
-
-
-if sys.hexversion >= 0x03000000:
-    long = int
-
-logger = logging.getLogger(__name__)
-
-
-class Restore(object):
-
-    def __init__(self, scope, role, datadir, connstring, env=None):
-        self.scope = scope
-        self.role = role
-        self.master_connection = Restore.parse_connstring(connstring)
-        self.data_dir = datadir
-        self.env = os.environ.copy() if not env else env
-
-    @staticmethod
-    def parse_connstring(connstring):
-        # the connection string is in the form host= port= user=
-        # return the dictionary with all components as separare keys
-        result = {}
-        if connstring:
-            for x in connstring.split():
-                if x and '=' in x:
-                    key, val = x.split('=')
-                result[key.strip()] = val.strip()
-        return result
-
-    def setup(self):
-        pass
-
-    def replica_method(self):
-        return self.create_replica_with_pg_basebackup
-
-    def replica_fallback_method(self):
-        return None
-
-    def run(self):
-        """ creates a new replica using either pg_basebackup or WAL-E """
-        method_fn = self.replica_method()
-        ret = method_fn() if method_fn else 1
-        if ret != 0 and self.replica_fallback_method() is not None:
-            ret = (self.replica_fallback_method())()
-        return ret
-
-    def create_replica_with_pg_basebackup(self):
-        try:
-            ret = subprocess.call(['pg_basebackup', '-R', '-D',
-                                   self.data_dir, '--host=' + self.master_connection['host'],
-                                   '--port=' + str(self.master_connection['port']),
-                                   '-U', self.master_connection['user']],
-                                  env=self.env)
-        except Exception as e:
-            logger.error('Error when fetching backup with pg_basebackup: {0}'.format(e))
-            return 1
-        return ret
-
-
-class WALERestore(Restore):
-
-    def __init__(self, scope, role, datadir, connstring, env=None):
-        super(WALERestore, self).__init__(scope, role, datadir, connstring, env)
-        # check the environment variables
-        self.init_error = False
-
-    def setup(self):
-        if (self.env.get('WAL_S3_BUCKET') and
-            self.env.get('WALE_BACKUP_THRESHOLD_PERCENTAGE') and
-           self.env.get('WALE_BACKUP_THRESHOLD_MEGABYTES')) is None:
-                self.init_error = True
-        else:
-            self.wal_e = namedtuple('WALE',
-                                    'threshold_megabytes threshold_backup_size_percentage s3_bucket cmd dir env_file')
-
-            self.wal_e.dir = self.env.get('WALE_ENV_DIR', '/home/postgres/etc/wal-e.d/env')
-            self.wal_e.env_file = os.path.join(self.wal_e.dir, 'WALE_S3_PREFIX')
-
-            self.wal_e.cmd = 'envdir {} wal-e --aws-instance-profile '.\
-                format(self.wal_e.dir)
-            self.wal_e.s3_bucket = self.env['WAL_S3_BUCKET']
-            self.wal_e.threshold_megabytes = self.env['WALE_BACKUP_THRESHOLD_MEGABYTES']
-            self.wal_e.threshold_backup_size_percentage = self.env['WALE_BACKUP_THRESHOLD_PERCENTAGE']
-
-            # check that the env file exists, create it otherwise
-            try:
-                if not os.path.exists(self.wal_e.dir):
-                    os.makedirs(self.wal_e.dir)
-                # if this is a directory - make sure we have full access there
-                elif not (os.path.isdir(self.wal_e.dir) and os.access(self.wal_e.dir, os.R_OK | os.W_OK | os.X_OK)):
-                    logger.error("Unable to access {} or not a directory".format(self.wal_e.dir))
-                    self.init_error = True
-                # if WAL_S3_PREFIX is not there - create it and write the full path to bucket
-                if not self.init_error and not os.path.exists(self.wal_e.env_file):
-                    with open(self.wal_e.env_file, 'w') as f:
-                        f.write("s3://{0}/spilo/{1}/wal/\n".format(self.wal_e.s3_bucket, self.scope))
-
-            except (os.error, IOError) as e:
-                logger.error("{0}: WAL-e archiving is disabled".format(e))
-                self.init_error = True
-
-    def replica_method(self):
-        if self.should_use_s3_to_create_replica():
-            return self.create_replica_with_s3
-        return None
-
-    def replica_fallback_method(self):
-        return self.create_replica_with_pg_basebackup
-
-    def should_use_s3_to_create_replica(self):
-        """ determine whether it makes sense to use S3 and not pg_basebackup """
-        if self.init_error:
-            return False
-
-        threshold_megabytes = self.wal_e.threshold_megabytes
-        threshold_backup_size_percentage = self.wal_e.threshold_backup_size_percentage
-
-        try:
-            latest_backup = subprocess.check_output(self.wal_e.cmd.split() + ['backup-list', '--detail', 'LATEST'],
-                                                    env=self.env)
-            # name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start
-            #                                                   wal_segment_backup_stop wal_segment_offset_backup_stop
-            # base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z
-            # 20310671    00000001000000000000007F    00000040
-            # 00000001000000000000007F    00000240
-            backup_strings = latest_backup.splitlines() if latest_backup else ()
-            if len(backup_strings) != 2:
-                return False
-
-            names = backup_strings[0].split()
-            vals = backup_strings[1].split()
-            if (len(names) != len(vals)) or (len(names) != 7):
-                return False
-
-            backup_info = dict(zip(names, vals))
-        except subprocess.CalledProcessError as e:
-            logger.error("could not query wal-e latest backup: {}".format(e))
-            return False
-
-        try:
-            backup_size = backup_info['expanded_size_bytes']
-            backup_start_segment = backup_info['wal_segment_backup_start']
-            backup_start_offset = backup_info['wal_segment_offset_backup_start']
-        except Exception as e:
-            logger.error("unable to get some of S3 backup parameters: {}".format(e))
-            return False
-
-        # WAL filename is XXXXXXXXYYYYYYYY000000ZZ, where X - timeline, Y - LSN logical log file,
-        # ZZ - 2 high digits of LSN offset. The rest of the offset is the provided decimal offset,
-        # that we have to convert to hex and 'prepend' to the high offset digits.
-
-        lsn_segment = backup_start_segment[8:16]
-        # first 2 characters of the result are 0x and the last one is L
-        lsn_offset = hex((long(backup_start_segment[16:32], 16) << 24) + long(backup_start_offset))[2:-1]
-
-        # construct the LSN from the segment and offset
-        backup_start_lsn = '{}/{}'.format(lsn_segment, lsn_offset)
-
-        conn = None
-        cursor = None
-        diff_in_bytes = long(backup_size)
-        try:
-            # get the difference in bytes between the current WAL location and the backup start offset
-            conn = psycopg2.connect(**(self.master_connection))
-            conn.autocommit = True
-            cursor = conn.cursor()
-            cursor.execute("SELECT pg_xlog_location_diff(pg_current_xlog_location(), %s)", (backup_start_lsn,))
-            diff_in_bytes = long(cursor.fetchone()[0])
-        except psycopg2.Error as e:
-            logger.error('could not determine difference with the master location: {}'.format(e))
-            return False
-        finally:
-            cursor and cursor.close()
-            conn and conn.close()
-
-        # if the size of the accumulated WAL segments is more than a certan percentage of the backup size
-        # or exceeds the pre-determined size - pg_basebackup is chosen instead.
-        return (diff_in_bytes < long(threshold_megabytes) * 1048576) and\
-               (diff_in_bytes < long(backup_size) * float(threshold_backup_size_percentage) / 100)
-
-    def create_replica_with_s3(self):
-        if self.init_error:
-            return 1
-        try:
-            ret = subprocess.call(self.wal_e.cmd + ' backup-fetch {} LATEST'.format(self.data_dir), env=self.env)
-        except Exception as e:
-            logger.error('Error when fetching backup with WAL-E: {0}'.format(e))
-            return 1
-        return ret
-
-
-if __name__ == '__main__':
-    if len(sys.argv) == 5:
-        # scope, role, datadir, connstring
-        restore = WALERestore(*(sys.argv[1:]))
-        restore.setup()
-        sys.exit(restore.run())
-    sys.exit("Usage: {0} scope role datadir connstring".format(sys.argv[0]))

patroni/scripts/wale_restore.py

@@ -0,0 +1,372 @@
+#!/usr/bin/env python
+
+# sample script to clone new replicas using WAL-E restore
+# falls back to pg_basebackup if WAL-E restore fails, or if
+# WAL-E backup is too far behind
+# note that pg_basebackup still expects to use restore from
+# WAL-E for transaction logs
+
+# theoretically should work with SWIFT, but not tested on it
+
+# arguments are:
+#   - cluster scope
+#   - cluster role
+#   - master connection string
+#   - number of retries
+#   - envdir for the WALE env
+# - WALE_BACKUP_THRESHOLD_MEGABYTES if WAL amount is above that - use pg_basebackup
+# - WALE_BACKUP_THRESHOLD_PERCENTAGE if WAL size exceeds a certain percentage of the
+
+# this script depends on an envdir defining the S3 bucket (or SWIFT dir),and login
+# credentials per WALE Documentation.
+
+# currently also requires that you configure the restore_command to use wal_e, example:
+#       recovery_conf:
+#               restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1
+import argparse
+import csv
+import logging
+import os
+import psycopg2
+import subprocess
+import sys
+import time
+
+from collections import namedtuple
+
+logger = logging.getLogger(__name__)
+
+RETRY_SLEEP_INTERVAL = 1
+si_prefixes = ['K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']
+
+
+# Meaningful names to the exit codes used by WALERestore
+ExitCode = type('Enum', (), {
+    'SUCCESS': 0,  #: Succeeded
+    'RETRY_LATER': 1,  #: External issue, retry later
+    'FAIL': 2  #: Don't try again unless configuration changes
+})
+
+
+# We need to know the current PG version in order to figure out the correct WAL directory name
+def get_major_version(data_dir):
+    version_file = os.path.join(data_dir, 'PG_VERSION')
+    if os.path.isfile(version_file):  # version file exists
+        try:
+            with open(version_file) as f:
+                return float(f.read())
+        except Exception:
+            logger.exception('Failed to read PG_VERSION from %s', data_dir)
+    return 0.0
+
+
+def repr_size(n_bytes):
+    """
+    >>> repr_size(1000)
+    '1000 Bytes'
+    >>> repr_size(8257332324597)
+    '7.5 TiB'
+    """
+    if n_bytes < 1024:
+        return '{0} Bytes'.format(n_bytes)
+    i = -1
+    while n_bytes > 1023:
+        n_bytes /= 1024.0
+        i += 1
+    return '{0} {1}iB'.format(round(n_bytes, 1), si_prefixes[i])
+
+
+def size_as_bytes(size_, prefix):
+    """
+    >>> size_as_bytes(7.5, 'T')
+    8246337208320
+    """
+    prefix = prefix.upper()
+
+    assert prefix in si_prefixes
+
+    exponent = si_prefixes.index(prefix) + 1
+
+    return int(size_ * (1024.0 ** exponent))
+
+
+WALEConfig = namedtuple(
+    'WALEConfig',
+    [
+        'env_dir',
+        'threshold_mb',
+        'threshold_pct',
+        'cmd',
+    ]
+)
+
+
+class WALERestore(object):
+    def __init__(self, scope, datadir, connstring, env_dir, threshold_mb,
+                 threshold_pct, use_iam, no_master, retries):
+        self.scope = scope
+        self.master_connection = connstring
+        self.data_dir = datadir
+        self.no_master = no_master
+
+        wale_cmd = [
+            'envdir',
+            env_dir,
+            'wal-e',
+        ]
+
+        if use_iam == 1:
+            wale_cmd += ['--aws-instance-profile']
+
+        self.wal_e = WALEConfig(
+            env_dir=env_dir,
+            threshold_mb=threshold_mb,
+            threshold_pct=threshold_pct,
+            cmd=wale_cmd,
+        )
+
+        self.init_error = (not os.path.exists(self.wal_e.env_dir))
+        self.retries = retries
+
+    def run(self):
+        """
+        Creates a new replica using WAL-E
+
+        Returns
+        -------
+        ExitCode
+            0 = Success
+            1 = Error, try again
+            2 = Error, don't try again
+
+        """
+        if self.init_error:
+            logger.error('init error: %r did not exist at initialization time',
+                         self.wal_e.env_dir)
+            return ExitCode.FAIL
+
+        try:
+            should_use_s3 = self.should_use_s3_to_create_replica()
+            if should_use_s3 is None:  # Need to retry
+                return ExitCode.RETRY_LATER
+            elif should_use_s3:
+                return self.create_replica_with_s3()
+            elif not should_use_s3:
+                return ExitCode.FAIL
+        except Exception:
+            logger.exception("Unhandled exception when running WAL-E restore")
+        return ExitCode.FAIL
+
+    def should_use_s3_to_create_replica(self):
+        """ determine whether it makes sense to use S3 and not pg_basebackup """
+
+        threshold_megabytes = self.wal_e.threshold_mb
+        threshold_percent = self.wal_e.threshold_pct
+
+        try:
+            cmd = self.wal_e.cmd + ['backup-list', '--detail', 'LATEST']
+
+            logger.debug('calling %r', cmd)
+            wale_output = subprocess.check_output(cmd)
+
+            reader = csv.DictReader(wale_output.decode('utf-8').splitlines(),
+                                    dialect='excel-tab')
+            rows = list(reader)
+            if not len(rows):
+                logger.warning('wal-e did not find any backups')
+                return False
+
+            # This check might not add much, it was performed in the previous
+            # version of this code. since the old version rolled CSV parsing the
+            # check may have been part of the CSV parsing.
+            if len(rows) > 1:
+                logger.warning(
+                    'wal-e returned more than one row of backups: %r',
+                    rows)
+                return False
+
+            backup_info = rows[0]
+        except subprocess.CalledProcessError:
+            logger.exception("could not query wal-e latest backup")
+            return None
+
+        try:
+            backup_size = int(backup_info['expanded_size_bytes'])
+            backup_start_segment = backup_info['wal_segment_backup_start']
+            backup_start_offset = backup_info['wal_segment_offset_backup_start']
+        except KeyError:
+            logger.exception("unable to get some of WALE backup parameters")
+            return None
+
+        # WAL filename is XXXXXXXXYYYYYYYY000000ZZ, where X - timeline, Y - LSN logical log file,
+        # ZZ - 2 high digits of LSN offset. The rest of the offset is the provided decimal offset,
+        # that we have to convert to hex and 'prepend' to the high offset digits.
+
+        lsn_segment = backup_start_segment[8:16]
+        # first 2 characters of the result are 0x and the last one is L
+        lsn_offset = hex((int(backup_start_segment[16:32], 16) << 24) + int(backup_start_offset))[2:-1]
+
+        # construct the LSN from the segment and offset
+        backup_start_lsn = '{0}/{1}'.format(lsn_segment, lsn_offset)
+
+        diff_in_bytes = backup_size
+        attempts_no = 0
+        while True:
+            if self.master_connection:
+                try:
+                    # get the difference in bytes between the current WAL location and the backup start offset
+                    with psycopg2.connect(self.master_connection) as con:
+                        if con.server_version >= 100000:
+                            wal_name = 'wal'
+                            lsn_name = 'lsn'
+                        else:
+                            wal_name = 'xlog'
+                            lsn_name = 'location'
+                        con.autocommit = True
+                        with con.cursor() as cur:
+                            cur.execute("""SELECT CASE WHEN pg_is_in_recovery()
+                                                       THEN GREATEST(
+                                                                pg_{0}_{1}_diff(COALESCE(
+                                                                  pg_last_{0}_receive_{1}(), '0/0'), %s)::bigint,
+                                                                pg_{0}_{1}_diff(pg_last_{0}_replay_{1}(), %s)::bigint)
+                                                       ELSE pg_{0}_{1}_diff(pg_current_{0}_{1}(), %s)::bigint
+                                                   END""".format(wal_name, lsn_name),
+                                        (backup_start_lsn, backup_start_lsn, backup_start_lsn))
+
+                            diff_in_bytes = int(cur.fetchone()[0])
+                except psycopg2.Error:
+                    logger.exception('could not determine difference with the master location')
+                    if attempts_no < self.retries:  # retry in case of a temporarily connection issue
+                        attempts_no = attempts_no + 1
+                        time.sleep(RETRY_SLEEP_INTERVAL)
+                        continue
+                    else:
+                        if not self.no_master:
+                            return False  # do no more retries on the outer level
+                        logger.info("continue with base backup from S3 since master is not available")
+                        diff_in_bytes = 0
+                        break
+            else:
+                # always try to use WAL-E if master connection string is not available
+                diff_in_bytes = 0
+            break
+
+        # if the size of the accumulated WAL segments is more than a certan percentage of the backup size
+        # or exceeds the pre-determined size - pg_basebackup is chosen instead.
+        is_size_thresh_ok = diff_in_bytes < int(threshold_megabytes) * 1048576
+        threshold_pct_bytes = backup_size * threshold_percent / 100.0
+        is_percentage_thresh_ok = float(diff_in_bytes) < int(threshold_pct_bytes)
+        are_thresholds_ok = is_size_thresh_ok and is_percentage_thresh_ok
+
+        class Size(object):
+            def __init__(self, n_bytes, prefix=None):
+                self.n_bytes = n_bytes
+                self.prefix = prefix
+
+            def __repr__(self):
+                if self.prefix is not None:
+                    n_bytes = size_as_bytes(self.n_bytes, self.prefix)
+                else:
+                    n_bytes = self.n_bytes
+                return repr_size(n_bytes)
+
+        class HumanContext(object):
+            def __init__(self, items):
+                self.items = items
+
+            def __repr__(self):
+                return ', '.join('{}={!r}'.format(key, value)
+                                 for key, value in self.items)
+
+        human_context = repr(HumanContext([
+            ('threshold_size', Size(threshold_megabytes, 'M')),
+            ('threshold_percent', threshold_percent),
+            ('threshold_percent_size', Size(threshold_pct_bytes)),
+            ('backup_size', Size(backup_size)),
+            ('backup_diff', Size(diff_in_bytes)),
+            ('is_size_thresh_ok', is_size_thresh_ok),
+            ('is_percentage_thresh_ok', is_percentage_thresh_ok),
+        ]))
+
+        if not are_thresholds_ok:
+            logger.info('wal-e backup size diff is over threshold, falling back '
+                        'to other means of restore: %s', human_context)
+        else:
+            logger.info('Thresholds are OK, using wal-e basebackup: %s', human_context)
+        return are_thresholds_ok
+
+    def fix_subdirectory_path_if_broken(self, dirname):
+        # in case it is a symlink pointing to a non-existing location, remove it and create the actual directory
+        path = os.path.join(self.data_dir, dirname)
+        if not os.path.exists(path):
+            if os.path.islink(path):  # broken xlog symlink, to remove
+                try:
+                    os.remove(path)
+                except OSError:
+                    logger.exception("could not remove broken %s symlink pointing to %s",
+                                     dirname, os.readlink(path))
+                    return False
+            try:
+                os.mkdir(path)
+            except OSError:
+                logger.exception("coud not create missing %s directory path", dirname)
+                return False
+        return True
+
+    def create_replica_with_s3(self):
+        # if we're set up, restore the replica using fetch latest
+        try:
+            cmd = self.wal_e.cmd + ['backup-fetch',
+                                    '{}'.format(self.data_dir),
+                                    'LATEST']
+            logger.debug('calling: %r', cmd)
+            exit_code = subprocess.call(cmd)
+        except Exception as e:
+            logger.error('Error when fetching backup with WAL-E: {0}'.format(e))
+            return ExitCode.RETRY_LATER
+
+        if (exit_code == 0 and not
+           self.fix_subdirectory_path_if_broken('pg_xlog' if get_major_version(self.data_dir) < 10 else 'pg_wal')):
+            return ExitCode.FAIL
+        return exit_code
+
+
+def main():
+    logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
+    parser = argparse.ArgumentParser(description='Script to image replicas using WAL-E')
+    parser.add_argument('--scope', required=True)
+    parser.add_argument('--role', required=False)
+    parser.add_argument('--datadir', required=True)
+    parser.add_argument('--connstring', required=True)
+    parser.add_argument('--retries', type=int, default=1)
+    parser.add_argument('--envdir', required=True)
+    parser.add_argument('--threshold_megabytes', type=int, default=10240)
+    parser.add_argument('--threshold_backup_size_percentage', type=int, default=30)
+    parser.add_argument('--use_iam', type=int, default=0)
+    parser.add_argument('--no_master', type=int, default=0)
+    args = parser.parse_args()
+
+    exit_code = None
+    assert args.retries >= 0
+
+    # Retry cloning in a loop. We do separate retries for the master
+    # connection attempt inside should_use_s3_to_create_replica,
+    # because we need to differentiate between the last attempt and
+    # the rest and make a decision when the last attempt fails on
+    # whether to use WAL-E or not depending on the no_master flag.
+    for _ in range(0, args.retries + 1):
+        restore = WALERestore(scope=args.scope, datadir=args.datadir, connstring=args.connstring,
+                              env_dir=args.envdir, threshold_mb=args.threshold_megabytes,
+                              threshold_pct=args.threshold_backup_size_percentage, use_iam=args.use_iam,
+                              no_master=args.no_master, retries=args.retries)
+        exit_code = restore.run()
+        if not exit_code == ExitCode.RETRY_LATER:  # only WAL-E failures lead to the retry
+            logger.debug('exit_code is %r, not retrying', exit_code)
+            break
+        time.sleep(RETRY_SLEEP_INTERVAL)
+
+    return exit_code
+
+
+if __name__ == '__main__':
+    sys.exit(main())

patroni/utils.py

@@ -1,95 +1,196 @@
-import datetime
-import os
 import random
-import re
-import signal
-import sys
 import time
 
+from dateutil import tz
 from patroni.exceptions import PatroniException
 
-ignore_sigterm = False
-interrupted_sleep = False
-reap_children = False
-
-_DATE_TIME_RE = re.compile(r'''^
-(?P<year>\d{4})\-(?P<month>\d{2})\-(?P<day>\d{2})  # date
-T
-(?P<hour>\d{2}):(?P<minute>\d{2}):(?P<second>\d{2})\.(?P<microsecond>\d{6})  # time
-\d*Z$''', re.X)
+tzutc = tz.tzutc()
 
 
-def parse_datetime(time_str):
+def deep_compare(obj1, obj2):
     """
-    >>> parse_datetime('2015-06-10T12:56:30.552539016Z')
-    datetime.datetime(2015, 6, 10, 12, 56, 30, 552539)
-    >>> parse_datetime('2015-06-10 12:56:30.552539016Z')
-    """
-    m = _DATE_TIME_RE.match(time_str)
-    if not m:
-        return None
-    p = dict((n, int(m.group(n))) for n in 'year month day hour minute second microsecond'.split(' '))
-    return datetime.datetime(**p)
-
-
-def calculate_ttl(expiration):
-    """
-    >>> calculate_ttl(None)
-    >>> calculate_ttl('2015-06-10 12:56:30.552539016Z')
-    >>> calculate_ttl('2015-06-10T12:56:30.552539016Z') < 0
+    >>> deep_compare({'1': None}, {})
+    False
+    >>> deep_compare({'1': {}}, {'1': None})
+    False
+    >>> deep_compare({'1': [1]}, {'1': [2]})
+    False
+    >>> deep_compare({'1': 2}, {'1': '2'})
+    True
+    >>> deep_compare({'1': {'2': [3, 4]}}, {'1': {'2': [3, 4]}})
     True
     """
-    if not expiration:
-        return None
-    expiration = parse_datetime(expiration)
-    if not expiration:
-        return None
-    now = datetime.datetime.utcnow()
-    return int((expiration - now).total_seconds())
+
+    if set(list(obj1.keys())) != set(list(obj2.keys())):  # Objects have different sets of keys
+        return False
+
+    for key, value in obj1.items():
+        if isinstance(value, dict):
+            if not (isinstance(obj2[key], dict) and deep_compare(value, obj2[key])):
+                return False
+        elif str(value) != str(obj2[key]):
+            return False
+    return True
 
 
-def sigterm_handler(signo, stack_frame):
-    global ignore_sigterm
-    if not ignore_sigterm:
-        ignore_sigterm = True
-        sys.exit()
+def patch_config(config, data):
+    """recursively 'patch' `config` with `data`
+    :returns: `!True` if the `config` was changed"""
+    is_changed = False
+    for name, value in data.items():
+        if value is None:
+            if config.pop(name, None) is not None:
+                is_changed = True
+        elif name in config:
+            if isinstance(value, dict):
+                if isinstance(config[name], dict):
+                    if patch_config(config[name], value):
+                        is_changed = True
+                else:
+                    config[name] = value
+                    is_changed = True
+            elif str(config[name]) != str(value):
+                config[name] = value
+                is_changed = True
+        else:
+            config[name] = value
+            is_changed = True
+    return is_changed
 
 
-def sigchld_handler(signo, stack_frame):
-    global interrupted_sleep, reap_children
-    reap_children = interrupted_sleep = True
+def parse_bool(value):
+    """
+    >>> parse_bool(1)
+    True
+    >>> parse_bool('off')
+    False
+    >>> parse_bool('foo')
+    """
+    value = str(value).lower()
+    if value in ('on', 'true', 'yes', '1'):
+        return True
+    if value in ('off', 'false', 'no', '0'):
+        return False
 
 
-def sleep(interval):
-    global interrupted_sleep
-    current_time = time.time()
-    end_time = current_time + interval
-    while current_time < end_time:
-        interrupted_sleep = False
-        time.sleep(end_time - current_time)
-        if not interrupted_sleep:  # we will ignore only sigchld
-            break
-        current_time = time.time()
-    interrupted_sleep = False
+def strtol(value, strict=True):
+    """As most as possible close equivalent of strtol(3) function (with base=0),
+       used by postgres to parse parameter values.
+    >>> strtol(0) == (0, '')
+    True
+    >>> strtol(1) == (1, '')
+    True
+    >>> strtol(9) == (9, '')
+    True
+    >>> strtol(' +0x400MB') == (1024, 'MB')
+    True
+    >>> strtol(' -070d') == (-56, 'd')
+    True
+    >>> strtol(' d ') == (None, 'd')
+    True
+    >>> strtol('9s', False) == (9, 's')
+    True
+    >>> strtol(' s ', False) == (1, 's')
+    True
+    """
+    value = str(value).strip()
+    ln = len(value)
+    i = 0
+    # skip sign:
+    if i < ln and value[i] in ('-', '+'):
+        i += 1
+
+    # we always expect to get digit in the beginning
+    if i < ln and value[i].isdigit():
+        if value[i] == '0':
+            i += 1
+            if i < ln and value[i] in ('x', 'X'):  # '0' followed by 'x': HEX
+                base = 16
+                i += 1
+            else:  # just starts with '0': OCT
+                base = 8
+        else:  # any other digit: DEC
+            base = 10
+
+        ret = None
+        while i <= ln:
+            try:  # try to find maximally long number
+                i += 1  # by giving to `int` longer and longer strings
+                ret = int(value[:i], base)
+            except ValueError:  # until we will not get an exception or end of the string
+                i -= 1
+                break
+        if ret is not None:  # yay! there is a number in the beginning of the string
+            return ret, value[i:].strip()  # return the number and the "rest"
+
+    return (None if strict else 1), value.strip()
 
 
-def setup_signal_handlers():
-    signal.signal(signal.SIGTERM, sigterm_handler)
-    signal.signal(signal.SIGCHLD, sigchld_handler)
+def parse_int(value, base_unit=None):
+    """
+    >>> parse_int('1') == 1
+    True
+    >>> parse_int(' 0x400 MB ', '16384kB') == 64
+    True
+    >>> parse_int('1MB', 'kB') == 1024
+    True
+    >>> parse_int('1000 ms', 's') == 1
+    True
+    >>> parse_int('1GB', 'MB') is None
+    True
+    >>> parse_int(0) == 0
+    True
+    """
+
+    convert = {
+        'kB': {'kB': 1, 'MB': 1024, 'GB': 1024 * 1024, 'TB': 1024 * 1024 * 1024},
+        'ms': {'ms': 1, 's': 1000, 'min': 1000 * 60, 'h': 1000 * 60 * 60, 'd': 1000 * 60 * 60 * 24},
+        's': {'ms': -1000, 's': 1, 'min': 60, 'h': 60 * 60, 'd': 60 * 60 * 24},
+        'min': {'ms': -1000 * 60, 's': -60, 'min': 1, 'h': 60, 'd': 60 * 24}
+    }
+
+    value, unit = strtol(value)
+    if value is not None:
+        if not unit:
+            return value
+
+        if base_unit and base_unit not in convert:
+            base_value, base_unit = strtol(base_unit, False)
+        else:
+            base_value = 1
+        if base_unit in convert and unit in convert[base_unit]:
+            multiplier = convert[base_unit][unit]
+            if multiplier < 0:
+                value /= -multiplier
+            else:
+                value *= multiplier
+            return int(value/base_value)
 
 
-def reap_children():
-    global reap_children
-    if reap_children:
-        try:
-            while True:
-                ret = os.waitpid(-1, os.WNOHANG)
-                if ret == (0, 0):
-                    break
-        except OSError:
-            pass
-        finally:
-            reap_children = False
+def compare_values(vartype, unit, old_value, new_value):
+    """
+    >>> compare_values('enum', None, 'remote_write', 'REMOTE_WRITE')
+    True
+    >>> compare_values('real', None, '1.23', 1.23)
+    True
+    """
+
+    # if the integer or bool new_value is not correct this function will return False
+    if vartype == 'bool':
+        old_value = parse_bool(old_value)
+        new_value = parse_bool(new_value)
+    elif vartype == 'integer':
+        old_value = parse_int(old_value)
+        new_value = parse_int(new_value, unit)
+    elif vartype == 'enum':
+        return str(old_value).lower() == str(new_value).lower()
+    else:  # ('string', 'real')
+        return str(old_value) == str(new_value)
+    return old_value is not None and new_value is not None and old_value == new_value
+
+
+def _sleep(interval):
+    time.sleep(interval)
 
 
 class RetryFailedError(PatroniException):
@@ -97,12 +198,12 @@ class RetryFailedError(PatroniException):
     """Raised when retrying an operation ultimately failed, after retrying the maximum number of attempts."""
 
 
-class Retry:
+class Retry(object):
 
     """Helper for retrying a method in the face of retry-able exceptions"""
 
     def __init__(self, max_tries=1, delay=0.1, backoff=2, max_jitter=0.8, max_delay=3600,
-                 sleep_func=sleep, deadline=None, retry_exceptions=PatroniException):
+                 sleep_func=_sleep, deadline=None, retry_exceptions=PatroniException):
         """Create a :class:`Retry` instance for retrying function calls
 
         :param max_tries: How many times to retry the command. -1 means infinite tries.
@@ -163,3 +264,20 @@ class Retry:
                 else:
                     self.sleep_func(sleeptime)
                 self._cur_delay = min(self._cur_delay * self.backoff, self.max_delay)
+
+
+def polling_loop(timeout, interval=1):
+    """Returns an iterator that returns values until timeout has passed. Timeout is measured from start of iteration."""
+    start_time = time.time()
+    iteration = 0
+    end_time = start_time + timeout
+    while time.time() < end_time:
+        yield iteration
+        iteration += 1
+        time.sleep(interval)
+
+
+def split_host_port(value, default_port):
+    t = value.rsplit(':', 1)
+    t.append(default_port)
+    return t[0], int(t[1])

patroni/version.py

@@ -1 +1 @@
-__version__ = '0.2'
+__version__ = '1.4.4'

patroni/watchdog/__init__.py

@@ -0,0 +1,2 @@
+from patroni.watchdog.base import WatchdogError, Watchdog
+__all__ = ['WatchdogError', 'Watchdog']

patroni/watchdog/base.py

@@ -0,0 +1,313 @@
+import abc
+import logging
+import platform
+import six
+import sys
+from threading import RLock
+
+from patroni.exceptions import WatchdogError
+
+__all__ = ['WatchdogError', 'Watchdog']
+
+logger = logging.getLogger(__name__)
+
+MODE_REQUIRED = 'required'    # Will not run if a watchdog is not available
+MODE_AUTOMATIC = 'automatic'  # Will use a watchdog if one is available
+MODE_OFF = 'off'              # Will not try to use a watchdog
+
+
+def parse_mode(mode):
+    if mode is False:
+        return MODE_OFF
+    mode = mode.lower()
+    if mode in ['require', 'required']:
+        return MODE_REQUIRED
+    elif mode in ['auto', 'automatic']:
+        return MODE_AUTOMATIC
+    else:
+        if mode not in ['off', 'disable', 'disabled']:
+            logger.warning("Watchdog mode {0} not recognized, disabling watchdog".format(mode))
+        return MODE_OFF
+
+
+def synchronized(func):
+    def wrapped(self, *args, **kwargs):
+        with self._lock:
+            return func(self, *args, **kwargs)
+    return wrapped
+
+
+class WatchdogConfig(object):
+    """Helper to contain a snapshot of configuration"""
+    def __init__(self, config):
+        self.mode = parse_mode(config['watchdog'].get('mode', 'automatic'))
+        self.ttl = config['ttl']
+        self.loop_wait = config['loop_wait']
+        self.safety_margin = config['watchdog'].get('safety_margin', 5)
+        self.driver = config['watchdog'].get('driver', 'default')
+        self.driver_config = dict((k, v) for k, v in config['watchdog'].items()
+                                  if k not in ['mode', 'safety_margin', 'driver'])
+
+    def __eq__(self, other):
+        return isinstance(other, WatchdogConfig) and \
+            all(getattr(self, attr) == getattr(other, attr) for attr in
+                ['mode', 'ttl', 'loop_wait', 'safety_margin', 'driver', 'driver_config'])
+
+    def __ne__(self, other):
+        return not self == other
+
+    def get_impl(self):
+        if self.driver == 'testing':
+            from patroni.watchdog.linux import TestingWatchdogDevice
+            return TestingWatchdogDevice.from_config(self.driver_config)
+        elif platform.system() == 'Linux' and self.driver == 'default':
+            from patroni.watchdog.linux import LinuxWatchdogDevice
+            return LinuxWatchdogDevice.from_config(self.driver_config)
+        else:
+            return NullWatchdog()
+
+    @property
+    def timeout(self):
+        if self.safety_margin == -1:
+            return int(self.ttl // 2)
+        else:
+            return self.ttl - self.safety_margin
+
+    @property
+    def timing_slack(self):
+        return self.timeout - self.loop_wait
+
+
+class Watchdog(object):
+    """Facade to dynamically manage watchdog implementations and handle config changes.
+
+    When activation fails underlying implementation will be switched to a Null implementation. To avoid log spam
+    activation will only be retried when watchdog configuration is changed."""
+    def __init__(self, config):
+        self.active_config = self.config = WatchdogConfig(config)
+        self._lock = RLock()
+        self.active = False
+
+        if self.config.mode == MODE_OFF:
+            self.impl = NullWatchdog()
+        else:
+            self.impl = self.config.get_impl()
+            if self.config.mode == MODE_REQUIRED and self.impl.is_null:
+                logger.error("Configuration requires a watchdog, but watchdog is not supported on this platform.")
+                sys.exit(1)
+
+    @synchronized
+    def reload_config(self, config):
+        self.config = WatchdogConfig(config)
+        # Turning a watchdog off can always be done immediately
+        if self.config.mode == MODE_OFF:
+            if self.active:
+                self._disable()
+            self.active_config = self.config
+            self.impl = NullWatchdog()
+        # If watchdog is not active we can apply config immediately to show any warnings early. Otherwise we need to
+        # delay until next time a keepalive is sent so timeout matches up with leader key update.
+        if not self.active:
+            if self.config.driver != self.active_config.driver or \
+               self.config.driver_config != self.active_config.driver_config:
+                self.impl = self.config.get_impl()
+            self.active_config = self.config
+
+    @synchronized
+    def activate(self):
+        """Activates the watchdog device with suitable timeouts. While watchdog is active keepalive needs
+        to be called every time loop_wait expires.
+
+        :returns False if a safe watchdog could not be configured, but is required.
+        """
+        self.active = True
+        return self._activate()
+
+    def _activate(self):
+        self.active_config = self.config
+
+        if self.config.timing_slack < 0:
+            logger.warning('Watchdog not supported because leader TTL {0} is less than 2x loop_wait {1}'
+                           .format(self.config.ttl, self.config.loop_wait))
+            self.impl = NullWatchdog()
+
+        try:
+            self.impl.open()
+            actual_timeout = self._set_timeout()
+        except WatchdogError as e:
+            logger.warning("Could not activate %s: %s", self.impl.describe(), e)
+            self.impl = NullWatchdog()
+
+        if self.impl.is_running and not self.impl.can_be_disabled:
+            logger.warning("Watchdog implementation can't be disabled."
+                           " Watchdog will trigger after Patroni loses leader key.")
+
+        if not self.impl.is_running or actual_timeout > self.config.timeout:
+            if self.config.mode == MODE_REQUIRED:
+                if self.impl.is_null:
+                    logger.error("Configuration requires watchdog, but watchdog could not be configured.")
+                else:
+                    logger.error("Configuration requires watchdog, but a safe watchdog timeout {0} could"
+                                 " not be configured. Watchdog timeout is {1}.".format(
+                                     self.config.timeout, actual_timeout))
+                return False
+            else:
+                if not self.impl.is_null:
+                    logger.warning("Watchdog timeout {0} seconds does not ensure safe termination within {1} seconds"
+                                   .format(actual_timeout, self.config.timeout))
+
+        if self.is_running:
+            logger.info("{0} activated with {1} second timeout, timing slack {2} seconds"
+                        .format(self.impl.describe(), actual_timeout, self.config.timing_slack))
+        else:
+            if self.config.mode == MODE_REQUIRED:
+                logger.error("Configuration requires watchdog, but watchdog could not be activated")
+                return False
+
+        return True
+
+    def _set_timeout(self):
+        if self.impl.has_set_timeout():
+            self.impl.set_timeout(self.config.timeout)
+
+        # Safety checks for watchdog implementations that don't support configurable timeouts
+        actual_timeout = self.impl.get_timeout()
+        if self.impl.is_running and actual_timeout < self.config.loop_wait:
+            logger.error('loop_wait of {0} seconds is too long for watchdog {1} second timeout'
+                         .format(self.config.loop_wait, actual_timeout))
+            if self.impl.can_be_disabled:
+                logger.info('Disabling watchdog due to unsafe timeout.')
+                self.impl.close()
+                self.impl = NullWatchdog()
+                return None
+        return actual_timeout
+
+    @synchronized
+    def disable(self):
+        self._disable()
+        self.active = False
+
+    def _disable(self):
+        try:
+            if self.impl.is_running and not self.impl.can_be_disabled:
+                # Give sysadmin some extra time to clean stuff up.
+                self.impl.keepalive()
+                logger.warning("Watchdog implementation can't be disabled. System will reboot after "
+                               "{0} seconds when watchdog times out.".format(self.impl.get_timeout()))
+            self.impl.close()
+        except WatchdogError as e:
+            logger.error("Error while disabling watchdog: %s", e)
+
+    @synchronized
+    def keepalive(self):
+        try:
+            if self.active:
+                self.impl.keepalive()
+            # In case there are any pending configuration changes apply them now.
+            if self.active and self.config != self.active_config:
+                if self.config.mode != MODE_OFF and self.active_config.mode == MODE_OFF:
+                    self.impl = self.config.get_impl()
+                    self._activate()
+                if self.config.driver != self.active_config.driver \
+                   or self.config.driver_config != self.active_config.driver_config:
+                    self._disable()
+                    self.impl = self.config.get_impl()
+                    self._activate()
+                if self.config.timeout != self.active_config.timeout:
+                    self.impl.set_timeout(self.config.timeout)
+        except WatchdogError as e:
+            logger.error("Error while sending keepalive: %s", e)
+
+    @property
+    @synchronized
+    def is_running(self):
+        return self.impl.is_running
+
+    @property
+    @synchronized
+    def is_healthy(self):
+        if self.config.mode != MODE_REQUIRED:
+            return True
+        return self.config.timing_slack >= 0 and self.impl.is_healthy
+
+
+@six.add_metaclass(abc.ABCMeta)
+class WatchdogBase(object):
+    """A watchdog object when opened requires periodic calls to keepalive.
+    When keepalive is not called within a timeout the system will be terminated."""
+    is_null = False
+
+    @property
+    def is_running(self):
+        """Returns True when watchdog is activated and capable of performing it's task."""
+        return False
+
+    @property
+    def is_healthy(self):
+        """Returns False when calling open() is known to fail."""
+        return False
+
+    @property
+    def can_be_disabled(self):
+        """Returns True when watchdog will be disabled by calling close(). Some watchdog devices
+        will keep running no matter what once activated. May raise WatchdogError if called without
+        calling open() first."""
+        return True
+
+    @abc.abstractmethod
+    def open(self):
+        """Open watchdog device.
+
+        When watchdog is opened keepalive must be called. Returns nothing on success
+        or raises WatchdogError if the device could not be opened."""
+
+    @abc.abstractmethod
+    def close(self):
+        """Gracefully close watchdog device."""
+
+    @abc.abstractmethod
+    def keepalive(self):
+        """Resets the watchdog timer.
+
+        Watchdog must be open when keepalive is called."""
+
+    @abc.abstractmethod
+    def get_timeout(self):
+        """Returns the current keepalive timeout in effect."""
+
+    @staticmethod
+    def has_set_timeout():
+        """Returns True if setting a timeout is supported."""
+        return False
+
+    def set_timeout(self, timeout):
+        """Set the watchdog timer timeout.
+
+        :param timeout: watchdog timeout in seconds"""
+        raise WatchdogError("Setting timeout is not supported on {0}".format(self.describe()))
+
+    def describe(self):
+        """Human readable name for this device"""
+        return self.__class__.__name__
+
+    @classmethod
+    def from_config(cls, config):
+        return cls()
+
+
+class NullWatchdog(WatchdogBase):
+    """Null implementation when watchdog is not supported."""
+    is_null = True
+
+    def open(self):
+        return
+
+    def close(self):
+        return
+
+    def keepalive(self):
+        return
+
+    def get_timeout(self):
+        # A big enough number to not matter
+        return 1000000000

patroni/watchdog/linux.py

@@ -0,0 +1,234 @@
+import collections
+import ctypes
+import fcntl
+import os
+import platform
+from patroni.watchdog.base import WatchdogBase, WatchdogError
+
+# Pythonification of linux/ioctl.h
+IOC_NONE = 0
+IOC_WRITE = 1
+IOC_READ = 2
+
+IOC_NRBITS = 8
+IOC_TYPEBITS = 8
+IOC_SIZEBITS = 14
+IOC_DIRBITS = 2
+
+# Non-generic platform special cases
+machine = platform.machine()
+if machine in ['mips', 'sparc', 'powerpc', 'ppc64']:
+    IOC_SIZEBITS = 13
+    IOC_DIRBITS = 3
+    IOC_NONE, IOC_WRITE, IOC_READ = 1, 2, 4
+elif machine == 'parisc':
+    IOC_WRITE, IOC_READ = 2, 1
+
+IOC_NRSHIFT = 0
+IOC_TYPESHIFT = IOC_NRSHIFT + IOC_NRBITS
+IOC_SIZESHIFT = IOC_TYPESHIFT + IOC_TYPEBITS
+IOC_DIRSHIFT = IOC_SIZESHIFT + IOC_SIZEBITS
+
+
+def IOW(type_, nr, size):
+    return IOC(IOC_WRITE, type_, nr, size)
+
+
+def IOR(type_, nr, size):
+    return IOC(IOC_READ, type_, nr, size)
+
+
+def IOWR(type_, nr, size):
+    return IOC(IOC_READ | IOC_WRITE, type_, nr, size)
+
+
+def IOC(dir_, type_, nr, size):
+    return (dir_ << IOC_DIRSHIFT) \
+        | (ord(type_) << IOC_TYPESHIFT) \
+        | (nr << IOC_NRSHIFT) \
+        | (size << IOC_SIZESHIFT)
+
+
+# Pythonification of linux/watchdog.h
+
+WATCHDOG_IOCTL_BASE = 'W'
+
+
+class watchdog_info(ctypes.Structure):
+    _fields_ = [
+        ('options', ctypes.c_uint32),           # Options the card/driver supports
+        ('firmware_version', ctypes.c_uint32),  # Firmware version of the card
+        ('identity', ctypes.c_uint8 * 32),      # Identity of the board
+    ]
+
+
+struct_watchdog_info_size = ctypes.sizeof(watchdog_info)
+int_size = ctypes.sizeof(ctypes.c_int)
+
+WDIOC_GETSUPPORT = IOR(WATCHDOG_IOCTL_BASE, 0, struct_watchdog_info_size)
+WDIOC_GETSTATUS = IOR(WATCHDOG_IOCTL_BASE, 1, int_size)
+WDIOC_GETBOOTSTATUS = IOR(WATCHDOG_IOCTL_BASE, 2, int_size)
+WDIOC_GETTEMP = IOR(WATCHDOG_IOCTL_BASE, 3, int_size)
+WDIOC_SETOPTIONS = IOR(WATCHDOG_IOCTL_BASE, 4, int_size)
+WDIOC_KEEPALIVE = IOR(WATCHDOG_IOCTL_BASE, 5, int_size)
+WDIOC_SETTIMEOUT = IOWR(WATCHDOG_IOCTL_BASE, 6, int_size)
+WDIOC_GETTIMEOUT = IOR(WATCHDOG_IOCTL_BASE, 7, int_size)
+WDIOC_SETPRETIMEOUT = IOWR(WATCHDOG_IOCTL_BASE, 8, int_size)
+WDIOC_GETPRETIMEOUT = IOR(WATCHDOG_IOCTL_BASE, 9, int_size)
+WDIOC_GETTIMELEFT = IOR(WATCHDOG_IOCTL_BASE, 10, int_size)
+
+
+WDIOF_UNKNOWN = -1  # Unknown flag error
+WDIOS_UNKNOWN = -1  # Unknown status error
+
+WDIOF = {
+    "OVERHEAT": 0x0001,       # Reset due to CPU overheat
+    "FANFAULT": 0x0002,       # Fan failed
+    "EXTERN1": 0x0004,        # External relay 1
+    "EXTERN2": 0x0008,        # External relay 2
+    "POWERUNDER": 0x0010,     # Power bad/power fault
+    "CARDRESET": 0x0020,      # Card previously reset the CPU
+    "POWEROVER": 0x0040,      # Power over voltage
+    "SETTIMEOUT": 0x0080,     # Set timeout (in seconds)
+    "MAGICCLOSE": 0x0100,     # Supports magic close char
+    "PRETIMEOUT": 0x0200,     # Pretimeout (in seconds), get/set
+    "ALARMONLY": 0x0400,      # Watchdog triggers a management or other external alarm not a reboot
+    "KEEPALIVEPING": 0x8000,  # Keep alive ping reply
+}
+
+WDIOS = {
+    "DISABLECARD": 0x0001,    # Turn off the watchdog timer
+    "ENABLECARD": 0x0002,     # Turn on the watchdog timer
+    "TEMPPANIC": 0x0004,      # Kernel panic on temperature trip
+}
+
+# Implementation
+
+
+class WatchdogInfo(collections.namedtuple('WatchdogInfo', 'options,version,identity')):
+    """Watchdog descriptor from the kernel"""
+    def __getattr__(self, name):
+        """Convenience has_XYZ attributes for checking WDIOF bits in options"""
+        if name.startswith('has_') and name[4:] in WDIOF:
+            return bool(self.options & WDIOF[name[4:]])
+
+        raise AttributeError("WatchdogInfo instance has no attribute '{0}'".format(name))
+
+
+class LinuxWatchdogDevice(WatchdogBase):
+    DEFAULT_DEVICE = '/dev/watchdog'
+
+    def __init__(self, device):
+        self.device = device
+        self._support_cache = None
+        self._fd = None
+
+    @classmethod
+    def from_config(cls, config):
+        device = config.get('device', cls.DEFAULT_DEVICE)
+        return cls(device)
+
+    @property
+    def is_running(self):
+        return self._fd is not None
+
+    @property
+    def is_healthy(self):
+        return os.path.exists(self.device) and os.access(self.device, os.W_OK)
+
+    def open(self):
+        try:
+            self._fd = os.open(self.device, os.O_WRONLY)
+        except OSError as e:
+            raise WatchdogError("Can't open watchdog device: {0}".format(e))
+
+    def close(self):
+        if self.is_running:
+            try:
+                os.write(self._fd, b'V')
+                os.close(self._fd)
+                self._fd = None
+            except OSError as e:
+                raise WatchdogError("Error while closing {0}: {1}".format(self.describe(), e))
+
+    @property
+    def can_be_disabled(self):
+        return self.get_support().has_MAGICCLOSE
+
+    def _ioctl(self, func, arg):
+        """Runs the specified ioctl on the underlying fd.
+
+        Raises WatchdogError if the device is closed.
+        Raises OSError or IOError (Python 2) when the ioctl fails."""
+        if self._fd is None:
+            raise WatchdogError("Watchdog device is closed")
+        fcntl.ioctl(self._fd, func, arg, True)
+
+    def get_support(self):
+        if self._support_cache is None:
+            info = watchdog_info()
+            try:
+                self._ioctl(WDIOC_GETSUPPORT, info)
+            except (WatchdogError, OSError, IOError) as e:
+                raise WatchdogError("Could not get information about watchdog device: {}".format(e))
+            self._support_cache = WatchdogInfo(info.options,
+                                               info.firmware_version,
+                                               bytearray(info.identity).decode(errors='ignore').rstrip('\x00'))
+        return self._support_cache
+
+    def describe(self):
+        dev_str = " at {0}".format(self.device) if self.device != self.DEFAULT_DEVICE else ""
+        ver_str = ""
+        identity = "Linux watchdog device"
+        if self._fd:
+            try:
+                _, version, identity = self.get_support()
+                ver_str = " (firmware {0})".format(version) if version else ""
+            except WatchdogError:
+                pass
+
+        return identity + ver_str + dev_str
+
+    def keepalive(self):
+        try:
+            os.write(self._fd, b'1')
+        except OSError as e:
+            raise WatchdogError("Could not send watchdog keepalive: {0}".format(e))
+
+    def has_set_timeout(self):
+        """Returns True if setting a timeout is supported."""
+        return self.get_support().has_SETTIMEOUT
+
+    def set_timeout(self, timeout):
+        timeout = int(timeout)
+        if not 0 < timeout < 0xFFFF:
+            raise WatchdogError("Invalid timeout {0}. Supported values are between 1 and 65535".format(timeout))
+        try:
+            self._ioctl(WDIOC_SETTIMEOUT, ctypes.c_int(timeout))
+        except (WatchdogError, OSError, IOError) as e:
+            raise WatchdogError("Could not set timeout on watchdog device: {}".format(e))
+
+    def get_timeout(self):
+        timeout = ctypes.c_int()
+        try:
+            self._ioctl(WDIOC_GETTIMEOUT, timeout)
+        except (WatchdogError, OSError, IOError) as e:
+            raise WatchdogError("Could not get timeout on watchdog device: {}".format(e))
+        return timeout.value
+
+
+class TestingWatchdogDevice(LinuxWatchdogDevice):
+    """Converts timeout ioctls to regular writes that can be intercepted from a named pipe."""
+    timeout = 60
+
+    def get_support(self):
+        return WatchdogInfo(WDIOF['MAGICCLOSE'] | WDIOF['SETTIMEOUT'], 0, "Watchdog test harness")
+
+    def set_timeout(self, timeout):
+        buf = "Ctimeout={0}\n".format(timeout).encode('utf8')
+        while len(buf):
+            buf = buf[os.write(self._fd, buf):]
+        self.timeout = timeout
+
+    def get_timeout(self):
+        return self.timeout

patroni/zookeeper.py

@@ -1,285 +0,0 @@
-import logging
-import random
-import requests
-import time
-
-from kazoo.client import KazooClient, KazooState
-from kazoo.exceptions import NoNodeError, NodeExistsError
-from patroni.dcs import AbstractDCS, Cluster, Failover, Leader, Member
-from patroni.exceptions import DCSError
-from patroni.utils import sleep
-from requests.exceptions import RequestException
-
-logger = logging.getLogger(__name__)
-
-
-class ZooKeeperError(DCSError):
-    pass
-
-
-class ExhibitorEnsembleProvider:
-
-    TIMEOUT = 3.1
-
-    def __init__(self, hosts, port, uri_path='/exhibitor/v1/cluster/list', poll_interval=300):
-        self._exhibitor_port = port
-        self._uri_path = uri_path
-        self._poll_interval = poll_interval
-        self._exhibitors = hosts
-        self._master_exhibitors = hosts
-        self._zookeeper_hosts = ''
-        self._next_poll = None
-        while not self.poll():
-            logger.info('waiting on exhibitor')
-            sleep(5)
-
-    def poll(self):
-        if self._next_poll and self._next_poll > time.time():
-            return False
-
-        json = self._query_exhibitors(self._exhibitors)
-        if not json:
-            json = self._query_exhibitors(self._master_exhibitors)
-
-        if isinstance(json, dict) and 'servers' in json and 'port' in json:
-            self._next_poll = time.time() + self._poll_interval
-            zookeeper_hosts = ','.join([h + ':' + str(json['port']) for h in sorted(json['servers'])])
-            if self._zookeeper_hosts != zookeeper_hosts:
-                logger.info('ZooKeeper connection string has changed: %s => %s', self._zookeeper_hosts, zookeeper_hosts)
-                self._zookeeper_hosts = zookeeper_hosts
-                self._exhibitors = json['servers']
-                return True
-        return False
-
-    def _query_exhibitors(self, exhibitors):
-        random.shuffle(exhibitors)
-        for host in exhibitors:
-            uri = 'http://{}:{}{}'.format(host, self._exhibitor_port, self._uri_path)
-            try:
-                response = requests.get(uri, timeout=self.TIMEOUT)
-                return response.json()
-            except RequestException:
-                pass
-        return None
-
-    @property
-    def zookeeper_hosts(self):
-        return self._zookeeper_hosts
-
-
-class ZooKeeper(AbstractDCS):
-
-    def __init__(self, name, config):
-        super(ZooKeeper, self).__init__(name, config)
-
-        hosts = config.get('hosts', [])
-        if isinstance(hosts, list):
-            hosts = ','.join(hosts)
-
-        self.exhibitor = None
-        if 'exhibitor' in config:
-            exhibitor = config['exhibitor']
-            interval = exhibitor.get('poll_interval', 300)
-            self.exhibitor = ExhibitorEnsembleProvider(exhibitor['hosts'], exhibitor['port'], poll_interval=interval)
-            hosts = self.exhibitor.zookeeper_hosts
-
-        self.client = KazooClient(hosts=hosts,
-                                  timeout=(config.get('session_timeout', None) or 30),
-                                  command_retry={
-                                      'deadline': (config.get('reconnect_timeout', None) or 10),
-                                      'max_delay': 1,
-                                      'max_tries': -1},
-                                  connection_retry={'max_delay': 1, 'max_tries': -1})
-        self.client.add_listener(self.session_listener)
-
-        self._my_member_data = None
-        self.fetch_cluster = True
-        self.last_leader_operation = 0
-
-        self.client.start(None)
-
-    def session_listener(self, state):
-        if state in [KazooState.SUSPENDED, KazooState.LOST]:
-            self.cluster_watcher(None)
-
-    def cluster_watcher(self, event):
-        self.fetch_cluster = True
-        self.event.set()
-
-    def get_node(self, key, watch=None):
-        try:
-            ret = self.client.get(key, watch)
-            return (ret[0].decode('utf-8'), ret[1])
-        except NoNodeError:
-            return None
-
-    @staticmethod
-    def member(name, value, znode):
-        return Member.from_node(znode.version, name, znode.ephemeralOwner, value)
-
-    def get_children(self, key, watch=None):
-        try:
-            return self.client.get_children(key, watch)
-        except NoNodeError:
-            return []
-
-    def load_members(self):
-        members = []
-        for member in self.get_children(self.members_path, self.cluster_watcher):
-            data = self.get_node(self.members_path + member)
-            if data is not None:
-                members.append(self.member(member, *data))
-        return members
-
-    def _inner_load_cluster(self):
-        self.fetch_cluster = False
-        self.event.clear()
-        nodes = set(self.get_children(self.client_path(''), self.cluster_watcher))
-        if not nodes:
-            self.fetch_cluster = True
-
-        # get initialize flag
-        initialize = self._INITIALIZE in nodes
-
-        # get list of members
-        members = self.load_members() if self._MEMBERS[:-1] in nodes else []
-
-        # get leader
-        leader = self.get_node(self.leader_path) if self._LEADER in nodes else None
-        if leader:
-            client_id = self.client.client_id
-            if leader[0] == self._name and client_id is not None and client_id[0] != leader[1].ephemeralOwner:
-                logger.info('I am leader but not owner of the session. Removing leader node')
-                self.client.delete(self.leader_path)
-                leader = None
-
-            if leader:
-                member = Member(-1, leader[0], None, {})
-                member = ([m for m in members if m.name == leader[0]] or [member])[0]
-                leader = Leader(leader[1].version, leader[1].ephemeralOwner, member)
-                self.fetch_cluster = member.index == -1
-
-        # failover key
-        failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
-        if failover:
-            failover = Failover.from_node(failover[1].version, failover[0])
-
-        # get last leader operation
-        optime = self.get_node(self.leader_optime_path) if self._OPTIME in nodes and self.fetch_cluster else None
-        self.last_leader_operation = 0 if optime is None else int(optime[0])
-        self._cluster = Cluster(initialize, leader, self.last_leader_operation, members, failover)
-
-    def _load_cluster(self):
-        if self.exhibitor and self.exhibitor.poll():
-            self.client.set_hosts(self.exhibitor.zookeeper_hosts)
-
-        if self.fetch_cluster:
-            try:
-                self.client.retry(self._inner_load_cluster)
-            except:
-                logger.exception('get_cluster')
-                self.session_listener(KazooState.LOST)
-                raise ZooKeeperError('ZooKeeper in not responding properly')
-
-    def _create(self, path, value, **kwargs):
-        try:
-            self.client.retry(self.client.create, path, value.encode('utf-8'), **kwargs)
-            return True
-        except:
-            return False
-
-    def attempt_to_acquire_leader(self):
-        ret = self._create(self.leader_path, self._name, makepath=True, ephemeral=True)
-        ret or logger.info('Could not take out TTL lock')
-        return ret
-
-    def set_failover_value(self, value, index=None):
-        try:
-            self.client.retry(self.client.set, self.failover_path, value.encode('utf-8'), version=index or -1)
-            return True
-        except NoNodeError:
-            return value == '' or (not index and self._create(self.failover_path, value.encode('utf-8')))
-        except:
-            logging.exception('set_failover_value')
-            return False
-
-    def initialize(self):
-        return self._create(self.initialize_path, self._name, makepath=True)
-
-    def touch_member(self, data, ttl=None):
-        cluster = self.cluster
-        me = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
-        path = self.member_path
-        data = data.encode('utf-8')
-        create = not me
-        if me and self.client.client_id is not None and me.session != self.client.client_id[0]:
-            try:
-                self.client.retry(self.client.delete, path)
-            except NoNodeError:
-                pass
-            except:
-                return False
-            create = True
-
-        if not create and data == self._my_member_data:
-            return True
-
-        try:
-            if create:
-                self.client.retry(self.client.create, path, data, makepath=True, ephemeral=True)
-            else:
-                self.client.retry(self.client.set, path, data)
-            self._my_member_data = data
-            return True
-        except NodeExistsError:
-            try:
-                self.client.retry(self.client.set, path, data)
-                self._my_member_data = data
-                return True
-            except:
-                logger.exception('touch_member')
-        except:
-            logger.exception('touch_member')
-        return False
-
-    def take_leader(self):
-        return self.attempt_to_acquire_leader()
-
-    def write_leader_optime(self, last_operation):
-        last_operation = last_operation.encode('utf-8')
-        if last_operation != self.last_leader_operation:
-            self.last_leader_operation = last_operation
-            path = self.leader_optime_path
-            try:
-                self.client.retry(self.client.set, path, last_operation)
-            except NoNodeError:
-                try:
-                    self.client.retry(self.client.create, path, last_operation, makepath=True)
-                except:
-                    logger.exception('Failed to create %s', path)
-            except:
-                logger.exception('Failed to update %s', path)
-
-    def update_leader(self):
-        return True
-
-    def delete_leader(self):
-        self.client.restart()
-        self._my_member_data = None
-        return True
-
-    def _cancel_initialization(self):
-        node = self.get_node(self.initialize_path)
-        if node and node[0] == self._name:
-            self.client.delete(self.initialize_path, version=node[1].version)
-
-    def cancel_initialization(self):
-        try:
-            self.client.retry(self._cancel_initialization)
-        except:
-            logger.exception("Unable to delete initialize key")
-
-    def watch(self, timeout):
-        if super(ZooKeeper, self).watch(timeout):
-            self.fetch_cluster = True
-        return self.fetch_cluster

patronictl.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+from patroni.ctl import ctl
+
+if __name__ == '__main__':
+    ctl(None)

postgres0.yml

@@ -1,69 +1,90 @@
-ttl: &ttl 30
-loop_wait: &loop_wait 10
-scope: &scope batman
+scope: batman
+#namespace: /service/
+name: postgresql0
+
 restapi:
   listen: 127.0.0.1:8008
   connect_address: 127.0.0.1:8008
-  auth: 'username:password'
 #  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
 #  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
+#  authentication:
+#    username: username
+#    password: password
+
 etcd:
-  scope: *scope
-  ttl: *ttl
-  host: 127.0.0.1:4001
-  #discovery_srv: my-etcd.domain
-#zookeeper:
-#  scope: *scope
-#  session_timeout: *ttl
-#  reconnect_timeout: *loop_wait
-#  hosts:
-#    - 127.0.0.1:2181
-#    - 127.0.0.2:2181
-#  exhibitor:
-#    poll_interval: 300
-#    port: 8181
-#    hosts:
-#      - host1
-#      - host2
-#      - host3
+  host: 127.0.0.1:2379
+
+bootstrap:
+  # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
+  # and all other cluster members will use it as a `global configuration`
+  dcs:
+    ttl: 30
+    loop_wait: 10
+    retry_timeout: 10
+    maximum_lag_on_failover: 1048576
+#    master_start_timeout: 300
+#    synchronous_mode: false
+    postgresql:
+      use_pg_rewind: true
+#      use_slots: true
+      parameters:
+#        wal_level: hot_standby
+#        hot_standby: "on"
+#        wal_keep_segments: 8
+#        max_wal_senders: 5
+#        max_replication_slots: 5
+#        wal_log_hints: "on"
+#        archive_mode: "on"
+#        archive_timeout: 1800s
+#        archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
+#      recovery_conf:
+#        restore_command: cp ../wal_archive/%f %p
+
+  # some desired options for 'initdb'
+  initdb:  # Note: It needs to be a list (some options need values, others are switches)
+  - encoding: UTF8
+  - data-checksums
+
+  pg_hba:  # Add following lines to pg_hba.conf after running 'initdb'
+  - host replication replicator 127.0.0.1/32 md5
+  - host all all 0.0.0.0/0 md5
+#  - hostssl all all 0.0.0.0/0 md5
+
+  # Additional script to be launched after initial cluster creation (will be passed the connection URL as parameter)
+# post_init: /usr/local/bin/setup_cluster.sh
+
+  # Some additional users users which needs to be created after initializing new cluster
+  users:
+    admin:
+      password: admin
+      options:
+        - createrole
+        - createdb
+
 postgresql:
-  name: postgresql0
-  scope: *scope
   listen: 127.0.0.1:5432
   connect_address: 127.0.0.1:5432
   data_dir: data/postgresql0
-  maximum_lag_on_failover: 1048576 # 1 megabyte in bytes
-  use_slots: True
-  pg_rewind:
-    username: postgres
-    password: zalando
-  pg_hba:
-  - host all all 0.0.0.0/0 md5
-  - hostssl all all 0.0.0.0/0 md5
-  replication:
-    username: replicator
-    password: rep-pass
-    network:  127.0.0.1/32
-  superuser:
-    username: postgres
-    password: zalando
-  admin:
-    username: admin
-    password: admin
-  wal_e:
-    env_dir: /home/postgres/etc/wal-e.d/env
-    threshold_megabytes: 10240
-    threshold_backup_size_percentage: 30
-  restore: patroni/scripts/restore.py
-  #recovery_conf:
-    #restore_command: cp ../wal_archive/%f %p
+#  bin_dir:
+#  config_dir:
+  pgpass: /tmp/pgpass0
+  authentication:
+    replication:
+      username: replicator
+      password: rep-pass
+    superuser:
+      username: postgres
+      password: zalando
   parameters:
-    archive_mode: "on"
-    wal_level: hot_standby
-    archive_command: mkdir -p ../wal_archive && cp %p ../wal_archive/%f
-    max_wal_senders: 5
-    wal_keep_segments: 8
-    archive_timeout: 1800s
-    max_replication_slots: 5
-    hot_standby: "on"
-    wal_log_hints: "on"
+    unix_socket_directories: '.'
+
+#watchdog:
+#  mode: automatic # Allowed values: off, automatic, required
+#  device: /dev/watchdog
+#  safety_margin: 5
+
+tags:
+    nofailover: false
+    noloadbalance: false
+    clonefrom: false
+    nosync: false

postgres1.yml

@@ -1,69 +1,84 @@
-ttl: &ttl 30
-loop_wait: &loop_wait 10
-scope: &scope batman
+scope: batman
+#namespace: /service/
+name: postgresql1
+
 restapi:
   listen: 127.0.0.1:8009
   connect_address: 127.0.0.1:8009
-  auth: 'username:password'
 #  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
 #  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
+#  authentication:
+#    username: username
+#    password: password
+
 etcd:
-  scope: *scope
-  ttl: *ttl
-  host: 127.0.0.1:4001
-  #discovery_srv: my-etcd.domain
-#zookeeper:
-#  scope: *scope
-#  session_timeout: *ttl
-#  reconnect_timeout: *loop_wait
-#  hosts:
-#    - 127.0.0.1:2181
-#    - 127.0.0.2:2181
-#  exhibitor:
-#    poll_interval: 300
-#    port: 8181
-#    hosts:
-#      - host1
-#      - host2
-#      - host3
+  host: 127.0.0.1:2379
+
+bootstrap:
+  # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
+  # and all other cluster members will use it as a `global configuration`
+  dcs:
+    ttl: 30
+    loop_wait: 10
+    retry_timeout: 10
+    maximum_lag_on_failover: 1048576
+    postgresql:
+      use_pg_rewind: true
+#      use_slots: true
+      parameters:
+#        wal_level: hot_standby
+#        hot_standby: "on"
+#        wal_keep_segments: 8
+#        max_wal_senders: 5
+#        max_replication_slots: 5
+#        wal_log_hints: "on"
+#        archive_mode: "on"
+#        archive_timeout: 1800s
+#        archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
+#      recovery_conf:
+#        restore_command: cp ../wal_archive/%f %p
+
+  # some desired options for 'initdb'
+  initdb:  # Note: It needs to be a list (some options need values, others are switches)
+  - encoding: UTF8
+  - data-checksums
+
+  pg_hba:  # Add following lines to pg_hba.conf after running 'initdb'
+  - host replication replicator 127.0.0.1/32 md5
+  - host all all 0.0.0.0/0 md5
+#  - hostssl all all 0.0.0.0/0 md5
+
+  # Additional script to be launched after initial cluster creation (will be passed the connection URL as parameter)
+# post_init: /usr/local/bin/setup_cluster.sh
+
+  # Some additional users users which needs to be created after initializing new cluster
+  users:
+    admin:
+      password: admin
+      options:
+        - createrole
+        - createdb
+
 postgresql:
-  name: postgresql1
-  scope: *scope
   listen: 127.0.0.1:5433
   connect_address: 127.0.0.1:5433
   data_dir: data/postgresql1
-  maximum_lag_on_failover: 1048576 # 1 megabyte in bytes
-  use_slots: True
-  pg_rewind:
-    username: postgres
-    password: zalando
-  pg_hba:
-  - host all all 0.0.0.0/0 md5
-  - hostssl all all 0.0.0.0/0 md5
-  replication:
-    username: replicator
-    password: rep-pass
-    network: 127.0.0.1/32
-  superuser:
-    user: postgres
-    password: zalando
-  admin:
-    username: admin
-    password: admin
-  #recovery_conf:
-    #restore_command: cp ../wal_archive/%f %p
-  wal_e:
-    env_dir: /home/postgres/etc/wal-e.d/env
-    threshold_megabytes: 10240
-    threshold_backup_size_percentage: 30
-  restore: patroni/scripts/restore.py
+#  bin_dir:
+#  config_dir:
+  pgpass: /tmp/pgpass1
+  authentication:
+    replication:
+      username: replicator
+      password: rep-pass
+    superuser:
+      username: postgres
+      password: zalando
   parameters:
-    archive_mode: "on"
-    wal_level: hot_standby
-    archive_command: mkdir -p ../wal_archive && cp %p ../wal_archive/%f
-    max_wal_senders: 5
-    wal_keep_segments: 8
-    archive_timeout: 1800s
-    max_replication_slots: 5
-    hot_standby: "on"
-    wal_log_hints: "on"
+    unix_socket_directories: '.'
+  basebackup:
+      - verbose
+      - max-rate: 100M
+tags:
+    nofailover: false
+    noloadbalance: false
+    clonefrom: false

postgres2.yml

@@ -0,0 +1,79 @@
+scope: batman
+#namespace: /service/
+name: postgresql2
+
+restapi:
+  listen: 127.0.0.1:8010
+  connect_address: 127.0.0.1:8010
+#  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
+#  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
+  authentication:
+    username: username
+    password: password
+
+etcd:
+  host: 127.0.0.1:2379
+
+bootstrap:
+  # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
+  # and all other cluster members will use it as a `global configuration`
+  dcs:
+    ttl: 30
+    loop_wait: 10
+    retry_timeout: 10
+    maximum_lag_on_failover: 1048576
+    postgresql:
+      use_pg_rewind: true
+#      use_slots: true
+      parameters:
+#        wal_level: hot_standby
+#        hot_standby: "on"
+#        wal_keep_segments: 8
+#        max_wal_senders: 5
+#        max_replication_slots: 5
+#        wal_log_hints: "on"
+#        archive_mode: "on"
+#        archive_timeout: 1800s
+#        archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
+#      recovery_conf:
+#        restore_command: cp ../wal_archive/%f %p
+
+  # some desired options for 'initdb'
+  initdb:  # Note: It needs to be a list (some options need values, others are switches)
+  - encoding: UTF8
+  - data-checksums
+
+  pg_hba:  # Add following lines to pg_hba.conf after running 'initdb'
+  - host replication replicator 127.0.0.1/32 md5
+  - host all all 0.0.0.0/0 md5
+#  - hostssl all all 0.0.0.0/0 md5
+
+  # Some additional users users which needs to be created after initializing new cluster
+  users:
+    admin:
+      password: admin
+      options:
+        - createrole
+        - createdb
+
+postgresql:
+  listen: 127.0.0.1:5434
+  connect_address: 127.0.0.1:5434
+  data_dir: data/postgresql2
+#  bin_dir:
+#  config_dir:
+  pgpass: /tmp/pgpass2
+  authentication:
+    replication:
+      username: replicator
+      password: rep-pass
+    superuser:
+      username: postgres
+      password: zalando
+  parameters:
+    unix_socket_directories: '.'
+tags:
+    nofailover: false
+    noloadbalance: false
+    clonefrom: false
+    replicatefrom: postgres1

requirements-py2.txt

@@ -1,9 +0,0 @@
-boto
-dnspython
-mock
-psycopg2
-PyYAML
-requests
-six >= 1.7
-kazoo>=2.2.1
-python-etcd>=0.4.1

requirements-py3.txt

@@ -1,9 +0,0 @@
-boto
-mock
-dnspython3
-psycopg2
-PyYAML
-requests
-six
-kazoo>=2.2.1
-python-etcd>=0.4.1

requirements.txt

@@ -0,0 +1,16 @@
+urllib3>=1.19.1,!=1.21
+boto
+psycopg2>=2.5.4
+PyYAML
+requests
+six >= 1.7
+kazoo>=1.3.1
+python-etcd>=0.4.3,<0.5
+python-consul>=0.7.0
+click>=4.1
+prettytable>=0.7
+tzlocal
+python-dateutil
+psutil
+cdiff
+kubernetes==3.0.0

setup.py

@@ -4,13 +4,12 @@
     Setup file for patroni
 """
 
-import sys
-import os
 import inspect
+import os
+import sys
 
-import setuptools
 from setuptools.command.test import test as TestCommand
-from setuptools import setup
+from setuptools import find_packages, setup
 
 if sys.version_info < (2, 7, 0):
     sys.stderr.write('FATAL: patroni needs to be run with Python 2.7+\n')
@@ -25,6 +24,7 @@ def read_version(package):
         exec(fd.read(), data)
     return data['__version__']
 
+
 NAME = 'patroni'
 MAIN_PACKAGE = NAME
 SCRIPTS = 'scripts'
@@ -32,9 +32,9 @@ VERSION = read_version(MAIN_PACKAGE)
 DESCRIPTION = 'PostgreSQL High-Available orchestrator and CLI'
 LICENSE = 'The MIT License'
 URL = 'https://github.com/zalando/patroni'
-AUTHOR = 'Alexander Kukushkin, Alexey Klyukin, Feike Steenbergen'
+AUTHOR = 'Alexander Kukushkin, Oleksii Kliukin, Feike Steenbergen'
 AUTHOR_EMAIL = '[email protected], [email protected], [email protected]'
-KEYWORDS = 'etcd governor patroni postgresql postgres ha zookeeper streaming replication'
+KEYWORDS = 'etcd governor patroni postgresql postgres ha haproxy confd zookeeper exhibitor consul streaming replication'
 
 COVERAGE_XML = True
 COVERAGE_HTML = False
@@ -51,12 +51,15 @@ CLASSIFIERS = [
     'Operating System :: POSIX :: Linux',
     'Programming Language :: Python',
     'Programming Language :: Python :: 2.7',
-    'Programming Language :: Python :: 3.3',
     'Programming Language :: Python :: 3.4',
+    'Programming Language :: Python :: 3.5',
     'Programming Language :: Python :: Implementation :: CPython',
 ]
 
-CONSOLE_SCRIPTS = ['patroni = patroni:main']
+CONSOLE_SCRIPTS = ['patroni = patroni:main',
+                   'patronictl = patroni.ctl:ctl',
+                   "patroni_wale_restore = patroni.scripts.wale_restore:main",
+                   "patroni_aws = patroni.scripts.aws:main"]
 
 
 class PyTest(TestCommand):
@@ -73,7 +76,7 @@ class PyTest(TestCommand):
     def finalize_options(self):
         TestCommand.finalize_options(self)
         if self.cov_xml or self.cov_html:
-            self.cov = ['--cov', MAIN_PACKAGE, '--cov', MAIN_PACKAGE, '--cov-report', 'term-missing']
+            self.cov = ['--cov', MAIN_PACKAGE, '--cov-report', 'term-missing']
             if self.cov_xml:
                 self.cov.extend(['--cov-report', 'xml'])
             if self.cov_html:
@@ -84,14 +87,19 @@ class PyTest(TestCommand):
     def run_tests(self):
         try:
             import pytest
-        except:
+        except Exception:
             raise RuntimeError('py.test is not installed, run: pip install pytest')
         params = {'args': self.test_args}
         if self.cov:
             params['args'] += self.cov
         if self.junitxml:
             params['args'] += self.junitxml
-        params['args'] += ['--doctest-modules', MAIN_PACKAGE, '-s', '-vv']
+        params['args'] += ['--doctest-modules', MAIN_PACKAGE, '-vv']
+
+        import logging
+        silence = logging.WARNING
+        logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=os.getenv('LOGLEVEL', silence))
+        params['args'] += ['-s' if logging.getLogger().getEffectiveLevel() < silence else '--capture=fd']
         errno = pytest.main(**params)
         sys.exit(errno)
 
@@ -107,14 +115,23 @@ def read(fname):
 
 def setup_package():
     # Assemble additional setup commands
-    cmdclass = {}
-    cmdclass['test'] = PyTest
+    cmdclass = {'test': PyTest}
 
     # Some helper variables
     version = os.getenv('GO_PIPELINE_LABEL', VERSION)
 
-    requirements = 'requirements-py2.txt' if sys.version_info[0] == 2 else 'requirements-py3.txt'
-    install_reqs = get_install_requirements(requirements)
+    install_requires = []
+    extras_require = {'aws': ['boto'], 'etcd': ['python-etcd'], 'consul': ['python-consul'],
+                      'exhibitor': ['kazoo'], 'zookeeper': ['kazoo'], 'kubernetes': ['kubernetes']}
+
+    for r in get_install_requirements('requirements.txt'):
+        extra = False
+        for e, v in extras_require.items():
+            if r.startswith(v[0]):
+                extras_require[e] = [r]
+                extra = True
+        if not extra:
+            install_requires.append(r)
 
     command_options = {'test': {'test_suite': ('setup.py', 'tests')}}
     if JUNIT_XML:
@@ -136,12 +153,12 @@ def setup_package():
         long_description=read('README.rst'),
         classifiers=CLASSIFIERS,
         test_suite='tests',
-        packages=setuptools.find_packages(exclude=['tests', 'tests.*']),
+        packages=find_packages(exclude=['tests', 'tests.*']),
         package_data={MAIN_PACKAGE: ["*.json"]},
-        install_requires=install_reqs,
-        setup_requires=['six', 'flake8'],
+        install_requires=install_requires,
+        extras_require=extras_require,
         cmdclass=cmdclass,
-        tests_require=['pytest-cov', 'pytest'],
+        tests_require=['flake8', 'mock>=2.0.0', 'pytest-cov', 'pytest'],
         command_options=command_options,
         entry_points={'console_scripts': CONSOLE_SCRIPTS},
     )

tests/test_api.py

@@ -1,119 +1,382 @@
+import datetime
+import json
 import psycopg2
 import unittest
 
-from mock import Mock, patch
+from mock import Mock, PropertyMock, patch
 from patroni.api import RestApiHandler, RestApiServer
+from patroni.dcs import ClusterConfig, Member
+from patroni.ha import _MemberStatus
+from patroni.utils import tzutc
 from six import BytesIO as IO
 from six.moves import BaseHTTPServer
 from test_postgresql import psycopg2_connect, MockCursor
 
 
-class MockPostgresql(Mock):
+future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5)
+postmaster_start_time = datetime.datetime.now(tzutc)
+
+
+class MockPostgresql(object):
 
     name = 'test'
     state = 'running'
     role = 'master'
+    server_version = '999999'
+    sysid = 'dummysysid'
+    scope = 'dummy'
+    pending_restart = True
+    wal_name = 'wal'
+    lsn_name = 'lsn'
 
-    def connection(self):
+    @staticmethod
+    def connection():
         return psycopg2_connect()
 
-    def is_running(self):
+    @staticmethod
+    def postmaster_start_time():
+        return str(postmaster_start_time)
+
+    @staticmethod
+    def replica_cached_timeline(_):
+        return 2
+
+
+class MockWatchdog(object):
+    is_healthy = False
+
+
+class MockHa(object):
+
+    state_handler = MockPostgresql()
+    watchdog = MockWatchdog()
+
+    @staticmethod
+    def reinitialize(_):
+        return 'reinitialize'
+
+    @staticmethod
+    def restart():
+        return (True, '')
+
+    @staticmethod
+    def restart_scheduled():
+        return False
+
+    @staticmethod
+    def delete_future_restart():
+        return True
+
+    @staticmethod
+    def fetch_nodes_statuses(members):
+        return [_MemberStatus(None, True, None, None, {}, False)]
+
+    @staticmethod
+    def schedule_future_restart(data):
+        return True
+
+    @staticmethod
+    def is_lagging(wal):
+        return False
+
+    @staticmethod
+    def get_effective_tags():
+        return {'nosync': True}
+
+    @staticmethod
+    def wakeup():
+        pass
+
+    @staticmethod
+    def is_paused():
         return True
 
 
-class MockHa(Mock):
+class MockPatroni(object):
 
-    dcs = Mock()
-    state_handler = MockPostgresql()
-
-    def schedule_restart(self):
-        return 'restart'
-
-    def schedule_reinitialize(self):
-        return 'reinitialize'
-
-    def restart(self):
-        return (True, '')
-
-    def restart_scheduled(self):
-        return False
-
-
-class MockPatroni:
-
-    postgresql = MockPostgresql()
     ha = MockHa()
+    config = Mock()
+    postgresql = ha.state_handler
     dcs = Mock()
+    tags = {}
+    version = '0.00'
+    noloadbalance = PropertyMock(return_value=False)
+    scheduled_restart = {'schedule': future_restart_time,
+                         'postmaster_start_time': postgresql.postmaster_start_time()}
+
+    @staticmethod
+    def sighup_handler():
+        pass
 
 
-class MockRequest:
+class MockRequest(object):
 
-    def __init__(self, path):
-        self.path = path
+    def __init__(self, request):
+        self.request = request.encode('utf-8')
 
     def makefile(self, *args, **kwargs):
-        return IO(self.path)
+        return IO(self.request)
+
+    def sendall(self, *args, **kwargs):
+        pass
 
 
 class MockRestApiServer(RestApiServer):
 
-    def __init__(self, Handler, path):
+    def __init__(self, Handler, request):
         self.socket = 0
+        self.serve_forever = Mock()
         BaseHTTPServer.HTTPServer.__init__ = Mock()
         MockRestApiServer._BaseServer__is_shut_down = Mock()
         MockRestApiServer._BaseServer__shutdown_request = True
-        config = {'listen': '127.0.0.1:8008', 'auth': 'test:test', 'certfile': 'dumb'}
+        config = {'listen': '127.0.0.1:8008', 'auth': 'test:test'}
         super(MockRestApiServer, self).__init__(MockPatroni(), config)
-        Handler(MockRequest(path), ('0.0.0.0', 8080), self)
+        config['certfile'] = 'dumb'
+        self.reload_config(config)
+        Handler(MockRequest(request), ('0.0.0.0', 8080), self)
 
 
 @patch('ssl.wrap_socket', Mock(return_value=0))
 class TestRestApiHandler(unittest.TestCase):
 
+    _authorization = '\nAuthorization: Basic dGVzdDp0ZXN0'
+
     def test_do_GET(self):
-        MockRestApiServer(RestApiHandler, b'GET /replica')
+        MockRestApiServer(RestApiHandler, 'GET /replica')
         with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={})):
-            MockRestApiServer(RestApiHandler, b'GET /replica')
+            MockRestApiServer(RestApiHandler, 'GET /replica')
         with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'master'})):
-            MockRestApiServer(RestApiHandler, b'GET /replica')
-        MockRestApiServer(RestApiHandler, b'GET /master')
+            MockRestApiServer(RestApiHandler, 'GET /replica')
+        MockRestApiServer(RestApiHandler, 'GET /master')
+        MockPatroni.dcs.cluster.sync.sync_standby = MockPostgresql.name
+        MockPatroni.dcs.cluster.is_synchronous_mode = Mock(return_value=True)
+        with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'replica'})):
+            MockRestApiServer(RestApiHandler, 'GET /synchronous')
+        with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'replica'})):
+            MockRestApiServer(RestApiHandler, 'GET /asynchronous')
         MockPatroni.dcs.cluster.leader.name = MockPostgresql.name
-        MockRestApiServer(RestApiHandler, b'GET /replica')
+        MockRestApiServer(RestApiHandler, 'GET /replica')
         MockPatroni.dcs.cluster = None
         with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'master'})):
-            MockRestApiServer(RestApiHandler, b'GET /master')
+            MockRestApiServer(RestApiHandler, 'GET /master')
         with patch.object(MockHa, 'restart_scheduled', Mock(return_value=True)):
-            MockRestApiServer(RestApiHandler, b'GET /master')
-        MockRestApiServer(RestApiHandler, b'GET /master')
+            MockRestApiServer(RestApiHandler, 'GET /master')
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /master'))
+        with patch.object(RestApiServer, 'query', Mock(return_value=[('', 1, '', '', '', '', False, '')])):
+            self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
 
+    def test_do_OPTIONS(self):
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'OPTIONS / HTTP/1.0'))
+
+    @patch.object(MockPostgresql, 'state', PropertyMock(return_value='stopped'))
     def test_do_GET_patroni(self):
-        MockRestApiServer(RestApiHandler, b'GET /patroni')
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
 
     def test_basicauth(self):
-        MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0')
-        MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0\nAuthorization:')
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0'))
+        MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0\nAuthorization:')
 
-    def test_do_POST_restart(self):
-        request = b'POST /restart HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0'
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_GET_config(self, mock_dcs):
+        mock_dcs.cluster.config.data = {}
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /config'))
+        mock_dcs.cluster.config = None
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /config'))
+
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_PATCH_config(self, mock_dcs):
+        config = {'postgresql': {'use_slots': False, 'use_pg_rewind': True, 'parameters': {'wal_level': 'logical'}}}
+        mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, json.dumps(config))
+        request = 'PATCH /config HTTP/1.0' + self._authorization
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
+        request += '\nContent-Length: '
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '34\n\n{"postgresql":{"use_slots":false}}'))
+        config['ttl'] = 5
+        config['postgresql'].update({'use_slots': {'foo': True}, "parameters": None})
+        config = json.dumps(config)
+        request += str(len(config)) + '\n\n' + config
         MockRestApiServer(RestApiHandler, request)
+        mock_dcs.set_config_value.return_value = False
+        MockRestApiServer(RestApiHandler, request)
+
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_PUT_config(self, mock_dcs):
+        mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, '{}')
+        request = 'PUT /config HTTP/1.0' + self._authorization + '\nContent-Length: '
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '2\n\n{}'))
+        config = '{"foo": "bar"}'
+        request += str(len(config)) + '\n\n' + config
+        MockRestApiServer(RestApiHandler, request)
+        mock_dcs.set_config_value.return_value = False
+        MockRestApiServer(RestApiHandler, request)
+        mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, config)
+        MockRestApiServer(RestApiHandler, request)
+
+    @patch.object(MockPatroni, 'sighup_handler', Mock(side_effect=Exception))
+    def test_do_POST_reload(self):
+        with patch.object(MockPatroni, 'config') as mock_config:
+            mock_config.reload_local_configuration.return_value = False
+            MockRestApiServer(RestApiHandler, 'POST /reload HTTP/1.0' + self._authorization)
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /reload HTTP/1.0' + self._authorization))
+
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_POST_restart(self, mock_dcs):
+        mock_dcs.get_cluster.return_value.is_paused.return_value = False
+        request = 'POST /restart HTTP/1.0' + self._authorization
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
+
         with patch.object(MockHa, 'restart', Mock(side_effect=Exception)):
             MockRestApiServer(RestApiHandler, request)
 
-    @patch.object(MockHa, 'dcs')
-    def test_do_POST_reinitialize(self, dcs):
-        cluster = dcs.get_cluster.return_value
-        request = b'POST /reinitialize HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0'
+        post = request + '\nContent-Length: '
+
+        def make_request(request=None, **kwargs):
+            request = json.dumps(kwargs) if request is None else request
+            return '{0}{1}\n\n{2}'.format(post, len(request), request)
+
+        # empty request
+        request = make_request('')
         MockRestApiServer(RestApiHandler, request)
-        cluster.is_unlocked.return_value = False
+        # invalid request
+        request = make_request('foobar=baz')
         MockRestApiServer(RestApiHandler, request)
-        with patch.object(MockHa, 'schedule_reinitialize', Mock(return_value=None)):
+        # wrong role
+        request = make_request(schedule=future_restart_time.isoformat(), role='unknown', postgres_version='9.5.3')
+        MockRestApiServer(RestApiHandler, request)
+        # wrong version
+        request = make_request(schedule=future_restart_time.isoformat(), role='master', postgres_version='9.5.3.1')
+        MockRestApiServer(RestApiHandler, request)
+        # unknown filter
+        request = make_request(schedule=future_restart_time.isoformat(), batman='lives')
+        MockRestApiServer(RestApiHandler, request)
+        # incorrect schedule
+        request = make_request(schedule='2016-08-42 12:45TZ+1', role='master')
+        MockRestApiServer(RestApiHandler, request)
+        # everything fine, but the schedule is missing
+        request = make_request(role='master', postgres_version='9.5.2')
+        MockRestApiServer(RestApiHandler, request)
+        for retval in (True, False):
+            with patch.object(MockHa, 'schedule_future_restart', Mock(return_value=retval)):
+                request = make_request(schedule=future_restart_time.isoformat())
+                MockRestApiServer(RestApiHandler, request)
+            with patch.object(MockHa, 'restart', Mock(return_value=(retval, "foo"))):
+                request = make_request(role='master', postgres_version='9.5.2')
+                MockRestApiServer(RestApiHandler, request)
+
+        mock_dcs.get_cluster.return_value.is_paused.return_value = True
+        MockRestApiServer(RestApiHandler, make_request(schedule='2016-08-42 12:45TZ+1', role='master'))
+        # Valid timeout
+        MockRestApiServer(RestApiHandler, make_request(timeout='60s'))
+        # Invalid timeout
+        MockRestApiServer(RestApiHandler, make_request(timeout='42towels'))
+
+    def test_do_DELETE_restart(self):
+        for retval in (True, False):
+            with patch.object(MockHa, 'delete_future_restart', Mock(return_value=retval)):
+                request = 'DELETE /restart HTTP/1.0' + self._authorization
+                self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
+
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_POST_reinitialize(self, mock_dcs):
+        cluster = mock_dcs.get_cluster.return_value
+        cluster.is_paused.return_value = False
+        request = 'POST /reinitialize HTTP/1.0' + self._authorization + '\nContent-Length: 15\n\n{"force": true}'
+        MockRestApiServer(RestApiHandler, request)
+        with patch.object(MockHa, 'reinitialize', Mock(return_value=None)):
             MockRestApiServer(RestApiHandler, request)
-        cluster.leader.name = 'test'
-        MockRestApiServer(RestApiHandler, request)
 
     @patch('time.sleep', Mock())
     def test_RestApiServer_query(self):
         with patch.object(MockCursor, 'execute', Mock(side_effect=psycopg2.OperationalError)):
-            MockRestApiServer(RestApiHandler, b'GET /patroni')
+            self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
         with patch.object(MockPostgresql, 'connection', Mock(side_effect=psycopg2.OperationalError)):
-            MockRestApiServer(RestApiHandler, b'GET /patroni')
+            self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
+
+    @patch('time.sleep', Mock())
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_POST_switchover(self, dcs):
+        dcs.loop_wait = 10
+        cluster = dcs.get_cluster.return_value
+        cluster.is_synchronous_mode.return_value = False
+        cluster.is_paused.return_value = False
+
+        post = 'POST /switchover HTTP/1.0' + self._authorization + '\nContent-Length: '
+
+        MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}')
+
+        request = post + '0\n\n'
+        MockRestApiServer(RestApiHandler, request)
+
+        cluster.leader.name = 'postgresql1'
+        MockRestApiServer(RestApiHandler, request)
+
+        request = post + '25\n\n{"leader": "postgresql1"}'
+
+        cluster.is_paused.return_value = True
+        MockRestApiServer(RestApiHandler, request)
+
+        cluster.is_paused.return_value = False
+        for cluster.is_synchronous_mode.return_value in (True, False):
+            MockRestApiServer(RestApiHandler, request)
+
+        cluster.leader.name = 'postgresql2'
+        request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
+        MockRestApiServer(RestApiHandler, request)
+
+        cluster.leader.name = 'postgresql1'
+        for cluster.is_synchronous_mode.return_value in (True, False):
+            MockRestApiServer(RestApiHandler, request)
+
+        cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
+                           Member(0, 'postgresql2', 30, {'api_url': 'http'})]
+        MockRestApiServer(RestApiHandler, request)
+
+        cluster.failover = None
+        MockRestApiServer(RestApiHandler, request)
+
+        dcs.get_cluster.side_effect = [cluster]
+        MockRestApiServer(RestApiHandler, request)
+
+        cluster2 = cluster.copy()
+        cluster2.leader.name = 'postgresql0'
+        cluster2.is_unlocked.return_value = False
+        dcs.get_cluster.side_effect = [cluster, cluster2]
+        MockRestApiServer(RestApiHandler, request)
+
+        cluster2.leader.name = 'postgresql2'
+        dcs.get_cluster.side_effect = [cluster, cluster2]
+        MockRestApiServer(RestApiHandler, request)
+
+        dcs.get_cluster.side_effect = None
+        dcs.manual_failover.return_value = False
+        MockRestApiServer(RestApiHandler, request)
+        dcs.manual_failover.return_value = True
+
+        with patch.object(MockHa, 'fetch_nodes_statuses', Mock(return_value=[])):
+            MockRestApiServer(RestApiHandler, request)
+
+        # Valid future date
+        request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2",' +\
+                         ' "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}'
+        MockRestApiServer(RestApiHandler, request)
+        with patch.object(MockPatroni, 'dcs') as d:
+            d.manual_failover.return_value = False
+            MockRestApiServer(RestApiHandler, request)
+
+        # Exception: No timezone specified
+        request = post + '97\n\n{"leader": "postgresql1", "member": "postgresql2",' +\
+                         ' "scheduled_at": "6016-02-15T18:13:30.568224"}'
+        MockRestApiServer(RestApiHandler, request)
+
+        # Exception: Scheduled in the past
+        request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "'
+        MockRestApiServer(RestApiHandler, request + '1016-02-15T18:13:30.568224+01:00"}')
+
+        # Invalid date
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}'))
+
+    @patch.object(MockPatroni, 'dcs', Mock())
+    def test_do_POST_failover(self):
+        post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
+        MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}')
+        MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}')

tests/test_async_executor.py

@@ -1,14 +1,14 @@
 import unittest
 
 from mock import Mock, patch
-from patroni.async_executor import AsyncExecutor
+from patroni.async_executor import AsyncExecutor, CriticalTask
 from threading import Thread
 
 
 class TestAsyncExecutor(unittest.TestCase):
 
     def setUp(self):
-        self.a = AsyncExecutor()
+        self.a = AsyncExecutor(Mock(), Mock())
 
     @patch.object(Thread, 'start', Mock())
     def test_run_async(self):
@@ -16,3 +16,17 @@ class TestAsyncExecutor(unittest.TestCase):
 
     def test_run(self):
         self.a.run(Mock(side_effect=Exception()))
+
+    def test_cancel(self):
+        self.a.cancel()
+        self.a.schedule('foo')
+        self.a.cancel()
+        self.a.run(Mock())
+
+
+class TestCriticalTask(unittest.TestCase):
+
+    def test_completed_task(self):
+        ct = CriticalTask()
+        ct.complete(1)
+        self.assertFalse(ct.cancel())

tests/test_aws.py

@@ -1,94 +1,71 @@
-import unittest
-import requests
 import boto.ec2
+import sys
+import unittest
+
+from mock import Mock, patch
 from collections import namedtuple
-from patroni.scripts.aws import AWSConnection
+from patroni.scripts.aws import AWSConnection, main as _main
 from requests.exceptions import RequestException
 
 
-class MockEc2Connection:
+class MockEc2Connection(object):
 
-    def __init__(self, error=False):
-        self.error = error
-
-    def get_all_volumes(self, filters):
-        if self.error:
-            raise Exception("get_all_volumes")
+    @staticmethod
+    def get_all_volumes(*args, **kwargs):
         oid = namedtuple('Volume', 'id')
         return [oid(id='a'), oid(id='b')]
 
-    def create_tags(self, objects, tags):
-        if self.error or len(objects) == 0:
-            raise Exception("create_tags")
+    @staticmethod
+    def create_tags(objects, *args, **kwargs):
+        if len(objects) == 0:
+            raise boto.exception.BotoServerError(503, 'Service Unavailable', 'Request limit exceeded')
         return True
 
 
-class MockResponse:
+class MockResponse(object):
+    ok = True
 
     def __init__(self, content):
         self.content = content
-        self.ok = True
 
     def json(self):
         return self.content
 
 
+def requests_get(url, **kwargs):
+    if url.split('/')[-1] == 'document':
+        result = {"instanceId": "012345", "region": "eu-west-1"}
+    else:
+        result = 'foo'
+    return MockResponse(result)
+
+
+@patch('boto.ec2.connect_to_region', Mock(return_value=MockEc2Connection()))
 class TestAWSConnection(unittest.TestCase):
 
-    def __init__(self, method_name='runTest'):
-        super(TestAWSConnection, self).__init__(method_name)
-
-    def set_error(self):
-        self.error = True
-
-    def set_json_error(self):
-        self.json_error = True
-
-    def boto_ec2_connect_to_region(self, region):
-        return MockEc2Connection(self.error)
-
-    def requests_get(self, url, **kwargs):
-        if self.error:
-            raise RequestException("foo")
-        result = namedtuple('Request', 'ok content')
-        result.ok = True
-        if url.split('/')[-1] == 'document' and not self.json_error:
-            result = {"instanceId": "012345", "region": "eu-west-1"}
-        else:
-            result = 'foo'
-        return MockResponse(result)
-
+    @patch('requests.get', requests_get)
     def setUp(self):
-        self.error = False
-        self.json_error = False
-        requests.get = self.requests_get
-        boto.ec2.connect_to_region = self.boto_ec2_connect_to_region
         self.conn = AWSConnection('test')
 
-    def test_aws_available(self):
-        self.assertTrue(self.conn.aws_available())
-
     def test_on_role_change(self):
-        self.assertTrue(self.conn._tag_ebs('master'))
-        self.assertTrue(self.conn._tag_ec2('master'))
         self.assertTrue(self.conn.on_role_change('master'))
+        with patch.object(MockEc2Connection, 'get_all_volumes', Mock(return_value=[])):
+            self.conn._retry.max_tries = 1
+            self.assertFalse(self.conn.on_role_change('master'))
 
+    @patch('requests.get', Mock(side_effect=RequestException('foo')))
     def test_non_aws(self):
-        self.set_error()
         conn = AWSConnection('test')
-        self.assertFalse(conn.aws_available())
-        self.assertFalse(conn._tag_ebs('master'))
-        self.assertFalse(conn._tag_ec2('master'))
+        self.assertFalse(conn.on_role_change("master"))
 
+    @patch('requests.get', Mock(return_value=MockResponse('foo')))
     def test_aws_bizare_response(self):
-        self.set_json_error()
         conn = AWSConnection('test')
         self.assertFalse(conn.aws_available())
 
-    def test_aws_tag_ebs_error(self):
-        self.set_error()
-        self.assertFalse(self.conn._tag_ebs("master"))
-
-    def test_aws_tag_ec2_error(self):
-        self.set_error()
-        self.assertFalse(self.conn._tag_ec2("master"))
+    @patch('requests.get', requests_get)
+    @patch('sys.exit', Mock())
+    def test_main(self):
+        self.assertIsNone(_main())
+        sys.argv = ['aws.py', 'on_start', 'replica', 'foo']
+        self.assertIsNone(_main())

tests/test_callback_executor.py

@@ -0,0 +1,24 @@
+import unittest
+
+from mock import Mock, patch
+from patroni.callback_executor import CallbackExecutor
+
+
+class TestCallbackExecutor(unittest.TestCase):
+
+    @patch('subprocess.Popen')
+    def test_callback_executor(self, mock_popen):
+        mock_popen.return_value.wait.side_effect = Exception
+        mock_popen.return_value.poll.return_value = None
+
+        ce = CallbackExecutor()
+        self.assertIsNone(ce.call([]))
+        ce.join()
+
+        self.assertIsNone(ce.call([]))
+
+        mock_popen.side_effect = Exception
+        ce = CallbackExecutor()
+        ce._callback_event.wait = Mock(side_effect=[None, Exception])
+        self.assertIsNone(ce.call([]))
+        ce.join()

tests/test_config.py

@@ -0,0 +1,85 @@
+import os
+import unittest
+import sys
+
+from mock import MagicMock, Mock, patch
+from patroni.config import Config
+from six.moves import builtins
+
+
+class TestConfig(unittest.TestCase):
+
+    @patch('os.path.isfile', Mock(return_value=True))
+    @patch('json.load', Mock(side_effect=Exception))
+    @patch.object(builtins, 'open', MagicMock())
+    def setUp(self):
+        sys.argv = ['patroni.py']
+        os.environ[Config.PATRONI_CONFIG_VARIABLE] = 'restapi: {}\npostgresql: {data_dir: foo}'
+        self.config = Config()
+
+    def test_no_config(self):
+        self.assertRaises(SystemExit, Config)
+
+    def test_set_dynamic_configuration(self):
+        with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
+            self.assertIsNone(self.config.set_dynamic_configuration({'foo': 'bar'}))
+        self.assertTrue(self.config.set_dynamic_configuration({'synchronous_mode': True}))
+
+    def test_reload_local_configuration(self):
+        os.environ.update({
+            'PATRONI_NAME': 'postgres0',
+            'PATRONI_NAMESPACE': '/patroni/',
+            'PATRONI_SCOPE': 'batman2',
+            'PATRONI_RESTAPI_USERNAME': 'username',
+            'PATRONI_RESTAPI_PASSWORD': 'password',
+            'PATRONI_RESTAPI_LISTEN': '0.0.0.0:8008',
+            'PATRONI_RESTAPI_CONNECT_ADDRESS': '127.0.0.1:8008',
+            'PATRONI_RESTAPI_CERTFILE': '/certfile',
+            'PATRONI_RESTAPI_KEYFILE': '/keyfile',
+            'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432',
+            'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432',
+            'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0',
+            'PATRONI_POSTGRESQL_CONFIG_DIR': 'data/postgres0',
+            'PATRONI_POSTGRESQL_PGPASS': '/tmp/pgpass0',
+            'PATRONI_ETCD_HOST': '127.0.0.1:2379',
+            'PATRONI_ETCD_URL': 'https://127.0.0.1:2379',
+            'PATRONI_ETCD_PROXY': 'http://127.0.0.1:2379',
+            'PATRONI_ETCD_SRV': 'test',
+            'PATRONI_ETCD_CACERT': '/cacert',
+            'PATRONI_ETCD_CERT': '/cert',
+            'PATRONI_ETCD_KEY': '/key',
+            'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
+            'PATRONI_KUBERNETES_LABELS': 'a:b:c',
+            'PATRONI_KUBERNETES_SCOPE_LABEL': 'a',
+            'PATRONI_KUBERNETES_PORTS': '[{"name": "postgresql"}]',
+            'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
+            'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',
+            'PATRONI_EXHIBITOR_PORT': '8181',
+            'PATRONI_foo_HOSTS': '[host1,host2',  # Exception in parse_list
+            'PATRONI_SUPERUSER_USERNAME': 'postgres',
+            'PATRONI_SUPERUSER_PASSWORD': 'zalando',
+            'PATRONI_REPLICATION_USERNAME': 'replicator',
+            'PATRONI_REPLICATION_PASSWORD': 'rep-pass',
+            'PATRONI_admin_PASSWORD': 'admin',
+            'PATRONI_admin_OPTIONS': 'createrole,createdb'
+        })
+        sys.argv = ['patroni.py', 'postgres0.yml']
+        config = Config()
+        with patch.object(Config, '_load_config_file', Mock(return_value={'restapi': {}})):
+            with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
+                self.assertRaises(Exception, config.reload_local_configuration, True)
+            self.assertTrue(config.reload_local_configuration(True))
+            self.assertTrue(config.reload_local_configuration())
+
+    @patch('tempfile.mkstemp', Mock(return_value=[3000, 'blabla']))
+    @patch('os.path.exists', Mock(return_value=True))
+    @patch('os.remove', Mock(side_effect=IOError))
+    @patch('os.close', Mock(side_effect=IOError))
+    @patch('os.rename', Mock(return_value=None))
+    @patch('json.dump', Mock())
+    def test_save_cache(self):
+        self.config.set_dynamic_configuration({'ttl': 30, 'postgresql': {'foo': 'bar'}})
+        with patch('os.fdopen', Mock(side_effect=IOError)):
+            self.config.save_cache()
+        with patch('os.fdopen', MagicMock()):
+            self.config.save_cache()

tests/test_consul.py

@@ -0,0 +1,178 @@
+import consul
+import unittest
+
+from consul import ConsulException, NotFound
+from mock import Mock, patch
+from patroni.dcs.consul import AbstractDCS, Cluster, Consul, ConsulInternalError, \
+                                ConsulError, HTTPClient, InvalidSessionTTL
+from test_etcd import SleepException
+
+
+def kv_get(self, key, **kwargs):
+    if key == 'service/test/members/postgresql1':
+        return '1', {'Session': 'fd4f44fe-2cac-bba5-a60b-304b51ff39b7'}
+    if key == 'service/test/':
+        return None, None
+    if key == 'service/good/leader':
+        return '1', None
+    if key == 'service/good/':
+        return ('6429',
+                [{'CreateIndex': 1334, 'Flags': 0, 'Key': key + 'failover', 'LockIndex': 0,
+                  'ModifyIndex': 1334, 'Value': b''},
+                 {'CreateIndex': 1334, 'Flags': 0, 'Key': key + 'initialize', 'LockIndex': 0,
+                  'ModifyIndex': 1334, 'Value': b'postgresql0'},
+                 {'CreateIndex': 2621, 'Flags': 0, 'Key': key + 'leader', 'LockIndex': 1,
+                  'ModifyIndex': 2621, 'Session': 'fd4f44fe-2cac-bba5-a60b-304b51ff39b7', 'Value': b'postgresql1'},
+                 {'CreateIndex': 6156, 'Flags': 0, 'Key': key + 'members/postgresql0', 'LockIndex': 1,
+                  'ModifyIndex': 6156, 'Session': '782e6da4-ed02-3aef-7963-99a90ed94b53',
+                  'Value': ('postgres://replicator:[email protected]:5432/postgres' +
+                            '?application_name=http://127.0.0.1:8008/patroni').encode('utf-8')},
+                 {'CreateIndex': 2630, 'Flags': 0, 'Key': key + 'members/postgresql1', 'LockIndex': 1,
+                  'ModifyIndex': 2630, 'Session': 'fd4f44fe-2cac-bba5-a60b-304b51ff39b7',
+                  'Value': ('postgres://replicator:[email protected]:5433/postgres' +
+                            '?application_name=http://127.0.0.1:8009/patroni').encode('utf-8')},
+                 {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'optime/leader', 'LockIndex': 0,
+                  'ModifyIndex': 6429, 'Value': b'4496294792'},
+                 {'CreateIndex': 1085, 'Flags': 0, 'Key': key + 'sync', 'LockIndex': 0,
+                  'ModifyIndex': 6429, 'Value': b'{"leader": "leader", "sync_standby": null}'}])
+    raise ConsulException
+
+
+class TestHTTPClient(unittest.TestCase):
+
+    def setUp(self):
+        self.client = HTTPClient('127.0.0.1', '8500', 'http', False)
+        self.client.http.request = Mock()
+
+    def test_get(self):
+        self.client.get(Mock(), '')
+        self.client.get(Mock(), '', {'wait': '1s', 'index': 1, 'token': 'foo'})
+        self.client.http.request.return_value.status = 500
+        self.client.http.request.return_value.data = b'Foo'
+        self.assertRaises(ConsulInternalError, self.client.get, Mock(), '')
+        self.client.http.request.return_value.data = b"Invalid Session TTL '3000000000', must be between [10s=24h0m0s]"
+        self.assertRaises(InvalidSessionTTL, self.client.get, Mock(), '')
+
+    def test_unknown_method(self):
+        try:
+            self.client.bla(Mock(), '')
+            self.assertFail()
+        except Exception as e:
+            self.assertTrue(isinstance(e, AttributeError))
+
+    def test_put(self):
+        self.client.put(Mock(), '/v1/session/create')
+        self.client.put(Mock(), '/v1/session/create', data='{"foo": "bar"}')
+
+
+@patch.object(consul.Consul.KV, 'get', kv_get)
+class TestConsul(unittest.TestCase):
+
+    @patch.object(consul.Consul.Session, 'create', Mock(return_value='fd4f44fe-2cac-bba5-a60b-304b51ff39b7'))
+    @patch.object(consul.Consul.Session, 'renew', Mock(side_effect=NotFound))
+    @patch.object(consul.Consul.KV, 'get', kv_get)
+    @patch.object(consul.Consul.KV, 'delete', Mock())
+    def setUp(self):
+        Consul({'ttl': 30, 'scope': 't', 'name': 'p', 'url': 'https://l:1', 'retry_timeout': 10,
+                'verify': 'on', 'key': 'foo', 'cert': 'bar', 'cacert': 'buz', 'token': 'asd', 'dc': 'dc1'})
+        Consul({'ttl': 30, 'scope': 't', 'name': 'p', 'url': 'https://l:1', 'retry_timeout': 10,
+                'verify': 'on', 'cert': 'bar', 'cacert': 'buz'})
+        self.c = Consul({'ttl': 30, 'scope': 'test', 'name': 'postgresql1', 'host': 'localhost:1', 'retry_timeout': 10})
+        self.c._base_path = '/service/good'
+        self.c._load_cluster()
+
+    @patch('time.sleep', Mock(side_effect=SleepException))
+    @patch.object(consul.Consul.Session, 'create', Mock(side_effect=ConsulException))
+    def test_create_session(self):
+        self.c._session = None
+        self.assertRaises(SleepException, self.c.create_session)
+
+    @patch.object(consul.Consul.Session, 'renew', Mock(side_effect=NotFound))
+    @patch.object(consul.Consul.Session, 'create', Mock(side_effect=[InvalidSessionTTL, ConsulException]))
+    @patch.object(consul.Consul.Agent, 'self', Mock(return_value={'Config': {'SessionTTLMin': 0}}))
+    @patch.object(HTTPClient, 'set_ttl', Mock(side_effect=ValueError))
+    def test_referesh_session(self):
+        self.c._session = '1'
+        self.assertFalse(self.c.refresh_session())
+        self.c._last_session_refresh = 0
+        self.assertRaises(ConsulError, self.c.refresh_session)
+
+    @patch.object(consul.Consul.KV, 'delete', Mock())
+    def test_get_cluster(self):
+        self.c._base_path = '/service/test'
+        self.assertIsInstance(self.c.get_cluster(), Cluster)
+        self.assertIsInstance(self.c.get_cluster(), Cluster)
+        self.c._base_path = '/service/fail'
+        self.assertRaises(ConsulError, self.c.get_cluster)
+        self.c._base_path = '/service/good'
+        self.c._session = 'fd4f44fe-2cac-bba5-a60b-304b51ff39b8'
+        self.assertIsInstance(self.c.get_cluster(), Cluster)
+
+    @patch.object(consul.Consul.KV, 'delete', Mock(side_effect=[ConsulException, True, True]))
+    @patch.object(consul.Consul.KV, 'put', Mock(side_effect=[True, ConsulException]))
+    def test_touch_member(self):
+        self.c.refresh_session = Mock(return_value=True)
+        self.c.touch_member({'balbla': 'blabla'})
+        self.c.touch_member({'balbla': 'blabla'})
+        self.c.touch_member({'balbla': 'blabla'})
+        self.c.refresh_session = Mock(return_value=False)
+        self.c.touch_member({'balbla': 'blabla'})
+
+    @patch.object(consul.Consul.KV, 'put', Mock(return_value=False))
+    def test_take_leader(self):
+        self.c.set_ttl(20)
+        self.c.refresh_session = Mock()
+        self.c.take_leader()
+
+    @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
+    def test_set_failover_value(self):
+        self.c.set_failover_value('')
+
+    @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
+    def test_set_config_value(self):
+        self.c.set_config_value('')
+
+    @patch.object(consul.Consul.KV, 'put', Mock(side_effect=ConsulException))
+    def test_write_leader_optime(self):
+        self.c.write_leader_optime('1')
+
+    @patch.object(consul.Consul.Session, 'renew', Mock())
+    def test_update_leader(self):
+        self.c.update_leader(None)
+
+    @patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
+    def test_delete_leader(self):
+        self.c.delete_leader()
+
+    @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
+    def test_initialize(self):
+        self.c.initialize()
+
+    @patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
+    def test_cancel_initialization(self):
+        self.c.cancel_initialization()
+
+    @patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
+    def test_delete_cluster(self):
+        self.c.delete_cluster()
+
+    @patch.object(AbstractDCS, 'watch', Mock())
+    def test_watch(self):
+        self.c.watch(None, 1)
+        self.c._name = ''
+        self.c.watch(6429, 1)
+        with patch.object(consul.Consul.KV, 'get', Mock(side_effect=ConsulException)):
+            self.c.watch(6429, 1)
+
+    def test_set_retry_timeout(self):
+        self.c.set_retry_timeout(10)
+
+    @patch.object(consul.Consul.KV, 'delete', Mock(return_value=True))
+    @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
+    def test_sync_state(self):
+        self.assertTrue(self.c.set_sync_state_value('{}'))
+        self.assertTrue(self.c.delete_sync_state())
+
+    @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
+    def test_set_history_value(self):
+        self.assertTrue(self.c.set_history_value('{}'))

tests/test_ctl.py

@@ -0,0 +1,554 @@
+import etcd
+import os
+import requests
+import sys
+import unittest
+
+from click.testing import CliRunner
+from datetime import datetime, timedelta
+from mock import patch, Mock
+from patroni.ctl import ctl, store_config, load_config, output_members, request_patroni, get_dcs, parse_dcs, \
+    get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException, apply_config_changes, \
+    format_config_for_editing, show_diff, invoke_editor, format_pg_version
+from patroni.dcs.etcd import Client, Failover
+from patroni.utils import tzutc
+from psycopg2 import OperationalError
+from test_etcd import etcd_read, requests_get, socket_getaddrinfo, MockResponse
+from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \
+    get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader, get_cluster, Member
+from test_postgresql import MockConnect, psycopg2_connect
+
+CONFIG_FILE_PATH = './test-ctl.yaml'
+
+
+def test_rw_config():
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        store_config({'etcd': {'host': 'localhost:2379'}}, CONFIG_FILE_PATH + '/dummy')
+        sys.argv = ['patronictl.py', '']
+        load_config(CONFIG_FILE_PATH + '/dummy', None)
+        load_config(CONFIG_FILE_PATH + '/dummy', '0.0.0.0')
+        os.remove(CONFIG_FILE_PATH + '/dummy')
+        os.rmdir(CONFIG_FILE_PATH)
+
+
+@patch('patroni.ctl.load_config',
+       Mock(return_value={'scope': 'alpha', 'postgresql': {'data_dir': '.', 'parameters': {}, 'retry_timeout': 5},
+                          'restapi': {'auth': 'u:p', 'listen': ''}, 'etcd': {'host': 'localhost:2379'}}))
+class TestCtl(unittest.TestCase):
+
+    @patch('socket.getaddrinfo', socket_getaddrinfo)
+    def setUp(self):
+        with patch.object(Client, 'machines') as mock_machines:
+            mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
+            self.runner = CliRunner()
+            self.e = get_dcs({'etcd': {'ttl': 30, 'host': 'ok:2379', 'retry_timeout': 10}}, 'foo')
+
+    @patch('psycopg2.connect', psycopg2_connect)
+    def test_get_cursor(self):
+        self.assertIsNone(get_cursor(get_cluster_initialized_without_leader(), {}, role='master'))
+
+        self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), {}, role='master'))
+
+        # MockCursor returns pg_is_in_recovery as false
+        self.assertIsNone(get_cursor(get_cluster_initialized_with_leader(), {}, role='replica'))
+
+        self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), {'database': 'foo'}, role='any'))
+
+    def test_parse_dcs(self):
+        assert parse_dcs(None) is None
+        assert parse_dcs('localhost') == {'etcd': {'host': 'localhost:2379'}}
+        assert parse_dcs('') == {'etcd': {'host': 'localhost:2379'}}
+        assert parse_dcs('localhost:8500') == {'consul': {'host': 'localhost:8500'}}
+        assert parse_dcs('zookeeper://localhost') == {'zookeeper': {'hosts': ['localhost:2181']}}
+        assert parse_dcs('exhibitor://dummy') == {'exhibitor': {'hosts': ['dummy'], 'port': 8181}}
+        assert parse_dcs('consul://localhost') == {'consul': {'host': 'localhost:8500'}}
+        self.assertRaises(PatroniCtlException, parse_dcs, 'invalid://test')
+
+    def test_output_members(self):
+        scheduled_at = datetime.now(tzutc) + timedelta(seconds=600)
+        cluster = get_cluster_initialized_with_leader(Failover(1, 'foo', 'bar', scheduled_at))
+        self.assertIsNone(output_members(cluster, name='abc', fmt='pretty'))
+        self.assertIsNone(output_members(cluster, name='abc', fmt='json'))
+        self.assertIsNone(output_members(cluster, name='abc', fmt='yaml'))
+        self.assertIsNone(output_members(cluster, name='abc', fmt='tsv'))
+
+    @patch('patroni.ctl.get_dcs')
+    @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
+    def test_switchover(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        mock_get_dcs.return_value.set_failover_value = Mock()
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+        assert 'leader' in result.output
+
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
+        assert result.exit_code == 0
+
+        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+            result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00'])
+            assert result.exit_code == 1
+
+        # Aborting switchover, as we anser NO to the confirmation
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\nN')
+        assert result.exit_code == 1
+
+        # Target and source are equal
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nleader\n\ny')
+        assert result.exit_code == 1
+
+        # Reality is not part of this cluster
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nReality\n\ny')
+        assert result.exit_code == 1
+
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force'])
+        assert 'Member' in result.output
+
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
+        assert result.exit_code == 0
+
+        # Invalid timestamp
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', 'invalid'])
+        assert result.exit_code != 0
+
+        # Invalid timestamp
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', '2115-02-30T12:00:00+01:00'])
+        assert result.exit_code != 0
+
+        # Specifying wrong leader
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='dummy')
+        assert result.exit_code == 1
+
+        with patch('patroni.ctl.request_patroni', Mock(side_effect=Exception)):
+            # Non-responding patroni
+            result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
+            assert 'falling back to DCS' in result.output
+
+        with patch('patroni.ctl.request_patroni') as mocked:
+            mocked.return_value.status_code = 500
+            result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+            assert 'Switchover failed' in result.output
+
+            mocked.return_value.status_code = 501
+            mocked.return_value.text = 'Server does not support this operation'
+            result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+            assert 'Switchover failed' in result.output
+
+        # No members available
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+        assert result.exit_code == 1
+
+        # No master available
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader
+        result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
+        assert result.exit_code == 1
+
+    @patch('patroni.ctl.get_dcs')
+    @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
+    def test_failover(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        mock_get_dcs.return_value.set_failover_value = Mock()
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='\n')
+        assert 'Failover could be performed only to a specific candidate' in result.output
+
+    def test_get_dcs(self):
+        self.assertRaises(PatroniCtlException, get_dcs, {'dummy': {}}, 'dummy')
+
+    @patch('psycopg2.connect', psycopg2_connect)
+    @patch('patroni.ctl.query_member', Mock(return_value=([['mock column']], None)))
+    @patch('patroni.ctl.get_dcs')
+    @patch.object(etcd.Client, 'read', etcd_read)
+    def test_query(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        # Mutually exclusive
+        result = self.runner.invoke(ctl, ['query', 'alpha', '--member', 'abc', '--role', 'master'])
+        assert result.exit_code == 1
+
+        with self.runner.isolated_filesystem():
+            with open('dummy', 'w') as dummy_file:
+                dummy_file.write('SELECT 1')
+
+            # Mutually exclusive
+            result = self.runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy', '--command', 'dummy'])
+            assert result.exit_code == 1
+
+            result = self.runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy'])
+            assert result.exit_code == 0
+
+            os.remove('dummy')
+
+        result = self.runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1'])
+        assert 'mock column' in result.output
+
+        # --command or --file is mandatory
+        result = self.runner.invoke(ctl, ['query', 'alpha'])
+        assert result.exit_code == 1
+
+        result = self.runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1', '--username', 'root',
+                                          '--password', '--dbname', 'postgres'], input='ab\nab')
+        assert 'mock column' in result.output
+
+    def test_query_member(self):
+        with patch('patroni.ctl.get_cursor', Mock(return_value=MockConnect().cursor())):
+            rows = query_member(None, None, None, 'master', 'SELECT pg_is_in_recovery()', {})
+            self.assertTrue('False' in str(rows))
+
+            rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()', {})
+            self.assertEquals(rows, (None, None))
+
+            with patch('test_postgresql.MockCursor.execute', Mock(side_effect=OperationalError('bla'))):
+                rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()', {})
+
+        with patch('patroni.ctl.get_cursor', Mock(return_value=None)):
+            rows = query_member(None, None, None, None, 'SELECT pg_is_in_recovery()', {})
+            self.assertTrue('No connection to' in str(rows))
+
+            rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()', {})
+            self.assertTrue('No connection to' in str(rows))
+
+        with patch('patroni.ctl.get_cursor', Mock(side_effect=OperationalError('bla'))):
+            rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()', {})
+
+    @patch('patroni.ctl.get_dcs')
+    def test_dsn(self, mock_get_dcs):
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        result = self.runner.invoke(ctl, ['dsn', 'alpha'])
+        assert 'host=127.0.0.1 port=5435' in result.output
+
+        # Mutually exclusive options
+        result = self.runner.invoke(ctl, ['dsn', 'alpha', '--role', 'master', '--member', 'dummy'])
+        assert result.exit_code == 1
+
+        # Non-existing member
+        result = self.runner.invoke(ctl, ['dsn', 'alpha', '--member', 'dummy'])
+        assert result.exit_code == 1
+
+    @patch('requests.post', requests_get)
+    @patch('patroni.ctl.get_dcs')
+    def test_restart_reinit(self, mock_get_dcs):
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        result = self.runner.invoke(ctl, ['restart', 'alpha'], input='y\n\nnow')
+        assert 'Failed: restart for' in result.output
+        assert result.exit_code == 0
+
+        result = self.runner.invoke(ctl, ['reinit', 'alpha'], input='y')
+        assert result.exit_code == 1
+
+        # successful reinit
+        result = self.runner.invoke(ctl, ['reinit', 'alpha', 'other'], input='y\ny')
+        assert result.exit_code == 0
+
+        # Aborted restart
+        result = self.runner.invoke(ctl, ['restart', 'alpha'], input='N')
+        assert result.exit_code == 1
+
+        result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force'])
+        assert result.exit_code == 0
+
+        # Not a member
+        result = self.runner.invoke(ctl, ['restart', 'alpha', 'dummy', '--any'], input='y')
+        assert result.exit_code == 1
+
+        # Wrong pg version
+        result = self.runner.invoke(ctl, ['restart', 'alpha', '--any', '--pg-version', '9.1'], input='y')
+        assert 'Error: Invalid PostgreSQL version format' in result.output
+        assert result.exit_code == 1
+
+        result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force', '--timeout', '10min'])
+        assert result.exit_code == 0
+
+        with patch('requests.delete', Mock(return_value=MockResponse(500))):
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force',
+                                              '--scheduled', '2300-10-01T14:30'])
+            assert 'Failed: flush scheduled restart' in result.output
+
+        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+            result = self.runner.invoke(ctl,
+                                        ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
+            assert result.exit_code == 1
+
+        with patch('requests.post', Mock(return_value=MockResponse())):
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0.0',
+                                        '--scheduled', '2300-10-01T14:30'], input='y')
+            assert result.exit_code == 0
+
+        with patch('requests.post', Mock(return_value=MockResponse(204))):
+            # get restart with the non-200 return code
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0',
+                                        '--scheduled', '2300-10-01T14:30'], input='y')
+            assert result.exit_code == 0
+
+        # force restart with restart already present
+        with patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse(204))):
+            result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force',
+                                              '--scheduled', '2300-10-01T14:30'])
+            assert result.exit_code == 0
+
+        with patch('requests.post', Mock(return_value=MockResponse(202))):
+            # get restart with the non-200 return code
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(
+                ctl, ['restart', 'alpha', '--pg-version', '99.0.0', '--scheduled', '2300-10-01T14:30'], input='y'
+            )
+            assert 'Success: restart scheduled' in result.output
+            assert result.exit_code == 0
+
+        with patch('requests.post', Mock(return_value=MockResponse(409))):
+            # get restart with the non-200 return code
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(
+                ctl, ['restart', 'alpha', '--pg-version', '99.0.0', '--scheduled', '2300-10-01T14:30'], input='y'
+            )
+            assert 'Failed: another restart is already' in result.output
+            assert result.exit_code == 0
+
+    @patch('patroni.ctl.get_dcs')
+    def test_remove(self, mock_get_dcs):
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        result = self.runner.invoke(ctl, ['remove', 'alpha'], input='alpha\nslave')
+        assert 'Please confirm' in result.output
+        assert 'You are about to remove all' in result.output
+        # Not typing an exact confirmation
+        assert result.exit_code == 1
+
+        # master specified does not match master of cluster
+        result = self.runner.invoke(ctl, ['remove', 'alpha'], input='alpha\nYes I am aware\nslave')
+        assert result.exit_code == 1
+
+        # cluster specified on cmdline does not match verification prompt
+        result = self.runner.invoke(ctl, ['remove', 'alpha'], input='beta\nleader')
+        assert result.exit_code == 1
+
+        result = self.runner.invoke(ctl, ['remove', 'alpha'], input='alpha\nYes I am aware\nleader')
+        assert result.exit_code == 0
+
+    @patch('requests.post', Mock(side_effect=requests.exceptions.ConnectionError('foo')))
+    def test_request_patroni(self):
+        member = get_cluster_initialized_with_leader().leader.member
+        self.assertRaises(requests.exceptions.ConnectionError, request_patroni, member, 'post', 'dummy', {})
+
+    def test_ctl(self):
+        self.runner.invoke(ctl, ['list'])
+
+        result = self.runner.invoke(ctl, ['--help'])
+        assert 'Usage:' in result.output
+
+    def test_get_any_member(self):
+        self.assertIsNone(get_any_member(get_cluster_initialized_without_leader(), role='master'))
+
+        m = get_any_member(get_cluster_initialized_with_leader(), role='master')
+        self.assertEquals(m.name, 'leader')
+
+    def test_get_all_members(self):
+        self.assertEquals(list(get_all_members(get_cluster_initialized_without_leader(), role='master')), [])
+
+        r = list(get_all_members(get_cluster_initialized_with_leader(), role='master'))
+        self.assertEquals(len(r), 1)
+        self.assertEquals(r[0].name, 'leader')
+
+        r = list(get_all_members(get_cluster_initialized_with_leader(), role='replica'))
+        self.assertEquals(len(r), 1)
+        self.assertEquals(r[0].name, 'other')
+
+        self.assertEquals(len(list(get_all_members(get_cluster_initialized_without_leader(), role='replica'))), 2)
+
+    @patch('patroni.ctl.get_dcs')
+    def test_members(self, mock_get_dcs):
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        result = self.runner.invoke(ctl, ['list'])
+        assert '127.0.0.1' in result.output
+        assert result.exit_code == 0
+        with patch('patroni.ctl.load_config', Mock(return_value={})):
+            self.runner.invoke(ctl, ['list'])
+
+    def test_configure(self):
+        result = self.runner.invoke(configure, ['--dcs', 'abc', '-c', 'dummy', '-n', 'bla'])
+        assert result.exit_code == 0
+
+    @patch('patroni.ctl.get_dcs')
+    def test_scaffold(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_not_initialized_without_leader
+        mock_get_dcs.return_value.initialize = Mock(return_value=True)
+        mock_get_dcs.return_value.touch_member = Mock(return_value=True)
+        mock_get_dcs.return_value.attempt_to_acquire_leader = Mock(return_value=True)
+        mock_get_dcs.return_value.delete_cluster = Mock()
+
+        with patch.object(self.e, 'initialize', return_value=False):
+            result = self.runner.invoke(ctl, ['scaffold', 'alpha'])
+            assert result.exception
+
+        with patch.object(mock_get_dcs.return_value, 'touch_member', Mock(return_value=False)):
+            result = self.runner.invoke(ctl, ['scaffold', 'alpha'])
+            assert result.exception
+
+        result = self.runner.invoke(ctl, ['scaffold', 'alpha'])
+        assert result.exit_code == 0
+
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        result = self.runner.invoke(ctl, ['scaffold', 'alpha'])
+        assert result.exception
+
+    @patch('patroni.ctl.get_dcs')
+    def test_list_extended(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
+        mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster)
+
+        result = self.runner.invoke(ctl, ['list', 'dummy', '--extended', '--timestamp'])
+        assert '2100' in result.output
+        assert 'Scheduled restart' in result.output
+
+    @patch('patroni.ctl.get_dcs')
+    @patch('requests.delete', Mock(return_value=MockResponse()))
+    def test_flush(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+
+        result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '-r', 'master'], input='y')
+        assert 'No scheduled restart' in result.output
+
+        result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '--force'])
+        assert 'Success: flush scheduled restart' in result.output
+        with patch.object(requests, 'delete', return_value=MockResponse(404)):
+            result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '--force'])
+            assert 'Failed: flush scheduled restart' in result.output
+
+    @patch('patroni.ctl.get_dcs')
+    @patch('patroni.ctl.polling_loop', Mock(return_value=[1]))
+    def test_pause_cluster(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+
+        with patch('requests.patch', Mock(return_value=MockResponse(500))):
+            result = self.runner.invoke(ctl, ['pause', 'dummy'])
+            assert 'Failed' in result.output
+
+        with patch('requests.patch', Mock(return_value=MockResponse(200))),\
+                patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+            result = self.runner.invoke(ctl, ['pause', 'dummy'])
+            assert 'Cluster is already paused' in result.output
+
+        with patch('requests.patch', Mock(return_value=MockResponse(200))):
+            result = self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
+            assert "'pause' request sent" in result.output
+            mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
+                                                                      get_cluster(None, None, [], None, None)])
+            self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
+            member = Member(1, 'other', 28, {})
+            mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
+                                                                      get_cluster(None, None, [member], None, None)])
+            self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
+
+    @patch('patroni.ctl.get_dcs')
+    def test_resume_cluster(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+
+        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+            with patch('requests.patch', Mock(return_value=MockResponse(200))):
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Success' in result.output
+
+            with patch('requests.patch', Mock(return_value=MockResponse(500))):
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Failed' in result.output
+
+            with patch('requests.patch', Mock(return_value=MockResponse(200))),\
+                    patch('patroni.dcs.Cluster.is_paused', Mock(return_value=False)):
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Cluster is not paused' in result.output
+
+            with patch('requests.patch', Mock(side_effect=Exception)):
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Can not find accessible cluster member' in result.output
+
+    def test_apply_config_changes(self):
+        config = {"postgresql": {"parameters": {"work_mem": "4MB"}, "use_pg_rewind": True}, "ttl": 30}
+
+        before_editing = format_config_for_editing(config)
+
+        # Spaces are allowed and stripped, numbers and booleans are interpreted
+        after_editing, changed_config = apply_config_changes(before_editing, config,
+                                                             ["postgresql.parameters.work_mem = 5MB",
+                                                              "ttl=15", "postgresql.use_pg_rewind=off", 'a.b=c'])
+        self.assertEquals(changed_config, {"a": {"b": "c"}, "postgresql": {"parameters": {"work_mem": "5MB"},
+                                                                           "use_pg_rewind": False}, "ttl": 15})
+
+        # postgresql.parameters namespace is flattened
+        after_editing, changed_config = apply_config_changes(before_editing, config,
+                                                             ["postgresql.parameters.work_mem.sub = x"])
+        self.assertEquals(changed_config, {"postgresql": {"parameters": {"work_mem": "4MB", "work_mem.sub": "x"},
+                                                          "use_pg_rewind": True}, "ttl": 30})
+
+        # Setting to null deletes
+        after_editing, changed_config = apply_config_changes(before_editing, config,
+                                                             ["postgresql.parameters.work_mem=null"])
+        self.assertEquals(changed_config, {"postgresql": {"use_pg_rewind": True}, "ttl": 30})
+        after_editing, changed_config = apply_config_changes(before_editing, config,
+                                                             ["postgresql.use_pg_rewind=null",
+                                                              "postgresql.parameters.work_mem=null"])
+        self.assertEquals(changed_config, {"ttl": 30})
+
+        self.assertRaises(PatroniCtlException, apply_config_changes, before_editing, config, ['a'])
+
+    @patch('sys.stdout.isatty', return_value=False)
+    @patch('cdiff.markup_to_pager')
+    def test_show_diff(self, mock_markup_to_pager, mock_isatty):
+        show_diff("foo:\n  bar: 1\n", "foo:\n  bar: 2\n")
+        mock_markup_to_pager.assert_not_called()
+
+        mock_isatty.return_value = True
+        show_diff("foo:\n  bar: 1\n", "foo:\n  bar: 2\n")
+        mock_markup_to_pager.assert_called_once()
+
+        # Test that unicode handling doesn't fail with an exception
+        show_diff(b"foo:\n  bar: \xc3\xb6\xc3\xb6\n".decode('utf-8'),
+                  b"foo:\n  bar: \xc3\xbc\xc3\xbc\n".decode('utf-8'))
+
+    def test_invoke_editor(self):
+        for e in ('', 'false'):
+            os.environ['EDITOR'] = e
+            self.assertRaises(PatroniCtlException, invoke_editor, 'foo: bar\n', 'test')
+
+    @patch('patroni.ctl.get_dcs')
+    def test_show_config(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        self.runner.invoke(ctl, ['show-config', 'dummy'])
+
+    @patch('patroni.ctl.get_dcs')
+    def test_edit_config(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        os.environ['EDITOR'] = 'true'
+        self.runner.invoke(ctl, ['edit-config', 'dummy'])
+        self.runner.invoke(ctl, ['edit-config', 'dummy', '-s', 'foo=bar'])
+        self.runner.invoke(ctl, ['edit-config', 'dummy', '--replace', 'postgres0.yml'])
+        self.runner.invoke(ctl, ['edit-config', 'dummy', '--apply', '-'], input='foo: bar')
+        self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar')
+        mock_get_dcs.return_value.set_config_value = Mock(return_value=True)
+        self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar')
+
+    @patch('patroni.ctl.get_dcs')
+    def test_version(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        with patch('patroni.ctl.request_patroni') as mocked:
+            result = self.runner.invoke(ctl, ['version'])
+            assert 'patronictl version' in result.output
+            mocked.return_value.json = lambda: {'patroni': {'version': '1.2.3'}, 'server_version': 100001}
+            result = self.runner.invoke(ctl, ['version', 'dummy'])
+            assert '1.2.3' in result.output
+        with patch('requests.get', Mock(side_effect=Exception)):
+            result = self.runner.invoke(ctl, ['version', 'dummy'])
+            assert 'failed to get version' in result.output
+
+    def test_format_pg_version(self):
+        self.assertEquals(format_pg_version(100001), '10.1')
+        self.assertEquals(format_pg_version(90605), '9.6.5')

tests/test_etcd.py

@@ -1,20 +1,21 @@
 import etcd
 import json
+import urllib3.util.connection
 import requests
-import urllib3
 import socket
 import unittest
 
 from dns.exception import DNSException
 from mock import Mock, patch
-from patroni.dcs import Cluster, DCSError, Leader
-from patroni.etcd import Client, Etcd
+from patroni.dcs.etcd import AbstractDCS, Client, Cluster, Etcd, EtcdError, DnsCachingResolver
+from patroni.exceptions import DCSError
+from urllib3.exceptions import ReadTimeoutError
 
 
-class MockResponse:
+class MockResponse(object):
 
-    def __init__(self):
-        self.status_code = 200
+    def __init__(self, status_code=200):
+        self.status_code = status_code
         self.content = '{}'
         self.ok = True
 
@@ -23,26 +24,21 @@ class MockResponse:
 
     @property
     def data(self):
-        if self.content == 'TimeoutError':
-            raise urllib3.exceptions.TimeoutError
-        if self.content == 'Exception':
-            raise Exception
+        return self.content.encode('utf-8')
+
+    @property
+    def text(self):
         return self.content
 
     @property
     def status(self):
         return self.status_code
 
+    @staticmethod
     def getheader(*args):
         return ''
 
 
-class MockPostgresql(Mock):
-
-    def last_operation(self):
-        return '0'
-
-
 def requests_get(url, **kwargs):
     members = '[{"id":14855829450254237642,"peerURLs":["http://localhost:2380","http://localhost:7001"],' +\
               '"name":"default","clientURLs":["http://localhost:2379","http://localhost:4001"]}]'
@@ -50,49 +46,52 @@ def requests_get(url, **kwargs):
     if url.startswith('http://local'):
         raise requests.exceptions.RequestException()
     elif ':8011/patroni' in url:
-        response.content = '{"role": "replica", "xlog": {"replayed_location": 0}}'
+        response.content = '{"role": "replica", "xlog": {"received_location": 0}, "tags": {}}'
     elif url.endswith('/members'):
-        if url.startswith('http://error'):
-            response.content = '[{}]'
-        else:
-            response.content = members
+        response.content = '[{}]' if url.startswith('http://error') else members
     elif url.startswith('http://exhibitor'):
         response.content = '{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}'
+    elif url.endswith(':8011/reinitialize'):
+        data = kwargs.get('data', '')
+        if ' false}' in data:
+            response.status_code = 503
+            response.ok = False
+            response.content = 'restarting after failure already in progress'
     else:
         response.status_code = 404
         response.ok = False
     return response
 
 
-def etcd_watch(key, index=None, timeout=None, recursive=None):
+def etcd_watch(self, key, index=None, timeout=None, recursive=None):
     if timeout == 2.0:
-        raise urllib3.exceptions.TimeoutError
+        raise etcd.EtcdWatchTimedOut
     elif timeout == 5.0:
         return etcd.EtcdResult('delete', {})
-    elif timeout == 10.0:
+    elif 5 < timeout <= 10.0:
         raise etcd.EtcdException
-    elif index == 20729:
-        return etcd.EtcdResult('set', {'value': 'postgresql1', 'modifiedIndex': index + 1})
-    elif index == 20731:
-        return etcd.EtcdResult('set', {'value': 'postgresql2', 'modifiedIndex': index + 1})
+    elif timeout == 20.0:
+        raise etcd.EtcdEventIndexCleared
 
 
-def etcd_write(key, value, **kwargs):
+def etcd_write(self, key, value, **kwargs):
     if key == '/service/exists/leader':
         raise etcd.EtcdAlreadyExist
-    if key == '/service/test/leader':
-        if kwargs.get('prevValue', None) == 'foo' or not kwargs.get('prevExist', True):
-            return True
+    if key in ['/service/test/leader', '/patroni/test/leader'] and \
+            (kwargs.get('prevValue') == 'foo' or not kwargs.get('prevExist', True)):
+        return True
     raise etcd.EtcdException
 
 
-def etcd_read(key, **kwargs):
+def etcd_read(self, key, **kwargs):
     if key == '/service/noleader/':
         raise DCSError('noleader')
     elif key == '/service/nocluster/':
         raise etcd.EtcdKeyNotFound
 
     response = {"action": "get", "node": {"key": "/service/batman5", "dir": True, "nodes": [
+                {"key": "/service/batman5/config", "value": '{"synchronous_mode": 0}',
+                 "modifiedIndex": 1582, "createdIndex": 1582},
                 {"key": "/service/batman5/failover", "value": "",
                  "modifiedIndex": 1582, "createdIndex": 1582},
                 {"key": "/service/batman5/initialize", "value": "postgresql0",
@@ -104,124 +103,188 @@ def etcd_read(key, **kwargs):
                     {"key": "/service/batman5/optime/leader", "value": "2164261704",
                      "modifiedIndex": 20729, "createdIndex": 20729}],
                  "modifiedIndex": 20437, "createdIndex": 20437},
+                {"key": "/service/batman5/sync", "value": '{"leader": "leader"}',
+                 "modifiedIndex": 1582, "createdIndex": 1582},
                 {"key": "/service/batman5/members", "dir": True, "nodes": [
                     {"key": "/service/batman5/members/postgresql1",
-                     "value": "postgres://replicator:[email protected]:5434/postgres"
-                        + "?application_name=http://127.0.0.1:8009/patroni",
+                     "value": "postgres://replicator:[email protected]:5434/postgres" +
+                        "?application_name=http://127.0.0.1:8009/patroni",
                      "expiration": "2015-05-15T09:10:59.949384522Z", "ttl": 21,
                      "modifiedIndex": 20727, "createdIndex": 20727},
                     {"key": "/service/batman5/members/postgresql0",
-                     "value": "postgres://replicator:[email protected]:5433/postgres"
-                        + "?application_name=http://127.0.0.1:8008/patroni",
+                     "value": "postgres://replicator:[email protected]:5433/postgres" +
+                        "?application_name=http://127.0.0.1:8008/patroni",
                      "expiration": "2015-05-15T09:11:09.611860899Z", "ttl": 30,
                      "modifiedIndex": 20730, "createdIndex": 20730}],
                  "modifiedIndex": 1581, "createdIndex": 1581}], "modifiedIndex": 1581, "createdIndex": 1581}}
-    return etcd.EtcdResult(**response)
+    result = etcd.EtcdResult(**response)
+    result.etcd_index = 0
+    return result
 
 
 class SleepException(Exception):
     pass
 
 
-class MockSRV:
-    port = 2380
-    target = '127.0.0.1'
-
-
-def dns_query(name, type):
+def dns_query(name, _):
+    if '-server' not in name or '-ssl' in name:
+        return []
     if name == '_etcd-server._tcp.blabla':
         return []
     elif name == '_etcd-server._tcp.exception':
         raise DNSException()
-    return [MockSRV()]
+    srv = Mock()
+    srv.port = 2380
+    srv.target.to_text.return_value = 'localhost' if name == '_etcd-server._tcp.foobar' else '127.0.0.1'
+    return [srv]
 
 
 def socket_getaddrinfo(*args):
-    if args[0] == 'ok':
-        return [(2, 1, 6, '', ('127.0.0.1', 2379)), (2, 1, 6, '', ('127.0.0.1', 2379))]
-    raise socket.error
+    if args[0] in ('ok', 'localhost', '127.0.0.1'):
+        return [(socket.AF_INET, 1, 6, '', ('127.0.0.1', 0)), (socket.AF_INET6, 1, 6, '', ('::1', 0))]
+    raise socket.gaierror
 
 
 def http_request(method, url, **kwargs):
+    if url == 'http://localhost:2379/timeout':
+        raise ReadTimeoutError(None, None, None)
+    if url == 'http://localhost:2379/v2/machines':
+        ret = MockResponse()
+        ret.content = 'http://localhost:2379,http://localhost:4001'
+        return ret
     if url == 'http://localhost:2379/':
         return MockResponse()
     raise socket.error
 
 
+class TestDnsCachingResolver(unittest.TestCase):
+
+    @patch('time.sleep', Mock(side_effect=SleepException))
+    @patch('socket.getaddrinfo', Mock(side_effect=socket.gaierror))
+    def test_run(self):
+        r = DnsCachingResolver()
+        self.assertIsNone(r.resolve_async('', 0))
+        r.join()
+
+
 @patch('dns.resolver.query', dns_query)
 @patch('socket.getaddrinfo', socket_getaddrinfo)
 @patch('requests.get', requests_get)
 class TestClient(unittest.TestCase):
 
     @patch('dns.resolver.query', dns_query)
+    @patch('socket.getaddrinfo', socket_getaddrinfo)
     @patch('requests.get', requests_get)
     def setUp(self):
-        with patch.object(etcd.Client, 'machines') as mock_machines:
+        with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
-            self.client = Client({'discovery_srv': 'test'})
+            self.client = Client({'srv': 'test', 'retry_timeout': 3}, DnsCachingResolver())
             self.client.http.request = http_request
+            self.client.http.request_encode_body = http_request
 
-    def test_api_execute(self):
+    def test_machines(self):
         self.client._base_uri = 'http://localhost:4001'
         self.client._machines_cache = ['http://localhost:2379']
-        self.client.api_execute('/', 'GET')
+        self.assertIsNotNone(self.client.machines)
+        self.client._base_uri = 'http://localhost:4001'
+        self.client._machines_cache = []
+        self.assertIsNotNone(self.client.machines)
+        self.client._update_machines_cache = True
+        machines = None
+        try:
+            machines = self.client.machines
+            self.assertFail()
+        except Exception:
+            self.assertIsNone(machines)
+
+    @patch.object(Client, 'machines')
+    def test_api_execute(self, mock_machines):
+        mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
+        self.assertRaises(ValueError, self.client.api_execute, '', '')
+        self.client._base_uri = 'http://localhost:4001'
+        self.client._machines_cache = ['http://localhost:2379']
+        self.client.api_execute('/', 'POST', timeout=0)
+        mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
+        self.client._machines_cache_updated = 0
+        self.client.api_execute('/', 'POST', timeout=0)
+        self.client._machines_cache = [self.client._base_uri]
+        self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
+        self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '')
+        self.client._update_machines_cache = True
+        with patch.object(Client, '_load_machines_cache', Mock(side_effect=etcd.EtcdException)):
+            self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'GET')
 
     def test_get_srv_record(self):
-        self.assertEquals(self.client.get_srv_record('blabla'), [])
-        self.assertEquals(self.client.get_srv_record('exception'), [])
-
-    def test__result_from_response(self):
-        response = MockResponse()
-        response.content = 'TimeoutError'
-        self.assertRaises(urllib3.exceptions.TimeoutError, self.client._result_from_response, response)
-        response.content = 'Exception'
-        self.assertRaises(etcd.EtcdException, self.client._result_from_response, response)
-        response.content = b'{}'
-        self.assertRaises(etcd.EtcdException, self.client._result_from_response, response)
+        self.assertEquals(self.client.get_srv_record('_etcd-server._tcp.blabla'), [])
+        self.assertEquals(self.client.get_srv_record('_etcd-server._tcp.exception'), [])
 
     def test__get_machines_cache_from_srv(self):
+        self.client._get_machines_cache_from_srv('foobar')
         self.client.get_srv_record = Mock(return_value=[('localhost', 2380)])
         self.client._get_machines_cache_from_srv('blabla')
 
     def test__get_machines_cache_from_dns(self):
-        self.client._get_machines_cache_from_dns('error:2379')
+        self.client._get_machines_cache_from_dns('error', 2379)
 
-    def test__load_machines_cache(self):
+    @patch.object(Client, 'machines')
+    def test__load_machines_cache(self, mock_machines):
+        mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
         self.client._config = {}
         self.assertRaises(Exception, self.client._load_machines_cache)
-        self.client._config = {'discovery_srv': 'blabla'}
+        self.client._config = {'srv': 'blabla'}
         self.assertRaises(etcd.EtcdException, self.client._load_machines_cache)
 
+    @patch.object(socket.socket, 'connect')
+    def test_create_connection_patched(self, mock_connect):
+        self.assertRaises(socket.error, urllib3.util.connection.create_connection, ('fail', 2379))
+        urllib3.util.connection.create_connection(('[localhost]', 2379))
+        mock_connect.side_effect = socket.error
+        self.assertRaises(socket.error, urllib3.util.connection.create_connection, ('[localhost]', 2379),
+                          timeout=1, source_address=('localhost', 53333),
+                          socket_options=[(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)])
+
 
 @patch('requests.get', requests_get)
+@patch('socket.getaddrinfo', socket_getaddrinfo)
+@patch.object(etcd.Client, 'write', etcd_write)
+@patch.object(etcd.Client, 'read', etcd_read)
+@patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
 class TestEtcd(unittest.TestCase):
 
+    @patch('socket.getaddrinfo', socket_getaddrinfo)
     def setUp(self):
         with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
-            self.etcd = Etcd('foo', {'ttl': 30, 'host': 'localhost:2379', 'scope': 'test'})
-            self.etcd.client.write = etcd_write
-            self.etcd.client.read = etcd_read
-            self.etcd.client.delete = Mock(side_effect=etcd.EtcdException())
+            self.etcd = Etcd({'namespace': '/patroni/', 'ttl': 30, 'retry_timeout': 10,
+                              'host': 'localhost:2379', 'scope': 'test', 'name': 'foo'})
+
+    def test_base_path(self):
+        self.assertEquals(self.etcd._base_path, '/patroni/test')
 
     @patch('dns.resolver.query', dns_query)
     def test_get_etcd_client(self):
-        with patch.object(etcd.Client, 'machines') as mock_machines:
+        with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(side_effect=etcd.EtcdException)
-            with patch('time.sleep', Mock(side_effect=SleepException())):
-                self.assertRaises(SleepException, self.etcd.get_etcd_client, {'discovery_srv': 'test'})
+            with patch('time.sleep', Mock(side_effect=SleepException)):
+                self.assertRaises(SleepException, self.etcd.get_etcd_client,
+                                  {'discovery_srv': 'test', 'retry_timeout': 10, 'cacert': '1', 'key': '1', 'cert': 1})
+                self.assertRaises(SleepException, self.etcd.get_etcd_client,
+                                  {'url': 'https://test:2379', 'retry_timeout': 10})
+                self.assertRaises(SleepException, self.etcd.get_etcd_client,
+                                  {'proxy': 'https://user:password@test:2379', 'retry_timeout': 10})
+                self.assertRaises(SleepException, self.etcd.get_etcd_client,
+                                  {'hosts': 'foo:4001,bar', 'retry_timeout': 10})
 
     def test_get_cluster(self):
-        self.assertIsInstance(self.etcd.get_cluster(), Cluster)
+        cluster = self.etcd.get_cluster()
+        self.assertIsInstance(cluster, Cluster)
+        self.assertFalse(cluster.is_synchronous_mode())
         self.etcd._base_path = '/service/nocluster'
         cluster = self.etcd.get_cluster()
         self.assertIsInstance(cluster, Cluster)
         self.assertIsNone(cluster.leader)
-
-    def test_current_leader(self):
-        self.assertIsInstance(self.etcd.current_leader(), Leader)
         self.etcd._base_path = '/service/noleader'
-        self.assertIsNone(self.etcd.current_leader())
+        self.assertRaises(EtcdError, self.etcd.get_cluster)
 
     def test_touch_member(self):
         self.assertFalse(self.etcd.touch_member('', ''))
@@ -239,7 +302,7 @@ class TestEtcd(unittest.TestCase):
         self.etcd.write_leader_optime('0')
 
     def test_update_leader(self):
-        self.assertTrue(self.etcd.update_leader())
+        self.assertTrue(self.etcd.update_leader(None))
 
     def test_initialize(self):
         self.assertFalse(self.etcd.initialize())
@@ -250,11 +313,31 @@ class TestEtcd(unittest.TestCase):
     def test_delete_leader(self):
         self.assertFalse(self.etcd.delete_leader())
 
+    def test_delete_cluster(self):
+        self.assertFalse(self.etcd.delete_cluster())
+
+    @patch('time.sleep', Mock(side_effect=SleepException))
+    @patch.object(etcd.Client, 'watch', etcd_watch)
     def test_watch(self):
-        self.etcd.client.watch = etcd_watch
-        self.etcd.watch(0)
+        self.etcd.watch(None, 0)
         self.etcd.get_cluster()
-        self.etcd.watch(1.5)
-        self.etcd.watch(4.5)
-        self.etcd.watch(9.5)
-        self.etcd.watch(100)
+        self.etcd.watch(20729, 1.5)
+        self.etcd.watch(20729, 4.5)
+        with patch.object(AbstractDCS, 'watch', Mock()):
+            self.assertTrue(self.etcd.watch(20729, 19.5))
+            self.assertRaises(SleepException, self.etcd.watch, 20729, 9.5)
+
+    def test_other_exceptions(self):
+        self.etcd.retry = Mock(side_effect=AttributeError('foo'))
+        self.assertRaises(EtcdError, self.etcd.cancel_initialization)
+
+    def test_set_ttl(self):
+        self.etcd.set_ttl(20)
+        self.assertTrue(self.etcd.watch(None, 1))
+
+    def test_sync_state(self):
+        self.assertFalse(self.etcd.write_sync_state('leader', None))
+        self.assertFalse(self.etcd.delete_sync_state())
+
+    def test_set_history_value(self):
+        self.assertFalse(self.etcd.set_history_value('{}'))

tests/test_exhibitor.py

@@ -0,0 +1,31 @@
+import unittest
+
+from mock import Mock, patch
+from patroni.dcs.exhibitor import ExhibitorEnsembleProvider, Exhibitor
+from patroni.dcs.zookeeper import ZooKeeperError
+from test_etcd import SleepException, requests_get
+from test_zookeeper import MockKazooClient
+
+
+@patch('requests.get', requests_get)
+@patch('time.sleep', Mock(side_effect=SleepException))
+class TestExhibitorEnsembleProvider(unittest.TestCase):
+
+    def test_init(self):
+        self.assertRaises(SleepException, ExhibitorEnsembleProvider, ['localhost'], 8181)
+
+    def test_poll(self):
+        self.assertFalse(ExhibitorEnsembleProvider(['exhibitor'], 8181).poll())
+
+
+class TestExhibitor(unittest.TestCase):
+
+    @patch('requests.get', requests_get)
+    @patch('patroni.dcs.zookeeper.KazooClient', MockKazooClient)
+    def setUp(self):
+        self.e = Exhibitor({'hosts': ['localhost', 'exhibitor'], 'port': 8181, 'scope': 'test',
+                            'name': 'foo', 'ttl': 30, 'retry_timeout': 10})
+
+    @patch.object(ExhibitorEnsembleProvider, 'poll', Mock(return_value=True))
+    def test_get_cluster(self):
+        self.assertRaises(ZooKeeperError, self.e.get_cluster)