Bumped version to 1.1

Feature/disable automatic failover

.gitignore

@@ -1,11 +1,48 @@
-data/*
-*.pyc
-*.egg/
-*.egg-info/
+*.py[cod]
+
+# vi(m) swap files:
+*.sw?
+
+# C extensions
+*.so
+
+# Packages
 .cache/
+*.egg
+*.eggs
+*.egg-info
+dist
+build
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+lib
+lib64
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
 .coverage
-.eggs/
-build/
+.tox
+nosetests.xml
 coverage.xml
+htmlcov
 junit.xml
+features/output
+dummy
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
 pgpass
+scm-source.json

.travis.yml

@@ -19,7 +19,7 @@ cache:
   directories:
   - $HOME/virtualenv/python2.7.9
   - $HOME/virtualenv/python3.4.2
-  - $HOME/virtualenv/python3.5.0
+  - $HOME/virtualenv/python3.5.2
 install:
   - |
     set -e
@@ -51,7 +51,7 @@ install:
     for pv in "2.7" "3.4" "3.5"; do
         source ~/virtualenv/python${pv}/bin/activate
         # explicitly install all needed python modules to cache them
-        for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8 mock pytest-cov pytest'; do
+        for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8==2.6.0 mock>=2.0.0 pytest-cov pytest'; do
             pip install $p
         done
     done

.zappr.yaml

@@ -0,0 +1,13 @@
+# for github.com
+approvals:
+  groups:
+    zalando:
+      minimum: 2
+      from:
+        orgs:
+          - "zalando"
+# team should be valid team id in team service https://teams.auth.zalando.com/api/teams/:id
+X-Zalando-Team: "acid"
+# type should be one of [code, doc, config, tools, secrets]
+# code will be the default value, if X-Zalando-Type is not found in .zappr.yml
+X-Zalando-Type: code

.zappr.yml

@@ -1,12 +0,0 @@
-approvals:
-  # PR needs at least 4 approvals
-  minimum: 1
-  # approval = comment that matches this regex
-  pattern: "^:?\\+1:?$"
-  from:
-    # commenter must be either one of:
-    # a public zalando org member
-    orgs:
-      - zalando
-    # a collaborator of the repo
-    collaborators: true

Dockerfile

@@ -1,41 +1,44 @@
 ## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine
 ## It has all the necessary components to play/debug with a single node appliance, running etcd
-FROM ubuntu:14.04
+FROM ubuntu:16.04
 MAINTAINER Feike Steenbergen <[email protected]>
 
-# We need curl
-RUN apt-get update -y && apt-get install curl -y
-
-# Add PGDG repositories
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
-RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-RUN apt-get update -y
-RUN apt-get upgrade -y
+RUN echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/01norecommend \
+    && echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/01norecommend
 
 ENV PGVERSION 9.5
-RUN apt-get install postgresql-${PGVERSION} postgresql-server-dev-${PGVERSION} -y
-RUN apt-get install python python-dev python-pip -y
-ADD requirements-py2.txt /requirements-py2.txt
-RUN pip install -r /requirements-py2.txt
-
 ENV PATH /usr/lib/postgresql/${PGVERSION}/bin:$PATH
+RUN apt-get update -y \
+    && apt-get upgrade -y \
+    && apt-get install -y curl jq haproxy zookeeper postgresql-${PGVERSION} python-psycopg2 python-yaml \
+        python-requests python-six python-click python-dateutil python-tzlocal python-urllib3 \
+        python-dnspython python-pip python-setuptools python-kazoo python-prettytable python \
+    && pip install python-etcd==0.4.3 python-consul==0.6.0 --upgrade \
+    && apt-get remove -y python-pip python-setuptools \
+    && apt-get autoremove -y \
+        # Clean up
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/* /root/.cache
 
-ADD patroni.py /patroni.py
-ADD patronictl.py /patronictl.py
-ADD patroni/ /patroni
+ENV ETCDVERSION 2.3.6
+RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz \
+    | tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl
 
-RUN ln -s /patroni.py /usr/local/bin/patroni
+ENV CONFDVERSION 0.11.0
+RUN curl -L https://github.com/kelseyhightower/confd/releases/download/v${CONFDVERSION}/confd-${CONFDVERSION}-linux-amd64 > /usr/local/bin/confd \
+    && chmod +x /usr/local/bin/confd
+
+ADD patronictl.py patroni.py docker/entrypoint.sh /
+ADD patroni /patroni/
+ADD extras/confd /etc/confd
 RUN ln -s /patronictl.py /usr/local/bin/patronictl
 
-ENV ETCDVERSION 2.2.5
-RUN curl -L https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-amd64.tar.gz | tar xz -C /bin --strip=1 --wildcards --no-anchored etcd etcdctl
-
 ### Setting up a simple script that will serve as an entrypoint
-RUN mkdir /data/ && touch /var/log/etcd.log /var/log/etcd.err /pgpass /patroni/postgres.yml
-RUN chown postgres:postgres -R /patroni/ /data/ /pgpass /var/log/etcd.* /patroni/postgres.yml
-ADD docker/entrypoint.sh /entrypoint.sh
+RUN mkdir /data/ && touch /pgpass /patroni.yml \
+    && chown postgres:postgres -R /patroni/ /data/ /pgpass /patroni.yml /etc/haproxy /var/run/ /var/lib/ /var/log/ \
+    && echo 1 > /etc/zookeeper/conf/myid
 
-EXPOSE 4001 5432 2380
+EXPOSE 2379 5432 8008
 
 ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
 USER postgres

README.rst

@@ -2,33 +2,67 @@
 
 Patroni: A Template for PostgreSQL HA with ZooKeeper, etcd or Consul
 ------------------------------------------------------------
+There are many ways to run high availability with PostgreSQL; for a list, see the `PostgreSQL Documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__.
 
-Patroni was previously known as Governor.
+Patroni is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like `ZooKeeper <https://zookeeper.apache.org/>`__, `etcd <https://github.com/coreos/etcd>`__ or `Consul <https://github.com/hashicorp/consul>`__. Database engineers, DBAs, DevOps engineers, and SREs who are looking to quickly deploy HA PostgreSQL in the datacenter-or anywhere else-will hopefully find it useful.
 
-*There are many ways to run high availability with PostgreSQL. Here, we
-present a template for you to create your own customized, high-availability 
-solution using Python and — for maximum accessibility — a distributed 
-configuration store like ZooKeeper, etcd or Consul.*
+We call Patroni a "template" because it is far from being a one-size-fits-all or plug-and-play replication system. It will have its own caveats. Use wisely.
 
-Getting Started
----------------
+**Note to Kubernetes users**: We're currently developing Patroni to be as useful as possible for teams running Kubernetes on top of Google Compute Engine; Patroni can be the HA solution for Postgres in such an environment. Please contact us via our Issues Tracker if this describes your team's current setup, and we'll follow up.
+
+.. contents::
+    :local:
+    :depth: 1
+    :backlinks: none
+
+==============
+How Patroni Works
+==============
+
+Patroni originated as a fork of `Governor <https://github.com/compose/governor>`__, the project from Compose. It includes plenty of new features. 
+
+For an example of a Docker-based deployment with Patroni, see `Spilo <https://github.com/zalando/spilo>`__, currently in use at Zalando.
+
+For additional background info, see:
+
+* `PostgreSQL HA with Kubernetes and Patroni <https://www.youtube.com/watch?v=iruaCgeG7qs>`__, talk by Josh Berkus at KubeCon 2016 (video)
+* `Feb. 2016 Zalando Tech blog post <https://tech.zalando.de/blog/zalandos-patroni-a-template-for-high-availability-postgresql/>`__
+
+================
+Development Status
+================
+
+Patroni is in active development and accepts contributions. See our `Contributing <https://github.com/zalando/patroni/blob/master/README.rst#contributing>`__ section below for more details. 
+
+===========================
+Technical Requirements/Installation
+===========================
+
+**For Mac**
+
+To install requirements on a Mac, run the following:
+
+::
+
+    brew install postgresql etcd haproxy libyaml python
+    pip install psycopg2 pyyaml
+
+===================
+Running and Configuring
+===================
 
 To get started, do the following from different terminals:
-
 ::
 
     > etcd --data-dir=data/etcd
     > ./patroni.py postgres0.yml
     > ./patroni.py postgres1.yml
 
-From there, you will see a high-availability cluster start up. Test
-different settings in the YAML files to see how its behavior changes. Kill
-some of the components to see how the system behaves.
+You will then see a high-availability cluster start up. Test different settings in the YAML files to see how the cluster's behavior changes. Kill some of the components to see how the system behaves.
 
 Add more ``postgres*.yml`` files to create an even larger cluster.
 
-We provide a haproxy configuration, which will give your application a
-single endpoint for connecting to the cluster's leader. To configure,
+Patroni provides an `HAProxy <http://www.haproxy.org/>`__ configuration, which will give your application a single endpoint for connecting to the cluster's leader. To configure,
 run:
 
 ::
@@ -39,167 +73,54 @@ run:
 
     > psql --host 127.0.0.1 --port 5000 postgres
 
-How Patroni Works
------------------
-
-For a diagram of the high availability decision loop, review this PDF:
-`postgres-ha.pdf <https://github.com/zalando/patroni/blob/master/postgres-ha.pdf>`__
-
+===============
 YAML Configuration
-------------------
+===============
 
-For an example file, see ``postgres0.yml``. Regarding settings:
+Go `here <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__. 
 
--  *ttl*: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process.
--  *loop\_wait*: the number of seconds the loop will sleep
+=========================
+Environment Configuration
+=========================
 
--  *restapi*:
-    -  *listen*: IP address + port that Patroni will listen to, to provide health-check information for haproxy.
-    -  *connect\_address*: IP address + port through which restapi is accessible.
-    -  *auth*: (optional) 'username:password' to protect dangerous REST API endpoints.
-    -  *certfile*: (optional) Specifies a file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL.
-    -  *keyfile*: (optional) Specifies a file with the secret key in the PEM format.
-
--  *etcd*:
-    -  *scope*: the relative path used on etcd's HTTP API for this deployment; makes it possible to run multiple HA deployments from a single etcd cluster.
-    -  *ttl*: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process.
-    -  *host*: the host:port for the etcd endpoint.
-
--  *consul*:
-    -  *scope*: the relative path used on Consul's HTTP API for this deployment; makes it possible to run multiple HA deployments from a single Consul cluster.
-    -  *ttl*: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process.
-    -  *host*: the host:port for the Consul endpoint.
-
--  *zookeeper*:
-    -  *scope*: the relative path used on ZooKeeper for this deployment; makes it possible to run multiple HA deployments from a single ZooKeeper cluster.
-    -  *session\_timeout*: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process.
-    -  *reconnect\_timeout*: how long we should try to reconnect to ZooKeeper after a connection loss. After this timeout, assume that you no longer have a lock and restart in read-only mode.
-    -  *hosts*: list of ZooKeeper cluster members in format: ['host1:port1', 'host2:port2', 'etc...']
-    -  *exhibitor*: if you are running a ZooKeeper cluster under the Exhibitor supervisory, the following section might interest you:
-        -  *poll\_interval*: how often the list of ZooKeeper and Exhibitor nodes should be updated from Exhibitor
-        -  *port*: Exhibitor port.
-        -  *hosts*: initial list of Exhibitor (ZooKeeper) nodes in format: ['host1', 'host2', 'etc...' ]. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes.
-
--  *postgresql*:
-    -  *name*: the name of the Postgres host. Must be unique for the cluster.
-    -  *listen*: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. The first address from this list will be used by Patroni to establish local connections to the PostgreSQL node. 
-    
-    -  *connect\_address*: IP address + port through which Postgres is accessible from other nodes and applications.
-    -  *data\_dir*: file path to initialize and store Postgres data files.
-    -  *maximum\_lag\_on\_failover*: the maximum bytes a follower may lag.
-    -  *use\_slots*: whether or not to use replication_slots. Must be False for PostgreSQL 9.3. You should comment out max_replication_slots before it becomes ineligible for leader status.
-
-    -  *initdb*:  List options to be passed on to initdb
-        -  *encoding*: default encoding for new databases
-        -  *locale*: default locale for new databases
-        -  *data-checksums*  # When pg_rewind is needed on 9.3, this needs to be enabled
-
-    -  *pg\_hba*: list of lines which should be added to pg\_hba.conf.
-        -  *- host all all 0.0.0.0/0 md5*.
-        -  *- host replication replicator 127.0.0.1/32 md5* # A line like this is required for replication
-
-    -  *replication*:
-        -  *username*: replication username; user will be created during initialization.
-        -  *password*: replication password; user will be created during initialization.
-
-    -  *callbacks* callback scripts to run on certain actions. Patroni will pass the action, role and cluster name. See scripts/aws.py as an example on how to write them.
-        -  *on\_start*: a script to run when the cluster starts.
-        -  *on\_stop*: a script to run when the cluster stops.
-        -  *on\_restart*: a script to run when the cluster restarts.
-        -  *on\_reload*: a script to run when configuration reload is triggered.
-        -  *on\_role\_change*: a script to run when the cluster is being promoted or demoted.
-
-    -  *superuser*:
-        -  *password*: password for the Postgres user, set during initialization.
-
-    -  *admin*:
-        -  *username*: admin username; user is created during initialization. It will have CREATEDB and CREATEROLE privileges.
-        -  *password*: admin password; user is created during initialization.
-
-    -  *recovery\_conf*: additional configuration settings written to recovery.conf when configuring follower.
-        -  *parameters*: list of configuration settings for Postgres.  Many of these are required for replication to work.
-
-    -  *create\_replica\_methods*: an ordered list of the create methods for turning a patroni node into a new replica.
-       "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured
-       as its own config item.
-
-    -  *replica\_method* for each create_replica_method other than basebackup, you would add a configuration section
-       of the same name.  At a minimum, this should include "command" with a full path to the actual script to be
-       executed.  Other configuration parameters will be passed along to the script in the form "parameter=value".
+Go `here <https://github.com/zalando/patroni/blob/master/docs/ENVIRONMENT.rst>`__ for comprehensive information about configuring(overriding) settings via environment variables.
 
+===============
 Replication Choices
--------------------
+===============
 
-Patroni uses Postgres' streaming replication. By default, this
-replication is asynchronous. For more information, see the `Postgres
-documentation on streaming
-replication <http://www.postgresql.org/docs/current/static/warm-standby.html#STREAMING-REPLICATION>`__.
+Patroni uses Postgres' streaming replication, which is asynchronous by default. For more information, see the `Postgres documentation on streaming replication <http://www.postgresql.org/docs/current/static/warm-standby.html#STREAMING-REPLICATION>`__.
 
-Patroni's asynchronous replication configuration allows for
-``maximum_lag_on_failover`` settings. This setting ensures failover will
-not occur if a follower is more than a certain number of bytes behind
-the follower. This setting should be increased or decreased based on
-business requirements.
+Patroni's asynchronous replication configuration allows for ``maximum_lag_on_failover`` settings. This setting ensures failover will not occur if a follower is more than a certain number of bytes behind the follower. This setting should be increased or decreased based on business requirements.
 
-When asynchronous replication is not optimal for your use case, investigate
-how Postgres's `synchronous
-replication <http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION>`__
-works. Synchronous replication ensures consistency across a cluster by
-confirming that writes are written to a secondary before returning to
-the connecting client with a success. The cost of synchronous
-replication: reduced throughput on writes. This throughput will
-be entirely based on network performance. In hosted datacenter
-environments (like AWS, Rackspace, or any network you do not control),
-synchrous replication significantly increases the variability of write 
-performance. If followers become inaccessible from the leader, the
-leader effectively becomes readonly.
+When asynchronous replication is not optimal for your use case, investigate Postgres's `synchronous replication <http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION>`__. Synchronous replication ensures consistency across a cluster by confirming that writes are written to a secondary before returning to the connecting client with a success. The cost of synchronous replication: reduced throughput on writes. This throughput will be entirely based on network performance. 
 
-To enable a simple synchronous replication test, add the follow lines to
-the ``parameters`` section of your YAML configuration files:
+In hosted datacenter environments (like AWS, Rackspace, or any network you do not control), synchronous replication significantly increases the variability of write performance. If followers become inaccessible from the leader, the leader effectively becomes read-only.
+
+To enable a simple synchronous replication test, add the follow lines to the ``parameters`` section of your YAML configuration files:
 
 .. code:: YAML
 
         synchronous_commit: "on"
         synchronous_standby_names: "*"
 
-When using synchronous replication, use at least three Postgres data nodes
-to ensure write availability if one host fails.
+When using synchronous replication, use at least three Postgres data nodes to ensure write availability if one host fails.
 
-Choosing your replication schema is dependent on your business
-considerations. Investigate both async and sync replication, as well as other
-HA solutions, to determine which solution is best for you.
+Choosing your replication schema is dependent on your business considerations. Investigate both async and sync replication, as well as other HA solutions, to determine which solution is best for you.
 
+===============================
 Applications Should Not Use Superusers
---------------------------------------
+===============================
 
-When connecting from an application, always use a non-superuser. Patroni
-requires access to the database to function properly. By using a
-superuser from an application, you can potentially use the entire
-connection pool, including the connections reserved for superusers with
-the ``superuser_reserved_connections`` setting. If Patroni cannot access
-the Primary because the connection pool is full, behavior will be
-undesireable.
+When connecting from an application, always use a non-superuser. Patroni requires access to the database to function properly. By using a superuser from an application, you can potentially use the entire connection pool, including the connections reserved for superusers, with the ``superuser_reserved_connections`` setting. If Patroni cannot access the Primary because the connection pool is full, behavior will be undesirable.
 
-Requirements on a Mac
----------------------
+================
+Contributing
+================
+Patroni accepts contributions from the open-source community; see the `Issues Tracker <https://github.com/zalando/patroni/issues>`__ for current needs. 
 
-Run the following on a Mac to install requirements:
-
-::
-
-    brew install postgresql etcd haproxy libyaml python
-    pip install psycopg2 pyyaml
-
-Notice
-------
-
-There are many different ways to do HA with PostgreSQL: See `the
-PostgreSQL
-documentation <https://wiki.postgresql.org/wiki/Replication,_Clustering,_and_Connection_Pooling>`__
-for a complete list.
-
-We call Patroni a "template" because it is far from being a one-size-fits-all
-or plug-and-play replication system. It will have its own caveats. Use wisely.
+Before making a contribution, please let us know by posting a comment to the relevant issue. 
+If you would like to propose a new feature, please first file a new issue explaining the feature you'd like to create.
 
 .. |Build Status| image:: https://travis-ci.org/zalando/patroni.svg?branch=master
    :target: https://travis-ci.org/zalando/patroni

TODO.md

@@ -1,15 +1,11 @@
 Failover
 ========
 - When determining who should become master, include the minor version of PostgreSQL in the decision.
-- Create a way to disable governance of a cluster, something like the existence of a "nogover" or "admin" file in PGDATA will stop patroni from changing the cluster state.
 
 Configuration
 ==============
-- Provide a way to change postgresql.conf and pg_hba.conf of a running cluster on the Patroni level, without changing individual nodes.
+- Provide a way to change pg_hba.conf of a running cluster on the Patroni level, without changing individual nodes.
 - Provide hooks to store and retrieve cluster-wide passwords without exposing them in a plain-text form to unauthorized users.
-- Implement patronictl command to create initial configuration of the cluster with leader and member keys fixed to the user-supplied values in order to simplify migrations.
-- Implement support for consul in addtion to etcd and zookeeper
-- Complete zookeeper support in patronictl
 
 Documentation
 ==============

docker-compose.yml

@@ -0,0 +1,58 @@
+# docker compose file for running a 3-node PostgreSQL cluster
+# with etcd as the SIS
+
+patroni_etcd:
+    container_name: patroni_etcd
+    image: patroni
+    command: --etcd
+
+dbnode1:
+    image: patroni
+    hostname: dbnode1
+    links:
+        - patroni_etcd:patroni_etcd
+    volumes:
+        - ./patroni:/patroni
+    env_file: docker/patroni-secrets.env
+    environment:
+        PATRONI_ETCD_HOST: patroni_etcd:2379
+        PATRONI_NAME: dbnode1
+        PATRONI_SCOPE: testcluster
+
+dbnode2:
+    image: patroni
+    hostname: dbnode2
+    links:
+        - patroni_etcd:patroni_etcd
+    volumes:
+        - ./patroni:/patroni
+    env_file: docker/patroni-secrets.env
+    environment:
+        PATRONI_ETCD_HOST: patroni_etcd:2379
+        PATRONI_NAME: dbnode2
+        PATRONI_SCOPE: testcluster
+
+dbnode3:
+    image: patroni
+    hostname: dbnode3
+    links:
+        - patroni_etcd:patroni_etcd
+    volumes:
+        - ./patroni:/patroni
+    env_file: docker/patroni-secrets.env
+    environment:
+        PATRONI_ETCD_HOST: patroni_etcd:2379
+        PATRONI_NAME: dbnode3
+        PATRONI_SCOPE: testcluster
+
+haproxy:
+    image: patroni
+    links:
+        - patroni_etcd:patroni_etcd
+    ports:
+        - "5000"
+        - "5001"
+    environment:
+        PATRONI_ETCD_HOST: patroni_etcd:2379
+        PATRONI_SCOPE: testcluster
+    command: --confd

docker/dev_patroni_cluster.sh

@@ -67,24 +67,38 @@ while getopts "$optspec" optchar; do
     esac
 done
 
-function random_name()
-{
-    cat /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9' | head -c 8
-}
-
-if [ -z ${PATRONI_SCOPE} ]
-then
-    PATRONI_SCOPE=$(random_name)
+if [ -z ${PATRONI_SCOPE} ]; then
+    PATRONI_SCOPE=$(cat /dev/urandom | LC_ALL=C tr -dc 'a-z0-9' | head -c 8)
 fi
 
-etcd_container=$(docker run -P -d --name="${PATRONI_SCOPE}_etcd" "${DOCKER_IMAGE}" --etcd-only)
-etcd_container_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${etcd_container})
-echo "The etcd container is ${etcd_container}, ip=${etcd_container_ip}"
+function docker_run()
+{
+    local name=$1
+    shift
+    container=$(docker run -d --name=$name $*)
+    container_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${container})
+    echo "Started container ${name}, ip=${container_ip}"
+}
 
-for i in $(seq 1 "${MEMBERS}")
-do
-    container_name=$(random_name)
-    patroni_container=$(docker run -P -d --name="${PATRONI_SCOPE}_${container_name}" "${DOCKER_IMAGE}" --etcd="${etcd_container_ip}:4001" --name="${PATRONI_SCOPE}")
-    patroni_container_ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${patroni_container})
-    echo "Started Patroni container ${patroni_container}, ip=${patroni_container_ip}"
+
+ETCD_CONTAINER="${PATRONI_SCOPE}_etcd"
+docker_run ${ETCD_CONTAINER} ${DOCKER_IMAGE} --etcd
+
+DOCKER_ARGS="--link=${ETCD_CONTAINER}:${ETCD_CONTAINER} -e PATRONI_SCOPE=${PATRONI_SCOPE} -e PATRONI_ETCD_HOST=${ETCD_CONTAINER}:2379"
+PATRONI_ENV=$(sed 's/#.*//g' docker/patroni-secrets.env | sed -n 's/^PATRONI_.*$/-e &/p' | tr '\n' ' ')
+PATRONI_VOLUME="-v $(dirname $(dirname $(realpath $0)))/patroni:/patroni"
+
+for i in $(seq 1 "${MEMBERS}"); do
+    container_name=postgres${i}
+    docker_run "${PATRONI_SCOPE}_${container_name}" \
+        $PATRONI_VOLUME \
+        $DOCKER_ARGS \
+        $PATRONI_ENV \
+        -e PATRONI_NAME=${container_name} \
+        ${DOCKER_IMAGE}
 done
+
+docker_run "${PATRONI_SCOPE}_haproxy" \
+    -p=5000 -p=5001 \
+    $DOCKER_ARGS \
+    ${DOCKER_IMAGE} --confd

docker/entrypoint.sh

@@ -7,49 +7,52 @@ Usage: $0
 
 Options:
 
-    --etcd      ETCD    Provide an external etcd to connect to
-    --name      NAME    Give the cluster a specific name
-    --etcd-only         Do not run Patroni, run a standalone etcd
+    --etcd      Do not run Patroni, run a standalone etcd
+    --confd     Do not run Patroni, run a standalone confd
+    --zookeeper Do not run Patroni, run a standalone zookeeper
 
 Examples:
 
-    $0 --etcd=127.17.0.84:4001
-    $0 --etcd-only
+    $0 --etcd
+    $0 --confd
+    $0 --zookeeper
     $0
-    $0 --name=true_scotsman
 __EOF__
 }
 
 DOCKER_IP=$(hostname --ip-address)
 PATRONI_SCOPE=${PATRONI_SCOPE:-batman}
+ETCD_ARGS="--data-dir /tmp/etcd.data -advertise-client-urls=http://${DOCKER_IP}:2379 -listen-client-urls=http://0.0.0.0:2379 -listen-peer-urls=http://0.0.0.0:2380"
 
 optspec=":vh-:"
 while getopts "$optspec" optchar; do
     case "${optchar}" in
         -)
             case "${OPTARG}" in
-                etcd-only)
-                    exec etcd --data-dir /tmp/etcd.data \
-                        -advertise-client-urls=http://${DOCKER_IP}:4001 \
-                        -listen-client-urls=http://0.0.0.0:4001 \
-                        -listen-peer-urls=http://0.0.0.0:2380
-                    exit 0
+                confd)
+                    haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D
+                    CONFD="confd -prefix=${PATRONI_NAMESPACE:-/service}/$PATRONI_SCOPE -interval=10 -backend"
+                    if [ ! -z ${PATRONI_ZOOKEEPER_HOSTS} ]; then
+                        while ! /usr/share/zookeeper/bin/zkCli.sh -server ${PATRONI_ZOOKEEPER_HOSTS} ls /; do
+                            sleep 1
+                        done
+                        exec $CONFD zookeeper -node ${PATRONI_ZOOKEEPER_HOSTS}
+                    else
+                        while ! curl -s ${PATRONI_ETCD_HOST}/v2/members | jq -r '.members[0].clientURLs[0]' | grep -q http; do
+                            sleep 1
+                        done
+                        exec $CONFD etcd -node $PATRONI_ETCD_HOST
+                    fi
+                    ;;
+                etcd)
+                    exec etcd $ETCD_ARGS
+                    ;;
+                zookeeper)
+                    exec /usr/share/zookeeper/bin/zkServer.sh start-foreground
                     ;;
                 cheat)
                     CHEAT=1
                     ;;
-                name)
-                    PATRONI_SCOPE="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
-                    ;;
-                name=*)
-                    PATRONI_SCOPE=${OPTARG#*=}
-                    ;;
-                etcd)
-                    ETCD_CLUSTER="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
-                    ;;
-                etcd=*)
-                    ETCD_CLUSTER=${OPTARG#*=}
-                    ;;
                 help)
                     usage
                     exit 0
@@ -70,76 +73,43 @@ while getopts "$optspec" optchar; do
     esac
 done
 
-if [ -z ${ETCD_CLUSTER} ]
-then
-    etcd --data-dir /tmp/etcd.data \
-        -advertise-client-urls=http://${DOCKER_IP}:4001 \
-        -listen-client-urls=http://0.0.0.0:4001 \
-        -listen-peer-urls=http://0.0.0.0:2380 > /var/log/etcd.log 2> /var/log/etcd.err &
-    ETCD_CLUSTER="127.0.0.1:4001"
+## We start an etcd
+if [[ -z ${PATRONI_ETCD_HOST} && -z ${PATRONI_ZOOKEEPER_HOSTS} ]]; then
+    etcd $ETCD_ARGS > /var/log/etcd.log 2> /var/log/etcd.err &
+    export PATRONI_ETCD_HOST="127.0.0.1:2379"
 fi
 
-mkdir -p ~postgres/.config/patroni
-cat > ~postgres/.config/patroni/patronictl.yaml <<__EOF__
-{dcs_api: 'etcd://${ETCD_CLUSTER}', namespace: /service/}
-__EOF__
+export PATRONI_SCOPE
+export PATRONI_NAME="${PATRONI_NAME:-${HOSTNAME}}"
+export PATRONI_RESTAPI_CONNECT_ADDRESS="${DOCKER_IP}:8008"
+export PATRONI_RESTAPI_LISTEN="0.0.0.0:8008"
+export PATRONI_admin_PASSWORD="${PATRONI_admin_PASSWORD:=admin}"
+export PATRONI_admin_OPTIONS="${PATRONI_admin_OPTIONS:-createdb, createrole}"
+export PATRONI_POSTGRESQL_CONNECT_ADDRESS="${DOCKER_IP}:5432"
+export PATRONI_POSTGRESQL_LISTEN="0.0.0.0:5432"
+export PATRONI_POSTGRESQL_DATA_DIR="data/${PATRONI_SCOPE}"
+export PATRONI_REPLICATION_USERNAME="${PATRONI_REPLICATION_USERNAME:-replicator}"
+export PATRONI_REPLICATION_PASSWORD="${PATRONI_REPLICATION_PASSWORD:-abcd}"
+export PATRONI_SUPERUSER_USERNAME="${PATRONI_SUPERUSER_USERNAME:-postgres}"
+export PATRONI_SUPERUSER_PASSWORD="${PATRONI_SUPERUSER_PASSWORD:-postgres}"
+export PATRONI_POSTGRESQL_PGPASS="$HOME/.pgpass"
 
-cat > /patroni/postgres.yaml <<__EOF__
+cat > /patroni.yml <<__EOF__
+bootstrap:
+  dcs:
+    postgresql:
+      use_pg_rewind: true
 
-ttl: &ttl 30
-loop_wait: &loop_wait 10
-scope: &scope '${PATRONI_SCOPE}'
-namespace: 'patroni'
-restapi:
-  listen: 0.0.0.0:8008
-  connect_address: ${DOCKER_IP}:8008
-etcd:
-  scope: *scope
-  ttl: *ttl
-  host: ${ETCD_CLUSTER}
-postgresql:
-  name: ${HOSTNAME}
-  scope: *scope
-  listen: 0.0.0.0:5432
-  connect_address: ${DOCKER_IP}:5432
-  data_dir: data/postgresql0
-  maximum_lag_on_failover: 1048576 # 1 megabyte in bytes
   pg_hba:
   - host all all 0.0.0.0/0 md5
-  - hostssl all all 0.0.0.0/0 md5
   - host replication replicator ${DOCKER_IP}/16    md5
-  replication:
-    username: replicator
-    password: rep-pass
-    network:  127.0.0.1/32
-  superuser:
-    password: zalando
-  restore: patroni/scripts/restore.py
-  admin:
-    username: admin
-    password: admin
-  parameters:
-    archive_mode: "on"
-    wal_level: hot_standby
-    archive_command: 'true'
-    max_wal_senders: 20
-    listen_addresses: 0.0.0.0
-    max_wal_size: 1GB
-    min_wal_size: 128MB
-    wal_keep_segments: 64
-    archive_timeout: 1800s
-    max_replication_slots: 20
-    hot_standby: "on"
 __EOF__
 
-cat /patroni/postgres.yaml
+mkdir -p "$HOME/.config/patroni"
+[ -h "$HOME/.config/patroni/patronictl.yaml" ] || ln -s /patroni.yml "$HOME/.config/patroni/patronictl.yaml"
 
-if [ ! -z $CHEAT ]
-then
-    while :
-    do
-        sleep 60
-    done
-else
-    exec python /patroni.py /patroni/postgres.yaml
-fi
+[ -z $CHEAT ] && exec python /patroni.py /patroni.yml
+
+while true; do
+    sleep 60
+done

docker/patroni-secrets.env

@@ -0,0 +1,8 @@
+PATRONI_RESTAPI_USERNAME=admin
+PATRONI_RESTAPI_PASSWORD=admin
+PATRONI_SUPERUSER_USERNAME=postgres
+PATRONI_SUPERUSER_PASSWORD=postgres
+PATRONI_REPLICATION_USERNAME=replicator
+PATRONI_REPLICATION_PASSWORD=replicate
+PATRONI_admin_PASSWORD=admin
+PATRONI_admin_OPTIONS=createdb,createrole

docs/ENVIRONMENT.rst

@@ -0,0 +1,59 @@
+==================================
+Environment Configuration Settings
+==================================
+
+It is possible to override some of the configuration parameters defined in the Patroni configuration file using the system environment variables. This document lists all environment variables handled by Patroni. The values set via those variables always take precedence over the ones set in the Patroni configuration file.
+
+Global/Universal
+----------------
+-  **PATRONI\_CONFIGURATION**: it is possible to set the entire configuration for the Patroni via ``PATRONI_CONFIGURATION`` environment variable. In this case any other environment variables will not be considered!
+-  **PATRONI\_NAME**: name of the node where the current instance of Patroni is running. Must be unique for the cluster.
+-  **PATRONI\_NAMESPACE**: path within the configuration store where Patroni will keep information about the cluster. Default value: "/service"
+-  **PATRONI\_SCOPE**: cluster name
+
+Bootstrap configuration
+-----------------------
+It is possible to create new database users right after the successful initialization of a new cluster. This process is defined by the following variables:
+
+-  **PATRONI\_<username>\_PASSWORD='<password>'**
+-  **PATRONI\_<username>\_OPTIONS='list,of,options'**
+
+Example: defining ``PATRONI_admin_PASSWORD=strongpasswd`` and ``PATRONI_admin_OPTIONS='createrole,createdb'`` will cause creation of the user **admin** with the password **strongpasswd** that is allowed to create other users and databases.
+
+Consul
+------
+-  **PATRONI\_CONSUL\_HOST**: the host:port for the Consul endpoint.
+
+Etcd
+----
+-  **PATRONI\_ETCD\_HOST**: the host:port for the etcd endpoint.
+
+Exhibitor
+---------
+-  **PATRONI\_EXHIBITOR\_HOSTS**: initial list of Exhibitor (ZooKeeper) nodes in format: 'host1,host2,etc...'. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes.
+-  **PATRONI\_EXHIBITOR\_PORT**: Exhibitor port.
+
+PostgreSQL
+----------
+-  **PATRONI\_POSTGRESQL\_LISTEN**: IP address + port that Postgres listens to. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
+-  **PATRONI\_POSTGRESQL\_CONNECT\_ADDRESS**: IP address + port through which Postgres is accessible from other nodes and applications.
+-  **PATRONI\_POSTGRESQL\_DATA\_DIR**: The location of the Postgres data directory, either existing or to be initialized by Patroni.
+-  **PATRONI\_POSTGRESQL\_BIN_DIR**: Path to PostgreSQL binaries. (pg_ctl, pg_rewind, pg_basebackup, postgres) The  default value is an empty string meaning that PATH environment variable will be used to find the executables.
+-  **PATRONI\_POSTGRESQL\_PGPASS**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni.
+-  **PATRONI\_REPLICATION\_USERNAME**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication
+-  **PATRONI\_REPLICATION\_PASSWORD**: replication password; the user will be created during initialization.
+-  **PATRONI\_SUPERUSER\_USERNAME**: name for the superuser, set during initialization (initdb) and later used by Patroni to connect to the postgres. Also this user is used by pg_rewind.
+-  **PATRONI\_SUPERUSER\_PASSWORD**: password for the superuser, set during initialization (initdb).
+
+REST API
+-------- 
+-  **PATRONI\_RESTAPI\_CONNECT\_ADDRESS**: IP address and port to access the REST API.
+-  **PATRONI\_RESTAPI\_LISTEN**: IP address and port that Patroni will listen to, to provide health-check information for HAProxy.
+-  **PATRONI\_RESTAPI\_USERNAME**: Basic-auth username to protect unsafe REST API endpoints.
+-  **PATRONI\_RESTAPI\_PASSWORD**: Basic-auth password to protect unsafe REST API endpoints.
+-  **PATRONI\_RESTAPI\_CERTFILE**: Specifies the file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL.
+-  **PATRONI\_RESTAPI\_KEYFILE**: Specifies the file with the secret key in the PEM format.
+
+ZooKeeper
+---------
+-  **PATRONI\_ZOOKEEPER\_HOSTS**: comma separated list of ZooKeeper cluster members: "'host1:port1','host2:port2','etc...'". It is important to quote every single entity!

docs/SETTINGS.rst

@@ -0,0 +1,93 @@
+===========================
+YAML Configuration Settings
+===========================
+
+Global/Universal
+----------------
+-  **name**: the name of the host. Must be unique for the cluster.
+-  **namespace**: path within the configuration store where Patroni will keep information about the cluster. Default value: "/service"
+-  **scope**: cluster name
+
+Bootstrap configuration
+-----------------------
+-  **dcs**: This section will be written into `/<namespace>/<scope>/config` of a given configuration store after initializing of new cluster. This is the global configuration for the cluster. If you want to change some parameters for all cluster nodes - just do it in DCS (or via Patroni API) and all nodes will apply this configuration.
+    -  **loop\_wait**: the number of seconds the loop will sleep. Default value: 10
+    -  **ttl**: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process. Default value: 30
+    -  **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election.
+    -  **postgresql**:
+        -  **use\_pg\_rewind**:whether or not to use pg_rewind
+        -  **use\_slots**: whether or not to use replication_slots. Must be False for PostgreSQL 9.3. You should comment out max_replication_slots before it becomes ineligible for leader status.
+        -  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower. 
+        -  **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work.
+-  **initdb**: List options to be passed on to initdb.
+        -  **- data-checksums**: Must be enabled when pg_rewind is needed on 9.3.
+        -  **- encoding: UTF8**: default encoding for new databases.
+        -  **- locale: UTF8**: default locale for new databases.
+-  **pg\_hba**: list of lines that you should add to pg\_hba.conf.
+        -  **- host all all 0.0.0.0/0 md5**.
+        -  **- host replication replicator 127.0.0.1/32 md5**: A line like this is required for replication.
+-  **users**: Some additional users users which needs to be created after initializing new cluster
+    -  **admin**: the name of user
+        -  **password: zalando**:
+        -  **options**: list of options for CREATE USER statement
+            -  **- createrole**
+            -  **- createdb**
+
+Consul
+------
+-  **host**: the host:port for the Consul endpoint.
+
+Etcd
+----
+-  **host**: the host:port for the etcd endpoint.
+
+Exhibitor
+---------
+-  **hosts**: initial list of Exhibitor (ZooKeeper) nodes in format: 'host1,host2,etc...'. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes.
+-  **poll\_interval**: how often the list of ZooKeeper and Exhibitor nodes should be updated from Exhibitor
+-  **port**: Exhibitor port.
+
+PostgreSQL
+----------
+-  **authentication**:
+    -  **superuser**:
+        -  **username**: name for the superuser, set during initialization (initdb) and later used by Patroni to connect to the postgres.
+        -  **password**: password for the superuser, set during initialization (initdb).
+    -  **replication**:
+        -  **username**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication
+        -  **password**: replication password; the user will be created during initialization.
+-  **callbacks**: callback scripts to run on certain actions. Patroni will pass the action, role and cluster name. (See scripts/aws.py as an example of how to write them.)
+        -  **on\_reload**: run this script when configuration reload is triggered.
+        -  **on\_restart**: run this script when the cluster restarts.
+        -  **on\_role\_change**: run this script when the cluster is being promoted or demoted.
+        -  **on\_start**: run this script when the cluster starts.
+        -  **on\_stop**: run this script when the cluster stops.
+-  **connect\_address**: IP address + port through which Postgres is accessible from other nodes and applications.
+-  **create\_replica\_methods**: an ordered list of the create methods for turning a Patroni node into a new replica. "basebackup" is the default method; other methods are assumed to refer to scripts, each of which is configured as its own config item.
+-  **data\_dir**: The location of the Postgres data directory, either existing or to be initialized by Patroni.
+-  **bin\_dir**: Path to PostgreSQL binaries. (pg_ctl, pg_rewind, pg_basebackup, postgres) The  default value is an empty string meaning that PATH environment variable will be used to find the executables.
+-  **listen**: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
+-  **pgpass**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni.
+-  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower.
+-  **custom_conf** : path to an optional custom ``postgresql.conf`` file, that will be used in place of ``postgresql.base.conf``. The file must exist on all cluster nodes, be readable by PostgreSQL and will be included from its location on the real ``postgresql.conf``. Note that Patroni will not monitor this file for changes, nor backup it. However, its settings can still be overriden by Patroni's own configuration facilities - see `dynamic configuration <https://github.com/zalando/patroni/blob/master/docs/dynamic_configuration.rst>`__ for details.
+-  **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work.
+-  **pg\_ctl\_timeout**: How long should pg_ctl wait when doing ``start``, ``stop`` or ``restart``. Default value is 60 seconds.
+-  **use\_pg\_rewind**: try to use pg\_rewind on the former leader when it joins cluster as a replica.
+-  **remove\_data\_directory\_on\_rewind\_failure**: If this option is enabled, Patroni will remove postgres data directory and recreate replica. Otherwise it will try to follow the new leader. Default value is **false**.
+-  **replica\_method** for each create_replica_method other than basebackup, you would add a configuration section of the same name. At a minimum, this should include "command" with a full path to the actual script to be executed.  Other configuration parameters will be passed along to the script in the form "parameter=value".
+
+REST API
+-------- 
+-  **connect\_address**: IP address and port to access the REST API.
+-  **listen**: IP address and port that Patroni will listen to, to provide health-check information for HAProxy.
+-  **Optional**:
+        -  **authentication**:
+            -  **username**: Basic-auth username to protect unsafe REST API endpoints.
+            -  **password**: Basic-auth password to protect unsafe REST API endpoints.
+
+        -  **certfile**: Specifies the file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL.
+        -  **keyfile**: Specifies the file with the secret key in the PEM format.
+
+ZooKeeper
+----------
+-  **hosts**: list of ZooKeeper cluster members in format: ['host1:port1', 'host2:port2', 'etc...'].

docs/dynamic_configuration.rst

@@ -0,0 +1,226 @@
+Patroni configuration
+=====================
+
+Patroni configuration is stored in the DCS (Distributed Configuration Store). There are 3 types of configuration:
+
+- Dynamic configuration.
+	These options can be set in DCS at any time. If the options changed are not part of the startup configuration,
+	they are applied asynchronously (upon the next wake up cycle) to every node, which gets subsequently reloaded.
+	If the node requires a restart to apply the configuration (for options with context postmaster, if their values
+	have changed), a special flag, ``pending_restart`` indicating this, is set in the members.data JSON.
+	Additionally, the node status also indicates this, by showing ``"restart_pending": true``.
+
+- Local `configuration <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ (patroni.yml).
+	These options are defined in the configuration file and take precedence over dynamic configuration.
+	patroni.yml could be changed and reload in runtime (without restart of Patroni) by sending SIGHUP to the Patroni process or by performing ``POST /reload`` REST-API request.
+
+- Environment `configuration <https://github.com/zalando/patroni/blob/master/docs/ENVIRONMENT.rst>`__ .
+	It is possible to set/override some of the "Local" configuration parameters with environment variables.
+	Environment configuration is very useful when you are running in a dynamic environment and you don't know some of the parameters in advance (for example it's not possible to know you external IP address when you are running inside ``docker``).
+
+Some of the PostgreSQL parameters must hold the same values on the master and the replicas. For those, values set either in the local patroni configuration files or via the environment variables take no effect. To alter or set their values one must change the shared configuration in the DCS. Below is the actual list of such parameters together with the default values:
+
+- max_connections: 100
+- max_locks_per_transaction: 64
+- max_worker_processes: 8
+- max_prepared_transactions: 0
+- wal_level: hot_standby
+- wal_log_hints: on
+- track_commit_timestamp: off
+
+For the parameters below, PostgreSQL does not require equal values among the master and all the replicas. However, considering the possibility of a replica to become the master at any time, it doesn't really make sense to set them differently; therefore, Patroni restricts setting their values to the Dynamic configuration
+
+- max_wal_senders: 5
+- max_replication_slots: 5
+- wal_keep_segments: 8
+
+These parameters are validated to ensure they are sane, or meet a minimum value.
+
+There are some other Postgres parameters controlled by Patroni:
+
+- listen_addresses - is set either from ``postgresql.listen`` or from ``PATRONI_POSTGRESQL_LISTEN`` environment variable
+- port - is set either from ``postgresql.listen`` or from ``PATRONI_POSTGRESQL_LISTEN`` environment variable
+- cluster_name - is set either from ``scope`` or from ``PATRRONI_SCOPE`` environment variable
+- hot_standby: on
+
+To be on the safe side parameters from the above lists are not written into ``postgresql.conf``, but passed as a list of arguments to the ``pg_ctl start`` which gives them the highest precedence, even above `ALTER SYSTEM <https://www.postgresql.org/docs/current/static/sql-altersystem.html>`__
+
+
+When applying the local or dynamic configuration options, the following actions are taken:
+
+- The node first checks if there is a postgresql.base.conf or if the ``custom_conf`` parameter is set.
+- If the `custom_conf` parameter is set, it will take the file specified on it as a base configuration, ignoring `postgresql.base.conf` and `postgresql.conf`.
+- If the `custom_conf` parameter is not set and `postgresql.base.conf` exists, it contains the renamed "original" configuration and it will be used as a base configuration.
+- If there is no `custom_conf` nor `postgresql.base.conf`, the original postgresql.conf is taken and renamed to postgresql.base.conf.
+- The dynamic options (with the exceptions above) are dumped into the postgresql.conf and an include is set in
+  postgresql.conf to the used base configuration (either postgresql.base.conf or what is on ``custom_conf``). Therefore, we would be able to apply new options without re-reading the configuration file to check if the include is present not.
+- Some parameters that are essential for Patroni to manage the cluster are overridden using the command line.
+- If some of the options that require restart are changed (we should look at the context in pg_settings and at the actual
+  values of those options), a pending_restart flag of a given node is set. This flag is reset on any restart.
+
+The parameters would be applied in the following order (run-time are given the highest priority):
+
+1. load parameters from file `postgresql.base.conf` (or from a `custom_conf` file, if set)
+2. load parameters from file `postgresql.conf`
+3. load parameters from file `postgresql.auto.conf`
+4. run-time parameter using `-o --name=value`
+
+This allows configuration for all the nodes (2), configuration for a specific node using `ALTER SYSTEM` (3) and ensures that parameters essential to the running of Patroni are enforced (4), as well as leaves room for configuration tools that manage `postgresql.conf` directly without involving Patroni (1).
+
+
+Also, the following Patroni configuration options can be changed only dynamically:
+
+- ttl: 30
+- loop_wait: 10
+- retry_timeouts: 10
+- maximum_lag_on_failover: 1048576
+- postgresql.use_slots: true
+
+Upon changing these options, Patroni will read the relevant section of the configuration stored in DCS and change its
+run-time values.
+
+Patroni nodes are dumping the state of the DCS options to disk upon for every change of the configuration into the file ``patroni.dynamic.json`` located in the Postgres data directory. Only the master is allowed to restore these options from the on-disk dump if these are completely absent from the DCS or if they are invalid.
+
+REST API
+========
+
+We provide a REST API endpoint for working with dynamic configuration.
+
+GET /config
+-----------
+Get current version of dynamic configuration.
+
+.. code-block:: bash
+
+	$ curl -s localhost:8008/config | jq .
+	{
+	  "ttl": 30,
+	  "loop_wait": 10,
+	  "retry_timeout": 10,
+	  "maximum_lag_on_failover": 1048576,
+	  "postgresql": {
+	    "use_slots": true,
+	    "use_pg_rewind": true,
+	    "parameters": {
+	      "hot_standby": "on",
+	      "wal_log_hints": "on",
+	      "wal_keep_segments": 8,
+	      "wal_level": "hot_standby",
+	      "max_wal_senders": 5,
+	      "max_replication_slots": 5,
+	      "max_connections": "100"
+	    }
+	  }
+	}
+
+PATCH /config
+-------------
+Change existing configuration.
+
+.. code-block:: bash
+
+	$ curl -s -XPATCH -d \
+		'{"loop_wait":5,"ttl":20,"postgresql":{"parameters":{"max_connections":"101"}}}' \
+		http://localhost:8008/config | jq .
+	{
+	  "ttl": 20,
+	  "loop_wait": 5,
+	  "maximum_lag_on_failover": 1048576,
+	  "retry_timeout": 10,
+	  "postgresql": {
+	    "use_slots": true,
+	    "use_pg_rewind": true,
+	    "parameters": {
+	      "hot_standby": "on",
+	      "wal_log_hints": "on",
+	      "wal_keep_segments": 8,
+	      "wal_level": "hot_standby",
+	      "max_wal_senders": 5,
+	      "max_replication_slots": 5,
+	      "max_connections": "101"
+	    }
+	  }
+	}
+
+The above REST API call patches the existing configuration and returns the new configuration.
+
+Let's check that the node processed this configuration. First of all it should start printing log lines every 5 seconds (loop_wait=5). The change of "max_connections" requires a restart, so the "restart_pending" flag should be exposed:
+
+.. code-block:: bash
+
+	$ curl -s http://localhost:8008/patroni | jq .
+	{
+	  "pending_restart": true,
+	  "database_system_identifier": "6287881213849985952",
+	  "postmaster_start_time": "2016-06-13 13:13:05.211 CEST",
+	  "xlog": {
+	    "location": 2197818976
+	  },
+	  "patroni": {
+	    "scope": "batman",
+	    "version": "1.0"
+	  },
+	  "state": "running",
+	  "role": "master",
+	  "server_version": 90503
+	}
+
+Removing parameters:
+
+If you want to remove (reset) some setting just patch it with ``null``:
+
+.. code-block:: bash
+
+	$ curl -s -XPATCH -d \
+		'{"postgresql":{"parameters":{"max_connections":null}}}' \
+		http://localhost:8008/config | jq .
+	{
+	  "ttl": 20,
+	  "loop_wait": 5,
+	  "retry_timeout": 10,
+	  "maximum_lag_on_failover": 1048576,
+	  "postgresql": {
+	    "use_slots": true,
+	    "use_pg_rewind": true,
+	    "parameters": {
+	      "hot_standby": "on",
+	      "unix_socket_directories": ".",
+	      "wal_keep_segments": 8,
+	      "wal_level": "hot_standby",
+	      "wal_log_hints": "on",
+	      "max_wal_senders": 5,
+	      "max_replication_slots": 5
+	    }
+	  }
+	}
+
+Above call removes ``postgresql.parameters.max_connections`` from the dynamic configuration.
+
+PUT /config
+-----------
+
+It's also possible to perform the full rewrite of an existing dynamic configuration unconditionally:
+
+.. code-block:: bash
+
+	$ curl -s -XPUT -d \
+		'{"maximum_lag_on_failover":1048576,"retry_timeout":10,"postgresql":{"use_slots":true,"use_pg_rewind":true,"parameters":{"hot_standby":"on","wal_log_hints":"on","wal_keep_segments":8,"wal_level":"hot_standby","unix_socket_directories":".","max_wal_senders":5}},"loop_wait":3,"ttl":20}' \
+		http://localhost:8008/config | jq .
+	{
+	  "ttl": 20,
+	  "maximum_lag_on_failover": 1048576,
+	  "retry_timeout": 10,
+	  "postgresql": {
+	    "use_slots": true,
+	    "parameters": {
+	      "hot_standby": "on",
+	      "unix_socket_directories": ".",
+	      "wal_keep_segments": 8,
+	      "wal_level": "hot_standby",
+	      "wal_log_hints": "on",
+	      "max_wal_senders": 5
+	    },
+	    "use_pg_rewind": true
+	  },
+	  "loop_wait": 3
+	}

docs/pause.rst

@@ -0,0 +1,33 @@
+Pause/Resume mode for the cluster
+=================================
+
+The goal
+--------
+
+Under certain circumstances Patroni needs to temporary step down from managing the cluster, while still retaining the cluster state in DCS. Possible use cases are uncommon activities on the cluster, such as major version upgrades or corruption recovery. During those activities nodes are often started and stopped for the reason unknown to Patroni, some nodes can be even temporary promoted, violating the assumption of running only one master. Therefore, Patroni needs to be able to "detach" from the running cluster, implementing an equivalent of the maintenance mode in Pacemaker.
+
+
+
+The implementation
+------------------
+
+When Patroni runs in a paused mode, it does not change the state of PostgreSQL, except for the following cases:
+
+- For each node, the member key in DCS is updated with the current information about the cluster. This causes Patroni to run read-only queries on a member node if the member is running.
+
+- For the Postgres master with the leader lock Patroni updates the lock. If the node with the leader lock stops being the master (i.e. is demoted manually), Patroni will release the lock instead of promoting the node back.
+
+- Manual unscheduled restart, reinitialize and manual failover are allowed. Manual failover is only allowed if the node to failover to is specified. In the paused mode, manual failover does not require a running master node.
+
+- If 'parallel' masters are detected by Patroni, it emits a warning, but does not demote the masters without the leader lock.
+
+- If there is no leader lock in the cluster, the running master acquires the lock. If there is more than one master node, then the first master to acquire the lock wins. If there are no masters altogether, Patroni does not try to promote any replicas. There is an exception in this rule: if there is no leader lock because the old master has demoted itself due to the manual promotion, then only the candidate node mentioned in the promotion request may take the leader lock. When the new leader lock is granted (i.e. after promoting a replica manually), Patroni makes sure the replicas that were streaming from the previous leader will switch to the new one.
+
+- When Postgres is stopped, Patroni does not try to start it. When Patroni is stopped, it does not to stop Postgres instance it is managing.
+
+User guide
+----------
+
+``patronictl`` supports ``pause`` and ``resume`` commands.
+
+One can also issue a ``PATCH`` request to the ``{namespace}/{cluster}/config`` key with ``{"pause": true/false/null}``

extras/README.md

@@ -0,0 +1,13 @@
+### confd
+
+`confd` directory contains haproxy template files for the [confd](https://github.com/kelseyhightower/confd) -- lightweight configuration management tool
+You need to copy content of `confd` directory into /etcd/confd and run confd service:
+```bash
+$ confd -prefix=/service/$PATRONI_SCOPE -backend etcd -node $PATRONI_ETCD_HOST -interval=10
+```
+It will periodically update haproxy.cfg with the actual list of Patroni nodes from `etcd` and "reload" haproxy when it is necessary.
+
+
+### startup-scripts
+
+`startup-scripts` directory contains startup scripts for various OSes and management tools for Patroni.

extras/confd/conf.d/haproxy.toml

@@ -0,0 +1,13 @@
+[template]
+#prefix = "/service/batman"
+#owner = "haproxy"
+#mode = "0644"
+src = "haproxy.tmpl"
+dest = "/etc/haproxy/haproxy.cfg"
+
+check_cmd = "/usr/sbin/haproxy -c -f {{ .src }}"
+reload_cmd = "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D -sf $(cat /var/run/haproxy.pid)"
+
+keys = [
+    "/members/",
+]

extras/confd/templates/haproxy.tmpl

@@ -0,0 +1,28 @@
+global
+	maxconn 100
+
+defaults
+	log	global
+	mode	tcp
+	retries 2
+	timeout client 30m
+	timeout connect 4s
+	timeout server 30m
+	timeout check 5s
+
+frontend master_postgresql
+	bind *:5000
+	default_backend backend_master
+
+frontend replicas_postgresql
+	bind *:5001
+	default_backend backend_replicas
+
+backend backend_master
+	option httpchk OPTIONS /master
+{{range gets "/members/*"}}	server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}}
+{{end}}
+backend backend_replicas
+	option httpchk OPTIONS /replica
+{{range gets "/members/*"}}	server {{base .Key}} {{$data := json .Value}}{{base (replace (index (split $data.conn_url "/") 2) "@" "/" -1)}} maxconn 100 check port {{index (split (index (split $data.api_url "/") 2) ":") 1}}
+{{end}}

features/environment.py

@@ -98,6 +98,13 @@ class PatroniController(AbstractController):
         except IOError:
             return None
 
+    def add_tag_to_config(self, tag, value):
+        with open(self._config) as r:
+            config = yaml.safe_load(r)
+            config['tags']['tag'] = value
+            with open(self._config, 'w') as w:
+                yaml.safe_dump(config, w, default_flow_style=False)
+
     def _start(self):
         return subprocess.Popen(['coverage', 'run', '--source=patroni', '-p', 'patroni.py', self._config],
                                 stdout=self._log, stderr=subprocess.STDOUT, cwd=self._work_directory)
@@ -110,40 +117,31 @@ class PatroniController(AbstractController):
         patroni_config_path = os.path.join(self._output_dir, patroni_config_name)
 
         with open(patroni_config_name) as f:
-            config = yaml.load(f)
+            config = yaml.safe_load(f)
+            config.pop('etcd')
 
         host = config['postgresql']['listen'].split(':')[0]
 
         config['postgresql']['listen'] = config['postgresql']['connect_address'] = '{0}:{1}'.format(host, self.__PORT)
 
-        user = config['postgresql'].get('superuser', {})
+        user = config['postgresql'].get('authentication', config['postgresql']).get('superuser', {})
         self._connkwargs = {k: user[n] for n, k in [('username', 'user'), ('password', 'password')] if n in user}
         self._connkwargs.update({'host': host, 'port': self.__PORT, 'database': 'postgres'})
 
-        config['postgresql'].update({'name': name, 'data_dir': self._data_dir})
+        config['name'] = name
+        config['postgresql']['data_dir'] = self._data_dir
         config['postgresql']['parameters'].update({
             'logging_collector': 'on', 'log_destination': 'csvlog', 'log_directory': self._output_dir,
             'log_filename': name + '.log', 'log_statement': 'all', 'log_min_messages': 'debug1'})
 
+        if 'bootstrap' in config and 'initdb' in config['bootstrap']:
+            config['bootstrap']['initdb'].extend([{'auth': 'md5'}, {'auth-host': 'md5'}])
+
         if tags:
             config['tags'] = tags
 
-        if dcs != 'etcd':
-            dcs_config = config.pop('etcd')
-            dcs_config.pop('host')
-
-            if dcs == 'consul':
-                config[dcs] = dcs_config
-            else:
-                dcs_config.update({'session_timeout': dcs_config.pop('ttl'), 'reconnect_timeout': config['loop_wait']})
-                if dcs == 'exhibitor':
-                    dcs_config['exhibitor'] = {'hosts': ['127.0.0.1'], 'port': 8181}
-                else:
-                    dcs_config['hosts'] = ['127.0.0.1:2181']
-                config['zookeeper'] = dcs_config
-
         with open(patroni_config_path, 'w') as f:
-            yaml.dump(config, f, default_flow_style=False)
+            yaml.safe_dump(config, f, default_flow_style=False)
 
         return patroni_config_path
 
@@ -182,7 +180,7 @@ class PatroniController(AbstractController):
 
 class AbstractDcsController(AbstractController):
 
-    _CLUSTER_NODE = 'service/batman'
+    _CLUSTER_NODE = '/service/batman'
 
     def _is_accessible(self):
         return self._is_running()
@@ -193,10 +191,17 @@ class AbstractDcsController(AbstractController):
         if self._work_directory:
             shutil.rmtree(self._work_directory)
 
+    def path(self, key=None):
+        return self._CLUSTER_NODE + (key and '/' + key or '')
+
     @abc.abstractmethod
     def query(self, key):
         """ query for a value of a given key """
 
+    @abc.abstractmethod
+    def set(self, key, value):
+        """ set a value to a given key """
+
     @abc.abstractmethod
     def cleanup_service_tree(self):
         """ clean all contents stored in the tree used for the tests """
@@ -206,6 +211,7 @@ class ConsulController(AbstractDcsController):
 
     def __init__(self, output_dir):
         super(ConsulController, self).__init__('consul', tempfile.mkdtemp(), output_dir)
+        os.environ['PATRONI_CONSUL_HOST'] = 'localhost:8500'
         self._client = consul.Consul()
 
     def _start(self):
@@ -218,12 +224,18 @@ class ConsulController(AbstractDcsController):
         except Exception:
             return False
 
+    def path(self, key=None):
+        return super(ConsulController, self).path(key)[1:]
+
     def query(self, key):
-        _, value = self._client.kv.get('{0}/{1}'.format(self._CLUSTER_NODE, key))
+        _, value = self._client.kv.get(self.path(key))
         return value and value['Value'].decode('utf-8')
 
+    def set(self, key, value):
+        self._client.kv.put(self.path(key), value)
+
     def cleanup_service_tree(self):
-        self._client.kv.delete(self._CLUSTER_NODE, recurse=True)
+        self._client.kv.delete(self.path(), recurse=True)
 
 
 class EtcdController(AbstractDcsController):
@@ -232,6 +244,7 @@ class EtcdController(AbstractDcsController):
 
     def __init__(self, output_dir):
         super(EtcdController, self).__init__('etcd', tempfile.mkdtemp(), output_dir)
+        os.environ['PATRONI_ETCD_HOST'] = 'localhost:4001'
         self._client = etcd.Client()
 
     def _start(self):
@@ -240,13 +253,16 @@ class EtcdController(AbstractDcsController):
 
     def query(self, key):
         try:
-            return self._client.get('/{0}/{1}'.format(self._CLUSTER_NODE, key)).value
+            return self._client.get(self.path(key)).value
         except etcd.EtcdKeyNotFound:
             return None
 
+    def set(self, key, value):
+        self._client.set(self.path(key), value)
+
     def cleanup_service_tree(self):
         try:
-            self._client.delete('/' + self._CLUSTER_NODE, recursive=True)
+            self._client.delete(self.path(), recursive=True)
         except (etcd.EtcdKeyNotFound, etcd.EtcdConnectionFailed):
             return
         except Exception as e:
@@ -264,8 +280,10 @@ class ZooKeeperController(AbstractDcsController):
 
     """ handles all zookeeper related tasks, used for the tests setup and cleanup """
 
-    def __init__(self, output_dir):
+    def __init__(self, output_dir, export_env=True):
         super(ZooKeeperController, self).__init__('zookeeper', None, output_dir)
+        if export_env:
+            os.environ['PATRONI_ZOOKEEPER_HOSTS'] = "'localhost:2181'"
         self._client = kazoo.client.KazooClient()
 
     def _start(self):
@@ -273,13 +291,16 @@ class ZooKeeperController(AbstractDcsController):
 
     def query(self, key):
         try:
-            return self._client.get('/{0}/{1}'.format(self._CLUSTER_NODE, key))[0].decode('utf-8')
+            return self._client.get(self.path(key))[0].decode('utf-8')
         except kazoo.exceptions.NoNodeError:
             return None
 
+    def set(self, key, value):
+        self._client.set(self.path(key), value.encode('utf-8'))
+
     def cleanup_service_tree(self):
         try:
-            self._client.delete('/' + self._CLUSTER_NODE, recursive=True)
+            self._client.delete(self.path(), recursive=True)
         except (kazoo.exceptions.NoNodeError):
             return
         except Exception as e:
@@ -295,10 +316,17 @@ class ZooKeeperController(AbstractDcsController):
             return False
 
 
+class ExhibitorController(ZooKeeperController):
+
+    def __init__(self, output_dir):
+        super(ExhibitorController, self).__init__(output_dir, False)
+        os.environ.update({'PATRONI_EXHIBITOR_HOSTS': 'localhost', 'PATRONI_EXHIBITOR_PORT': '8181'})
+
+
 class PatroniPoolController(object):
 
     KNOWN_DCS = {'consul': ConsulController, 'etcd': EtcdController,
-                 'zookeeper': ZooKeeperController, 'exhibitor': ZooKeeperController}
+                 'zookeeper': ZooKeeperController, 'exhibitor': ExhibitorController}
 
     def __init__(self):
         self._dcs = None
@@ -328,7 +356,7 @@ class PatroniPoolController(object):
         self._processes[pg_name].start(max_wait_limit)
 
     def __getattr__(self, func):
-        if func not in ['stop', 'query', 'write_label', 'read_label', 'check_role_has_changed_to']:
+        if func not in ['stop', 'query', 'write_label', 'read_label', 'check_role_has_changed_to', 'add_tag_to_config']:
             raise AttributeError("PatroniPoolController instance has no attribute '{0}'".format(func))
 
         def wrapper(pg_name, *args, **kwargs):
@@ -350,8 +378,8 @@ class PatroniPoolController(object):
     @property
     def dcs(self):
         if self._dcs is None:
-            self._dcs = os.environ.get('DCS', 'etcd')
-            assert self._dcs in self.KNOWN_DCS, 'Unsupported dcs: ' + self.dcs
+            self._dcs = os.environ.pop('DCS', 'etcd')
+            assert self._dcs in self.KNOWN_DCS, 'Unsupported dcs: ' + self._dcs
         return self._dcs
 
 

features/patroni_api.feature

@@ -10,44 +10,90 @@ Scenario: check API requests on a stand-alone server
 	And I receive a response role master
 	When I issue a GET request to http://127.0.0.1:8008/replica
 	Then I receive a response code 503
-	When I issue an empty POST request to http://127.0.0.1:8008/reinitialize
-	Then I receive a response code 503
-	And I receive a response text "I am the leader, can not reinitialize"
-	When I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0
+	When I run patronictl.py reinit batman postgres0 --force
+	Then I receive a response returncode 0
+	And I receive a response output "Failed: reinitialize for member postgres0, status code=503, (I am the leader, can not reinitialize)"
+	When I run patronictl.py failover batman --master postgres0 --force
+	Then I receive a response returncode 1
+	And I receive a response output "Error: No candidates found to failover to"
+	When I issue a POST request to http://127.0.0.1:8008/failover with {"leader": "postgres0"}
 	Then I receive a response code 500
-	And I receive a response text "failover is not possible: cluster does not have members except leader"
+	And I receive a response text failover is not possible: cluster does not have members except leader
 	When I issue an empty POST request to http://127.0.0.1:8008/failover
 	Then I receive a response code 400
+	When I issue a POST request to http://127.0.0.1:8008/failover with {"foo": "bar"}
+	Then I receive a response code 400
 	And I receive a response text "No values given for required parameters leader and candidate"
 
-Scenario: check API requests for the primary-replica pair
-	Given I start postgres1
-	And replication works from postgres0 to postgres1 after 20 seconds
+Scenario: check local configuration reload
+	Given I issue an empty POST request to http://127.0.0.1:8008/reload
+	Then I receive a response code 200
+	And I receive a response text nothing changed
+	When I add tag new_tag new_value to postgres0 config
+	And I issue an empty POST request to http://127.0.0.1:8008/reload
+	Then I receive a response code 202
+
+Scenario: check dynamic configuration change via DCS
+	Given I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "loop_wait": 2, "postgresql": {"parameters": {"max_connections": 101}}}
+	Then I receive a response code 200
+	And I receive a response loop_wait 2
+	And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 11 seconds
+	When I issue a GET request to http://127.0.0.1:8008/config
+	Then I receive a response code 200
+	And I receive a response loop_wait 2
+	When I issue a GET request to http://127.0.0.1:8008/patroni
+	Then I receive a response code 200
+	And I receive a response tags {'tag': 'new_value'}
+
+Scenario: check API requests for the primary-replica pair in the pause mode
+	Given I run patronictl.py pause batman
+	Then I receive a response returncode 0
+	When I start postgres1
+	Then replication works from postgres0 to postgres1 after 20 seconds
 	When I issue a GET request to http://127.0.0.1:8009/replica
 	Then I receive a response code 200
 	And I receive a response state running
 	And I receive a response role replica
-	When I issue an empty POST request to http://127.0.0.1:8009/reinitialize
-	Then I receive a response code 200
-	When I issue an empty POST request to http://127.0.0.1:8008/restart
-	Then I receive a response code 200
-        And postgres0 role is the primary after 5 seconds
-        When I sleep for 10 seconds
-        Then postgres1 role is the secondary after 15 seconds
+	When I run patronictl.py reinit batman postgres1 --force
+	Then I receive a response returncode 0
+	And I receive a response output "Success: reinitialize for member postgres1"
+	When I run patronictl.py restart batman postgres0 --force
+	Then I receive a response returncode 0
+	And I receive a response output "Success: restart on member postgres0"
+	And postgres0 role is the primary after 5 seconds
+	When I sleep for 10 seconds
+	Then postgres1 role is the secondary after 15 seconds
 
-Scenario: check the failover via the API
-	Given I issue a POST request to http://127.0.0.1:8008/failover with leader=postgres0,candidate=postgres1
-	Then I receive a response code 200
+Scenario: check the failover via the API in the pause mode
+	Given I run patronictl.py failover batman --master postgres0 --candidate postgres1 --force
+	Then I receive a response returncode 0
 	And postgres1 is a leader after 5 seconds
-        And postgres1 role is the primary after 5 seconds
-        And postgres0 role is the secondary after 10 seconds
+	And postgres1 role is the primary after 10 seconds
+	And postgres0 role is the secondary after 10 seconds
 	And replication works from postgres1 to postgres0 after 20 seconds
 
 Scenario: check the scheduled failover
-	Given I issue a scheduled failover at http://127.0.0.1:8009 from postgres1 to postgres0 in 1 seconds
-	Then I receive a response code 200
+	Given I issue a scheduled failover from postgres1 to postgres0 in 1 seconds
+	Then I receive a response returncode 1
+	And I receive a response output "Can't schedule failover in the paused state"
+	When I run patronictl.py resume batman
+	Then I receive a response returncode 0
+	Given I issue a scheduled failover from postgres1 to postgres0 in 1 seconds
+	Then I receive a response returncode 0
 	And postgres0 is a leader after 20 seconds
-        And postgres0 role is the primary after 5 seconds
-        And postgres1 role is the secondary after 10 seconds
+	And postgres0 role is the primary after 10 seconds
+	And postgres1 role is the secondary after 10 seconds
 	And replication works from postgres0 to postgres1 after 25 seconds
 
+Scenario: check the scheduled restart
+	Given I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"superuser_reserved_connections": "6"}}}
+	Then I receive a response code 200
+		And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 5 seconds
+	Given I issue a scheduled restart at http://127.0.0.1:8008 in 1 seconds with {"role": "replica"}
+	Then I receive a response code 202
+		And I sleep for 2 seconds
+		And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 10 seconds
+	Given I issue a scheduled restart at http://127.0.0.1:8008 in 1 seconds with {"restart_pending": "True"}
+	Then I receive a response code 202
+		And Response on GET http://127.0.0.1:8008/patroni does not contain pending_restart after 10 seconds
+

features/steps/patroni_api.py

@@ -1,7 +1,11 @@
+import json
 import parse
 import pytz
 import requests
+import shlex
+import subprocess
 import time
+import yaml
 
 from behave import register_type, step, then
 from datetime import datetime, timedelta
@@ -12,12 +16,7 @@ def parse_url(text):
     return text
 
 
-@parse.with_pattern(r'(?:\w+=(?:\w|\.|:|-|\+|\s)+,?)+')
-def parse_data(text):
-    return text
-
-
-register_type(url=parse_url, data=parse_data)
+register_type(url=parse_url)
 
 
 # there is no way we can find out if the node has already
@@ -39,6 +38,23 @@ def sleep_for_n_seconds(context, value):
     time.sleep(int(value))
 
 
+def _set_response(context, response):
+    context.status_code = response.status_code
+    data = response.content.decode('utf-8')
+    ct = response.headers.get('content-type', '')
+    if ct.startswith('application/json') or\
+            ct.startswith('text/yaml') or\
+            ct.startswith('text/x-yaml') or\
+            ct.startswith('application/yaml') or\
+            ct.startswith('application/x-yaml'):
+        try:
+            context.response = yaml.safe_load(data)
+        except ValueError:
+            context.response = data
+    else:
+        context.response = data
+
+
 @step('I issue a GET request to {url:url}')
 def do_get(context, url):
     try:
@@ -47,54 +63,88 @@ def do_get(context, url):
         context.status_code = None
         context.response = None
     else:
-        context.status_code = r.status_code
-        try:
-            context.response = r.json()
-        except ValueError:
-            context.response = r.content.decode('utf-8')
+        _set_response(context, r)
 
 
 @step('I issue an empty POST request to {url:url}')
 def do_post_empty(context, url):
-    do_post(context, url, None)
+    do_request(context, 'POST', url, None)
 
 
-@step('I issue a POST request to {url:url} with {data:data}')
-def do_post(context, url, data):
-    post_data = {}
-    if data:
-        post_components = data.split(',')
-        for pc in post_components:
-            if '=' in pc:
-                k, v = pc.split('=', 2)
-                post_data[k.strip()] = v.strip()
+@step('I issue a {request_method:w} request to {url:url} with {data}')
+def do_request(context, request_method, url, data):
+    data = data and json.loads(data) or {}
     try:
-        r = requests.post(url, json=post_data)
+        if request_method == 'PATCH':
+            r = requests.patch(url, json=data)
+        else:
+            r = requests.post(url, json=data)
     except requests.exceptions.RequestException:
         context.status_code = None
         context.response = None
     else:
-        context.status_code = r.status_code
-        try:
-            context.response = r.json()
-        except ValueError:
-            context.response = r.content.decode('utf-8')
+        _set_response(context, r)
+
+
+@step('I run {cmd}')
+def do_run(context, cmd):
+    cmd = ['coverage', 'run', '--source=patroni', '-p'] + shlex.split(cmd)
+    try:
+        response = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+        context.status_code = 0
+    except subprocess.CalledProcessError as e:
+        response = e.output
+        context.status_code = e.returncode
+    context.response = response.decode('utf-8').strip()
 
 
 @then('I receive a response {component:w} {data}')
 def check_response(context, component, data):
     if component == 'code':
         assert context.status_code == int(data),\
-            "status code {0} != {1}, response: {2}".format(context.status_code, int(data), context.response)
+            "status code {0} != {1}, response: {2}".format(context.status_code, data, context.response)
+    elif component == 'returncode':
+        assert context.status_code == int(data), "return code {0} != {1}".format(context.status_code, data)
     elif component == 'text':
         assert context.response == data.strip('"'), "response {0} does not contain {1}".format(context.response, data)
+    elif component == 'output':
+        assert data.strip('"') in context.response, "response {0} does not contain {1}".format(context.response, data)
     else:
         assert component in context.response, "{0} is not part of the response".format(component)
-        assert context.response[component] == data, "{0} does not contain {1}".format(component, data)
+        assert str(context.response[component]) == str(data), "{0} does not contain {1}".format(component, data)
 
 
-@step('I issue a scheduled failover at {at_url:url} from {from_host:w} to {to_host:w} in {in_seconds:d} seconds')
-def scheduled_failover(context, at_url, from_host, to_host, in_seconds):
+@step('I issue a scheduled failover from {from_host:w} to {to_host:w} in {in_seconds:d} seconds')
+def scheduled_failover(context, from_host, to_host, in_seconds):
     context.execute_steps(u"""
-        Given I issue a POST request to {0}/failover with leader={1},candidate={2},scheduled_at={3}
-    """.format(at_url, from_host, to_host, datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds))))
+        Given I run patronictl.py failover batman --master {0} --candidate {1} --scheduled "{2}" --force
+    """.format(from_host, to_host, datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds))))
+
+
+@step('I issue a scheduled restart at {url:url} in {in_seconds:d} seconds with {data}')
+def scheduled_restart(context, url, in_seconds, data):
+    data = data and json.loads(data) or {}
+    data.update(schedule='{0}'.format((datetime.now(pytz.utc) + timedelta(seconds=int(in_seconds))).isoformat()))
+    context.execute_steps(u"""Given I issue a POST request to {0}/restart with {1}""".format(url, json.dumps(data)))
+
+
+@step('I add tag {tag:w} {value:w} to {pg_name:w} config')
+def add_tag_to_config(context, tag, value, pg_name):
+    context.pctl.add_tag_to_config(pg_name, tag, value)
+
+
+@then('Response on GET {url} contains {value} after {timeout:d} seconds')
+def check_http_response(context, url, value, timeout, negate=False):
+    for _ in range(int(timeout)):
+        r = requests.get(url)
+        if (value in r.content.decode('utf-8')) != negate:
+            break
+        time.sleep(1)
+    else:
+        assert False,\
+            "Value {0} is {1} present in response after {2} seconds".format(value, "not" if not negate else "", timeout)
+
+
+@then('Response on GET {url} does not contain {value} after {timeout:d} seconds')
+def check_not_in_http_response(context, url, value, timeout):
+    check_http_response(context, url, value, timeout, negate=True)

mkbinary.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -e
+
+pip install --ignore-installed setuptools==19.2 pyinstaller
+pyinstaller --clean --onefile patroni.spec

patroni.spec

@@ -0,0 +1,39 @@
+# -*- mode: python -*-
+
+block_cipher = None
+
+
+def hiddenimports():
+    import sys
+    sys.path.insert(0, '.')
+    try:
+        import patroni.dcs
+        return patroni.dcs.dcs_modules()
+    finally:
+        sys.path.pop(0)
+
+
+a = Analysis(['patroni/__main__.py'],
+             pathex=[],
+             binaries=None,
+             datas=None,
+             hiddenimports=hiddenimports(),
+             hookspath=[],
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher)
+
+pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
+
+exe = EXE(pyz,
+    a.scripts,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    name='patroni',
+    debug=False,
+    strip=False,
+    upx=True,
+    console=True)

patroni/__init__.py

@@ -1,64 +1,95 @@
 import logging
-import os
+import signal
 import sys
 import time
-import yaml
 
 from patroni.api import RestApiServer
-from patroni.exceptions import PatroniException
+from patroni.config import Config
+from patroni.dcs import get_dcs
+from patroni.exceptions import DCSError
 from patroni.ha import Ha
 from patroni.postgresql import Postgresql
-from patroni.utils import reap_children, set_ignore_sigterm, setup_signal_handlers
+from patroni.utils import reap_children, sigchld_handler
 from patroni.version import __version__
 
 logger = logging.getLogger(__name__)
 
 
 class Patroni(object):
-    PATRONI_CONFIG_VARIABLE = 'PATRONI_CONFIGURATION'
 
-    def __init__(self, config):
-        self.nap_time = config['loop_wait']
-        self.tags = {tag: value for tag, value in config.get('tags', {}).items()
-                     if tag not in ('clonefrom', 'nofailover', 'noloadbalance') or value}
-        self.postgresql = Postgresql(config['postgresql'])
-        self.dcs = self.get_dcs(self.postgresql.name, config)
+    def __init__(self):
+        self.setup_signal_handlers()
+
         self.version = __version__
-        self.api = RestApiServer(self, config['restapi'])
-        self.ha = Ha(self)
-        self.next_run = time.time()
+        self.config = Config()
+        self.dcs = get_dcs(self.config)
+        self.load_dynamic_configuration()
 
-    @property
-    def noloadbalance(self):
-        return self.tags.get('noloadbalance', False)
+        self.postgresql = Postgresql(self.config['postgresql'])
+        self.api = RestApiServer(self, self.config['restapi'])
+        self.ha = Ha(self)
+
+        self.tags = self.get_tags()
+        self.next_run = time.time()
+        self.scheduled_restart = {}
+
+    def load_dynamic_configuration(self):
+        while True:
+            try:
+                cluster = self.dcs.get_cluster()
+                if cluster and cluster.config:
+                    if self.config.set_dynamic_configuration(cluster.config):
+                        self.dcs.reload_config(self.config)
+                elif not self.config.dynamic_configuration and 'bootstrap' in self.config:
+                    if self.config.set_dynamic_configuration(self.config['bootstrap']['dcs']):
+                        self.dcs.reload_config(self.config)
+                break
+            except DCSError:
+                logger.warning('Can not get cluster from dcs')
+
+    def get_tags(self):
+        return {tag: value for tag, value in self.config.get('tags', {}).items()
+                if tag not in ('clonefrom', 'nofailover', 'noloadbalance') or value}
 
     @property
     def nofailover(self):
-        return self.tags.get('nofailover', False)
+        return bool(self.tags.get('nofailover', False))
+
+    def reload_config(self):
+        try:
+            self.tags = self.get_tags()
+            self.dcs.reload_config(self.config)
+            self.api.reload_config(self.config['restapi'])
+            self.postgresql.reload_config(self.config['postgresql'])
+        except Exception:
+            logger.exception('Failed to reload config_file=%s', self.config.config_file)
 
     @property
     def replicatefrom(self):
         return self.tags.get('replicatefrom')
 
-    @staticmethod
-    def get_dcs(name, config):
-        if 'etcd' in config:
-            from patroni.etcd import Etcd
-            return Etcd(name, config['etcd'])
-        if 'zookeeper' in config:
-            from patroni.zookeeper import ZooKeeper
-            return ZooKeeper(name, config['zookeeper'])
-        if 'consul' in config:
-            from patroni.consul import Consul
-            return Consul(name, config['consul'])
-        raise PatroniException('Can not find suitable configuration of distributed configuration store')
+    def sighup_handler(self, *args):
+        self._received_sighup = True
+
+    def sigterm_handler(self, *args):
+        if not self._received_sigterm:
+            self._received_sigterm = True
+            sys.exit()
+
+    @property
+    def noloadbalance(self):
+        return bool(self.tags.get('noloadbalance', False))
 
     def schedule_next_run(self):
-        self.next_run += self.nap_time
+        self.next_run += self.dcs.loop_wait
         current_time = time.time()
         nap_time = self.next_run - current_time
         if nap_time <= 0:
             self.next_run = current_time
+            # Release the GIL so we don't starve anyone waiting on async_executor lock
+            time.sleep(0.001)
+            # Warn user that Patroni is not keeping up
+            logger.warning("Loop time exceeded, rescheduling immediately.")
         elif self.dcs.watch(nap_time):
             self.next_run = time.time()
 
@@ -66,41 +97,45 @@ class Patroni(object):
         self.api.start()
         self.next_run = time.time()
 
-        while True:
+        while not self._received_sigterm:
+            if self._received_sighup:
+                self._received_sighup = False
+                if self.config.reload_local_configuration():
+                    self.reload_config()
+
             logger.info(self.ha.run_cycle())
+
+            cluster = self.dcs.cluster
+            if cluster and cluster.config and self.config.set_dynamic_configuration(cluster.config):
+                self.reload_config()
+
+            if not self.postgresql.data_directory_empty():
+                self.config.save_cache()
+
             reap_children()
             self.schedule_next_run()
 
+    def setup_signal_handlers(self):
+        self._received_sighup = False
+        self._received_sigterm = False
+        signal.signal(signal.SIGHUP, self.sighup_handler)
+        signal.signal(signal.SIGTERM, self.sigterm_handler)
+        signal.signal(signal.SIGCHLD, sigchld_handler)
+
 
 def main():
     logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
     logging.getLogger('requests').setLevel(logging.WARNING)
-    setup_signal_handlers()
 
-    # Patroni reads the configuration from the command-line argument if it exists, and from the environment otherwise.
-    use_env = False
-    use_file = (len(sys.argv) >= 2 and os.path.isfile(sys.argv[1]))
-    if not use_file:
-        config_env = os.environ.get(Patroni.PATRONI_CONFIG_VARIABLE)
-        use_env = config_env is not None
-        if not use_env:
-            print('Usage: {0} config.yml'.format(sys.argv[0]))
-            print('\tPatroni may also read the configuration from the {} environment variable'.
-                  format(Patroni.PATRONI_CONFIG_VARIABLE))
-            return
-
-    if use_file:
-        with open(sys.argv[1], 'r') as f:
-            config = yaml.load(f)
-    elif use_env:
-        config = yaml.load(config_env)
-
-    patroni = Patroni(config)
+    patroni = Patroni()
     try:
         patroni.run()
     except KeyboardInterrupt:
-        set_ignore_sigterm()
+        pass
     finally:
         patroni.api.shutdown()
-        patroni.postgresql.stop(checkpoint=False)
-        patroni.dcs.delete_leader()
+        if patroni.ha.is_paused():
+            logger.info('Leader key is not deleted and Postgresql is not stopped due paused state')
+        else:
+            patroni.postgresql.stop(checkpoint=False)
+            patroni.dcs.delete_leader()

patroni/api.py

@@ -3,14 +3,13 @@ import fcntl
 import json
 import logging
 import psycopg2
-import socket
 import time
-import dateutil
+import dateutil.parser
 import datetime
 import pytz
 
 from patroni.exceptions import PostgresConnectionException
-from patroni.utils import Retry, RetryFailedError
+from patroni.utils import deep_compare, patch_config, Retry, RetryFailedError, is_valid_pg_version
 from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 from six.moves.socketserver import ThreadingMixIn
 from threading import Thread
@@ -34,47 +33,43 @@ def check_auth(func):
 
 class RestApiHandler(BaseHTTPRequestHandler):
 
-    def _write_response(self, status_code, body, headers=None):
+    def _write_response(self, status_code, body, content_type='text/html', headers=None):
         self.send_response(status_code)
-        if body is not None:
-            headers = headers or {}
-            if 'Content-Type' not in headers:
-                headers['Content-Type'] = 'text/html'
-            for name, value in (headers or {}).items():
-                self.send_header(name, value)
-            self.end_headers()
-            self.wfile.write(body.encode('utf-8'))
+        headers = headers or {}
+        if content_type:
+            headers['Content-Type'] = content_type
+        for name, value in headers.items():
+            self.send_header(name, value)
+        self.end_headers()
+        self.wfile.write(body.encode('utf-8'))
+
+    def _write_json_response(self, status_code, response):
+        self._write_response(status_code, json.dumps(response), content_type='application/json')
 
     def send_auth_request(self, body):
-        self._write_response(401, body, {'WWW-Authenticate': 'Basic realm=\"Patroni\"'})
-
-    def finish(self, *args, **kwargs):
-        try:
-            if not self.wfile.closed:
-                self.wfile.flush()
-                self.wfile.close()
-        except socket.error:
-            pass
-        self.rfile.close()
+        headers = {'WWW-Authenticate': 'Basic realm="' + self.server.patroni.__class__.__name__ + '"'}
+        self._write_response(401, body, headers=headers)
 
     def check_auth_header(self):
         auth_header = self.headers.get('Authorization')
         status = self.server.check_auth_header(auth_header)
         return not status or self.send_auth_request(status)
 
-    def _write_status_response(self, status_code, response, options=False):
-        if options:
-            body = None
-        else:
-            patroni = self.server.patroni
-            response.update({'tags': patroni.tags} if patroni.tags else {})
-            if patroni.postgresql.sysid:
-                response['database_system_identifier'] = patroni.postgresql.sysid
-            response['patroni'] = {'version': patroni.version, 'scope': patroni.postgresql.scope}
-            body = json.dumps(response)
-        self._write_response(status_code, body, {'Content-Type': 'application/json'})
+    def _write_status_response(self, status_code, response):
+        patroni = self.server.patroni
+        response.update({'tags': patroni.tags} if patroni.tags else {})
+        if patroni.postgresql.sysid:
+            response['database_system_identifier'] = patroni.postgresql.sysid
+        if patroni.postgresql.pending_restart:
+            response['pending_restart'] = True
+        response['patroni'] = {'version': patroni.version, 'scope': patroni.postgresql.scope}
+        if patroni.scheduled_restart and isinstance(patroni.scheduled_restart, dict):
+            response['scheduled_restart'] = patroni.scheduled_restart.copy()
+            del response['scheduled_restart']['postmaster_start_time']
+            response['scheduled_restart']['schedule'] = (response['scheduled_restart']['schedule']).isoformat()
+        self._write_json_response(status_code, response)
 
-    def do_GET(self, options=False):
+    def do_GET(self, write_status_code_only=False):
         """Default method for processing all GET requests which can not be routed to other methods"""
 
         path = '/master' if self.path == '/' else self.path
@@ -100,48 +95,177 @@ class RestApiHandler(BaseHTTPRequestHandler):
             status_code = 200
         else:
             status_code = 503
-        self._write_status_response(status_code, response, options)
+
+        if write_status_code_only:  # when haproxy sends OPTIONS request it reads only status code and nothing more
+            message = self.responses[status_code][0]
+            self.wfile.write('{0} {1} {2}\r\n'.format(self.protocol_version, status_code, message).encode('utf-8'))
+        else:
+            self._write_status_response(status_code, response)
 
     def do_OPTIONS(self):
-        self.do_GET(options=True)
+        self.do_GET(write_status_code_only=True)
 
     def do_GET_patroni(self):
         response = self.get_postgresql_status(True)
         self._write_status_response(200, response)
 
+    def do_GET_config(self):
+        cluster = self.server.patroni.dcs.cluster or self.server.patroni.dcs.get_cluster()
+        if cluster.config:
+            self._write_json_response(200, cluster.config.data)
+        else:
+            self.send_error(502)
+
+    def _read_json_content(self, body_is_optional=False):
+        if 'content-length' not in self.headers:
+            return self.send_error(411) if not body_is_optional else {}
+        try:
+            content_length = int(self.headers.get('content-length'))
+            if content_length == 0 and body_is_optional:
+                return {}
+            request = json.loads(self.rfile.read(content_length).decode('utf-8'))
+            if isinstance(request, dict) and (request or body_is_optional):
+                return request
+        except Exception:
+            logger.exception('Bad request')
+        self.send_error(400)
+
+    @check_auth
+    def do_PATCH_config(self):
+        request = self._read_json_content()
+        if request:
+            cluster = self.server.patroni.dcs.get_cluster()
+            data = cluster.config.data.copy()
+            if patch_config(data, request):
+                value = json.dumps(data, separators=(',', ':'))
+                if not self.server.patroni.dcs.set_config_value(value, cluster.config.index):
+                    return self.send_error(409)
+            self._write_json_response(200, data)
+
+    @check_auth
+    def do_PUT_config(self):
+        request = self._read_json_content()
+        if request:
+            cluster = self.server.patroni.dcs.get_cluster()
+            if not deep_compare(request, cluster.config.data):
+                value = json.dumps(request, separators=(',', ':'))
+                if not self.server.patroni.dcs.set_config_value(value):
+                    return self.send_error(502)
+            self._write_json_response(200, request)
+
+    @check_auth
+    def do_POST_reload(self):
+        try:
+            if self.server.patroni.config.reload_local_configuration(True):
+                status_code = 202
+                response = 'reload scheduled'
+                self.server.patroni.sighup_handler()
+            else:
+                status_code = 200
+                response = 'nothing changed'
+        except Exception as e:
+            status_code = 500
+            response = str(e)
+        self._write_response(status_code, response)
+
+    @staticmethod
+    def parse_schedule(schedule, action):
+        """ parses the given schedule and validates at """
+        error = None
+        scheduled_at = None
+        try:
+            scheduled_at = dateutil.parser.parse(schedule)
+            if scheduled_at.tzinfo is None:
+                error = 'Timezone information is mandatory for the scheduled {0}'.format(action)
+                status_code = 400
+            elif scheduled_at < datetime.datetime.now(pytz.utc):
+                error = 'Cannot schedule {0} in the past'.format(action)
+                status_code = 422
+            else:
+                status_code = None
+        except (ValueError, TypeError):
+            logger.exception('Invalid scheduled %s time: %s', action, schedule)
+            error = 'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601'
+            status_code = 422
+        return (status_code, error, scheduled_at)
+
     @check_auth
     def do_POST_restart(self):
         status_code = 500
         data = 'restart failed'
-        try:
-            status, data = self.server.patroni.ha.restart()
-            status_code = 200 if status else 503
-        except Exception:
-            logger.exception('Exception during restart')
+        request = self._read_json_content(body_is_optional=True)
+        cluster = self.server.patroni.dcs.get_cluster()
+        if request is None:
+            # failed to parse the json
+            return
+        if request:
+            logger.debug("received restart request: {0}".format(request))
+
+        if cluster.is_paused() and 'schedule' in request:
+            self._write_response(status_code, "Can't schedule restart in the paused state")
+            return
+
+        for k in request:
+            if k == 'schedule':
+                (_, data, request[k]) = self.parse_schedule(request[k], "restart")
+                if _:
+                    status_code = _
+                    break
+            elif k == 'role':
+                if request[k] not in ('master', 'replica'):
+                    status_code = 400
+                    data = "PostgreSQL role should be either master or replica"
+                    break
+            elif k == 'postgres_version':
+                if not is_valid_pg_version(request[k]):
+                    status_code = 400
+                    data = "PostgreSQL version should be in the first.major.minor format"
+                    break
+            elif k != 'restart_pending':
+                status_code = 400
+                data = "Unknown filter for the scheduled restart: {0}".format(k)
+                break
+        else:
+            if 'schedule' not in request:
+                try:
+                    status, data = self.server.patroni.ha.restart(request)
+                    status_code = 200 if status else 503
+                except Exception:
+                    logger.exception('Exception during restart')
+                    status_code = 400
+            else:
+                request['postmaster_start_time'] = self.server.patroni.ha.state_handler.postmaster_start_time()
+                if self.server.patroni.ha.schedule_future_restart(request):
+                    data = "Restart scheduled"
+                    status_code = 202
+                else:
+                    data = "Another restart is already scheduled"
+                    status_code = 409
         self._write_response(status_code, data)
 
     @check_auth
-    def do_POST_reinitialize(self):
-        ha = self.server.patroni.ha
-        cluster = ha.dcs.get_cluster()
-        if cluster.is_unlocked():
-            status_code = 503
-            data = 'Cluster has no leader, can not reinitialize'
-        elif cluster.leader.name == ha.state_handler.name:
-            status_code = 503
-            data = 'I am the leader, can not reinitialize'
+    def do_DELETE_restart(self):
+        if self.server.patroni.ha.delete_future_restart():
+            data = "scheduled restart deleted"
+            code = 200
         else:
-            action = ha.schedule_reinitialize()
-            if action is not None:
-                status_code = 503
-                data = action + ' already in progress'
-            else:
-                status_code = 200
-                data = 'reinitialize scheduled'
+            data = "no restarts are scheduled"
+            code = 404
+        self._write_response(code, data)
+
+    @check_auth
+    def do_POST_reinitialize(self):
+        data = self.server.patroni.ha.reinitialize()
+        if data is None:
+            status_code = 200
+            data = 'reinitialize started'
+        else:
+            status_code = 503
         self._write_response(status_code, data)
 
     def poll_failover_result(self, leader, candidate):
-        for _ in range(0, 15):
+        timeout = max(10, self.server.patroni.dcs.loop_wait)
+        for _ in range(0, timeout*2):
             time.sleep(1)
             try:
                 cluster = self.server.patroni.dcs.get_cluster()
@@ -157,7 +281,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         return 503, 'Failover status unknown'
 
     def is_failover_possible(self, cluster, leader, candidate):
-        if leader and not cluster.leader or cluster.leader.name != leader:
+        if leader and (not cluster.leader or cluster.leader.name != leader):
             return 'leader name does not match'
         if candidate:
             members = [m for m in cluster.members if m.name == candidate]
@@ -174,16 +298,18 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
     @check_auth
     def do_POST_failover(self):
-        content_length = int(self.headers.get('content-length', 0))
-        try:
-            request = json.loads(self.rfile.read(content_length).decode('utf-8'))
-        except ValueError:
-            request = {}
+        request = self._read_json_content()
+        status_code = 500
+        if not request:
+            return
+
         leader = request.get('leader')
         candidate = request.get('candidate') or request.get('member')
         scheduled_at = request.get('scheduled_at')
-        cluster = self.server.patroni.ha.dcs.get_cluster()
-        status_code = 500
+        cluster = self.server.patroni.dcs.get_cluster()
+
+        if scheduled_at and cluster.is_paused():
+            self._write_response(status_code, "Can't schedule failover in the paused state")
 
         logger.info("received failover request with leader=%s candidate=%s scheduled_at=%s",
                     leader, candidate, scheduled_at)
@@ -191,25 +317,16 @@ class RestApiHandler(BaseHTTPRequestHandler):
         data = ''
         if leader or candidate:
             if scheduled_at:
-                try:
-                    scheduled_at = dateutil.parser.parse(scheduled_at)
-                    if scheduled_at.tzinfo is None:
-                        data = 'Timezone information is mandatory for scheduled_at'
-                        status_code = 400
-                    elif scheduled_at < datetime.datetime.now(pytz.utc):
-                        data = 'Cannot schedule failover in the past'
-                        status_code = 422
-                    elif self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
-                        self.server.patroni.dcs.event.set()
-                        data = 'Failover scheduled'
-                        status_code = 200
-                    else:
-                        data = 'failed to write failover key into DCS'
-                        status_code = 503
-                except (ValueError, TypeError):
-                    logger.exception('Invalid scheduled failover time: %s', request['scheduled_at'])
-                    data = 'Unable to parse scheduled timestamp. It should be in an unambiguous format, e.g. ISO 8601'
-                    status_code = 422
+                (_, data, scheduled_at) = self.parse_schedule(scheduled_at, "failover")
+                if _:
+                    status_code = _
+                elif self.server.patroni.dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at):
+                    self.server.patroni.dcs.event.set()
+                    data = 'Failover scheduled'
+                    status_code = 202
+                else:
+                    data = 'failed to write failover key into DCS'
+                    status_code = 503
             else:
                 data = self.is_failover_possible(cluster, leader, candidate)
                 if not data:
@@ -242,12 +359,6 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 self.command = mname
         return ret
 
-    def handle_one_request(self):
-        try:
-            BaseHTTPRequestHandler.handle_one_request(self)
-        except socket.error:
-            pass
-
     def query(self, sql, *params, **kwargs):
         if not kwargs.get('retry', False):
             return self.server.query(sql, *params)
@@ -256,7 +367,11 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
     def get_postgresql_status(self, retry=False):
         try:
-            row = self.query("""SELECT to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
+            row = self.query("""WITH replication_info AS (
+                                    SELECT usename, application_name, client_addr, state, sync_state, sync_priority
+                                      FROM pg_stat_replication
+                                )
+                                SELECT to_char(pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
                                        pg_is_in_recovery(),
                                        CASE WHEN pg_is_in_recovery()
                                             THEN 0
@@ -265,8 +380,10 @@ class RestApiHandler(BaseHTTPRequestHandler):
                                        pg_xlog_location_diff(pg_last_xlog_receive_location(), '0/0')::bigint,
                                        pg_xlog_location_diff(pg_last_xlog_replay_location(), '0/0')::bigint,
                                        to_char(pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),
-                                       pg_is_in_recovery() AND pg_is_xlog_replay_paused()""", retry=retry)[0]
-            return {
+                                       pg_is_in_recovery() AND pg_is_xlog_replay_paused(),
+                                       (SELECT json_agg(row_to_json(ri)) FROM replication_info ri)""", retry=retry)[0]
+
+            result = {
                 'state': self.server.patroni.postgresql.state,
                 'postmaster_start_time': row[0],
                 'role': 'replica' if row[1] else 'master',
@@ -279,12 +396,17 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     'location': row[2]
                 })
             }
+
+            if row[7]:
+                result['replication'] = row[7]
+
+            return result
         except (psycopg2.Error, RetryFailedError, PostgresConnectionException):
             state = self.server.patroni.postgresql.state
             if state == 'running':
                 logger.exception('get_postgresql_status')
                 state = 'unknown'
-            return {'state': state}
+            return {'state': state, 'role': self.server.patroni.postgresql.role}
 
     def log_message(self, fmt, *args):
         logger.debug("API thread: %s - - [%s] %s", self.client_address[0], self.log_date_time_string(), fmt % args)
@@ -293,25 +415,9 @@ class RestApiHandler(BaseHTTPRequestHandler):
 class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
 
     def __init__(self, patroni, config):
-        self._auth_key = base64.b64encode(config['auth'].encode('utf-8')).decode('utf-8') if 'auth' in config else None
-        host, port = config['listen'].split(':')
-        HTTPServer.__init__(self, (host, int(port)), RestApiHandler)
-        Thread.__init__(self, target=self.serve_forever)
-        self._set_fd_cloexec(self.socket)
-
-        protocol = 'http'
-
-        # wrap socket with ssl if 'certfile' is defined in a config.yaml
-        # Sometime it's also needed to pass reference to a 'keyfile'.
-        options = {option: config[option] for option in ['certfile', 'keyfile'] if option in config}
-        if options.get('certfile'):
-            import ssl
-            self.socket = ssl.wrap_socket(self.socket, server_side=True, **options)
-            protocol = 'https'
-
-        self.connection_string = '{0}://{1}/patroni'.format(protocol, config.get('connect_address', config['listen']))
-
         self.patroni = patroni
+        self.__initialize(config)
+        self.__set_config_parameters(config)
         self.daemon = True
 
     def query(self, sql, *params):
@@ -331,11 +437,47 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC)
 
     def check_basic_auth_key(self, key):
-        return self._auth_key == key
+        return self.__auth_key == key
 
     def check_auth_header(self, auth_header):
-        if self._auth_key:
+        if self.__auth_key:
             if auth_header is None:
                 return 'no auth header received'
             if not auth_header.startswith('Basic ') or not self.check_basic_auth_key(auth_header[6:]):
                 return 'not authenticated'
+
+    @staticmethod
+    def __get_ssl_options(config):
+        return {option: config[option] for option in ['certfile', 'keyfile'] if option in config}
+
+    def __set_connection_string(self, connect_address):
+        self.connection_string = '{0}://{1}/patroni'.format(self.__protocol, connect_address or self.__listen)
+
+    def __set_config_parameters(self, config):
+        self.__auth_key = base64.b64encode(config['auth'].encode('utf-8')).decode('utf-8') if 'auth' in config else None
+        self.__set_connection_string(config.get('connect_address'))
+
+    def __initialize(self, config):
+        self.__ssl_options = self.__get_ssl_options(config)
+        self.__listen = config['listen']
+        host, port = config['listen'].split(':')
+        HTTPServer.__init__(self, (host, int(port)), RestApiHandler)
+        Thread.__init__(self, target=self.serve_forever)
+        self._set_fd_cloexec(self.socket)
+
+        self.__protocol = 'http'
+
+        # wrap socket with ssl if 'certfile' is defined in a config.yaml
+        # Sometime it's also needed to pass reference to a 'keyfile'.
+        if self.__ssl_options.get('certfile'):
+            import ssl
+            self.socket = ssl.wrap_socket(self.socket, server_side=True, **self.__ssl_options)
+            self.__protocol = 'https'
+        self.__set_connection_string(config.get('connect_address'))
+
+    def reload_config(self, config):
+        self.__set_config_parameters(config)
+        if self.__listen != config['listen'] or self.__ssl_options != self.__get_ssl_options(config):
+            self.shutdown()
+            self.__initialize(config)
+            self.start()

patroni/async_executor.py

@@ -1,5 +1,5 @@
 import logging
-from threading import Lock, Thread
+from threading import RLock, Thread
 
 logger = logging.getLogger(__name__)
 
@@ -7,21 +7,19 @@ logger = logging.getLogger(__name__)
 class AsyncExecutor(object):
 
     def __init__(self):
-        self._busy = False
-        self._thread_lock = Lock()
+        self._thread_lock = RLock()
         self._scheduled_action = None
-        self._scheduled_action_lock = Lock()
+        self._scheduled_action_lock = RLock()
 
     @property
     def busy(self):
-        return self._busy
+        return self.scheduled_action is not None
 
     def schedule(self, action, immediately=False):
         with self._scheduled_action_lock:
             if self._scheduled_action is not None:
                 return self._scheduled_action
             self._scheduled_action = action
-            self._busy = immediately
         return None
 
     @property
@@ -40,11 +38,9 @@ class AsyncExecutor(object):
             logger.exception('Exception during execution of long running task %s', self.scheduled_action)
         finally:
             with self:
-                self._busy = False
                 self.reset_scheduled_action()
 
     def run_async(self, func, args=()):
-        self._busy = True
         Thread(target=self.run, args=(func, args)).start()
 
     def __enter__(self):

patroni/config.py

@@ -0,0 +1,312 @@
+import json
+import logging
+import os
+import sys
+import tempfile
+import yaml
+
+from collections import defaultdict
+from copy import deepcopy
+from patroni.dcs import ClusterConfig
+from patroni.postgresql import Postgresql
+from patroni.utils import deep_compare, parse_int, patch_config
+
+logger = logging.getLogger(__name__)
+
+
+class Config(object):
+    """
+    This class is responsible for:
+
+      1) Building and giving access to `effective_configuration` from:
+         * `Config.__DEFAULT_CONFIG` -- some sane default values
+         * `dynamic_configuration` -- configuration stored in DCS
+         * `local_configuration` -- configuration from `config.yml` or environment
+
+      2) Saving and loading `dynamic_configuration` into 'patroni.dynamic.json' file
+         located in local_configuration['postgresql']['data_dir'] directory.
+         This is necessary to be able to restore `dynamic_configuration`
+         if DCS was accidentally wiped
+
+      3) Loading of configuration file in the old format and converting it into new format
+
+      4) Mimicking some of the `dict` interfaces to make it possible
+         to work with it as with the old `config` object.
+    """
+
+    PATRONI_ENV_PREFIX = 'PATRONI_'
+    PATRONI_CONFIG_VARIABLE = PATRONI_ENV_PREFIX + 'CONFIGURATION'
+
+    __CACHE_FILENAME = 'patroni.dynamic.json'
+    __DEFAULT_CONFIG = {
+        'ttl': 30, 'loop_wait': 10, 'retry_timeout': 10,
+        'maximum_lag_on_failover': 1048576,
+        'postgresql': {
+            'bin_dir': '',
+            'use_slots': True,
+            'parameters': {p: v[0] for p, v in Postgresql.CMDLINE_OPTIONS.items()}
+        }
+    }
+
+    def __init__(self):
+        self._modify_index = -1
+        self._dynamic_configuration = {}
+
+        self.__environment_configuration = self._build_environment_configuration()
+
+        # Patroni reads the configuration from the command-line argument if it exists, otherwise from the environment
+        self._config_file = len(sys.argv) >= 2 and os.path.isfile(sys.argv[1]) and sys.argv[1]
+        if self._config_file:
+            self._local_configuration = self._load_config_file()
+        else:
+            config_env = os.environ.pop(self.PATRONI_CONFIG_VARIABLE, None)
+            self._local_configuration = config_env and yaml.safe_load(config_env) or self.__environment_configuration
+            if not self._local_configuration:
+                print('Usage: {0} config.yml'.format(sys.argv[0]))
+                print('\tPatroni may also read the configuration from the {0} environment variable'.
+                      format(self.PATRONI_CONFIG_VARIABLE))
+                sys.exit(1)
+
+        self.__effective_configuration = self._build_effective_configuration({}, self._local_configuration)
+        self._data_dir = self.__effective_configuration['postgresql']['data_dir']
+        self._cache_file = os.path.join(self._data_dir, self.__CACHE_FILENAME)
+        self._load_cache()
+        self._cache_needs_saving = False
+
+    @property
+    def config_file(self):
+        return self._config_file
+
+    @property
+    def dynamic_configuration(self):
+        return deepcopy(self._dynamic_configuration)
+
+    def _load_config_file(self):
+        """Loads config.yaml from filesystem and applies some values which were set via ENV"""
+        with open(self._config_file) as f:
+            config = yaml.safe_load(f)
+            patch_config(config, self.__environment_configuration)
+            return config
+
+    def _load_cache(self):
+        if os.path.isfile(self._cache_file):
+            try:
+                with open(self._cache_file) as f:
+                    self.set_dynamic_configuration(json.load(f))
+            except Exception:
+                logger.exception('Exception when loading file: %s', self._cache_file)
+
+    def save_cache(self):
+        if self._cache_needs_saving:
+            tmpfile = fd = None
+            try:
+                (fd, tmpfile) = tempfile.mkstemp(prefix=self.__CACHE_FILENAME, dir=self._data_dir)
+                with os.fdopen(fd, 'w') as f:
+                    fd = None
+                    json.dump(self.dynamic_configuration, f)
+                tmpfile = os.rename(tmpfile, self._cache_file)
+                self._cache_needs_saving = False
+            except Exception:
+                logger.exception('Exception when saving file: %s', self._cache_file)
+                if fd:
+                    try:
+                        os.close(fd)
+                    except Exception:
+                        logger.error('Can not close temporary file %s', tmpfile)
+                if tmpfile and os.path.exists(tmpfile):
+                    try:
+                        os.remove(tmpfile)
+                    except Exception:
+                        logger.error('Can not remove temporary file %s', tmpfile)
+
+    # configuration could be either ClusterConfig or dict
+    def set_dynamic_configuration(self, configuration):
+        if isinstance(configuration, ClusterConfig):
+            if self._modify_index == configuration.modify_index:
+                return False  # If the index didn't changed there is nothing to do
+            self._modify_index = configuration.modify_index
+            configuration = configuration.data
+
+        if not deep_compare(self._dynamic_configuration, configuration):
+            try:
+                self.__effective_configuration = self._build_effective_configuration(configuration,
+                                                                                     self._local_configuration)
+                self._dynamic_configuration = configuration
+                self._cache_needs_saving = True
+                return True
+            except Exception:
+                logger.exception('Exception when setting dynamic_configuration')
+
+    def reload_local_configuration(self, dry_run=False):
+        if self.config_file:
+            try:
+                configuration = self._load_config_file()
+                if not deep_compare(self._local_configuration, configuration):
+                    new_configuration = self._build_effective_configuration(self._dynamic_configuration, configuration)
+                    if dry_run:
+                        return not deep_compare(new_configuration, self.__effective_configuration)
+                    self._local_configuration = configuration
+                    self.__effective_configuration = new_configuration
+                    return True
+            except Exception:
+                logger.exception('Exception when reloading local configuration from %s', self.config_file)
+                if dry_run:
+                    raise
+
+    @staticmethod
+    def _process_postgresql_parameters(parameters, is_local=False):
+        ret = {}
+        for name, value in (parameters or {}).items():
+            if name not in Postgresql.CMDLINE_OPTIONS or not is_local and Postgresql.CMDLINE_OPTIONS[name][1](value):
+                ret[name] = value
+        return ret
+
+    def _safe_copy_dynamic_configuration(self, dynamic_configuration):
+        config = deepcopy(self.__DEFAULT_CONFIG)
+
+        for name, value in dynamic_configuration.items():
+            if name == 'postgresql':
+                for name, value in (value or {}).items():
+                    if name == 'parameters':
+                        config['postgresql'][name].update(self._process_postgresql_parameters(value))
+                    elif name not in ('connect_address', 'listen', 'data_dir', 'pgpass', 'authentication'):
+                        config['postgresql'][name] = deepcopy(value)
+            elif name in config:  # only variables present in __DEFAULT_CONFIG allowed to be overriden from DCS
+                config[name] = int(value)
+        return config
+
+    @staticmethod
+    def _build_environment_configuration():
+        ret = defaultdict(dict)
+
+        def _popenv(name):
+            return os.environ.pop(Config.PATRONI_ENV_PREFIX + name.upper(), None)
+
+        for param in ('name', 'namespace', 'scope'):
+            value = _popenv(param)
+            if value:
+                ret[param] = value
+
+        def _set_section_values(section, params):
+            for param in params:
+                value = _popenv(section + '_' + param)
+                if value:
+                    ret[section][param] = value
+
+        _set_section_values('restapi', ['listen', 'connect_address', 'certfile', 'keyfile'])
+        _set_section_values('postgresql', ['listen', 'connect_address', 'data_dir', 'pgpass', 'bin_dir'])
+
+        def _get_auth(name):
+            ret = {}
+            for param in ('username', 'password'):
+                value = _popenv(name + '_' + param)
+                if value:
+                    ret[param] = value
+            return ret
+
+        restapi_auth = _get_auth('restapi')
+        if restapi_auth:
+            ret['restapi']['authentication'] = restapi_auth
+
+        authentication = {}
+        for user_type in ('replication', 'superuser'):
+            entry = _get_auth(user_type)
+            if entry:
+                authentication[user_type] = entry
+
+        if authentication:
+            ret['postgresql']['authentication'] = authentication
+
+        users = {}
+
+        def _parse_list(value):
+            if not (value.strip().startswith('-') or '[' in value):
+                value = '[{0}]'.format(value)
+            try:
+                return yaml.safe_load(value)
+            except Exception:
+                logger.exception('Exception when parsing list %s', value)
+                return None
+
+        for param in list(os.environ.keys()):
+            if param.startswith(Config.PATRONI_ENV_PREFIX):
+                name, suffix = (param[8:].rsplit('_', 1) + [''])[:2]
+                if name and suffix:
+                    # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT)
+                    if suffix in ('HOST', 'HOSTS', 'PORT') and '_' not in name:
+                        value = os.environ.pop(param)
+                        if suffix == 'PORT':
+                            value = value and parse_int(value)
+                        elif suffix == 'HOSTS':
+                            value = value and _parse_list(value)
+                        if value:
+                            ret[name.lower()][suffix.lower()] = value
+                    # PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>
+                    # CREATE USER "<username>" WITH <OPTIONS> PASSWORD '<password>'
+                    elif suffix == 'PASSWORD':
+                        password = os.environ.pop(param)
+                        if password:
+                            users[name] = {'password': password}
+                            options = os.environ.pop(param[:-9] + '_OPTIONS', None)
+                            options = options and _parse_list(options)
+                            if options:
+                                users[name]['options'] = options
+        if users:
+            ret['bootstrap']['users'] = users
+
+        return ret
+
+    def _build_effective_configuration(self, dynamic_configuration, local_configuration):
+        config = self._safe_copy_dynamic_configuration(dynamic_configuration)
+        for name, value in local_configuration.items():
+            if name == 'postgresql':
+                for name, value in (value or {}).items():
+                    if name == 'parameters':
+                        config['postgresql'][name].update(self._process_postgresql_parameters(value, True))
+                    elif name != 'use_slots':  # replication slots must be enabled/disabled globally
+                        config['postgresql'][name] = deepcopy(value)
+            elif name not in config:
+                config[name] = deepcopy(value) if value else {}
+
+        # restapi server expects to get restapi.auth = 'username:password'
+        if 'authentication' in config['restapi']:
+            restapi = config['restapi']
+            auth = restapi['authentication']
+            restapi['auth'] = '{0}:{1}'.format(auth['username'], auth['password'])
+
+        # special treatment for old config
+
+        # 'exhibitor' inside 'zookeeper':
+        if 'zookeeper' in config and 'exhibitor' in config['zookeeper']:
+            config['exhibitor'] = config['zookeeper'].pop('exhibitor')
+            config.pop('zookeeper')
+
+        pg_config = config['postgresql']
+        # no 'authentication' in 'postgresql', but 'replication' and 'superuser'
+        if 'authentication' not in pg_config:
+            pg_config['use_pg_rewind'] = 'pg_rewind' in pg_config
+            pg_config['authentication'] = {u: pg_config[u] for u in ('replication', 'superuser') if u in pg_config}
+        # no 'superuser' in 'postgresql'.'authentication'
+        if 'superuser' not in pg_config['authentication'] and 'pg_rewind' in pg_config:
+            pg_config['authentication']['superuser'] = pg_config['pg_rewind']
+
+        # no 'name' in config
+        if 'name' not in config and 'name' in pg_config:
+            config['name'] = pg_config['name']
+
+        pg_config.update({p: config[p] for p in ('name', 'scope', 'retry_timeout',
+                          'maximum_lag_on_failover') if p in config})
+
+        return config
+
+    def get(self, key, default=None):
+        return self.__effective_configuration.get(key, default)
+
+    def __contains__(self, key):
+        return key in self.__effective_configuration
+
+    def __getitem__(self, key):
+        return self.__effective_configuration[key]
+
+    def copy(self):
+        return deepcopy(self.__effective_configuration)

patroni/ctl.py

@@ -2,22 +2,27 @@
 Patroni Control
 '''
 
+import base64
 import click
 import datetime
-import dateutil
+import dateutil.parser
 import json
 import logging
 import os
 import psycopg2
 import random
 import requests
+import sys
 import time
 import tzlocal
 import yaml
 
 from click import ClickException
-from patroni import Patroni, PatroniException
-from patroni.postgresql import parseurl
+from patroni.config import Config
+from patroni.dcs import get_dcs as _get_dcs
+from patroni.exceptions import PatroniException
+from patroni.postgresql import Postgresql
+from patroni.utils import is_valid_pg_version
 from prettytable import PrettyTable
 from six.moves.urllib_parse import urlparse
 
@@ -25,7 +30,7 @@ CONFIG_DIR_PATH = click.get_app_dir('patroni')
 CONFIG_FILE_PATH = os.path.join(CONFIG_DIR_PATH, 'patronictl.yaml')
 LOGLEVEL = 'WARNING'
 DCS_DEFAULTS = {'zookeeper': {'port': 2181, 'template': "zookeeper:\n hosts: ['{host}:{port}']"},
-                'exhibitor': {'port': 8181, 'template': "zookeeper:\n exhibitor:\n  hosts: [{host}]\n  port: {port}"},
+                'exhibitor': {'port': 8181, 'template': "exhibitor:\n hosts: [{host}]\n port: {port}"},
                 'consul': {'port': 8500, 'template': "consul:\n host: '{host}:{port}'"},
                 'etcd': {'port': 4001, 'template': "etcd:\n host: '{host}:{port}'"}}
 
@@ -55,15 +60,23 @@ def parse_dcs(dcs):
 
 def load_config(path, dcs):
     logging.debug('Loading configuration from file %s', path)
-    config = dict()
+    config = {}
+    old_argv = list(sys.argv)
     try:
-        with open(path, 'rb') as fd:
-            config = yaml.safe_load(fd)
-    except (IOError, yaml.YAMLError):
-        logging.exception('Could not load configuration file')
-
-    config.update(parse_dcs(dcs) or parse_dcs(config.get('dcs_api')) or {})
+        sys.argv[1] = path
+        if Config.PATRONI_CONFIG_VARIABLE not in os.environ:
+            for p in ('PATRONI_RESTAPI_LISTEN', 'PATRONI_POSTGRESQL_DATA_DIR'):
+                if p not in os.environ:
+                    os.environ[p] = '.'
+        config = Config().copy()
+    finally:
+        sys.argv = old_argv
 
+    dcs = parse_dcs(dcs) or parse_dcs(config.get('dcs_api')) or {}
+    if dcs:
+        for d in DCS_DEFAULTS:
+            config.pop(d, None)
+        config.update(dcs)
     return config
 
 
@@ -93,20 +106,30 @@ def ctl(ctx):
 
 
 def get_dcs(config, scope):
-    for k in set(DCS_DEFAULTS.keys()) & set(config.keys()):
-        config[k].setdefault('scope', scope)
+    config['scope'] = scope
+    config.setdefault('name', scope)
     try:
-        return Patroni.get_dcs(scope, config)
+        return _get_dcs(config)
     except PatroniException as e:
         raise PatroniCtlException(str(e))
 
 
-def post_patroni(member, endpoint, content, headers=None):
-    url = urlparse(member.api_url)
-    logging.debug(url)
-    return requests.post('{0}://{1}/{2}'.format(url.scheme, url.netloc, endpoint),
-                         headers=headers or {'Content-Type': 'application/json'},
-                         data=json.dumps(content), timeout=60)
+def auth_header(config):
+    if config.get('restapi', {}).get('auth', ''):
+        return {'Authorization': 'Basic ' + base64.b64encode(config['restapi']['auth'].encode('utf-8')).decode('utf-8')}
+
+
+def request_patroni(member, request_type, endpoint, content=None, headers=None):
+    headers = headers or {}
+    url_parts = urlparse(member.api_url)
+    logging.debug(url_parts)
+    if 'Content-Type' not in headers:
+        headers['Content-Type'] = 'application/json'
+
+    url = '{0}://{1}/{2}'.format(url_parts.scheme, url_parts.netloc, endpoint)
+
+    return getattr(requests, request_type)(url, headers=headers,
+                                           data=json.dumps(content) if content else None, timeout=60)
 
 
 def print_output(columns, rows=None, alignment=None, fmt='pretty', header=True, delimiter='\t'):
@@ -121,10 +144,7 @@ def print_output(columns, rows=None, alignment=None, fmt='pretty', header=True,
         return
 
     if fmt == 'json':
-        elements = list()
-        for r in rows:
-            elements.append(dict(zip(columns, r)))
-
+        elements = [dict(zip(columns, r)) for r in rows]
         click.echo(json.dumps(elements))
 
     if fmt == 'tsv':
@@ -164,17 +184,6 @@ def watching(w, watch, max_count=None, clear=True):
         yield 0
 
 
-def build_connect_parameters(conn_url, connect_parameters=None):
-    params = (connect_parameters or {}).copy()
-    parsed = parseurl(conn_url)
-    params['host'] = parsed['host']
-    params['port'] = parsed['port']
-    params['fallback_application_name'] = 'Patroni ctl'
-    params['connect_timeout'] = '5'
-
-    return params
-
-
 def get_all_members(cluster, role='master'):
     if role == 'master':
         if cluster.leader is not None:
@@ -194,12 +203,17 @@ def get_any_member(cluster, role='master', member=None):
             return m
 
 
-def get_cursor(cluster, role='master', member=None, connect_parameters=None):
+def get_cursor(cluster, connect_parameters, role='master', member=None):
     member = get_any_member(cluster, role=role, member=member)
     if member is None:
         return None
 
-    params = build_connect_parameters(member.conn_url, connect_parameters)
+    params = member.conn_kwargs(connect_parameters)
+    params.update({'fallback_application_name': 'Patroni ctl', 'connect_timeout': '5'})
+    if 'database' in connect_parameters:
+        params['database'] = connect_parameters['database']
+    else:
+        params.pop('database')
 
     conn = psycopg2.connect(**params)
     conn.autocommit = True
@@ -218,6 +232,37 @@ def get_cursor(cluster, role='master', member=None, connect_parameters=None):
     return None
 
 
+def get_members(cluster, cluster_name, member_names, role, force, action):
+    candidates = {m.name: m for m in cluster.members}
+
+    if not force or role:
+        output_members(cluster, cluster_name)
+
+    if role:
+        role_names = [m.name for m in get_all_members(cluster, role)]
+        if member_names:
+            member_names = list(set(member_names) & set(role_names))
+            if not member_names:
+                raise PatroniCtlException('No {0} among provided members'.format(role))
+        else:
+            member_names = role_names
+
+    if not member_names and not force:
+        member_names = [click.prompt('Which member do you want to {0} [{1}]?'.format(action,
+                        ', '.join(candidates.keys())), type=str, default='')]
+
+    for mn in member_names:
+        if mn not in candidates:
+            raise PatroniCtlException('{0} is not a member of cluster'.format(mn))
+
+    if not force:
+        confirm = click.confirm('Are you sure you want to {0} members {1}?'.format(action, ', '.join(member_names)))
+        if not confirm:
+            raise PatroniCtlException('Aborted {0}'.format(action))
+
+    return [candidates[n] for n in member_names]
+
+
 @ctl.command('dsn', help='Generate a dsn for the provided member, defaults to a dsn of the master')
 @click.option('--role', '-r', help='Give a dsn of any member with this role', type=click.Choice(['master', 'replica',
               'any']), default=None)
@@ -236,7 +281,7 @@ def dsn(cluster_name, config_file, dcs, role, member):
     if m is None:
         raise PatroniCtlException('Can not find a suitable member')
 
-    params = build_connect_parameters(m.conn_url)
+    params = m.conn_kwargs()
     click.echo('host={host} port={port}'.format(**params))
 
 
@@ -286,7 +331,7 @@ def query(
 
     connect_parameters = dict()
     if username:
-        connect_parameters['user'] = username
+        connect_parameters['username'] = username
     if password:
         connect_parameters['password'] = click.prompt('Password', hide_input=True, type=str)
     if dbname:
@@ -307,10 +352,10 @@ def query(
             cluster = dcs.get_cluster()
 
 
-def query_member(cluster, cursor, member, role, command, connect_parameters=None):
+def query_member(cluster, cursor, member, role, command, connect_parameters):
     try:
         if cursor is None:
-            cursor = get_cursor(cluster, role=role, member=member, connect_parameters=connect_parameters)
+            cursor = get_cursor(cluster, connect_parameters, role=role, member=member)
 
         if cursor is None:
             if role is None:
@@ -346,7 +391,7 @@ def query_member(cluster, cursor, member, role, command, connect_parameters=None
 def remove(config_file, cluster_name, fmt, dcs):
     _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
 
-    output_members(cluster, cluster_name, fmt)
+    output_members(cluster, cluster_name, fmt=fmt)
 
     confirm = click.prompt('Please confirm the cluster name to remove', type=str)
     if confirm != cluster_name:
@@ -381,32 +426,6 @@ def wait_for_leader(dcs, timeout=30):
     raise PatroniCtlException('Timeout occured')
 
 
-def empty_post_to_members(cluster, member_names, force, endpoint):
-    candidates = dict()
-    for m in cluster.members:
-        candidates[m.name] = m
-
-    if not member_names:
-        member_names = [click.prompt('Which member do you want to {0} [{1}]?'.format(endpoint,
-                        ', '.join(candidates.keys())), type=str, default='')]
-
-    for mn in member_names:
-        if mn not in candidates.keys():
-            raise PatroniCtlException('{0} is not a member of cluster'.format(mn))
-
-    if not force:
-        confirm = click.confirm('Are you sure you want to {0} members {1}?'.format(endpoint, ', '.join(member_names)))
-        if not confirm:
-            raise PatroniCtlException('Aborted {0}'.format(endpoint))
-
-    for mn in member_names:
-        r = post_patroni(candidates[mn], endpoint, '')
-        if r.status_code != 200:
-            click.echo('{0} failed for member {1}, status code={2}, ({3})'.format(endpoint, mn, r.status_code, r.text))
-        else:
-            click.echo('Succesful {0} on member {1}'.format(endpoint, mn))
-
-
 def ctl_load_config(cluster_name, config_file, dcs):
     config = load_config(config_file, dcs)
     dcs = get_dcs(config, cluster_name)
@@ -415,31 +434,92 @@ def ctl_load_config(cluster_name, config_file, dcs):
     return config, dcs, cluster
 
 
+def check_response(response, member_name, action_name, silent_success=False):
+    if response.status_code >= 400:
+        click.echo('Failed: {0} for member {1}, status code={2}, ({3})'.format(
+            action_name, member_name, response.status_code, response.text
+        ))
+    elif not silent_success:
+        click.echo('Success: {0} for member {1}'.format(action_name, member_name))
+
+
+def parse_scheduled(scheduled):
+    if (scheduled or 'now') != 'now':
+        try:
+            scheduled_at = dateutil.parser.parse(scheduled)
+            if scheduled_at.tzinfo is None:
+                scheduled_at = tzlocal.get_localzone().localize(scheduled_at)
+        except (ValueError, TypeError):
+            message = 'Unable to parse scheduled timestamp ({0}). It should be in an unambiguous format (e.g. ISO 8601)'
+            raise PatroniCtlException(message.format(scheduled))
+        return scheduled_at
+
+    return None
+
+
 @ctl.command('restart', help='Restart cluster member')
 @click.argument('cluster_name')
 @click.argument('member_names', nargs=-1)
 @click.option('--role', '-r', help='Restart only members with this role', default='any',
               type=click.Choice(['master', 'replica', 'any']))
 @click.option('--any', 'p_any', help='Restart a single member only', is_flag=True)
+@click.option('--scheduled', help='Timestamp of a scheduled restart in unambiguous format (e.g. ISO 8601)',
+              default=None)
+@click.option('--pg-version', 'version', help='Restart if the PostgreSQL version is less than provided (e.g. 9.5.2)',
+              default=None)
+@click.option('--pending', help='Restart if pending', is_flag=True)
 @option_config_file
 @option_force
 @option_dcs
-def restart(cluster_name, member_names, config_file, dcs, force, role, p_any):
-    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
-
-    role_names = [m.name for m in get_all_members(cluster, role)]
-
-    if member_names:
-        member_names = list(set(member_names) & set(role_names))
-    else:
-        member_names = role_names
+def restart(cluster_name, member_names, config_file, dcs, force, role, p_any, scheduled, version, pending):
+    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
 
+    members = get_members(cluster, cluster_name, member_names, role, force, 'restart')
     if p_any:
-        random.shuffle(member_names)
-        member_names = member_names[:1]
+        random.shuffle(members)
+        members = members[:1]
 
-    output_members(cluster, cluster_name)
-    empty_post_to_members(cluster, member_names, force, 'restart')
+    if version is None and not force:
+        version = click.prompt('Restart if the PostgreSQL version is less than provided (e.g. 9.5.2) ',
+                               type=str, default='')
+
+    content = {}
+    if pending:
+        content['restart_pending'] = True
+
+    if version:
+        if not is_valid_pg_version(version):
+            message = 'PostgreSQL version should be in the first.major.minor format'
+            raise PatroniCtlException(message)
+        else:
+            content['postgres_version'] = version
+
+    if scheduled is None and not force:
+        scheduled = click.prompt('When should the restart take place (e.g. 2015-10-01T14:30) ', type=str, default='now')
+
+    scheduled_at = parse_scheduled(scheduled)
+    if scheduled_at:
+        if cluster.is_paused():
+            raise PatroniCtlException("Can't schedule restart in the paused state")
+        content['schedule'] = scheduled_at.isoformat()
+
+    for member in members:
+        if 'schedule' in content:
+            if force and member.data.get('scheduled_restart'):
+                r = request_patroni(member, 'delete', 'restart', headers=auth_header(config))
+                check_response(r, member.name, 'flush scheduled restart', True)
+
+        r = request_patroni(member, 'post', 'restart', content, auth_header(config))
+        if r.status_code == 200:
+            click.echo('Success: restart on member {0}'.format(member.name))
+        elif r.status_code == 202:
+            click.echo('Success: restart scheduled on member {0}'.format(member.name))
+        elif r.status_code == 409:
+            click.echo('Failed: another restart is already scheduled on member {0}'.format(member.name))
+        else:
+            click.echo('Failed: restart for member {0}, status code={1}, ({2})'.format(
+                member.name, r.status_code, r.text)
+            )
 
 
 @ctl.command('reinit', help='Reinitialize cluster member')
@@ -449,8 +529,12 @@ def restart(cluster_name, member_names, config_file, dcs, force, role, p_any):
 @option_force
 @option_dcs
 def reinit(cluster_name, member_names, config_file, dcs, force):
-    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
-    empty_post_to_members(cluster, member_names, force, 'reinitialize')
+    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    members = get_members(cluster, cluster_name, member_names, None, force, 'reinitialize')
+
+    for member in members:
+        r = request_patroni(member, 'post', 'reinitialize', headers=auth_header(config))
+        check_response(r, member.name, 'reinitialize')
 
 
 @ctl.command('failover', help='Failover to a replica')
@@ -459,7 +543,7 @@ def reinit(cluster_name, member_names, config_file, dcs, force):
 @click.option('--candidate', help='The name of the candidate', default=None)
 @click.option('--scheduled', help='Timestamp of a scheduled failover in unambiguous format (e.g. ISO 8601)',
               default=None)
-@click.option('--force', is_flag=True)
+@option_force
 @option_config_file
 @option_dcs
 def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled):
@@ -470,18 +554,18 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
         If so, we trigger a failover and keep the client up to date.
     """
 
-    _, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
 
-    if cluster.leader is None:
+    if cluster.leader is None and not cluster.is_paused():
         raise PatroniCtlException('This cluster has no master')
 
-    if master is None:
+    if master is None and (not cluster.is_paused() or cluster.leader):
         if force:
             master = cluster.leader.member.name
         else:
             master = click.prompt('Master', type=str, default=cluster.leader.member.name)
 
-    if cluster.leader.member.name != master:
+    if master is not None and cluster.leader and cluster.leader.member.name != master:
         raise PatroniCtlException('Member {0} is not the leader of cluster {1}'.format(master, cluster_name))
 
     candidate_names = [str(m.name) for m in cluster.members if m.name != master]
@@ -504,19 +588,15 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
         scheduled = click.prompt('When should the failover take place (e.g. 2015-10-01T14:30) ', type=str,
                                  default='now')
 
-    if (scheduled or 'now') == 'now':
-        scheduled_at = None
-    else:
-        try:
-            scheduled_at = dateutil.parser.parse(scheduled)
-            if scheduled_at.tzinfo is None:
-                scheduled_at = tzlocal.get_localzone().localize(scheduled_at)
-        except (ValueError, TypeError):
-            message = 'Unable to parse scheduled timestamp ({0}). It should be in an unambiguous format (e.g. ISO 8601)'
-            raise PatroniCtlException(message.format(scheduled))
+    scheduled_at = parse_scheduled(scheduled)
+
+    if scheduled_at:
+        if cluster.is_paused():
+            raise PatroniCtlException("Can't schedule failover in the paused state")
         scheduled_at = scheduled_at.isoformat()
 
     failover_value = {'leader': master, 'candidate': candidate, 'scheduled_at': scheduled_at}
+
     logging.debug(failover_value)
 
     # By now we have established that the leader exists and the candidate exists
@@ -532,8 +612,10 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
 
     r = None
     try:
-        r = post_patroni(cluster.leader.member, 'failover', failover_value)
-        if r.status_code == 200:
+        member = cluster.leader.member if cluster.leader else [m for m in cluster.members if m.name == candidate][0]
+
+        r = request_patroni(member, 'post', 'failover', failover_value, auth_header(config))
+        if r.status_code in (200, 202):
             logging.debug(r)
             cluster = dcs.get_cluster()
             logging.debug(cluster)
@@ -551,7 +633,7 @@ def failover(config_file, cluster_name, master, candidate, force, dcs, scheduled
     output_members(cluster, cluster_name)
 
 
-def output_members(cluster, name, fmt='pretty'):
+def output_members(cluster, name, extended=False, fmt='pretty'):
     rows = []
     logging.debug(cluster)
     leader_name = None
@@ -569,21 +651,32 @@ def output_members(cluster, name, fmt='pretty'):
         if m.name == leader_name:
             leader = '*'
 
-        host = build_connect_parameters(m.conn_url)['host']
+        host = m.conn_kwargs()['host']
 
         xlog_location = m.data.get('xlog_location') or 0
         lag = ''
-        if (xlog_location_cluster >= xlog_location):
+        if xlog_location_cluster >= xlog_location:
             lag = round((xlog_location_cluster - xlog_location)/1024/1024)
 
-        rows.append([
+        row = [
             name,
             m.name,
             host,
             leader,
             m.data.get('state', ''),
-            lag
-        ])
+            lag,
+        ]
+        if extended:
+            value = ''
+            scheduled_restart = m.data.get('scheduled_restart')
+            if scheduled_restart:
+                value = scheduled_restart['schedule']
+                if 'postgres_version' in scheduled_restart:
+                    value += ' if version < {0}'.format(scheduled_restart['postgres_version'])
+
+            row.append(value)
+
+        rows.append(row)
 
     columns = [
         'Cluster',
@@ -595,17 +688,22 @@ def output_members(cluster, name, fmt='pretty'):
     ]
     alignment = {'Cluster': 'l', 'Member': 'l', 'Host': 'l', 'Lag in MB': 'r'}
 
+    if extended:
+        columns.append('Scheduled restart')
+        alignment['Scheduled restart'] = 'l'
+
     print_output(columns, rows, alignment, fmt)
 
 
 @ctl.command('list', help='List the Patroni members for a given Patroni')
 @click.argument('cluster_names', nargs=-1)
+@click.option('--extended', '-e', help='Show some extra information', is_flag=True)
 @option_config_file
 @option_format
 @option_watch
 @option_watchrefresh
 @option_dcs
-def members(config_file, cluster_names, fmt, watch, w, dcs):
+def members(config_file, cluster_names, fmt, watch, w, dcs, extended):
     if not cluster_names:
         logging.warning('Listing members: No cluster names were provided')
         return
@@ -615,7 +713,8 @@ def members(config_file, cluster_names, fmt, watch, w, dcs):
         dcs = get_dcs(config, cluster_name)
 
         for _ in watching(w, watch):
-            output_members(dcs.get_cluster(), cluster_name, fmt)
+            cluster = dcs.get_cluster()
+            output_members(cluster, cluster_name, extended, fmt)
 
 
 def timestamp(precision=6):
@@ -631,3 +730,110 @@ def configure(config_file, dcs, namespace):
     config['dcs_api'] = str(dcs)
     config['namespace'] = str(namespace)
     store_config(config, config_file)
+
+
+def touch_member(config, dcs):
+    ''' Rip-off of the ha.touch_member without inter-class dependencies '''
+    p = Postgresql(config['postgresql'])
+    p.set_state('running')
+    p.set_role('master')
+
+    def restapi_connection_string(config):
+        protocol = 'https' if config.get('certfile') else 'http'
+        connect_address = config.get('connect_address')
+        listen = config['listen']
+        return '{0}://{1}/patroni'.format(protocol, connect_address or listen)
+
+    data = {
+        'conn_url': p.connection_string,
+        'api_url': restapi_connection_string(config['restapi']),
+        'state': p.state,
+        'role': p.role
+    }
+
+    return dcs.touch_member(json.dumps(data, separators=(',', ':')), permanent=True)
+
+
+def set_defaults(config, cluster_name):
+    """fill-in some basic configuration parameters if config file is not set """
+    config['postgresql'].setdefault('name', cluster_name)
+    config['postgresql'].setdefault('scope', cluster_name)
+    config['postgresql'].setdefault('listen', '127.0.0.1')
+    config['postgresql']['authentication'] = {'replication': None}
+    config['restapi']['listen'] = ':' in config['restapi']['listen'] and config['restapi']['listen'] or '127.0.0.1:8008'
+
+
+@ctl.command('scaffold', help='Create a structure for the cluster in DCS')
+@click.argument('cluster_name')
+@click.option('--sysid', '-s', help='System ID of the cluster to put into the initialize key', default="")
+@option_config_file
+@option_dcs
+def scaffold(cluster_name, config_file, dcs, sysid):
+    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    if cluster and cluster.initialize is not None:
+        raise PatroniCtlException("This cluster is already initialized")
+
+    if not dcs.initialize(create_new=True, sysid=sysid):
+        # initialize key already exists, don't touch this cluster
+        raise PatroniCtlException("Initialize key for cluster {0} already exists".format(cluster_name))
+
+    set_defaults(config, cluster_name)
+
+    # make sure the leader keys will never expire
+    if not (touch_member(config, dcs) and dcs.attempt_to_acquire_leader(permanent=True)):
+        # we did initialize this cluster, but failed to write the leader or member keys, wipe it down completely.
+        dcs.delete_cluster()
+        raise PatroniCtlException("Unable to install permanent leader for cluster {0}".format(cluster_name))
+    click.echo("Cluster {0} has been created successfully".format(cluster_name))
+
+
+@ctl.command('flush', help='Flush scheduled events')
+@click.argument('cluster_name')
+@click.argument('member_names', nargs=-1)
+@click.argument('target', type=click.Choice(['restart']))
+@click.option('--role', '-r', help='Flush only members with this role', default='any',
+              type=click.Choice(['master', 'replica', 'any']))
+@option_config_file
+@option_force
+@option_dcs
+def flush(cluster_name, member_names, config_file, dcs, force, role, target):
+    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+
+    members = get_members(cluster, cluster_name, member_names, role, force, 'flush')
+    for member in members:
+        if target == 'restart':
+            if member.data.get('scheduled_restart'):
+                r = request_patroni(member, 'delete', 'restart', None, auth_header(config))
+                check_response(r, member.name, 'flush scheduled restart')
+            else:
+                click.echo('No scheduled restart for member {0}'.format(member.name))
+
+
+def toggle_pause(config_file, cluster_name, dcs, paused):
+    config, dcs, cluster = ctl_load_config(cluster_name, config_file, dcs)
+    if cluster.is_paused() == paused:
+        raise PatroniCtlException('Cluster is {0} paused'.format(paused and 'already' or 'not'))
+
+    r = request_patroni(cluster.leader.member, 'patch', 'config', {'pause': paused or None}, auth_header(config))
+
+    if r.status_code == 200:
+        click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
+    else:
+        click.echo('Failed: {0} cluster management status code={1}, ({2})'.format(
+                   paused and 'pause' or 'resume', r.status_code, r.text))
+
+
+@ctl.command('pause', help='Disable auto failover')
+@click.argument('cluster_name')
+@option_config_file
+@option_dcs
+def pause(config_file, cluster_name, dcs):
+    return toggle_pause(config_file, cluster_name, dcs, True)
+
+
+@ctl.command('resume', help='Resume auto failover')
+@click.argument('cluster_name')
+@option_config_file
+@option_dcs
+def resume(config_file, cluster_name, dcs):
+    return toggle_pause(config_file, cluster_name, dcs, False)

patroni/dcs/__init__.py

@@ -1,9 +1,15 @@
 import abc
 import dateutil
+import importlib
+import inspect
 import json
+import os
+import pkgutil
 import six
+import sys
 
 from collections import namedtuple
+from patroni.exceptions import PatroniException
 from random import randint
 from six.moves.urllib_parse import urlparse, urlunparse, parse_qsl
 from threading import Event, Lock
@@ -26,6 +32,41 @@ def parse_connection_string(value):
     return conn_url, api_url
 
 
+def dcs_modules():
+    """Get names of DCS modules, depending on execution environment. If being packaged with PyInstaller,
+    modules aren't discoverable dynamically by scanning source directory because `FrozenImporter` doesn't
+    implement `iter_modules` method. But it is still possible to find all potential DCS modules by
+    iterating through `toc`, which contains list of all "frozen" resources."""
+
+    dcs_dirname = os.path.dirname(__file__)
+    module_prefix = __package__ + '.'
+
+    if getattr(sys, 'frozen', False):
+        importer = pkgutil.get_importer(dcs_dirname)
+        return [module for module in list(importer.toc) if module.startswith(module_prefix) and module.count('.') == 2]
+    else:
+        return [module_prefix + name for _, name, is_pkg in pkgutil.iter_modules([dcs_dirname]) if not is_pkg]
+
+
+def get_dcs(config):
+    available_implementations = set()
+    for module_name in dcs_modules():
+        module = importlib.import_module(module_name)
+        for name in filter(lambda name: not name.startswith('__'), dir(module)):  # iterate through module content
+            value = getattr(module, name)
+            name = name.lower()
+            # try to find implementation of AbstractDCS interface, class name must match with module_name
+            if inspect.isclass(value) and issubclass(value, AbstractDCS) and __package__ + '.' + name == module_name:
+                available_implementations.add(name)
+                if name in config:  # which has configuration section in the config file
+                    # propagate some parameters
+                    config[name].update({p: config[p] for p in ('namespace', 'name', 'scope',
+                                         'loop_wait', 'ttl', 'retry_timeout') if p in config})
+                    return value(config[name])
+    raise PatroniException("""Can not find suitable configuration of distributed configuration store
+Available implementations: """ + ', '.join(available_implementations))
+
+
 class Member(namedtuple('Member', 'index,name,session,data')):
 
     """Immutable object (namedtuple) which represents single member of PostgreSQL cluster.
@@ -61,6 +102,26 @@ class Member(namedtuple('Member', 'index,name,session,data')):
     def conn_url(self):
         return self.data.get('conn_url')
 
+    def conn_kwargs(self, auth=None):
+        ret = self.data.get('conn_kwargs')
+        if ret:
+            ret = ret.copy()
+        else:
+            r = urlparse(self.conn_url)
+            ret = {
+                'host': r.hostname,
+                'port': r.port or 5432,
+                'database': r.path[1:]
+            }
+            self.data['conn_kwargs'] = ret.copy()
+
+        if auth and isinstance(auth, dict):
+            if 'username' in auth:
+                ret['user'] = auth['username']
+            if 'password' in auth:
+                ret['password'] = auth['password']
+        return ret
+
     @property
     def api_url(self):
         return self.data.get('api_url')
@@ -79,7 +140,7 @@ class Member(namedtuple('Member', 'index,name,session,data')):
 
     @property
     def clonefrom(self):
-        return self.tags.get('clonefrom', False)
+        return self.tags.get('clonefrom', False) and bool(self.conn_url)
 
 
 class Leader(namedtuple('Leader', 'index,session,member')):
@@ -94,6 +155,9 @@ class Leader(namedtuple('Leader', 'index,session,member')):
     def name(self):
         return self.member.name
 
+    def conn_kwargs(self, auth=None):
+        return self.member.conn_kwargs(auth)
+
     @property
     def conn_url(self):
         return self.member.conn_url
@@ -138,12 +202,32 @@ class Failover(namedtuple('Failover', 'index,leader,candidate,scheduled_at')):
 
         return Failover(index, data.get('leader'), data.get('member'), data.get('scheduled_at'))
 
+    def __len__(self):
+        return int(bool(self.leader)) + int(bool(self.candidate))
 
-class Cluster(namedtuple('Cluster', 'initialize,leader,last_leader_operation,members,failover')):
+
+class ClusterConfig(namedtuple('ClusterConfig', 'index,data,modify_index')):
+
+    @staticmethod
+    def from_node(index, data, modify_index=None):
+        """
+        >>> ClusterConfig.from_node(1, '{') is None
+        True
+        """
+
+        try:
+            data = json.loads(data)
+        except (TypeError, ValueError):
+            return None
+        return ClusterConfig(index, data, modify_index or index)
+
+
+class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operation,members,failover')):
 
     """Immutable object (namedtuple) which represents PostgreSQL cluster.
     Consists of the following fields:
-    :param initialize: boolean, shows whether this cluster has initialization key stored in DC or not.
+    :param initialize: shows whether this cluster has initialization key stored in DC or not.
+    :param config: global dynamic configuration, reference to `ClusterConfig` object
     :param leader: `Leader` object which represents current leader of the cluster
     :param last_leader_operation: int or long object containing position of last known leader operation.
         This value is stored in `/optime/leader` key
@@ -163,26 +247,30 @@ class Cluster(namedtuple('Cluster', 'initialize,leader,last_leader_operation,mem
         candidates = [m for m in self.members if m.clonefrom and (not self.leader or m.name != self.leader.name)]
         return candidates[randint(0, len(candidates) - 1)] if candidates else self.leader
 
+    def is_paused(self):
+        return self.config and self.config.data.get('pause', False) or False
+
 
 @six.add_metaclass(abc.ABCMeta)
 class AbstractDCS(object):
 
     _INITIALIZE = 'initialize'
+    _CONFIG = 'config'
     _LEADER = 'leader'
     _FAILOVER = 'failover'
     _MEMBERS = 'members/'
     _OPTIME = 'optime'
     _LEADER_OPTIME = _OPTIME + '/' + _LEADER
 
-    def __init__(self, name, config):
+    def __init__(self, config):
         """
-        :param name: name of current instance (the same value as `~Postgresql.name`)
         :param config: dict, reference to config section of selected DCS.
             i.e.: `zookeeper` for zookeeper, `etcd` for etcd, etc...
         """
-        self._name = name
+        self._name = config['name']
         self._namespace = '/{0}'.format(config.get('namespace', '/service/').strip('/'))
         self._base_path = '/'.join([self._namespace, config['scope']])
+        self._set_loop_wait(config.get('loop_wait', 10))
 
         self._cluster = None
         self._cluster_thread_lock = Lock()
@@ -195,6 +283,10 @@ class AbstractDCS(object):
     def initialize_path(self):
         return self.client_path(self._INITIALIZE)
 
+    @property
+    def config_path(self):
+        return self.client_path(self._CONFIG)
+
     @property
     def members_path(self):
         return self.client_path(self._MEMBERS)
@@ -215,6 +307,26 @@ class AbstractDCS(object):
     def leader_optime_path(self):
         return self.client_path(self._LEADER_OPTIME)
 
+    @abc.abstractmethod
+    def set_ttl(self, ttl):
+        """Set the new ttl value for leader key"""
+
+    @abc.abstractmethod
+    def set_retry_timeout(self, retry_timeout):
+        """Set the new value for retry_timeout"""
+
+    def _set_loop_wait(self, loop_wait):
+        self._loop_wait = loop_wait
+
+    def reload_config(self, config):
+        self._set_loop_wait(config['loop_wait'])
+        self.set_ttl(config['ttl'])
+        self.set_retry_timeout(config['retry_timeout'])
+
+    @property
+    def loop_wait(self):
+        return self._loop_wait
+
     @abc.abstractmethod
     def _load_cluster(self):
         """Internally this method should build  `Cluster` object which
@@ -259,9 +371,11 @@ class AbstractDCS(object):
         for example for etcd `prevValue` parameter must be used."""
 
     @abc.abstractmethod
-    def attempt_to_acquire_leader(self):
+    def attempt_to_acquire_leader(self, permanent=False):
         """Attempt to acquire leader lock
         This method should create `/leader` key with value=`~self._name`
+        :param permanent: if set to `!True`, the leader key will never expire.
+         Used in patronictl for the external master
         :returns: `!True` if key has been created successfully.
 
         Key must be created atomically. In case if key already exists it should not be
@@ -282,16 +396,22 @@ class AbstractDCS(object):
         if scheduled_at:
             failover_value['scheduled_at'] = scheduled_at.isoformat()
 
-        return self.set_failover_value(json.dumps(failover_value), index)
+        return self.set_failover_value(json.dumps(failover_value, separators=(',', ':')), index)
 
     @abc.abstractmethod
-    def touch_member(self, connection_string, ttl=None):
+    def set_config_value(self, value, index=None):
+        """Create or update `/config` key"""
+
+    @abc.abstractmethod
+    def touch_member(self, data, ttl=None, permanent=False):
         """Update member key in DCS.
         This method should create or update key with the name = '/members/' + `~self._name`
-        and value = connection_string in a given DCS.
+        and value = data in a given DCS.
 
-        :param connection_string: how this instance can be accessed by other instances
+        :param data: json serialized information about instance (including connection strings)
         :param ttl: ttl for member key, optional parameter. If it is None `~self.member_ttl will be used`
+        :param permanent: if set to `!True`, the member key will never expire.
+         Used in patronictl for the external master.
         :returns: `!True` on success otherwise `!False`
         """
 

patroni/dcs/consul.py

@@ -5,7 +5,7 @@ import time
 import six
 
 from consul import ConsulException, NotFound, base, std
-from patroni.dcs import AbstractDCS, Cluster, Failover, Leader, Member
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member
 from patroni.exceptions import DCSError
 from patroni.utils import sleep
 from requests.exceptions import RequestException
@@ -21,9 +21,8 @@ class HTTPClient(std.HTTPClient):
 
     def __init__(self, *args, **kwargs):
         super(HTTPClient, self).__init__(*args, **kwargs)
-        self._patch_default_timeout()
 
-    def _patch_default_timeout(self):
+    def patch_default_timeout(self, timeout):
         # Set a default timeout for the `request.session.request` method, that is used
         # internally  by the methods request.session.get, request.session.post and
         # others. We monkey-patch here to avoid reimplementing each individual method from
@@ -35,7 +34,7 @@ class HTTPClient(std.HTTPClient):
         defaults_attr_name = '__defaults__' if six.PY3 else 'func_defaults'
         defaults = list(getattr(request_func, defaults_attr_name))
         code = request_func.__code__ if six.PY3 else request_func.func_code
-        defaults[code.co_varnames[code.co_argcount - len(defaults):code.co_argcount].index('timeout')] = 5
+        defaults[code.co_varnames[code.co_argcount - len(defaults):code.co_argcount].index('timeout')] = timeout
         setattr(request_func, defaults_attr_name, tuple(defaults))  # monkeypatching
 
     def get(self, callback, path, params=None):
@@ -70,27 +69,37 @@ def catch_consul_errors(func):
 
 class Consul(AbstractDCS):
 
-    def __init__(self, name, config):
-        super(Consul, self).__init__(name, config)
-        self.ttl = int((config.get('ttl') or 30)/2)  # My experiments have shown that session expires after 2*ttl time
-        host, port = config.get('host', '127.0.0.1:8500').split(':')
-        self._client = ConsulClient(host=host, port=port)
-        self._scope = config['scope']
+    def __init__(self, config):
+        super(Consul, self).__init__(config)
+        self._ttl = None
         self._session = None
         self._my_member_data = None
-        self.create_or_restore_session()
+        self.set_ttl(config.get('ttl') or 30)
+        host, port = config.get('host', '127.0.0.1:8500').split(':')
+        self._client = ConsulClient(host=host, port=port)
+        self._client.http.patch_default_timeout(config['retry_timeout']/2.0)
+        self._scope = config['scope']
+        self.create_session()
+        self.__do_not_watch = False
 
-    def create_or_restore_session(self):
+    def create_session(self):
         while not self._session:
             try:
-                _, member = self._client.kv.get(self.member_path)
-                self._session = (member or {}).get('Session')
-                if self.refresh_session():
-                    self._client.kv.delete(self.member_path)
-            except (ConsulException, RequestException):
+                self.refresh_session()
+            except ConsulError:
                 logger.info('waiting on consul')
                 sleep(5)
 
+    def set_ttl(self, ttl):
+        ttl = ttl/2.0  # My experiments have shown that session expires after 2*ttl time
+        if self._ttl != ttl:
+            self._session = None
+            self.__do_not_watch = True
+        self._ttl = ttl
+
+    def set_retry_timeout(self, retry_timeout):
+        self._client.http.patch_default_timeout(retry_timeout/2.0)
+
     def refresh_session(self):
         """:returns: `!True` if it had to create new session"""
         if self._session:
@@ -101,7 +110,7 @@ class Consul(AbstractDCS):
         if not self._session:
             name = self._scope + '-' + self._name
             try:
-                self._session = self._client.session.create(name=name, lock_delay=0, behavior='delete', ttl=self.ttl)
+                self._session = self._client.session.create(name=name, lock_delay=0, behavior='delete', ttl=self._ttl)
             except (ConsulException, RequestException):
                 logger.exception('session.create')
         if not self._session:
@@ -132,6 +141,10 @@ class Consul(AbstractDCS):
             initialize = nodes.get(self._INITIALIZE)
             initialize = initialize and initialize['Value']
 
+            # get global dynamic configuration
+            config = nodes.get(self._CONFIG)
+            config = config and ClusterConfig.from_node(config['ModifyIndex'], config['Value'])
+
             # get last leader operation
             last_leader_operation = nodes.get(self._LEADER_OPTIME)
             last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation['Value'])
@@ -156,20 +169,21 @@ class Consul(AbstractDCS):
             if failover:
                 failover = Failover.from_node(failover['ModifyIndex'], failover['Value'])
 
-            self._cluster = Cluster(initialize, leader, last_leader_operation, members, failover)
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover)
         except NotFound:
-            self._cluster = Cluster(False, None, None, [], None)
+            self._cluster = Cluster(None, None, None, None, [], None)
         except:
             logger.exception('get_cluster')
             raise ConsulError('Consul is not responding properly')
 
     def touch_member(self, data, **kwargs):
-        create_member = self.refresh_session()
         cluster = self.cluster
         member = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
-        if create_member and member:
+        create_member = self.refresh_session()
+        if member and (create_member or member.session != self._session):
             try:
                 self._client.kv.delete(self.member_path)
+                create_member = True
             except Exception:
                 return False
 
@@ -177,7 +191,8 @@ class Consul(AbstractDCS):
             return True
 
         try:
-            self._client.kv.put(self.member_path, data, acquire=self._session)
+            args = {} if kwargs.get('permanent', False) else {'acquire': self._session}
+            self._client.kv.put(self.member_path, data, **args)
             self._my_member_data = data
             return True
         except Exception:
@@ -185,8 +200,9 @@ class Consul(AbstractDCS):
         return False
 
     @catch_consul_errors
-    def attempt_to_acquire_leader(self):
-        ret = self._client.kv.put(self.leader_path, self._name, acquire=self._session)
+    def attempt_to_acquire_leader(self, permanent=False):
+        args = {} if permanent else {'acquire': self._session}
+        ret = self._client.kv.put(self.leader_path, self._name, **args)
         if not ret:
             logger.info('Could not take out TTL lock')
         return ret
@@ -198,6 +214,10 @@ class Consul(AbstractDCS):
     def set_failover_value(self, value, index=None):
         return self._client.kv.put(self.failover_path, value, cas=index)
 
+    @catch_consul_errors
+    def set_config_value(self, value, index=None):
+        return self._client.kv.put(self.config_path, value, cas=index)
+
     @catch_consul_errors
     def write_leader_optime(self, last_operation):
         return self._client.kv.put(self.leader_optime_path, last_operation)
@@ -226,6 +246,10 @@ class Consul(AbstractDCS):
             return self._client.kv.delete(self.leader_path, cas=cluster.leader.index)
 
     def watch(self, timeout):
+        if self.__do_not_watch:
+            self.__do_not_watch = False
+            return True
+
         cluster = self.cluster
         if cluster and cluster.leader and cluster.leader.name != self._name and cluster.leader.index:
             end_time = time.time() + timeout

patroni/dcs/etcd.py

@@ -9,7 +9,7 @@ import time
 
 from dns.exception import DNSException
 from dns import resolver
-from patroni.dcs import AbstractDCS, Cluster, Failover, Leader, Member
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member
 from patroni.exceptions import DCSError
 from patroni.utils import Retry, RetryFailedError, sleep
 from urllib3.exceptions import HTTPError, ReadTimeoutError
@@ -26,29 +26,65 @@ class EtcdError(DCSError):
 class Client(etcd.Client):
 
     def __init__(self, config):
-        super(Client, self).__init__(read_timeout=5)
+        super(Client, self).__init__(read_timeout=config['retry_timeout'])
         self._config = config
         self._load_machines_cache()
         self._allow_reconnect = True
 
+    def _build_request_parameters(self):
+        kwargs = {'headers': self._get_headers(), 'redirect': self.allow_redirect}
+
+        # calculate the number of retries and timeout *per node*
+        # actual number of retries depends on the number of nodes
+        etcd_nodes = len(self._machines_cache) + 1
+        kwargs['retries'] = 0 if etcd_nodes > 3 else (1 if etcd_nodes > 1 else 2)
+
+        # if etcd_nodes > 3:
+        #     kwargs.update({'retries': 0, 'timeout': float(self.read_timeout)/etcd_nodes})
+        # elif etcd_nodes > 1:
+        #     kwargs.update({'retries': 1, 'timeout': self.read_timeout/2.0/etcd_nodes})
+        # else:
+        #     kwargs.update({'retries': 2, 'timeout': self.read_timeout/3.0})
+        kwargs['timeout'] = self.read_timeout/float(kwargs['retries'] + 1)/etcd_nodes
+        return kwargs
+
     @property
     def machines(self):
         """Original `machines` method(property) of `etcd.Client` class raise exception
         when it failed to get list of etcd cluster members. This method is being called
         only when request failed on one of the etcd members during `api_execute` call.
-        For us it's more important to execute original request rather then get new
-        topology of etcd cluster. So we will catch this exception and return valid list
-        of machines with setting flag `self._update_machines_cache` to `!True`.
-        Later, during next `api_execute` call we will forcefully update machines_cache"""
-        try:
-            ret = super(Client, self).machines
-            random.shuffle(ret)
-            return ret
-        except etcd.EtcdException:
-            if self._update_machines_cache:  # We are updating machines_cache
-                raise  # This exception is fatal, we should re-raise it.
-            self._update_machines_cache = True
-            return [self._base_uri]
+        For us it's more important to execute original request rather then get new topology
+        of etcd cluster. So we will catch this exception and return empty list of machines.
+        Later, during next `api_execute` call we will forcefully update machines_cache.
+
+        Also this method implements the same timeout-retry logic as `api_execute`, because
+        the original method was retrying 2 times with the `read_timeout` on each node."""
+
+        kwargs = self._build_request_parameters()
+
+        while True:
+            try:
+                response = self.http.request(self._MGET, self._base_uri + self.version_prefix + '/machines', **kwargs)
+                machines = [n.strip() for n in self._handle_server_response(response).data.decode('utf-8').split(',')]
+                logger.debug("Retrieved list of machines: %s", machines)
+                random.shuffle(machines)
+                return machines
+            except Exception as e:
+                # We can't get the list of machines, if one server is in the
+                # machines cache, try on it
+                logger.error("Failed to get list of machines from %s%s: %r", self._base_uri, self.version_prefix, e)
+                if self._machines_cache:
+                    self._base_uri = self._machines_cache.pop(0)
+                    logger.info("Retrying on %s", self._base_uri)
+                elif self._update_machines_cache:
+                    raise etcd.EtcdException("Could not get the list of servers, "
+                                             "maybe you provided the wrong "
+                                             "host(s) to connect to?")
+                else:
+                    return []
+
+    def set_read_timeout(self, timeout):
+        self._read_timeout = timeout
 
     def _do_http_request(self, request_executor, method, url, fields=None, **kwargs):
         try:
@@ -70,14 +106,7 @@ class Client(etcd.Client):
         if not path.startswith('/'):
             raise ValueError('Path does not start with /')
 
-        if timeout is None:
-            timeout = self.read_timeout
-
-        if timeout == 0:
-            timeout = None
-
-        kwargs = {'timeout': timeout, 'fields': params, 'redirect': self.allow_redirect,
-                  'headers': self._get_headers(), 'preload_content': False}
+        kwargs = {'fields': params, 'preload_content': False}
 
         if method in [self._MGET, self._MDELETE]:
             request_executor = self.http.request
@@ -91,19 +120,29 @@ class Client(etcd.Client):
         if self._update_machines_cache:
             self._load_machines_cache()
 
+        kwargs.update(self._build_request_parameters())
+
+        if timeout is not None:
+            kwargs.update({'retries': 0, 'timeout': timeout})
+
         response = False
 
         try:
+            some_request_failed = False
             while not response:
                 response = self._do_http_request(request_executor, method, self._base_uri + path, **kwargs)
 
-                if response is False and not self._use_proxies:
-                    self._machines_cache = self.machines
+                if response is False:
+                    some_request_failed = True
+            if some_request_failed and not self._use_proxies:
+                self._machines_cache = self.machines
+                if self._base_uri in self._machines_cache:
                     self._machines_cache.remove(self._base_uri)
-            return self._handle_server_response(response)
         except etcd.EtcdConnectionFailed:
             self._update_machines_cache = True
-            raise
+            if not response:
+                raise
+        return self._handle_server_response(response)
 
     @staticmethod
     def get_srv_record(host):
@@ -122,7 +161,7 @@ class Client(etcd.Client):
         for host, port in self.get_srv_record(discovery_srv):
             url = '{0}://{1}:{2}/members'.format(self._protocol, host, port)
             try:
-                response = requests.get(url, timeout=5)
+                response = requests.get(url, timeout=self.read_timeout)
                 if response.ok:
                     for member in response.json():
                         ret.extend(member['clientURLs'])
@@ -191,15 +230,15 @@ def catch_etcd_errors(func):
 
 class Etcd(AbstractDCS):
 
-    def __init__(self, name, config):
-        super(Etcd, self).__init__(name, config)
-        self.ttl = config.get('ttl', 30)
-        self._retry = Retry(deadline=10, max_delay=1, max_tries=-1,
-                            retry_exceptions=(etcd.EtcdConnectionFailed,
-                                              etcd.EtcdLeaderElectionInProgress,
+    def __init__(self, config):
+        super(Etcd, self).__init__(config)
+        self._ttl = int(config.get('ttl') or 30)
+        self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
+                            retry_exceptions=(etcd.EtcdLeaderElectionInProgress,
                                               etcd.EtcdWatcherCleared,
                                               etcd.EtcdEventIndexCleared))
         self._client = self.get_etcd_client(config)
+        self.__do_not_watch = False
 
     def retry(self, *args, **kwargs):
         return self._retry.copy()(*args, **kwargs)
@@ -215,6 +254,15 @@ class Etcd(AbstractDCS):
                 sleep(5)
         return client
 
+    def set_ttl(self, ttl):
+        ttl = int(ttl)
+        self.__do_not_watch = self._ttl != ttl
+        self._ttl = ttl
+
+    def set_retry_timeout(self, retry_timeout):
+        self._retry.deadline = retry_timeout
+        self._client.set_read_timeout(retry_timeout)
+
     @staticmethod
     def member(node):
         return Member.from_node(node.modifiedIndex, os.path.basename(node.key), node.ttl, node.value)
@@ -228,6 +276,10 @@ class Etcd(AbstractDCS):
             initialize = nodes.get(self._INITIALIZE)
             initialize = initialize and initialize.value
 
+            # get global dynamic configuration
+            config = nodes.get(self._CONFIG)
+            config = config and ClusterConfig.from_node(config.modifiedIndex, config.value)
+
             # get last leader operation
             last_leader_operation = nodes.get(self._LEADER_OPTIME)
             last_leader_operation = 0 if last_leader_operation is None else int(last_leader_operation.value)
@@ -240,31 +292,36 @@ class Etcd(AbstractDCS):
             if leader:
                 member = Member(-1, leader.value, None, {})
                 member = ([m for m in members if m.name == leader.value] or [member])[0]
-                leader = Leader(leader.modifiedIndex, leader.ttl, member)
+                index = result.etcd_index if result.etcd_index > leader.modifiedIndex else leader.modifiedIndex + 1
+                leader = Leader(index, leader.ttl, member)
 
             # failover key
             failover = nodes.get(self._FAILOVER)
             if failover:
                 failover = Failover.from_node(failover.modifiedIndex, failover.value)
 
-            self._cluster = Cluster(initialize, leader, last_leader_operation, members, failover)
+            self._cluster = Cluster(initialize, config, leader, last_leader_operation, members, failover)
         except etcd.EtcdKeyNotFound:
-            self._cluster = Cluster(False, None, None, [], None)
+            self._cluster = Cluster(None, None, None, None, [], None)
         except:
             logger.exception('get_cluster')
             raise EtcdError('Etcd is not responding properly')
 
     @catch_etcd_errors
-    def touch_member(self, connection_string, ttl=None):
-        return self.retry(self._client.set, self.member_path, connection_string, ttl or self.ttl)
+    def touch_member(self, data, ttl=None, permanent=False):
+        return self.retry(self._client.set, self.member_path, data, None if permanent else ttl or self._ttl)
 
     @catch_etcd_errors
     def take_leader(self):
-        return self.retry(self._client.set, self.leader_path, self._name, self.ttl)
+        return self.retry(self._client.set, self.leader_path, self._name, self._ttl)
 
-    def attempt_to_acquire_leader(self):
+    def attempt_to_acquire_leader(self, permanent=False):
         try:
-            return bool(self.retry(self._client.write, self.leader_path, self._name, ttl=self.ttl, prevExist=False))
+            return bool(self.retry(self._client.write,
+                                   self.leader_path,
+                                   self._name,
+                                   ttl=None if permanent else self._ttl,
+                                   prevExist=False))
         except etcd.EtcdAlreadyExist:
             logger.info('Could not take out TTL lock')
         except (RetryFailedError, etcd.EtcdException):
@@ -275,13 +332,17 @@ class Etcd(AbstractDCS):
     def set_failover_value(self, value, index=None):
         return self._client.write(self.failover_path, value, prevIndex=index or 0)
 
+    @catch_etcd_errors
+    def set_config_value(self, value, index=None):
+        return self._client.write(self.config_path, value, prevIndex=index or 0)
+
     @catch_etcd_errors
     def write_leader_optime(self, last_operation):
         return self._client.set(self.leader_optime_path, last_operation)
 
     @catch_etcd_errors
     def update_leader(self):
-        return self.retry(self._client.test_and_set, self.leader_path, self._name, self._name, self.ttl)
+        return self.retry(self._client.test_and_set, self.leader_path, self._name, self._name, self._ttl)
 
     @catch_etcd_errors
     def initialize(self, create_new=True, sysid=""):
@@ -300,6 +361,10 @@ class Etcd(AbstractDCS):
         return self.retry(self._client.delete, self.client_path(''), recursive=True)
 
     def watch(self, timeout):
+        if self.__do_not_watch:
+            self.__do_not_watch = False
+            return True
+
         cluster = self.cluster
         # watch on leader key changes if it is defined and current node is not lock owner
         if cluster and cluster.leader and cluster.leader.name != self._name and cluster.leader.index:
@@ -307,7 +372,7 @@ class Etcd(AbstractDCS):
 
             while timeout >= 1:  # when timeout is too small urllib3 doesn't have enough time to connect
                 try:
-                    self._client.watch(self.leader_path, index=cluster.leader.index + 1, timeout=timeout + 0.5)
+                    self._client.watch(self.leader_path, index=cluster.leader.index, timeout=timeout + 0.5)
                     # Synchronous work of all cluster members with etcd is less expensive
                     # than reestablishing http connection every time from every replica.
                     return True

patroni/dcs/exhibitor.py

@@ -0,0 +1,75 @@
+import logging
+import random
+import requests
+import time
+
+from patroni.dcs.zookeeper import ZooKeeper
+from patroni.utils import sleep
+from requests.exceptions import RequestException
+
+logger = logging.getLogger(__name__)
+
+
+class ExhibitorEnsembleProvider(object):
+
+    TIMEOUT = 3.1
+
+    def __init__(self, hosts, port, uri_path='/exhibitor/v1/cluster/list', poll_interval=300):
+        self._exhibitor_port = port
+        self._uri_path = uri_path
+        self._poll_interval = poll_interval
+        self._exhibitors = hosts
+        self._master_exhibitors = hosts
+        self._zookeeper_hosts = ''
+        self._next_poll = None
+        while not self.poll():
+            logger.info('waiting on exhibitor')
+            sleep(5)
+
+    def poll(self):
+        if self._next_poll and self._next_poll > time.time():
+            return False
+
+        json = self._query_exhibitors(self._exhibitors)
+        if not json:
+            json = self._query_exhibitors(self._master_exhibitors)
+
+        if isinstance(json, dict) and 'servers' in json and 'port' in json:
+            self._next_poll = time.time() + self._poll_interval
+            zookeeper_hosts = ','.join([h + ':' + str(json['port']) for h in sorted(json['servers'])])
+            if self._zookeeper_hosts != zookeeper_hosts:
+                logger.info('ZooKeeper connection string has changed: %s => %s', self._zookeeper_hosts, zookeeper_hosts)
+                self._zookeeper_hosts = zookeeper_hosts
+                self._exhibitors = json['servers']
+                return True
+        return False
+
+    def _query_exhibitors(self, exhibitors):
+        random.shuffle(exhibitors)
+        for host in exhibitors:
+            uri = 'http://{0}:{1}{2}'.format(host, self._exhibitor_port, self._uri_path)
+            try:
+                response = requests.get(uri, timeout=self.TIMEOUT)
+                return response.json()
+            except RequestException:
+                pass
+        return None
+
+    @property
+    def zookeeper_hosts(self):
+        return self._zookeeper_hosts
+
+
+class Exhibitor(ZooKeeper):
+
+    def __init__(self, config):
+        interval = config.get('poll_interval', 300)
+        self._ensemble_provider = ExhibitorEnsembleProvider(config['hosts'], config['port'], poll_interval=interval)
+        config = config.copy()
+        config['hosts'] = self._ensemble_provider.zookeeper_hosts
+        super(Exhibitor, self).__init__(config)
+
+    def _load_cluster(self):
+        if self._ensemble_provider.poll():
+            self._client.set_hosts(self._ensemble_provider.zookeeper_hosts)
+        return super(Exhibitor, self)._load_cluster()

patroni/dcs/zookeeper.py

@@ -0,0 +1,313 @@
+import logging
+
+from kazoo.client import KazooClient, KazooState
+from kazoo.exceptions import NoNodeError, NodeExistsError
+from kazoo.handlers.threading import SequentialThreadingHandler
+from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member
+from patroni.exceptions import DCSError
+
+logger = logging.getLogger(__name__)
+
+
+class ZooKeeperError(DCSError):
+    pass
+
+
+class PatroniSequentialThreadingHandler(SequentialThreadingHandler):
+
+    def __init__(self, connect_timeout):
+        super(PatroniSequentialThreadingHandler, self).__init__()
+        self.set_connect_timeout(connect_timeout)
+
+    def set_connect_timeout(self, connect_timeout):
+        self._connect_timeout = max(1.0, connect_timeout/2.0)  # try to connect to zookeeper node during loop_wait/2
+
+    def create_connection(self, *args, **kwargs):
+        """This method is trying to establish connection with one of the zookeeper nodes.
+           Somehow strategy "fail earlier and retry more often" works way better comparing to
+           the original strategy "try to connect with specified timeout".
+           Since we want to try connect to zookeeper more often (with the smaller connect_timeout),
+           he have to override `create_connection` method in the `SequentialThreadingHandler`
+           class (which is used by `kazoo.Client`).
+
+        :param args: always contains `tuple(host, port)` as the first element and could contain
+                     `connect_timeout` (negotiated session timeout) as the second element."""
+
+        args = list(args)
+        if len(args) == 1:
+            args.append(self._connect_timeout)
+        else:
+            args[1] = max(self._connect_timeout, args[1]/10.0)
+        return super(PatroniSequentialThreadingHandler, self).create_connection(*args, **kwargs)
+
+
+class ZooKeeper(AbstractDCS):
+
+    def __init__(self, config):
+        super(ZooKeeper, self).__init__(config)
+
+        hosts = config.get('hosts', [])
+        if isinstance(hosts, list):
+            hosts = ','.join(hosts)
+
+        self._client = KazooClient(hosts, handler=PatroniSequentialThreadingHandler(config['retry_timeout']),
+                                   timeout=config['ttl'], connection_retry={'max_delay': 1, 'max_tries': -1},
+                                   command_retry={'deadline': config['retry_timeout'], 'max_delay': 1, 'max_tries': -1})
+        self._client.add_listener(self.session_listener)
+
+        self._my_member_data = None
+        self._fetch_cluster = True
+        self._last_leader_operation = 0
+
+        self._orig_kazoo_connect = self._client._connection._connect
+        self._client._connection._connect = self._kazoo_connect
+
+        self._client.start()
+
+    def _kazoo_connect(self, host, port):
+
+        """Kazoo is using Ping's to determine health of connection to zookeeper. If there is no
+        response on Ping after Ping interval (1/2 from read_timeout) it will consider current
+        connection dead and try to connect to another node. Without this "magic" it was taking
+        up to 2/3 from session timeout (ttl) to figure out that connection was dead and we had
+        only small time for reconnect and retry.
+
+        This method is needed to return different value of read_timeout, which is not calculated
+        from negotiated session timeout but from value of `loop_wait`. And it is 2 sec smaller
+        than loop_wait, because we can spend up to 2 seconds when calling `touch_member()` and
+        `write_leader_optime()` methods, which also may hang..."""
+
+        ret = self._orig_kazoo_connect(host, port)
+        return max(self.loop_wait - 2, 2)*1000, ret[1]
+
+    def session_listener(self, state):
+        if state in [KazooState.SUSPENDED, KazooState.LOST]:
+            self.cluster_watcher(None)
+
+    def cluster_watcher(self, event):
+        self._fetch_cluster = True
+        self.event.set()
+
+    def reload_config(self, config):
+        self.set_retry_timeout(config['retry_timeout'])
+
+        loop_wait = config['loop_wait']
+
+        loop_wait_changed = self._loop_wait != loop_wait
+        self._loop_wait = loop_wait
+        self._client.handler.set_connect_timeout(loop_wait)
+
+        # We need to reestablish connection to zookeeper if we want to change
+        # read_timeout (and Ping interval respectively), because read_timeout
+        # is calculated in `_kazoo_connect` method. If we are changing ttl at
+        # the same time, set_ttl method will reestablish connection and return
+        # `!True`, otherwise we will close existing connection and let kazoo
+        # open the new one.
+        if not self.set_ttl(int(config['ttl'] * 1000)) and loop_wait_changed:
+            self._client._connection._socket.close()
+
+    def set_ttl(self, ttl):
+        """It is not possible to change ttl (session_timeout) in zookeeper without
+        destroying old session and creating the new one. This method returns `!True`
+        if session_timeout has been changed (`restart()` has been called)."""
+        if self._client._session_timeout != ttl:
+            self._client._session_timeout = ttl
+            self._client.restart()
+            return True
+
+    def set_retry_timeout(self, retry_timeout):
+        self._client._retry.deadline = retry_timeout
+
+    def get_node(self, key, watch=None):
+        try:
+            ret = self._client.get(key, watch)
+            return (ret[0].decode('utf-8'), ret[1])
+        except NoNodeError:
+            return None
+
+    @staticmethod
+    def member(name, value, znode):
+        return Member.from_node(znode.version, name, znode.ephemeralOwner, value)
+
+    def get_children(self, key, watch=None):
+        try:
+            return self._client.get_children(key, watch)
+        except NoNodeError:
+            return []
+
+    def load_members(self):
+        members = []
+        for member in self.get_children(self.members_path, self.cluster_watcher):
+            data = self.get_node(self.members_path + member)
+            if data is not None:
+                members.append(self.member(member, *data))
+        return members
+
+    def _inner_load_cluster(self):
+        self._fetch_cluster = False
+        self.event.clear()
+        nodes = set(self.get_children(self.client_path(''), self.cluster_watcher))
+        if not nodes:
+            self._fetch_cluster = True
+
+        # get initialize flag
+        initialize = (self.get_node(self.initialize_path) or [None])[0] if self._INITIALIZE in nodes else None
+
+        # get global dynamic configuration
+        config = self.get_node(self.config_path, watch=self.cluster_watcher) if self._CONFIG in nodes else None
+        config = config and ClusterConfig.from_node(config[1].version, config[0], config[1].mzxid)
+
+        # get list of members
+        members = self.load_members() if self._MEMBERS[:-1] in nodes else []
+
+        # get leader
+        leader = self.get_node(self.leader_path) if self._LEADER in nodes else None
+        if leader:
+            client_id = self._client.client_id
+            if leader[0] == self._name and client_id is not None and client_id[0] != leader[1].ephemeralOwner:
+                logger.info('I am leader but not owner of the session. Removing leader node')
+                self._client.delete(self.leader_path)
+                leader = None
+
+            if leader:
+                member = Member(-1, leader[0], None, {})
+                member = ([m for m in members if m.name == leader[0]] or [member])[0]
+                leader = Leader(leader[1].version, leader[1].ephemeralOwner, member)
+                self._fetch_cluster = member.index == -1
+
+        # failover key
+        failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
+        failover = failover and Failover.from_node(failover[1].version, failover[0])
+
+        # get last leader operation
+        optime = self.get_node(self.leader_optime_path) if self._OPTIME in nodes and self._fetch_cluster else None
+        self._last_leader_operation = 0 if optime is None else int(optime[0])
+        self._cluster = Cluster(initialize, config, leader, self._last_leader_operation, members, failover)
+
+    def _load_cluster(self):
+        if self._fetch_cluster or self._cluster is None:
+            try:
+                self._client.retry(self._inner_load_cluster)
+            except Exception:
+                logger.exception('get_cluster')
+                self.cluster_watcher(None)
+                raise ZooKeeperError('ZooKeeper in not responding properly')
+
+    def _create(self, path, value, **kwargs):
+        try:
+            self._client.retry(self._client.create, path, value.encode('utf-8'), **kwargs)
+            return True
+        except:
+            return False
+
+    def attempt_to_acquire_leader(self, permanent=False):
+        ret = self._create(self.leader_path, self._name, makepath=True, ephemeral=not permanent)
+        if not ret:
+            logger.info('Could not take out TTL lock')
+        return ret
+
+    def set_failover_value(self, value, index=None):
+        try:
+            self._client.retry(self._client.set, self.failover_path, value.encode('utf-8'), version=index or -1)
+            return True
+        except NoNodeError:
+            return value == '' or (index is None and self._create(self.failover_path, value))
+        except:
+            logging.exception('set_failover_value')
+            return False
+
+    def set_config_value(self, value, index=None):
+        try:
+            self._client.retry(self._client.set, self.config_path, value.encode('utf-8'), version=index or -1)
+            return True
+        except NoNodeError:
+            return index is None and self._create(self.config_path, value)
+        except Exception:
+            logging.exception('set_config_value')
+            return False
+
+    def initialize(self, create_new=True, sysid=""):
+        return self._create(self.initialize_path, sysid, makepath=True) if create_new \
+            else self._client.retry(self._client.set, self.initialize_path,  sysid.encode("utf-8"))
+
+    def touch_member(self, data, ttl=None, permanent=False):
+        cluster = self.cluster
+        member = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
+        data = data.encode('utf-8')
+        if member and self._client.client_id is not None and member.session != self._client.client_id[0]:
+            try:
+                self._client.delete_async(self.member_path).get(timeout=1)
+            except NoNodeError:
+                pass
+            except:
+                return False
+            member = None
+
+        if member:
+            if data == self._my_member_data:
+                return True
+        else:
+            try:
+                self._client.create_async(self.member_path, data, makepath=True, ephemeral=not permanent).get(timeout=1)
+                self._my_member_data = data
+                return True
+            except Exception as e:
+                if not isinstance(e, NodeExistsError):
+                    logger.exception('touch_member')
+                    return False
+        try:
+            self._client.set_async(self.member_path, data).get(timeout=1)
+            self._my_member_data = data
+            return True
+        except:
+            logger.exception('touch_member')
+
+        return False
+
+    def take_leader(self):
+        return self.attempt_to_acquire_leader()
+
+    def write_leader_optime(self, last_operation):
+        last_operation = last_operation.encode('utf-8')
+        if last_operation != self._last_leader_operation:
+            try:
+                self._client.set_async(self.leader_optime_path, last_operation).get(timeout=1)
+                self._last_leader_operation = last_operation
+            except NoNodeError:
+                try:
+                    self._client.create_async(self.leader_optime_path, last_operation, makepath=True).get(timeout=1)
+                    self._last_leader_operation = last_operation
+                except:
+                    logger.exception('Failed to create %s', self.leader_optime_path)
+            except:
+                logger.exception('Failed to update %s', self.leader_optime_path)
+
+    def update_leader(self):
+        return True
+
+    def delete_leader(self):
+        self._client.restart()
+        self._my_member_data = None
+        return True
+
+    def _cancel_initialization(self):
+        node = self.get_node(self.initialize_path)
+        if node:
+            self._client.delete(self.initialize_path, version=node[1].version)
+
+    def cancel_initialization(self):
+        try:
+            self._client.retry(self._cancel_initialization)
+        except:
+            logger.exception("Unable to delete initialize key")
+
+    def delete_cluster(self):
+        try:
+            return self._client.retry(self._client.delete, self.client_path(''), recursive=True)
+        except NoNodeError:
+            return True
+
+    def watch(self, timeout):
+        if super(ZooKeeper, self).watch(timeout):
+            self._fetch_cluster = True
+        return self._fetch_cluster

patroni/ha.py

@@ -9,6 +9,7 @@ import pytz
 from multiprocessing.pool import ThreadPool
 from patroni.async_executor import AsyncExecutor
 from patroni.exceptions import DCSError, PostgresConnectionException
+from patroni.postgresql import ACTION_ON_START
 from patroni.utils import sleep
 
 logger = logging.getLogger(__name__)
@@ -25,6 +26,9 @@ class Ha(object):
         self.recovering = False
         self._async_executor = AsyncExecutor()
 
+    def is_paused(self):
+        return self.cluster and self.cluster.is_paused()
+
     def load_cluster_from_dcs(self):
         cluster = self.dcs.get_cluster()
 
@@ -59,19 +63,28 @@ class Ha(object):
         }
         if self.patroni.tags:
             data['tags'] = self.patroni.tags
+        if self.state_handler.pending_restart:
+            data['pending_restart'] = True
         if not self._async_executor.busy and data['state'] in ['running', 'restarting', 'starting']:
             try:
                 data['xlog_location'] = self.state_handler.xlog_position()
             except:
                 pass
+        if self.patroni.scheduled_restart:
+            scheduled_restart_data = self.patroni.scheduled_restart.copy()
+            scheduled_restart_data['schedule'] = scheduled_restart_data['schedule'].isoformat()
+            data['scheduled_restart'] = scheduled_restart_data
+
         self.dcs.touch_member(json.dumps(data, separators=(',', ':')))
 
     def clone(self, clone_member=None, msg='(without leader)'):
-        if self.state_handler.bootstrap(cluster_initialized=True, clone_member=clone_member):
+        if self.state_handler.clone(clone_member):
             logger.info('bootstrapped %s', msg)
+            cluster = self.dcs.get_cluster()
+            node_to_follow = self._get_node_to_follow(cluster)
+            self.state_handler.follow(node_to_follow, cluster.leader, True)
         else:
             logger.error('failed to bootstrap %s', msg)
-            self.state_handler.stop('immediate')
             self.state_handler.remove_data_directory()
 
     def bootstrap(self):
@@ -82,10 +95,11 @@ class Ha(object):
             self._async_executor.schedule('bootstrap {0}'.format(msg))
             self._async_executor.run_async(self.clone, args=(clone_member, msg))
             return 'trying to bootstrap {0}'.format(msg)
-        elif not self.cluster.initialize and not self.patroni.nofailover:  # no initialize key
+        # no initialize key and node is allowed to be master and has 'bootstrap' section in a configuration file
+        elif self.cluster.initialize is None and not self.patroni.nofailover and 'bootstrap' in self.patroni.config:
             if self.dcs.initialize(create_new=True):  # race for initialization
                 try:
-                    self.state_handler.bootstrap()
+                    self.state_handler.bootstrap(self.patroni.config['bootstrap'])
                     self.dcs.initialize(create_new=False, sysid=self.state_handler.sysid)
                 except:  # initdb or start failed
                     # remove initialization key and give a chance to other members
@@ -94,6 +108,7 @@ class Ha(object):
                     self.state_handler.stop('immediate')
                     self.state_handler.move_data_directory()
                     raise
+                self.dcs.set_config_value(json.dumps(self.patroni.config.dynamic_configuration, separators=(',', ':')))
                 self.dcs.take_leader()
                 self.load_cluster_from_dcs()
                 return 'initialized a new cluster'
@@ -109,29 +124,44 @@ class Ha(object):
         self.recovering = True
         return self.follow("starting as readonly because i had the session lock", "starting as a secondary", True, True)
 
-    def follow(self, demote_reason, follow_reason, refresh=True, recovery=False):
+    def _get_node_to_follow(self, cluster):
+        # determine the node to follow. If replicatefrom tag is set,
+        # try to follow the node mentioned there, otherwise, follow the leader.
+        if not self.patroni.replicatefrom or self.patroni.replicatefrom == self.state_handler.name:
+            node_to_follow = cluster.leader
+        else:
+            node_to_follow = cluster.get_member(self.patroni.replicatefrom)
+
+        return node_to_follow if node_to_follow and node_to_follow.name != self.state_handler.name else None
+
+    def follow(self, demote_reason, follow_reason, refresh=True, recovery=False, need_rewind=None):
         if refresh:
             self.load_cluster_from_dcs()
 
-        ret = demote_reason if not recovery and self.state_handler.is_leader() else follow_reason
-
-        # determine the node to follow. If replicatefrom tag is set,
-        # try to follow the node mentioned there, otherwise, follow the leader.
-
-        if self.patroni.replicatefrom:
-            node_to_follow = self.cluster.get_member(self.patroni.replicatefrom, fallback_to_leader=True)
+        if recovery:
+            ret = demote_reason if self.has_lock() else follow_reason
         else:
-            node_to_follow = self.cluster.leader
-        if node_to_follow and node_to_follow.name == self.state_handler.name:
-            ret = demote_reason
-            node_to_follow = None
-        if not self.state_handler.check_recovery_conf(node_to_follow) or recovery:
-            self._async_executor.schedule('changing primary_conninfo and restarting')
-            self._async_executor.run_async(self.state_handler.follow, (node_to_follow, recovery))
+            is_leader = self.state_handler.is_leader()
+            ret = demote_reason if is_leader else follow_reason
+
+        node_to_follow = self._get_node_to_follow(self.cluster)
+
+        if self.is_paused() and not self.state_handler.need_rewind:
+            self.state_handler.set_role('master' if is_leader else 'replica')
+            if is_leader:
+                return 'continue to run as master without lock'
+            elif not node_to_follow:
+                return 'no action'
+
+        self.state_handler.follow(node_to_follow, self.cluster.leader, recovery, self._async_executor, need_rewind)
+
         return ret
 
     def enforce_master_role(self, message, promote_message):
         if self.state_handler.is_leader() or self.state_handler.role == 'master':
+            # Inform the state handler about its master role.
+            # It may be unaware of it if postgres is promoted manually.
+            self.state_handler.set_role('master')
             return message
         else:
             self.state_handler.promote()
@@ -154,11 +184,10 @@ class Ha(object):
             logger.info('Got response from %s %s: %s', member.name, member.api_url, response.content)
             json = response.json()
             is_master = json['role'] == 'master'
-            xlog_location = json['xlog']['location' if is_master else 'replayed_location']
-            tags = json.get('tags', dict())
-            return (member, True, not is_master, xlog_location, tags)
-        except:
-            logging.exception('request failed: GET %s', member.api_url)
+            xlog_location = None if is_master else json['xlog']['replayed_location']
+            return (member, True, not is_master, xlog_location, json.get('tags', {}))
+        except Exception as e:
+            logger.warning("request failed: GET %s (%s)", member.api_url, e)
         return (member, False, None, 0, {})
 
     def fetch_nodes_statuses(self, members):
@@ -171,12 +200,6 @@ class Ha(object):
     def _is_healthiest_node(self, members, check_replication_lag=True):
         """This method tries to determine whether I am healthy enough to became a new leader candidate or not."""
 
-        if self.state_handler.is_leader():
-            return True
-
-        if self.patroni.nofailover is True:
-            return False
-
         if check_replication_lag and not self.state_handler.check_replication_lag(self.cluster.last_leader_operation):
             return False  # Too far behind last reported xlog location on master
 
@@ -214,6 +237,15 @@ class Ha(object):
         if failover.candidate:  # manual failover to specific member
             if failover.candidate == self.state_handler.name:  # manual failover to me
                 return True
+            elif self.is_paused():
+                # Remove failover key if the node to failover has terminated to avoid waiting for it indefinitely
+                # In order to avoid attempts to delete this key from all nodes only the master is allowed to do it.
+                if (not self.cluster.get_member(failover.candidate, fallback_to_leader=False) and
+                   self.state_handler.is_leader()):
+                    logger.warning("manual failover: removing failover key because failover candidate is not running")
+                    self.dcs.manual_failover('', '', index=self.cluster.failover.index)
+                    return None
+                return False
 
             # find specific node and check that it is healthy
             member = self.cluster.get_member(failover.candidate, fallback_to_leader=False)
@@ -230,6 +262,8 @@ class Ha(object):
 
             # at this point we should consider all members as a candidates for failover
             # i.e. we assume that failover.candidate is None
+        elif self.is_paused():
+            return False
 
         # try to pick some other members to failover and check that they are healthy
         if failover.leader:
@@ -248,10 +282,18 @@ class Ha(object):
         return self._is_healthiest_node(members, check_replication_lag=False)
 
     def is_healthiest_node(self):
+        if self.is_paused() and not self.patroni.nofailover and \
+                self.cluster.failover and not self.cluster.failover.scheduled_at:
+            ret = self.manual_failover_process_no_leader()
+            if ret is not None:  # continue if we just deleted the stale failover key as a master
+                return ret
 
         if self.state_handler.is_leader():  # leader is always the healthiest
             return True
 
+        if self.is_paused():
+            return False
+
         if self.patroni.nofailover:  # nofailover tag makes node always unhealthy
             return False
 
@@ -265,64 +307,91 @@ class Ha(object):
     def demote(self, delete_leader=True):
         if delete_leader:
             self.state_handler.stop()
+            self.state_handler.set_role('demoted')
             self.dcs.delete_leader()
             self.touch_member()
             self.dcs.reset_cluster()
-            sleep(2)  # Give a time to somebody to promote
-            self.recover()
+            sleep(2)  # Give a time to somebody to take the leader lock
+            cluster = self.dcs.get_cluster()
+            node_to_follow = self._get_node_to_follow(cluster)
+            self.state_handler.follow(node_to_follow, cluster.leader, recovery=True, need_rewind=True)
         else:
-            self.state_handler.follow(None)
+            self.state_handler.follow(None, None)
+
+    def should_run_scheduled_action(self, action_name, scheduled_at, cleanup_fn):
+        if scheduled_at and not self.is_paused():
+            # If the scheduled action is in the far future, we shouldn't do anything and just return.
+            # If the scheduled action is in the past, we consider the value to be stale and we remove
+            # the value.
+            # If the value is close to now, we initiate the scheduled action
+            # Additionally, if the scheduled action cannot be executed altogether, i.e. there is an error
+            # or the action is in the past - we take care of cleaning it up.
+            now = datetime.datetime.now(pytz.utc)
+            try:
+                delta = (scheduled_at - now).total_seconds()
+
+                if delta > self.dcs.loop_wait:
+                    logger.info('Awaiting %s at %s (in %.0f seconds)',
+                                action_name, scheduled_at.isoformat(), delta)
+                    return False
+                elif delta < - int(self.dcs.loop_wait * 1.5):
+                    logger.warning('Found a stale %s value, cleaning up: %s',
+                                   action_name, scheduled_at.isoformat())
+                    cleanup_fn()
+                    return False
+
+                # The value is very close to now
+                sleep(max(delta, 0))
+                logger.info('Manual scheduled {0} at %s'.format(action_name), scheduled_at.isoformat())
+                return True
+            except TypeError:
+                logger.warning('Incorrect value of scheduled_at: %s', scheduled_at)
+                cleanup_fn()
+        return False
 
     def process_manual_failover_from_leader(self):
         failover = self.cluster.failover
 
-        if failover.scheduled_at:
-            # If the failover is in the far future, we shouldn't do anything and just return.
-            # If the failover is in the past, we consider the value to be stale and we remove
-            # the value.
-            # If the value is close to now, we initiate the failover
-            now = datetime.datetime.now(pytz.utc)
-            try:
-                delta = (failover.scheduled_at - now).total_seconds()
-
-                if delta > self.patroni.nap_time:
-                    logging.info('Awaiting failover at %s (in %.0f seconds)', failover.scheduled_at.isoformat(), delta)
-                    return
-                elif delta < - int(self.patroni.nap_time * 1.5):
-                    logger.warning('Found a stale failover value, cleaning up: %s', failover.scheduled_at)
-                    self.dcs.manual_failover('', '', index=self.cluster.failover.index)
-                    return
-
-                # The value is very close to now
-                sleep(max(delta, 0))
-                logger.info('Manual scheduled failover at {}'.format(failover.scheduled_at.isoformat()))
-            except TypeError:
-                logger.warning('Incorrect value in of scheduled_at: %s', failover.scheduled_at)
+        if (failover.scheduled_at and not
+            self.should_run_scheduled_action("failover", failover.scheduled_at, lambda:
+                                             self.dcs.manual_failover('', '', index=failover.index))):
+            return
 
         if not failover.leader or failover.leader == self.state_handler.name:
             if not failover.candidate or failover.candidate != self.state_handler.name:
-                members = [m for m in self.cluster.members if not failover.candidate or m.name == failover.candidate]
-                if self.is_failover_possible(members):  # check that there are healthy members
-                    self._async_executor.schedule('manual failover: demote')
-                    self._async_executor.run_async(self.demote)
-                    return 'manual failover: demoting myself'
+                if not failover.candidate and self.is_paused():
+                    logger.warning('Failover is possible only to a specific candidate in a paused state')
                 else:
-                    logger.warning('manual failover: no healthy members found, failover is not possible')
+                    members = [m for m in self.cluster.members
+                               if not failover.candidate or m.name == failover.candidate]
+                    if self.is_failover_possible(members):  # check that there are healthy members
+                        self._async_executor.schedule('manual failover: demote')
+                        self._async_executor.run_async(self.demote)
+                        return 'manual failover: demoting myself'
+                    else:
+                        logger.warning('manual failover: no healthy members found, failover is not possible')
             else:
                 logger.warning('manual failover: I am already the leader, no need to failover')
         else:
             logger.warning('manual failover: leader name does not match: %s != %s',
-                           self.cluster.failover.leader, self.state_handler.name)
+                           failover.leader, self.state_handler.name)
 
-        logger.info('Trying to clean up failover key')
-        self.dcs.manual_failover('', '', index=self.cluster.failover.index)
+        logger.info('Cleaning up failover key')
+        self.dcs.manual_failover('', '', index=failover.index)
 
     def process_unhealthy_cluster(self):
+        """Cluster has no leader key"""
+
         if self.is_healthiest_node():
             if self.acquire_lock():
-                if self.cluster.failover:
-                    logger.info('Cleaning up failover key after acquiring leader lock...')
-                    self.dcs.manual_failover('', '')
+                failover = self.cluster.failover
+                if failover:
+                    if self.is_paused() and failover.leader and failover.candidate:
+                        logger.info('Updating failover key after acquiring leader lock...')
+                        self.dcs.manual_failover('', failover.candidate, failover.scheduled_at, failover.index)
+                    else:
+                        logger.info('Cleaning up failover key after acquiring leader lock...')
+                        self.dcs.manual_failover('', '')
                 self.load_cluster_from_dcs()
                 return self.enforce_master_role('acquired session lock as a leader',
                                                 'promoted self to leader by acquiring session lock')
@@ -330,19 +399,35 @@ class Ha(object):
                 return self.follow('demoted self after trying and failing to obtain lock',
                                    'following new leader after trying and failing to obtain lock')
         else:
+            # when we are doing manual failover there is no guaranty that new leader is ahead of any other node
+            # node tagged as nofailover can be ahead of the new leader either, but it is always excluded from elections
+            need_rewind = bool(self.cluster.failover) or self.patroni.nofailover
+            if need_rewind:
+                sleep(2)  # Give a time to somebody to take the leader lock
+
             if self.patroni.nofailover:
                 return self.follow('demoting self because I am not allowed to become master',
-                                   'following a different leader because I am not allowed to promote')
+                                   'following a different leader because I am not allowed to promote',
+                                   need_rewind=need_rewind)
             return self.follow('demoting self because i am not the healthiest node',
-                               'following a different leader because i am not the healthiest node')
+                               'following a different leader because i am not the healthiest node',
+                               need_rewind=need_rewind)
 
     def process_healthy_cluster(self):
         if self.has_lock():
-            if self.cluster.failover:
+            if self.cluster.failover and (not self.is_paused() or self.state_handler.is_leader()):
                 msg = self.process_manual_failover_from_leader()
                 if msg is not None:
                     return msg
 
+            if self.is_paused() and not self.state_handler.is_leader():
+                if self.cluster.failover and self.cluster.failover.candidate == self.state_handler.name:
+                    return 'waiting to become master after promote...'
+
+                self.dcs.delete_leader()
+                self.dcs.reset_cluster()
+                return 'removed leader lock because postgres is not running as master'
+
             if self.update_lock():
                 return self.enforce_master_role('no action.  i am the leader with the lock',
                                                 'promoted self to leader because i had the session lock')
@@ -355,48 +440,117 @@ class Ha(object):
         return self.follow('demoting self because i do not have the lock and i was a leader',
                            'no action.  i am a secondary and i am following a leader', False)
 
-    def schedule(self, action):
+    def evaluate_scheduled_restart(self):
+        # restart if we need to
+        restart_data = self.future_restart_scheduled()
+        if restart_data:
+            recent_time = self.state_handler.postmaster_start_time()
+            request_time = restart_data['postmaster_start_time']
+            # check if postmaster start time has changed since the last restart
+            if recent_time and request_time and recent_time != request_time:
+                logger.info("Cancelling scheduled restart: postgres restart has already happened at %s", recent_time)
+                self.delete_future_restart()
+                return None
+
+        if (restart_data and
+           self.should_run_scheduled_action('restart', restart_data['schedule'], self.delete_future_restart)):
+            try:
+                ret, message = self.restart(restart_data, run_async=True)
+                if not ret:
+                    logger.warning("Scheduled restart: %s", message)
+                    return None
+                return message
+            finally:
+                self.delete_future_restart()
+
+    def restart_matches(self, role, postgres_version, pending_restart):
+        reason_to_cancel = ""
+        # checking the restart filters here seem to be less ugly than moving them into the
+        # run_scheduled_action.
+        if role and role != self.state_handler.role:
+            reason_to_cancel = "host role mismatch"
+
+        if (postgres_version and
+           self.state_handler.postgres_version_to_int(postgres_version) <= int(self.state_handler.server_version)):
+            reason_to_cancel = "postgres version mismatch"
+
+        if pending_restart and not self.state_handler.pending_restart:
+            reason_to_cancel = "pending restart flag is not set"
+
+        if not reason_to_cancel:
+            return True
+        else:
+            logger.info("not proceeding with the restart: %s", reason_to_cancel)
+        return False
+
+    def schedule_future_restart(self, restart_data):
         with self._async_executor:
-            return self._async_executor.schedule(action)
+            if not self.patroni.scheduled_restart:
+                self.patroni.scheduled_restart = restart_data
+                self.touch_member()
+                return True
+        return False
+
+    def delete_future_restart(self):
+        ret = False
+        with self._async_executor:
+            if self.patroni.scheduled_restart:
+                self.patroni.scheduled_restart = {}
+                self.touch_member()
+                ret = True
+        return ret
+
+    def future_restart_scheduled(self):
+        return self.patroni.scheduled_restart.copy() if (self.patroni.scheduled_restart and
+                                                         isinstance(self.patroni.scheduled_restart, dict)) else None
 
     def restart_scheduled(self):
         return self._async_executor.scheduled_action == 'restart'
 
-    def schedule_reinitialize(self):
-        return self.schedule('reinitialize')
+    def restart(self, restart_data=None, run_async=False):
+        """ conditional and unconditional restart """
+        if (restart_data and isinstance(restart_data, dict) and
+            not self.restart_matches(restart_data.get('role'),
+                                     restart_data.get('postgres_version'),
+                                     ('restart_pending' in restart_data))):
+            return (False, "restart conditions are not satisfied")
 
-    def reinitialize_scheduled(self):
-        return self._async_executor.scheduled_action == 'reinitialize'
-
-    def restart(self):
         with self._async_executor:
-            prev = self._async_executor.schedule('restart', True)
+            prev = self._async_executor.schedule('restart')
             if prev is not None:
                 return (False, prev + ' already in progress')
-        if self._async_executor.run(self.state_handler.restart):
+
+        if run_async:
+            self._async_executor.run_async(self.state_handler.restart)
+            return (True, 'restart initiated')
+        elif self._async_executor.run(self.state_handler.restart):
             return (True, 'restarted successfully')
         else:
             return (False, 'restart failed')
 
-    def reinitialize(self, cluster):
+    def _do_reinitialize(self, cluster):
         self.state_handler.stop('immediate')
         self.state_handler.remove_data_directory()
 
-        clone_member = cluster.get_clone_member()
-        member_role = 'leader' if clone_member == cluster.leader else 'replica'
+        clone_member = self.cluster.get_clone_member()
+        member_role = 'leader' if clone_member == self.cluster.leader else 'replica'
         self.clone(clone_member, "from {0} '{1}'".format(member_role, clone_member.name))
 
-    def process_scheduled_action(self):
-        if self.reinitialize_scheduled():
+    def reinitialize(self):
+        with self._async_executor:
+            self.load_cluster_from_dcs()
+
             if self.cluster.is_unlocked():
-                logger.error('Cluster has no leader, can not reinitialize')
-                self._async_executor.reset_scheduled_action()
-            elif self.has_lock():
-                logger.error('I am the leader, can not reinitialize')
-                self._async_executor.reset_scheduled_action()
-            else:
-                self._async_executor.run_async(self.reinitialize, args=(self.cluster, ))
-                return 'reinitialize started'
+                return 'Cluster has no leader, can not reinitialize'
+
+            if self.cluster.leader.name == self.state_handler.name:
+                return 'I am the leader, can not reinitialize'
+
+            action = self._async_executor.schedule('reinitialize', immediately=True)
+            if action is not None:
+                return '{0} already in progress'.format(action)
+
+        self._async_executor.run_async(self._do_reinitialize, args=(self.cluster, ))
 
     def handle_long_action_in_progress(self):
         if self.has_lock():
@@ -413,7 +567,8 @@ class Ha(object):
     def sysid_valid(sysid):
         # sysid does tv_sec << 32, where tv_sec is the number of seconds sine 1970,
         # so even 1 << 32 would have 10 digits.
-        return str(sysid) and len(str(sysid)) >= 10 and str(sysid).isdigit()
+        sysid = str(sysid)
+        return len(sysid) >= 10 and sysid.isdigit()
 
     def post_recover(self):
         if not self.state_handler.is_running():
@@ -431,29 +586,27 @@ class Ha(object):
             self.touch_member()
 
             # cluster has leader key but not initialize key
-            if not self.cluster.is_unlocked() and not self.sysid_valid(self.cluster.initialize) and self.has_lock():
+            if not (self.cluster.is_unlocked() or self.sysid_valid(self.cluster.initialize)) and self.has_lock():
                 self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=self.state_handler.sysid)
 
+            if not (self.cluster.is_unlocked() or self.cluster.config and self.cluster.config.data) and self.has_lock():
+                self.dcs.set_config_value(json.dumps(self.patroni.config.dynamic_configuration, separators=(',', ':')))
+
             if self._async_executor.busy:
                 return self.handle_long_action_in_progress()
 
             # we've got here, so any async action has finished. Check if we tried to recover and failed
-            if self.recovering:
+            if self.recovering and not self.state_handler.need_rewind:
                 self.recovering = False
                 msg = self.post_recover()
                 if msg is not None:
                     return msg
 
-            # currently it can trigger only reinitialize
-            msg = self.process_scheduled_action()
-            if msg is not None:
-                return msg
-
             # is data directory empty?
             if self.state_handler.data_directory_empty():
                 return self.bootstrap()  # new node
             # "bootstrap", but data directory is not empty
-            elif not self.sysid_valid(self.cluster.initialize) and self.cluster.is_unlocked():
+            elif not self.sysid_valid(self.cluster.initialize) and self.cluster.is_unlocked() and not self.is_paused():
                 self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=self.state_handler.sysid)
             else:
                 # check if we are allowed to join
@@ -462,31 +615,44 @@ class Ha(object):
                                  self.state_handler.name, self.cluster.initialize, self.state_handler.sysid)
                     sys.exit(1)
 
-            # try to start dead postgres
             if not self.state_handler.is_healthy():
-                msg = self.recover()
-                if msg is not None:
-                    return msg
+                if self.is_paused():
+                    if self.has_lock():
+                        self.dcs.delete_leader()
+                        self.dcs.reset_cluster()
+                        return 'removed leader lock because postgres is not running'
+                    elif not self.state_handler.need_rewind:
+                        return 'postgres is not running'
+
+                # try to start dead postgres
+                return self.recover()
 
             try:
                 if self.cluster.is_unlocked():
                     return self.process_unhealthy_cluster()
                 else:
+                    msg = self.evaluate_scheduled_restart()
+                    if msg is not None:
+                        return msg
                     return self.process_healthy_cluster()
             finally:
                 # we might not have a valid PostgreSQL connection here if another thread
                 # stops PostgreSQL, therefore, we only reload replication slots if no
                 # asynchronous processes are running (should be always the case for the master)
                 if not self._async_executor.busy:
+                    if not self.state_handler.cb_called:
+                        self.state_handler.call_nowait(ACTION_ON_START)
                     self.state_handler.sync_replication_slots(self.cluster)
         except DCSError:
             logger.error('Error communicating with DCS')
-            if self.state_handler.is_running() and self.state_handler.is_leader():
+            if not self.is_paused() and self.state_handler.is_running() and self.state_handler.is_leader():
                 self.demote(delete_leader=False)
                 return 'demoted self because DCS is not accessible and i was a leader'
+            return 'DCS is not accessible'
         except (psycopg2.Error, PostgresConnectionException):
-            logger.exception('Error communicating with PostgreSQL. Will try again later')
+            return 'Error communicating with PostgreSQL. Will try again later'
 
     def run_cycle(self):
         with self._async_executor:
-            return self._run_cycle()
+            info = self._run_cycle()
+            return (self.is_paused() and 'PAUSE: ' or '') + info

patroni/scripts/wale_restore.py

@@ -33,7 +33,7 @@ import sys
 import argparse
 
 
-if sys.hexversion >= 0x03000000:
+if sys.hexversion >= 0x0300000:
     long = int
 
 logger = logging.getLogger(__name__)
@@ -73,7 +73,7 @@ class WALERestore(object):
             # base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z
             # 20310671    00000001000000000000007F    00000040
             # 00000001000000000000007F    00000240
-            backup_strings = latest_backup.splitlines() if latest_backup else ()
+            backup_strings = latest_backup.decode('utf-8').splitlines() if latest_backup else ()
             if len(backup_strings) != 2:
                 return False
 

patroni/utils.py

@@ -1,47 +1,198 @@
-import datetime
 import os
 import random
-import signal
 import sys
 import time
-import pytz
-import dateutil.parser
+import re
 
 from patroni.exceptions import PatroniException
 
-__ignore_sigterm = False
+if sys.hexversion >= 0x0300000:
+    long = int
+
 __interrupted_sleep = False
 __reap_children = False
 
 
-def calculate_ttl(expiration):
+def deep_compare(obj1, obj2):
     """
-    >>> calculate_ttl(None)
-    >>> calculate_ttl('2015-06-10 12:56:30.552539016Z') < 0
+    >>> deep_compare({'1': None}, {})
+    False
+    >>> deep_compare({'1': {}}, {'1': None})
+    False
+    >>> deep_compare({'1': [1]}, {'1': [2]})
+    False
+    >>> deep_compare({'1': 2}, {'1': '2'})
     True
-    >>> calculate_ttl('2015-06-10T12:56:30.552539016Z') < 0
+    >>> deep_compare({'1': {'2': [3, 4]}}, {'1': {'2': [3, 4]}})
     True
-    >>> calculate_ttl('fail-06-10T12:56:30.552539016Z')
     """
-    if not expiration:
-        return None
-    try:
-        expiration = dateutil.parser.parse(expiration)
-    except (ValueError, TypeError):
-        return None
-    now = datetime.datetime.now(pytz.utc)
-    return int((expiration - now).total_seconds())
+
+    if set(list(obj1.keys())) != set(list(obj2.keys())):  # Objects have different sets of keys
+        return False
+
+    for key, value in obj1.items():
+        if isinstance(value, dict):
+            if not (isinstance(obj2[key], dict) and deep_compare(value, obj2[key])):
+                return False
+        elif str(value) != str(obj2[key]):
+            return False
+    return True
 
 
-def set_ignore_sigterm(value=True):
-    global __ignore_sigterm
-    __ignore_sigterm = value
+def patch_config(config, data):
+    """recursively 'patch' `config` with `data`
+    :returns: `!True` if the `config` was changed"""
+    is_changed = False
+    for name, value in data.items():
+        if value is None:
+            if config.pop(name, None) is not None:
+                is_changed = True
+        elif name in config:
+            if isinstance(value, dict):
+                if isinstance(config[name], dict):
+                    if patch_config(config[name], value):
+                        is_changed = True
+                else:
+                    config[name] = value
+                    is_changed = True
+            elif str(config[name]) != str(value):
+                config[name] = value
+                is_changed = True
+        else:
+            config[name] = value
+            is_changed = True
+    return is_changed
 
 
-def sigterm_handler(signo, stack_frame):
-    if not __ignore_sigterm:
-        set_ignore_sigterm()
-        sys.exit()
+def parse_bool(value):
+    """
+    >>> parse_bool(1)
+    True
+    >>> parse_bool('off')
+    False
+    >>> parse_bool('foo')
+    """
+    value = str(value).lower()
+    if value in ('on', 'true', 'yes', '1'):
+        return True
+    if value in ('off', 'false', 'no', '0'):
+        return False
+
+
+def strtol(value, strict=True):
+    """As most as possible close equivalent of strtol(3) function (with base=0),
+       used by postgres to parse parameter values.
+    >>> strtol(0) == (0, '')
+    True
+    >>> strtol(1) == (1, '')
+    True
+    >>> strtol(9) == (9, '')
+    True
+    >>> strtol(' +0x400MB') == (1024, 'MB')
+    True
+    >>> strtol(' -070d') == (-56, 'd')
+    True
+    >>> strtol(' d ') == (None, 'd')
+    True
+    >>> strtol('9s', False) == (9, 's')
+    True
+    >>> strtol(' s ', False) == (1, 's')
+    True
+    """
+    value = str(value).strip()
+    l = len(value)
+    i = 0
+    # skip sign:
+    if i < l and value[i] in ('-', '+'):
+        i += 1
+
+    # we always expect to get digit in the beginning
+    if i < l and value[i].isdigit():
+        if value[i] == '0':
+            i += 1
+            if i < l and value[i] in ('x', 'X'):  # '0' followed by 'x': HEX
+                base = 16
+                i += 1
+            else:  # just starts with '0': OCT
+                base = 8
+        else:  # any other digit: DEC
+            base = 10
+
+        ret = None
+        while i <= l:
+            try:  # try to find maximally long number
+                i += 1  # by giving to `int` longer and longer strings
+                ret = long(value[:i], base)
+            except ValueError:  # until we will not get an exception or end of the string
+                i -= 1
+                break
+        if ret is not None:  # yay! there is a number in the beginning of the string
+            return ret, value[i:].strip()  # return the number and the "rest"
+
+    return (None if strict else 1), value.strip()
+
+
+def parse_int(value, base_unit=None):
+    """
+    >>> parse_int('1') == 1
+    True
+    >>> parse_int(' 0x400 MB ', '16384kB') == 64
+    True
+    >>> parse_int('1MB', 'kB') == 1024
+    True
+    >>> parse_int('1000 ms', 's') == 1
+    True
+    >>> parse_int('1GB', 'MB') is None
+    True
+    >>> parse_int(0) == 0
+    True
+    """
+
+    convert = {
+        'kB': {'kB': 1, 'MB': 1024, 'GB': 1024 * 1024, 'TB': 1024 * 1024 * 1024},
+        'ms': {'ms': 1, 's': 1000, 'min': 1000 * 60, 'h': 1000 * 60 * 60, 'd': 1000 * 60 * 60 * 24},
+        's': {'ms': -1000, 's': 1, 'min': 60, 'h': 60 * 60, 'd': 60 * 60 * 24},
+        'min': {'ms': -1000 * 60, 's': -60, 'min': 1, 'h': 60, 'd': 60 * 24}
+    }
+
+    value, unit = strtol(value)
+    if value is not None:
+        if not unit:
+            return value
+
+        if base_unit and base_unit not in convert:
+            base_value, base_unit = strtol(base_unit, False)
+        else:
+            base_value = 1
+        if base_unit in convert and unit in convert[base_unit]:
+            multiplier = convert[base_unit][unit]
+            if multiplier < 0:
+                value /= -multiplier
+            else:
+                value *= multiplier
+            return int(value/base_value)
+
+
+def compare_values(vartype, unit, old_value, new_value):
+    """
+    >>> compare_values('enum', None, 'remote_write', 'REMOTE_WRITE')
+    True
+    >>> compare_values('real', None, '1.23', 1.23)
+    True
+    """
+
+    # if the integer or bool new_value is not correct this function will return False
+    if vartype == 'bool':
+        old_value = parse_bool(old_value)
+        new_value = parse_bool(new_value)
+    elif vartype == 'integer':
+        old_value = parse_int(old_value)
+        new_value = parse_int(new_value, unit)
+    elif vartype == 'enum':
+        return str(old_value).lower() == str(new_value).lower()
+    else:  # ('string', 'real')
+        return str(old_value) == str(new_value)
+    return old_value is not None and new_value is not None and old_value == new_value
 
 
 def sigchld_handler(signo, stack_frame):
@@ -62,11 +213,6 @@ def sleep(interval):
     __interrupted_sleep = False
 
 
-def setup_signal_handlers():
-    signal.signal(signal.SIGTERM, sigterm_handler)
-    signal.signal(signal.SIGCHLD, sigchld_handler)
-
-
 def reap_children():
     global __reap_children
     if __reap_children:
@@ -81,6 +227,10 @@ def reap_children():
             __reap_children = False
 
 
+def is_valid_pg_version(version):
+    return re.match(r'[1-9][0-9]?(\.(0|([1-9][0-9]?))){2}$', version)
+
+
 class RetryFailedError(PatroniException):
 
     """Raised when retrying an operation ultimately failed, after retrying the maximum number of attempts."""

patroni/version.py

@@ -1 +1 @@
-__version__ = '0.90'
+__version__ = '1.1'

patroni/zookeeper.py

@@ -1,289 +0,0 @@
-import logging
-import random
-import requests
-import time
-
-from kazoo.client import KazooClient, KazooState
-from kazoo.exceptions import NoNodeError, NodeExistsError
-from patroni.dcs import AbstractDCS, Cluster, Failover, Leader, Member
-from patroni.exceptions import DCSError
-from patroni.utils import sleep
-from requests.exceptions import RequestException
-
-logger = logging.getLogger(__name__)
-
-
-class ZooKeeperError(DCSError):
-    pass
-
-
-class ExhibitorEnsembleProvider(object):
-
-    TIMEOUT = 3.1
-
-    def __init__(self, hosts, port, uri_path='/exhibitor/v1/cluster/list', poll_interval=300):
-        self._exhibitor_port = port
-        self._uri_path = uri_path
-        self._poll_interval = poll_interval
-        self._exhibitors = hosts
-        self._master_exhibitors = hosts
-        self._zookeeper_hosts = ''
-        self._next_poll = None
-        while not self.poll():
-            logger.info('waiting on exhibitor')
-            sleep(5)
-
-    def poll(self):
-        if self._next_poll and self._next_poll > time.time():
-            return False
-
-        json = self._query_exhibitors(self._exhibitors)
-        if not json:
-            json = self._query_exhibitors(self._master_exhibitors)
-
-        if isinstance(json, dict) and 'servers' in json and 'port' in json:
-            self._next_poll = time.time() + self._poll_interval
-            zookeeper_hosts = ','.join([h + ':' + str(json['port']) for h in sorted(json['servers'])])
-            if self._zookeeper_hosts != zookeeper_hosts:
-                logger.info('ZooKeeper connection string has changed: %s => %s', self._zookeeper_hosts, zookeeper_hosts)
-                self._zookeeper_hosts = zookeeper_hosts
-                self._exhibitors = json['servers']
-                return True
-        return False
-
-    def _query_exhibitors(self, exhibitors):
-        random.shuffle(exhibitors)
-        for host in exhibitors:
-            uri = 'http://{0}:{1}{2}'.format(host, self._exhibitor_port, self._uri_path)
-            try:
-                response = requests.get(uri, timeout=self.TIMEOUT)
-                return response.json()
-            except RequestException:
-                pass
-        return None
-
-    @property
-    def zookeeper_hosts(self):
-        return self._zookeeper_hosts
-
-
-class ZooKeeper(AbstractDCS):
-
-    def __init__(self, name, config):
-        super(ZooKeeper, self).__init__(name, config)
-
-        hosts = config.get('hosts', [])
-        if isinstance(hosts, list):
-            hosts = ','.join(hosts)
-
-        self.exhibitor = None
-        if 'exhibitor' in config:
-            exhibitor = config['exhibitor']
-            interval = exhibitor.get('poll_interval', 300)
-            self.exhibitor = ExhibitorEnsembleProvider(exhibitor['hosts'], exhibitor['port'], poll_interval=interval)
-            hosts = self.exhibitor.zookeeper_hosts
-
-        self._client = KazooClient(hosts=hosts, timeout=(config.get('session_timeout') or 30),
-                                   command_retry={'deadline': (config.get('reconnect_timeout') or 10),
-                                                  'max_delay': 1, 'max_tries': -1},
-                                   connection_retry={'max_delay': 1, 'max_tries': -1})
-        self._client.add_listener(self.session_listener)
-
-        self._my_member_data = None
-        self._fetch_cluster = True
-        self._last_leader_operation = 0
-
-        self._client.start()
-
-    def session_listener(self, state):
-        if state in [KazooState.SUSPENDED, KazooState.LOST]:
-            self.cluster_watcher(None)
-
-    def cluster_watcher(self, event):
-        self._fetch_cluster = True
-        self.event.set()
-
-    def get_node(self, key, watch=None):
-        try:
-            ret = self._client.get(key, watch)
-            return (ret[0].decode('utf-8'), ret[1])
-        except NoNodeError:
-            return None
-
-    @staticmethod
-    def member(name, value, znode):
-        return Member.from_node(znode.version, name, znode.ephemeralOwner, value)
-
-    def get_children(self, key, watch=None):
-        try:
-            return self._client.get_children(key, watch)
-        except NoNodeError:
-            return []
-
-    def load_members(self):
-        members = []
-        for member in self.get_children(self.members_path, self.cluster_watcher):
-            data = self.get_node(self.members_path + member)
-            if data is not None:
-                members.append(self.member(member, *data))
-        return members
-
-    def _inner_load_cluster(self):
-        self._fetch_cluster = False
-        self.event.clear()
-        nodes = set(self.get_children(self.client_path(''), self.cluster_watcher))
-        if not nodes:
-            self._fetch_cluster = True
-
-        # get initialize flag
-        initialize = (self.get_node(self.initialize_path) or [None])[0] if self._INITIALIZE in nodes else None
-
-        # get list of members
-        members = self.load_members() if self._MEMBERS[:-1] in nodes else []
-
-        # get leader
-        leader = self.get_node(self.leader_path) if self._LEADER in nodes else None
-        if leader:
-            client_id = self._client.client_id
-            if leader[0] == self._name and client_id is not None and client_id[0] != leader[1].ephemeralOwner:
-                logger.info('I am leader but not owner of the session. Removing leader node')
-                self._client.delete(self.leader_path)
-                leader = None
-
-            if leader:
-                member = Member(-1, leader[0], None, {})
-                member = ([m for m in members if m.name == leader[0]] or [member])[0]
-                leader = Leader(leader[1].version, leader[1].ephemeralOwner, member)
-                self._fetch_cluster = member.index == -1
-
-        # failover key
-        failover = self.get_node(self.failover_path, watch=self.cluster_watcher) if self._FAILOVER in nodes else None
-        if failover:
-            failover = Failover.from_node(failover[1].version, failover[0])
-
-        # get last leader operation
-        optime = self.get_node(self.leader_optime_path) if self._OPTIME in nodes and self._fetch_cluster else None
-        self._last_leader_operation = 0 if optime is None else int(optime[0])
-        self._cluster = Cluster(initialize, leader, self._last_leader_operation, members, failover)
-
-    def _load_cluster(self):
-        if self.exhibitor and self.exhibitor.poll():
-            self._client.set_hosts(self.exhibitor.zookeeper_hosts)
-
-        if self._fetch_cluster or self._cluster is None:
-            try:
-                self._client.retry(self._inner_load_cluster)
-            except:
-                logger.exception('get_cluster')
-                self.session_listener(KazooState.LOST)
-                raise ZooKeeperError('ZooKeeper in not responding properly')
-
-    def _create(self, path, value, **kwargs):
-        try:
-            self._client.retry(self._client.create, path, value.encode('utf-8'), **kwargs)
-            return True
-        except:
-            return False
-
-    def attempt_to_acquire_leader(self):
-        ret = self._create(self.leader_path, self._name, makepath=True, ephemeral=True)
-        if not ret:
-            logger.info('Could not take out TTL lock')
-        return ret
-
-    def set_failover_value(self, value, index=None):
-        try:
-            self._client.retry(self._client.set, self.failover_path, value.encode('utf-8'), version=index or -1)
-            return True
-        except NoNodeError:
-            return value == '' or (not index and self._create(self.failover_path, value))
-        except:
-            logging.exception('set_failover_value')
-            return False
-
-    def initialize(self, create_new=True, sysid=""):
-        return self._create(self.initialize_path, sysid, makepath=True) if create_new \
-            else self._client.retry(self._client.set, self.initialize_path,  sysid.encode("utf-8"))
-
-    def touch_member(self, data, ttl=None):
-        cluster = self.cluster
-        member = cluster and ([m for m in cluster.members if m.name == self._name] or [None])[0]
-        path = self.member_path
-        data = data.encode('utf-8')
-        if member and self._client.client_id is not None and member.session != self._client.client_id[0]:
-            try:
-                self._client.retry(self._client.delete, path)
-            except NoNodeError:
-                pass
-            except:
-                return False
-            member = None
-
-        if member and data == self._my_member_data:
-            return True
-
-        try:
-            if member:
-                self._client.retry(self._client.set, path, data)
-            else:
-                self._client.retry(self._client.create, path, data, makepath=True, ephemeral=True)
-            self._my_member_data = data
-            return True
-        except NodeExistsError:
-            try:
-                self._client.retry(self._client.set, path, data)
-                self._my_member_data = data
-                return True
-            except:
-                logger.exception('touch_member')
-        except:
-            logger.exception('touch_member')
-        return False
-
-    def take_leader(self):
-        return self.attempt_to_acquire_leader()
-
-    def write_leader_optime(self, last_operation):
-        last_operation = last_operation.encode('utf-8')
-        if last_operation != self._last_leader_operation:
-            self._last_leader_operation = last_operation
-            path = self.leader_optime_path
-            try:
-                self._client.retry(self._client.set, path, last_operation)
-            except NoNodeError:
-                try:
-                    self._client.retry(self._client.create, path, last_operation, makepath=True)
-                except:
-                    logger.exception('Failed to create %s', path)
-            except:
-                logger.exception('Failed to update %s', path)
-
-    def update_leader(self):
-        return True
-
-    def delete_leader(self):
-        self._client.restart()
-        self._my_member_data = None
-        return True
-
-    def _cancel_initialization(self):
-        node = self.get_node(self.initialize_path)
-        if node:
-            self._client.delete(self.initialize_path, version=node[1].version)
-
-    def cancel_initialization(self):
-        try:
-            self._client.retry(self._cancel_initialization)
-        except:
-            logger.exception("Unable to delete initialize key")
-
-    def delete_cluster(self):
-        try:
-            return self._client.retry(self._client.delete, self.client_path(''), recursive=True)
-        except NoNodeError:
-            return True
-
-    def watch(self, timeout):
-        if super(ZooKeeper, self).watch(timeout):
-            self._fetch_cluster = True
-        return self._fetch_cluster

postgres0.yml

@@ -1,104 +1,77 @@
-ttl: &ttl 30
-loop_wait: &loop_wait 10
-scope: &scope batman
+scope: batman
+#namespace: /service/
+name: postgresql0
+
 restapi:
   listen: 127.0.0.1:8008
   connect_address: 127.0.0.1:8008
-#  auth: 'username:password'
 #  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
 #  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
+#  authentication:
+#    username: username
+#    password: password
+
 etcd:
-  scope: *scope
-  ttl: *ttl
   host: 127.0.0.1:4001
-  #discovery_srv: my-etcd.domain
-#consul:
-#  scope: *scope
-#  ttl: *ttl
-#  host: 127.0.0.1:8500
-#zookeeper:
-#  scope: *scope
-#  session_timeout: *ttl
-#  reconnect_timeout: *loop_wait
-#  hosts:
-#    - 127.0.0.1:2181
-#    - 127.0.0.2:2181
-#  exhibitor:
-#    poll_interval: 300
-#    port: 8181
-#    hosts:
-#      - host1
-#      - host2
-#      - host3
+
+bootstrap:
+  # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
+  # and all other cluster members will use it as a `global configuration`
+  dcs:
+    ttl: 30
+    loop_wait: 10
+    retry_timeout: 10
+    maximum_lag_on_failover: 1048576
+    postgresql:
+      use_pg_rewind: true
+#      use_slots: true
+      parameters:
+#        wal_level: hot_standby
+#        hot_standby: "on"
+#        wal_keep_segments: 8
+#        max_wal_senders: 5
+#        max_replication_slots: 5
+#        wal_log_hints: "on"
+#        archive_mode: "on"
+#        archive_timeout: 1800s
+#        archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
+#      recovery_conf:
+#        restore_command: cp ../wal_archive/%f %p
+
+  # some desired options for 'initdb'
+  initdb:  # Note: It needs to be a list (some options need values, others are switches)
+  - encoding: UTF8
+  - data-checksums
+
+  pg_hba:  # Add following lines to pg_hba.conf after running 'initdb'
+  - host replication replicator 127.0.0.1/32 md5
+  - host all all 0.0.0.0/0 md5
+#  - hostssl all all 0.0.0.0/0 md5
+
+  # Some additional users users which needs to be created after initializing new cluster
+  users:
+    admin:
+      password: admin
+      options:
+        - createrole
+        - createdb
+
 postgresql:
-  name: postgresql0
-  scope: *scope
   listen: 127.0.0.1:5432
   connect_address: 127.0.0.1:5432
   data_dir: data/postgresql0
-  maximum_lag_on_failover: 1048576 # 1 megabyte in bytes
-  use_slots: True
+#  bin_dir:
   pgpass: /tmp/pgpass0
-  initdb:  ## We allow the following options to be passed on to initdb
-  # - auth: authmethod
-  # - auth-host: authmethod
-  # - auth-local: authmethod
-  - encoding: UTF8
-  # - data-checksums  # When pg_rewind is needed on 9.3, this needs to be enabled
-  # - locale: locale
-  # - lc-collate: locale
-  # - lc-ctype: locale
-  # - lc-messages: locale
-  # - lc-monetary: locale
-  # - lc-numeric: locale
-  # - lc-time: locale
-  # - text-search-config: CFG
-  # - xlogdir: directory
-  # - debug
-  # - noclean
-  pg_rewind:
-    username: postgres
-    password: zalando
-  pg_hba:
-  - host replication replicator 127.0.0.1/32 md5
-  - host all all 0.0.0.0/0 md5
-  # - hostssl all all 0.0.0.0/0 md5
-  replication:
-    username: replicator
-    password: rep-pass
-  superuser:
-    username: postgres
-    password: zalando
-  admin:
-    username: admin
-    password: admin
-  create_replica_method:
-    - basebackup
-#   - wal_e
-# commented-out example for wal-e provisioning
-  #wal_e:
-    #command: /patroni/scripts/wale_restore.py
-    #env_dir: /etc/wal-e.d/env
-    #threshold_megabytes: 10240
-    #threshold_backup_size_percentage: 30
-    #retries: 2
-    #use_iam: 1
-  #recovery_conf:
-    #restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1
-  recovery_conf:
-    restore_command: cp ../wal_archive/%f %p
+  authentication:
+    replication:
+      username: replicator
+      password: rep-pass
+    superuser:
+      username: postgres
+      password: zalando
   parameters:
-    archive_mode: "on"
-    wal_level: hot_standby
-    archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
-    max_wal_senders: 10
-    wal_keep_segments: 8
-    archive_timeout: 1800s
-    max_replication_slots: 10
-    hot_standby: "on"
-    wal_log_hints: "on"
     unix_socket_directories: '.'
 tags:
-    nofailover: False
-    noloadbalance: False
-    clonefrom: False
+    nofailover: false
+    noloadbalance: false
+    clonefrom: false

postgres1.yml

@@ -1,105 +1,77 @@
-ttl: &ttl 30
-loop_wait: &loop_wait 10
-scope: &scope batman
+scope: batman
+#namespace: /service/
+name: postgresql1
+
 restapi:
   listen: 127.0.0.1:8009
   connect_address: 127.0.0.1:8009
-#  auth: 'username:password'
 #  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
 #  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
+#  authentication:
+#    username: username
+#    password: password
+
 etcd:
-  scope: *scope
-  ttl: *ttl
   host: 127.0.0.1:4001
-  #discovery_srv: my-etcd.domain
-#consul:
-#  scope: *scope
-#  ttl: *ttl
-#  host: 127.0.0.1:8500
-#zookeeper:
-#  scope: *scope
-#  session_timeout: *ttl
-#  reconnect_timeout: *loop_wait
-#  hosts:
-#    - 127.0.0.1:2181
-#    - 127.0.0.2:2181
-#  exhibitor:
-#    poll_interval: 300
-#    port: 8181
-#    hosts:
-#      - host1
-#      - host2
-#      - host3
+
+bootstrap:
+  # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
+  # and all other cluster members will use it as a `global configuration`
+  dcs:
+    ttl: 30
+    loop_wait: 10
+    retry_timeout: 10
+    maximum_lag_on_failover: 1048576
+    postgresql:
+      use_pg_rewind: true
+#      use_slots: true
+      parameters:
+#        wal_level: hot_standby
+#        hot_standby: "on"
+#        wal_keep_segments: 8
+#        max_wal_senders: 5
+#        max_replication_slots: 5
+#        wal_log_hints: "on"
+#        archive_mode: "on"
+#        archive_timeout: 1800s
+#        archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
+#      recovery_conf:
+#        restore_command: cp ../wal_archive/%f %p
+
+  # some desired options for 'initdb'
+  initdb:  # Note: It needs to be a list (some options need values, others are switches)
+  - encoding: UTF8
+  - data-checksums
+
+  pg_hba:  # Add following lines to pg_hba.conf after running 'initdb'
+  - host replication replicator 127.0.0.1/32 md5
+  - host all all 0.0.0.0/0 md5
+#  - hostssl all all 0.0.0.0/0 md5
+
+  # Some additional users users which needs to be created after initializing new cluster
+  users:
+    admin:
+      password: admin
+      options:
+        - createrole
+        - createdb
+
 postgresql:
-  name: postgresql1
-  scope: *scope
   listen: 127.0.0.1:5433
   connect_address: 127.0.0.1:5433
   data_dir: data/postgresql1
-  maximum_lag_on_failover: 1048576 # 1 megabyte in bytes
-  use_slots: True
+#  bin_dir:
   pgpass: /tmp/pgpass1
-  initdb:  ## We allow the following options to be passed on to initdb
-  # - auth: authmethod
-  # - auth-host: authmethod
-  # - auth-local: authmethod
-  - encoding: UTF8
-  # - data-checksums  # When pg_rewind is needed on 9.3, this needs to be enabled
-  # - locale: locale
-  # - lc-collate: locale
-  # - lc-ctype: locale
-  # - lc-messages: locale
-  # - lc-monetary: locale
-  # - lc-numeric: locale
-  # - lc-time: locale
-  # - text-search-config: CFG
-  # - xlogdir: directory
-  # - debug
-  # - noclean
-  pg_rewind:
-    username: postgres
-    password: zalando
-  pg_hba:
-  - host replication replicator 127.0.0.1/32 md5
-  - host all all 0.0.0.0/0 md5
-  # - hostssl all all 0.0.0.0/0 md5
-  replication:
-    username: replicator
-    password: rep-pass
-  superuser:
-    username: postgres
-    password: zalando
-  admin:
-    username: admin
-    password: admin
-# commented-out example for wal-e provisioning
-  create_replica_method:
-    - basebackup
-#   - wal_e
-# commented-out example for wal-e provisioning
-  #wal_e:
-    #command: /patroni/scripts/wale_restore.py
-    #env_dir: /home/postgres/etc/wal-e.d/env
-    #threshold_megabytes: 10240
-    #threshold_backup_size_percentage: 30
-    #retries: 2
-    #use_iam: 1
-  #recovery_conf:
-    #restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1
-  recovery_conf:
-    restore_command: cp ../wal_archive/%f %p
+  authentication:
+    replication:
+      username: replicator
+      password: rep-pass
+    superuser:
+      username: postgres
+      password: zalando
   parameters:
-    archive_mode: "on"
-    wal_level: hot_standby
-    archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
-    max_wal_senders: 10
-    wal_keep_segments: 8
-    archive_timeout: 1800s
-    max_replication_slots: 10
-    hot_standby: "on"
-    wal_log_hints: "on"
     unix_socket_directories: '.'
 tags:
-    nofailover: False
-    noloadbalance: False
-    clonefrom: False
+    nofailover: false
+    noloadbalance: false
+    clonefrom: false

postgres2.yml

@@ -1,106 +1,78 @@
-ttl: &ttl 30
-loop_wait: &loop_wait 10
-scope: &scope batman
+scope: batman
+#namespace: /service/
+name: postgresql2
+
 restapi:
   listen: 127.0.0.1:8010
   connect_address: 127.0.0.1:8010
-  auth: 'username:password'
 #  certfile: /etc/ssl/certs/ssl-cert-snakeoil.pem
 #  keyfile: /etc/ssl/private/ssl-cert-snakeoil.key
+  authentication:
+    username: username
+    password: password
+
 etcd:
-  scope: *scope
-  ttl: *ttl
   host: 127.0.0.1:4001
-  #discovery_srv: my-etcd.domain
-#consul:
-#  scope: *scope
-#  ttl: *ttl
-#  host: 127.0.0.1:8500
-#zookeeper:
-#  scope: *scope
-#  session_timeout: *ttl
-#  reconnect_timeout: *loop_wait
-#  hosts:
-#    - 127.0.0.1:2181
-#    - 127.0.0.2:2181
-#  exhibitor:
-#    poll_interval: 300
-#    port: 8181
-#    hosts:
-#      - host1
-#      - host2
-#      - host3
+
+bootstrap:
+  # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
+  # and all other cluster members will use it as a `global configuration`
+  dcs:
+    ttl: 30
+    loop_wait: 10
+    retry_timeout: 10
+    maximum_lag_on_failover: 1048576
+    postgresql:
+      use_pg_rewind: true
+#      use_slots: true
+      parameters:
+#        wal_level: hot_standby
+#        hot_standby: "on"
+#        wal_keep_segments: 8
+#        max_wal_senders: 5
+#        max_replication_slots: 5
+#        wal_log_hints: "on"
+#        archive_mode: "on"
+#        archive_timeout: 1800s
+#        archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
+#      recovery_conf:
+#        restore_command: cp ../wal_archive/%f %p
+
+  # some desired options for 'initdb'
+  initdb:  # Note: It needs to be a list (some options need values, others are switches)
+  - encoding: UTF8
+  - data-checksums
+
+  pg_hba:  # Add following lines to pg_hba.conf after running 'initdb'
+  - host replication replicator 127.0.0.1/32 md5
+  - host all all 0.0.0.0/0 md5
+#  - hostssl all all 0.0.0.0/0 md5
+
+  # Some additional users users which needs to be created after initializing new cluster
+  users:
+    admin:
+      password: admin
+      options:
+        - createrole
+        - createdb
+
 postgresql:
-  name: postgresql2
-  scope: *scope
   listen: 127.0.0.1:5434
   connect_address: 127.0.0.1:5434
   data_dir: data/postgresql2
-  maximum_lag_on_failover: 1048576 # 1 megabyte in bytes
-  use_slots: True
+#  bin_dir:
   pgpass: /tmp/pgpass2
-  initdb:  ## We allow the following options to be passed on to initdb
-  # - auth: authmethod
-  # - auth-host: authmethod
-  # - auth-local: authmethod
-  - encoding: UTF8
-  # - data-checksums  # When pg_rewind is needed on 9.3, this needs to be enabled
-  # - locale: locale
-  # - lc-collate: locale
-  # - lc-ctype: locale
-  # - lc-messages: locale
-  # - lc-monetary: locale
-  # - lc-numeric: locale
-  # - lc-time: locale
-  # - text-search-config: CFG
-  # - xlogdir: directory
-  # - debug
-  # - noclean
-  pg_rewind:
-    username: postgres
-    password: zalando
-  pg_hba:
-  - host replication replicator 127.0.0.1/32 md5
-  - host all all 0.0.0.0/0 md5
-  # - hostssl all all 0.0.0.0/0 md5
-  replication:
-    username: replicator
-    password: rep-pass
-  superuser:
-    username: postgres
-    password: zalando
-  admin:
-    username: admin
-    password: admin
-# commented-out example for wal-e provisioning
-  create_replica_method:
-    - basebackup
-#   - wal_e
-# commented-out example for wal-e provisioning
-  #wal_e:
-    #command: /patroni/scripts/wale_restore.py
-    #env_dir: /home/postgres/etc/wal-e.d/env
-    #threshold_megabytes: 10240
-    #threshold_backup_size_percentage: 30
-    #retries: 2
-    #use_iam: 1
-  #recovery_conf:
-    #restore_command: envdir /etc/wal-e.d/env wal-e wal-fetch "%f" "%p" -p 1
-  recovery_conf:
-    restore_command: cp ../wal_archive/%f %p
+  authentication:
+    replication:
+      username: replicator
+      password: rep-pass
+    superuser:
+      username: postgres
+      password: zalando
   parameters:
-    archive_mode: "on"
-    wal_level: hot_standby
-    archive_command: mkdir -p ../wal_archive && test ! -f ../wal_archive/%f && cp %p ../wal_archive/%f
-    max_wal_senders: 10
-    wal_keep_segments: 8
-    archive_timeout: 1800s
-    max_replication_slots: 10
-    hot_standby: "on"
-    wal_log_hints: "on"
     unix_socket_directories: '.'
 tags:
-    nofailover: False
-    noloadbalance: False
-    clonefrom: False
-    replicatefrom: postgresql1
+    nofailover: false
+    noloadbalance: false
+    clonefrom: false
+    replicatefrom: postgres1

requirements.txt

@@ -3,10 +3,10 @@ psycopg2>=2.6.1
 PyYAML
 requests
 six >= 1.7
-kazoo>=2.2.1
+kazoo==2.2.1
 python-etcd==0.4.3
 python-consul==0.6.0
 click>=4.1
 prettytable>=0.7
 tzlocal
-python-dateutil
+python-dateutil

setup.py

@@ -33,7 +33,7 @@ LICENSE = 'The MIT License'
 URL = 'https://github.com/zalando/patroni'
 AUTHOR = 'Alexander Kukushkin, Oleksii Kliukin, Feike Steenbergen'
 AUTHOR_EMAIL = '[email protected], [email protected], [email protected]'
-KEYWORDS = 'etcd governor patroni postgresql postgres ha zookeeper streaming replication'
+KEYWORDS = 'etcd governor patroni postgresql postgres ha haproxy confd zookeeper exhibitor consul streaming replication'
 
 COVERAGE_XML = True
 COVERAGE_HTML = False
@@ -147,7 +147,7 @@ def setup_package():
         install_requires=install_reqs,
         setup_requires=['flake8'],
         cmdclass=cmdclass,
-        tests_require=['mock', 'pytest-cov', 'pytest'],
+        tests_require=['mock>=2.0.0', 'pytest-cov', 'pytest'],
         command_options=command_options,
         entry_points={'console_scripts': CONSOLE_SCRIPTS},
     )

tests/test_api.py

@@ -1,16 +1,21 @@
+import datetime
+import json
 import psycopg2
+import pytz
 import unittest
 
 from mock import Mock, patch
 from patroni.api import RestApiHandler, RestApiServer
-from patroni.dcs import Member
+from patroni.dcs import ClusterConfig, Member
 from six import BytesIO as IO
 from six.moves import BaseHTTPServer
-from six.moves.BaseHTTPServer import BaseHTTPRequestHandler
-import socket
 from test_postgresql import psycopg2_connect, MockCursor
 
 
+future_restart_time = datetime.datetime.now(pytz.utc) + datetime.timedelta(days=5)
+postmaster_start_time = datetime.datetime.now(pytz.utc)
+
+
 class MockPostgresql(object):
 
     name = 'test'
@@ -19,19 +24,23 @@ class MockPostgresql(object):
     server_version = '999999'
     sysid = 'dummysysid'
     scope = 'dummy'
+    pending_restart = True
 
     @staticmethod
     def connection():
         return psycopg2_connect()
 
+    @staticmethod
+    def postmaster_start_time():
+        return str(postmaster_start_time)
+
 
 class MockHa(object):
 
-    dcs = Mock()
     state_handler = MockPostgresql()
 
     @staticmethod
-    def schedule_reinitialize():
+    def reinitialize():
         return 'reinitialize'
 
     @staticmethod
@@ -42,126 +51,223 @@ class MockHa(object):
     def restart_scheduled():
         return False
 
+    @staticmethod
+    def delete_future_restart():
+        return True
+
     @staticmethod
     def fetch_nodes_statuses(members):
         return [[None, True, None, None, {}]]
 
+    @staticmethod
+    def schedule_future_restart(data):
+        return True
+
 
 class MockPatroni(object):
 
-    postgresql = MockPostgresql()
     ha = MockHa()
+    config = Mock()
+    postgresql = ha.state_handler
     dcs = Mock()
     tags = {}
     version = '0.00'
     noloadbalance = Mock(return_value=False)
+    scheduled_restart = {'schedule': future_restart_time,
+                         'postmaster_start_time': postgresql.postmaster_start_time()}
+
+    @staticmethod
+    def sighup_handler():
+        pass
 
 
 class MockRequest(object):
 
-    def __init__(self, path):
-        self.path = path
+    def __init__(self, request):
+        self.request = request.encode('utf-8')
 
     def makefile(self, *args, **kwargs):
-        return IO(self.path)
+        return IO(self.request)
 
 
 class MockRestApiServer(RestApiServer):
 
-    def __init__(self, Handler, path):
+    def __init__(self, Handler, request):
         self.socket = 0
+        self.serve_forever = Mock()
         BaseHTTPServer.HTTPServer.__init__ = Mock()
         MockRestApiServer._BaseServer__is_shut_down = Mock()
         MockRestApiServer._BaseServer__shutdown_request = True
-        config = {'listen': '127.0.0.1:8008', 'auth': 'test:test', 'certfile': 'dumb'}
+        config = {'listen': '127.0.0.1:8008', 'auth': 'test:test'}
         super(MockRestApiServer, self).__init__(MockPatroni(), config)
-        Handler(MockRequest(path), ('0.0.0.0', 8080), self)
+        config['certfile'] = 'dumb'
+        self.reload_config(config)
+        Handler(MockRequest(request), ('0.0.0.0', 8080), self)
 
 
 @patch('ssl.wrap_socket', Mock(return_value=0))
 class TestRestApiHandler(unittest.TestCase):
 
+    _authorization = '\nAuthorization: Basic dGVzdDp0ZXN0'
+
     def test_do_GET(self):
-        MockRestApiServer(RestApiHandler, b'GET /replica')
+        MockRestApiServer(RestApiHandler, 'GET /replica')
         with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={})):
-            MockRestApiServer(RestApiHandler, b'GET /replica')
+            MockRestApiServer(RestApiHandler, 'GET /replica')
         with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'master'})):
-            MockRestApiServer(RestApiHandler, b'GET /replica')
-        MockRestApiServer(RestApiHandler, b'GET /master')
+            MockRestApiServer(RestApiHandler, 'GET /replica')
+        MockRestApiServer(RestApiHandler, 'GET /master')
         MockPatroni.dcs.cluster.leader.name = MockPostgresql.name
-        MockRestApiServer(RestApiHandler, b'GET /replica')
+        MockRestApiServer(RestApiHandler, 'GET /replica')
         MockPatroni.dcs.cluster = None
         with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'master'})):
-            MockRestApiServer(RestApiHandler, b'GET /master')
+            MockRestApiServer(RestApiHandler, 'GET /master')
         with patch.object(MockHa, 'restart_scheduled', Mock(return_value=True)):
-            MockRestApiServer(RestApiHandler, b'GET /master')
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /master'))
+            MockRestApiServer(RestApiHandler, 'GET /master')
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /master'))
 
     def test_do_OPTIONS(self):
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0'))
-
-        with patch.object(BaseHTTPRequestHandler, 'handle_one_request') as mock_handle_request:
-            mock_handle_request.side_effect = socket.error("foo")
-            MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0')
-
-        # make sure socket.error gets propagated via wfile object in finalize()
-        with patch.object(MockRequest, 'makefile') as makefile:
-            makefile.return_value.closed = False
-            makefile.return_value.readline = Mock(return_value=b'foo')
-            makefile.return_value.flush = Mock(side_effect=socket.error('foo'))
-            MockRestApiServer(RestApiHandler, b'OPTIONS / HTTP/1.0')
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'OPTIONS / HTTP/1.0'))
 
     def test_do_GET_patroni(self):
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /patroni'))
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
 
     def test_basicauth(self):
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0'))
-        MockRestApiServer(RestApiHandler, b'POST /restart HTTP/1.0\nAuthorization:')
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0'))
+        MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0\nAuthorization:')
 
-    def test_do_POST_restart(self):
-        request = b'POST /restart HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0'
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_GET_config(self, mock_dcs):
+        mock_dcs.cluster.config.data = {}
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /config'))
+        mock_dcs.cluster.config = None
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /config'))
+
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_PATCH_config(self, mock_dcs):
+        config = {'postgresql': {'use_slots': False, 'use_pg_rewind': True, 'parameters': {'wal_level': 'logical'}}}
+        mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, json.dumps(config))
+        request = 'PATCH /config HTTP/1.0' + self._authorization
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
+        request += '\nContent-Length: '
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '34\n\n{"postgresql":{"use_slots":false}}'))
+        config['ttl'] = 5
+        config['postgresql'].update({'use_slots': {'foo': True}, "parameters": None})
+        config = json.dumps(config)
+        request += str(len(config)) + '\n\n' + config
+        MockRestApiServer(RestApiHandler, request)
+        mock_dcs.set_config_value.return_value = False
+        MockRestApiServer(RestApiHandler, request)
+
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_PUT_config(self, mock_dcs):
+        mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, '{}')
+        request = 'PUT /config HTTP/1.0' + self._authorization + '\nContent-Length: '
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '2\n\n{}'))
+        config = '{"foo": "bar"}'
+        request += str(len(config)) + '\n\n' + config
+        MockRestApiServer(RestApiHandler, request)
+        mock_dcs.set_config_value.return_value = False
+        MockRestApiServer(RestApiHandler, request)
+        mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, config)
+        MockRestApiServer(RestApiHandler, request)
+
+    @patch.object(MockPatroni, 'sighup_handler', Mock(side_effect=Exception))
+    def test_do_POST_reload(self):
+        with patch.object(MockPatroni, 'config') as mock_config:
+            mock_config.reload_local_configuration.return_value = False
+            MockRestApiServer(RestApiHandler, 'POST /reload HTTP/1.0' + self._authorization)
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /reload HTTP/1.0' + self._authorization))
+
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_POST_restart(self, mock_dcs):
+        mock_dcs.get_cluster.return_value.is_paused.return_value = False
+        request = 'POST /restart HTTP/1.0' + self._authorization
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
+
         with patch.object(MockHa, 'restart', Mock(side_effect=Exception)):
             MockRestApiServer(RestApiHandler, request)
 
-    @patch.object(MockHa, 'dcs')
-    def test_do_POST_reinitialize(self, dcs):
-        cluster = dcs.get_cluster.return_value
-        request = b'POST /reinitialize HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0'
+        post = request + '\nContent-Length: '
+
+        def make_request(request=None, **kwargs):
+            request = json.dumps(kwargs) if request is None else request
+            return '{0}{1}\n\n{2}'.format(post, len(request), request)
+
+        # empty request
+        request = make_request('')
         MockRestApiServer(RestApiHandler, request)
-        cluster.is_unlocked.return_value = False
+        # invalid request
+        request = make_request('foobar=baz')
         MockRestApiServer(RestApiHandler, request)
-        with patch.object(MockHa, 'schedule_reinitialize', Mock(return_value=None)):
+        # wrong role
+        request = make_request(schedule=future_restart_time.isoformat(), role='unknown', postgres_version='9.5.3')
+        MockRestApiServer(RestApiHandler, request)
+        # wrong version
+        request = make_request(schedule=future_restart_time.isoformat(), role='master', postgres_version='9.5.3.1')
+        MockRestApiServer(RestApiHandler, request)
+        # unknown filter
+        request = make_request(schedule=future_restart_time.isoformat(), batman='lives')
+        MockRestApiServer(RestApiHandler, request)
+        # incorrect schedule
+        request = make_request(schedule='2016-08-42 12:45TZ+1', role='master')
+        MockRestApiServer(RestApiHandler, request)
+        # everything fine, but the schedule is missing
+        request = make_request(role='master', postgres_version='9.5.2')
+        MockRestApiServer(RestApiHandler, request)
+        for retval in (True, False):
+            with patch.object(MockHa, 'schedule_future_restart', Mock(return_value=retval)):
+                request = make_request(schedule=future_restart_time.isoformat())
+                MockRestApiServer(RestApiHandler, request)
+            with patch.object(MockHa, 'restart', Mock(return_value=(retval, "foo"))):
+                request = make_request(role='master', postgres_version='9.5.2')
+                MockRestApiServer(RestApiHandler, request)
+
+        mock_dcs.get_cluster.return_value.is_paused.return_value = True
+        MockRestApiServer(RestApiHandler, make_request(schedule='2016-08-42 12:45TZ+1', role='master'))
+
+    def test_do_DELETE_restart(self):
+        for retval in (True, False):
+            with patch.object(MockHa, 'delete_future_restart', Mock(return_value=retval)):
+                request = 'DELETE /restart HTTP/1.0' + self._authorization
+                self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
+
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_POST_reinitialize(self, mock_dcs):
+        cluster = mock_dcs.get_cluster.return_value
+        cluster.is_paused.return_value = False
+        request = 'POST /reinitialize HTTP/1.0' + self._authorization
+        MockRestApiServer(RestApiHandler, request)
+        with patch.object(MockHa, 'reinitialize', Mock(return_value=None)):
             MockRestApiServer(RestApiHandler, request)
-        cluster.leader.name = 'test'
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
 
     @patch('time.sleep', Mock())
     def test_RestApiServer_query(self):
         with patch.object(MockCursor, 'execute', Mock(side_effect=psycopg2.OperationalError)):
-            self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /patroni'))
+            self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
         with patch.object(MockPostgresql, 'connection', Mock(side_effect=psycopg2.OperationalError)):
-            self.assertIsNotNone(MockRestApiServer(RestApiHandler, b'GET /patroni'))
+            self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
 
     @patch('time.sleep', Mock())
-    @patch.object(MockHa, 'dcs')
+    @patch.object(MockPatroni, 'dcs')
     def test_do_POST_failover(self, dcs):
+        dcs.loop_wait = 10
         cluster = dcs.get_cluster.return_value
 
-        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\
-                  b'Content-Length: 0\n\n'
+        post = 'POST /failover HTTP/1.0' + self._authorization + '\nContent-Length: '
+
+        MockRestApiServer(RestApiHandler, post + '7\n\n{"1":2}')
+
+        request = post + '0\n\n'
         MockRestApiServer(RestApiHandler, request)
 
         cluster.leader.name = 'postgresql1'
         MockRestApiServer(RestApiHandler, request)
 
-        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\
-                  b'Content-Length: 25\n\n{"leader": "postgresql1"}'
-        MockRestApiServer(RestApiHandler, request)
+        MockRestApiServer(RestApiHandler, post + '25\n\n{"leader": "postgresql1"}')
 
         cluster.leader.name = 'postgresql2'
-        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\n' +\
-                  b'Content-Length: 53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
+        request = post + '53\n\n{"leader": "postgresql1", "candidate": "postgresql2"}'
         MockRestApiServer(RestApiHandler, request)
 
         cluster.leader.name = 'postgresql1'
@@ -170,41 +276,46 @@ class TestRestApiHandler(unittest.TestCase):
         cluster.members = [Member(0, 'postgresql0', 30, {'api_url': 'http'}),
                            Member(0, 'postgresql2', 30, {'api_url': 'http'})]
         MockRestApiServer(RestApiHandler, request)
-        with patch.object(MockPatroni, 'dcs') as d:
-            cluster = d.get_cluster.return_value
-            cluster.leader.name = 'postgresql0'
-            MockRestApiServer(RestApiHandler, request)
-            cluster.leader.name = 'postgresql2'
-            MockRestApiServer(RestApiHandler, request)
-            cluster.leader.name = 'postgresql1'
-            cluster.failover = None
-            MockRestApiServer(RestApiHandler, request)
-            d.get_cluster = Mock(side_effect=Exception)
-            MockRestApiServer(RestApiHandler, request)
-            d.manual_failover.return_value = False
-            MockRestApiServer(RestApiHandler, request)
+
+        cluster.failover = None
+        MockRestApiServer(RestApiHandler, request)
+
+        dcs.get_cluster.side_effect = [cluster]
+        MockRestApiServer(RestApiHandler, request)
+
+        cluster2 = cluster.copy()
+        cluster2.leader.name = 'postgresql0'
+        dcs.get_cluster.side_effect = [cluster, cluster2]
+        MockRestApiServer(RestApiHandler, request)
+
+        cluster2.leader.name = 'postgresql2'
+        dcs.get_cluster.side_effect = [cluster, cluster2]
+        MockRestApiServer(RestApiHandler, request)
+
+        dcs.get_cluster.side_effect = None
+        dcs.manual_failover.return_value = False
+        MockRestApiServer(RestApiHandler, request)
+        dcs.manual_failover.return_value = True
+
         with patch.object(MockHa, 'fetch_nodes_statuses', Mock(return_value=[])):
             MockRestApiServer(RestApiHandler, request)
 
         # Valid future date
-        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\
-                  b'"postgresql1", "member": "postgresql2", "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}'
+        request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2",' +\
+                         ' "scheduled_at": "6016-02-15T18:13:30.568224+01:00"}'
         MockRestApiServer(RestApiHandler, request)
         with patch.object(MockPatroni, 'dcs') as d:
             d.manual_failover.return_value = False
             MockRestApiServer(RestApiHandler, request)
 
         # Exception: No timezone specified
-        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 97\n\n{"leader": ' +\
-                  b'"postgresql1", "member": "postgresql2", "scheduled_at": "6016-02-15T18:13:30.568224"}'
+        request = post + '97\n\n{"leader": "postgresql1", "member": "postgresql2",' +\
+                         ' "scheduled_at": "6016-02-15T18:13:30.568224"}'
         MockRestApiServer(RestApiHandler, request)
 
         # Exception: Scheduled in the past
-        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\
-                  b'"postgresql1", "member": "postgresql2", "scheduled_at": "1016-02-15T18:13:30.568224+01:00"}'
-        MockRestApiServer(RestApiHandler, request)
+        request = post + '103\n\n{"leader": "postgresql1", "member": "postgresql2", "scheduled_at": "'
+        MockRestApiServer(RestApiHandler, request + '1016-02-15T18:13:30.568224+01:00"}')
 
         # Invalid date
-        request = b'POST /failover HTTP/1.0\nAuthorization: Basic dGVzdDp0ZXN0\nContent-Length: 103\n\n{"leader": ' +\
-                  b'"postgresql1", "member": "postgresql2", "scheduled_at": "2010-02-29T18:13:30.568224+01:00"}'
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request))
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, request + '2010-02-29T18:13:30.568224+01:00"}'))

tests/test_config.py

@@ -0,0 +1,74 @@
+import os
+import unittest
+import sys
+
+from mock import MagicMock, Mock, patch
+from patroni.config import Config
+from six.moves import builtins
+
+
+class TestConfig(unittest.TestCase):
+
+    @patch('os.path.isfile', Mock(return_value=True))
+    @patch('json.load', Mock(side_effect=Exception))
+    @patch.object(builtins, 'open', MagicMock())
+    def setUp(self):
+        sys.argv = ['patroni.py']
+        os.environ[Config.PATRONI_CONFIG_VARIABLE] = 'restapi: {}\npostgresql: {data_dir: foo}'
+        self.config = Config()
+
+    def test_no_config(self):
+        self.assertRaises(SystemExit, Config)
+
+    @patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception))
+    def test_set_dynamic_configuration(self):
+        self.assertIsNone(self.config.set_dynamic_configuration({'foo': 'bar'}))
+
+    def test_reload_local_configuration(self):
+        os.environ.update({
+            'PATRONI_NAME': 'postgres0',
+            'PATRONI_NAMESPACE': '/patroni/',
+            'PATRONI_SCOPE': 'batman2',
+            'PATRONI_RESTAPI_USERNAME': 'username',
+            'PATRONI_RESTAPI_PASSWORD': 'password',
+            'PATRONI_RESTAPI_LISTEN': '0.0.0.0:8008',
+            'PATRONI_RESTAPI_CONNECT_ADDRESS': '127.0.0.1:8008',
+            'PATRONI_RESTAPI_CERTFILE': '/certfile',
+            'PATRONI_RESTAPI_KEYFILE': '/keyfile',
+            'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432',
+            'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432',
+            'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0',
+            'PATRONI_POSTGRESQL_PGPASS': '/tmp/pgpass0',
+            'PATRONI_ETCD_HOST': '127.0.0.1:2379',
+            'PATRONI_CONSUL_HOST': '127.0.0.1:8500',
+            'PATRONI_ZOOKEEPER_HOSTS': "'host1:2181','host2:2181'",
+            'PATRONI_EXHIBITOR_HOSTS': 'host1,host2',
+            'PATRONI_EXHIBITOR_PORT': '8181',
+            'PATRONI_foo_HOSTS': '[host1,host2',  # Exception in parse_list
+            'PATRONI_SUPERUSER_USERNAME': 'postgres',
+            'PATRONI_SUPERUSER_PASSWORD': 'zalando',
+            'PATRONI_REPLICATION_USERNAME': 'replicator',
+            'PATRONI_REPLICATION_PASSWORD': 'rep-pass',
+            'PATRONI_admin_PASSWORD': 'admin',
+            'PATRONI_admin_OPTIONS': 'createrole,createdb'
+        })
+        sys.argv = ['patroni.py', 'postgres0.yml']
+        config = Config()
+        with patch.object(Config, '_load_config_file', Mock(return_value={'restapi': {}})):
+            with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
+                self.assertRaises(Exception, config.reload_local_configuration, True)
+            self.assertTrue(config.reload_local_configuration(True))
+            self.assertTrue(config.reload_local_configuration())
+
+    @patch('tempfile.mkstemp', Mock(return_value=[3000, 'blabla']))
+    @patch('os.path.exists', Mock(return_value=True))
+    @patch('os.remove', Mock(side_effect=IOError))
+    @patch('os.close', Mock(side_effect=IOError))
+    @patch('os.rename', Mock(return_value=None))
+    @patch('json.dump', Mock())
+    def test_save_cache(self):
+        self.config.set_dynamic_configuration({'ttl': 30, 'postgresql': {'foo': 'bar'}})
+        with patch('os.fdopen', Mock(side_effect=IOError)):
+            self.config.save_cache()
+        with patch('os.fdopen', MagicMock()):
+            self.config.save_cache()

tests/test_consul.py

@@ -1,9 +1,8 @@
 import consul
 import unittest
 
-from patroni.dcs import AbstractDCS
 from mock import Mock, patch
-from patroni.consul import Cluster, Consul, ConsulError, ConsulException, HTTPClient, NotFound
+from patroni.dcs.consul import AbstractDCS, Cluster, Consul, ConsulError, ConsulException, HTTPClient, NotFound
 from test_etcd import SleepException
 
 
@@ -52,14 +51,15 @@ class TestConsul(unittest.TestCase):
     @patch.object(consul.Consul.KV, 'get', kv_get)
     @patch.object(consul.Consul.KV, 'delete', Mock())
     def setUp(self):
-        self.c = Consul('postgresql1', {'ttl': 30, 'scope': 'test', 'host': 'localhost:1'})
+        self.c = Consul({'ttl': 30, 'scope': 'test', 'name': 'postgresql1', 'host': 'localhost:1', 'retry_timeout': 10})
         self.c._base_path = '/service/good'
         self.c._load_cluster()
 
     @patch('time.sleep', Mock(side_effect=SleepException))
-    def test_create_or_restore_session(self):
+    @patch.object(consul.Consul.Session, 'create', Mock(side_effect=ConsulException))
+    def test_create_session(self):
         self.c._session = None
-        self.assertRaises(SleepException, self.c.create_or_restore_session)
+        self.assertRaises(SleepException, self.c.create_session)
 
     @patch.object(consul.Consul.Session, 'renew', Mock(side_effect=NotFound))
     @patch.object(consul.Consul.Session, 'create', Mock(side_effect=ConsulException))
@@ -97,6 +97,10 @@ class TestConsul(unittest.TestCase):
     def test_set_failover_value(self):
         self.c.set_failover_value('')
 
+    @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
+    def test_set_config_value(self):
+        self.c.set_config_value('')
+
     @patch.object(consul.Consul.KV, 'put', Mock(side_effect=ConsulException))
     def test_write_leader_optime(self):
         self.c.write_leader_optime('')
@@ -126,3 +130,11 @@ class TestConsul(unittest.TestCase):
         self.c.watch(1)
         with patch.object(consul.Consul.KV, 'get', Mock(side_effect=ConsulException)):
             self.c.watch(1)
+
+    @patch.object(consul.Consul.Session, 'destroy', Mock(side_effect=ConsulException))
+    def test_set_ttl(self):
+        self.c.set_ttl(20)
+        self.assertTrue(self.c.watch(1))
+
+    def test_set_retry_timeout(self):
+        self.c.set_retry_timeout(10)

tests/test_ctl.py

@@ -1,18 +1,18 @@
 import etcd
 import os
-import pytest
-import requests.exceptions
+import requests
+import sys
 import unittest
 
 from click.testing import CliRunner
 from mock import patch, Mock
-from patroni.ctl import ctl, members, store_config, load_config, output_members, post_patroni, get_dcs, parse_dcs, \
+from patroni.ctl import ctl, members, store_config, load_config, output_members, request_patroni, get_dcs, parse_dcs, \
     wait_for_leader, get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException
-from patroni.etcd import Etcd, Client
+from patroni.dcs.etcd import Client
 from psycopg2 import OperationalError
 from test_etcd import etcd_read, requests_get, socket_getaddrinfo, MockResponse
 from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \
-    get_cluster_initialized_with_only_leader
+    get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader
 from test_postgresql import MockConnect, psycopg2_connect
 
 CONFIG_FILE_PATH = './test-ctl.yaml'
@@ -20,50 +20,37 @@ CONFIG_FILE_PATH = './test-ctl.yaml'
 
 def test_rw_config():
     runner = CliRunner()
-    config = {'a': 'b'}
     with runner.isolated_filesystem():
-        store_config(config, CONFIG_FILE_PATH + '/dummy')
+        store_config({'etcd': {'host': 'localhost:2379'}}, CONFIG_FILE_PATH + '/dummy')
+        sys.argv = ['patronictl.py', '']
+        load_config(CONFIG_FILE_PATH + '/dummy', None)
+        load_config(CONFIG_FILE_PATH + '/dummy', '0.0.0.0')
         os.remove(CONFIG_FILE_PATH + '/dummy')
         os.rmdir(CONFIG_FILE_PATH)
 
-        with pytest.raises(Exception):
-            result = load_config(CONFIG_FILE_PATH, None)
-            assert 'Could not load configuration file' in result.output
 
-        os.mkdir(CONFIG_FILE_PATH)
-        with pytest.raises(Exception):
-            store_config(config, CONFIG_FILE_PATH)
-
-        os.rmdir(CONFIG_FILE_PATH)
-
-    store_config(config, CONFIG_FILE_PATH)
-    load_config(CONFIG_FILE_PATH, None)
-    load_config(CONFIG_FILE_PATH, '0.0.0.0')
-
-    store_config({'dcs_api': None}, CONFIG_FILE_PATH)
-    load_config(CONFIG_FILE_PATH, None)
-
-
-@patch('patroni.ctl.load_config', Mock(return_value={'etcd': {'host': 'localhost:4001'}}))
+@patch('patroni.ctl.load_config',
+       Mock(return_value={'postgresql': {'data_dir': '.', 'parameters': {}, 'retry_timeout': 5},
+                          'restapi': {'auth': 'u:p', 'listen': ''}, 'etcd': {'host': 'localhost:4001'}}))
 class TestCtl(unittest.TestCase):
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
     def setUp(self):
-        self.runner = CliRunner()
         with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
-            self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'})
+            self.runner = CliRunner()
+            self.e = get_dcs({'etcd': {'ttl': 30, 'host': 'ok:2379', 'retry_timeout': 10}}, 'foo')
 
     @patch('psycopg2.connect', psycopg2_connect)
     def test_get_cursor(self):
-        self.assertIsNone(get_cursor(get_cluster_initialized_without_leader(), role='master'))
+        self.assertIsNone(get_cursor(get_cluster_initialized_without_leader(), {}, role='master'))
 
-        self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), role='master'))
+        self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), {}, role='master'))
 
         # MockCursor returns pg_is_in_recovery as false
-        self.assertIsNone(get_cursor(get_cluster_initialized_with_leader(), role='replica'))
+        self.assertIsNone(get_cursor(get_cluster_initialized_with_leader(), {}, role='replica'))
 
-        self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), role='any'))
+        self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), {'database': 'foo'}, role='any'))
 
     def test_parse_dcs(self):
         assert parse_dcs(None) is None
@@ -71,7 +58,7 @@ class TestCtl(unittest.TestCase):
         assert parse_dcs('') == {'etcd': {'host': 'localhost:4001'}}
         assert parse_dcs('localhost:8500') == {'consul': {'host': 'localhost:8500'}}
         assert parse_dcs('zookeeper://localhost') == {'zookeeper': {'hosts': ['localhost:2181']}}
-        assert parse_dcs('exhibitor://dummy') == {'zookeeper': {'exhibitor': {'hosts': ['dummy'], 'port': 8181}}}
+        assert parse_dcs('exhibitor://dummy') == {'exhibitor': {'hosts': ['dummy'], 'port': 8181}}
         assert parse_dcs('consul://localhost') == {'consul': {'host': 'localhost:8500'}}
         self.assertRaises(PatroniCtlException, parse_dcs, 'invalid://test')
 
@@ -81,29 +68,35 @@ class TestCtl(unittest.TestCase):
         self.assertIsNone(output_members(cluster, name='abc', fmt='json'))
         self.assertIsNone(output_members(cluster, name='abc', fmt='tsv'))
 
-    @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
-    @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
-    @patch('patroni.ctl.post_patroni', Mock(return_value=MockResponse()))
-    def test_failover(self):
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n\ny''')
+    @patch('patroni.ctl.get_dcs')
+    @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
+    def test_failover(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
         assert 'leader' in result.output
 
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n2100-01-01T12:23:00\ny''')
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2100-01-01T12:23:00\ny')
         assert result.exit_code == 0
 
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n2030-01-01T12:23:00\ny''')
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n2030-01-01T12:23:00\ny')
         assert result.exit_code == 0
 
+        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+            result = self.runner.invoke(ctl,
+                                        ['failover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00+01:00'])
+            assert result.exit_code == 1
+
         # Aborting failover,as we anser NO to the confirmation
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n\nN''')
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\nN')
         assert result.exit_code == 1
 
         # Target and source are equal
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nleader\n\ny''')
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nleader\n\ny')
         assert result.exit_code == 1
 
         # Reality is not part of this cluster
-        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nReality\n\ny''')
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nReality\n\ny')
         assert result.exit_code == 1
 
         result = self.runner.invoke(ctl, ['failover', 'dummy', '--force'])
@@ -124,145 +117,199 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['failover', 'dummy'], input='dummy')
         assert result.exit_code == 1
 
-        with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_only_leader())):
-            # No members available
-            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n\ny''')
-            assert result.exit_code == 1
-
-        with patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_without_leader())):
-            # No master available
-            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n\ny''')
-            assert result.exit_code == 1
-
-        with patch('patroni.ctl.post_patroni', Mock(side_effect=Exception)):
+        with patch('patroni.ctl.request_patroni', Mock(side_effect=Exception)):
             # Non-responding patroni
-            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n\ny''')
+            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
             assert 'falling back to DCS' in result.output
 
-        with patch('patroni.ctl.post_patroni') as mocked:
+        with patch('patroni.ctl.request_patroni') as mocked:
             mocked.return_value.status_code = 500
-            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='''leader\nother\n\ny''')
+            result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
             assert 'Failover failed' in result.output
 
+        # No members available
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
+        assert result.exit_code == 1
+
+        # No master available
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader
+        result = self.runner.invoke(ctl, ['failover', 'dummy'], input='leader\nother\n\ny')
+        assert result.exit_code == 1
+
     def test_get_dcs(self):
         self.assertRaises(PatroniCtlException, get_dcs, {'dummy': {}}, 'dummy')
-        with patch('patroni.Patroni.get_dcs', Mock(return_value=self.e)):
-            assert get_dcs({'etcd': {'host': 'none'}}, 'dummy').client_path('') == '/service/test/'
 
     @patch('psycopg2.connect', psycopg2_connect)
     @patch('patroni.ctl.query_member', Mock(return_value=([['mock column']], None)))
+    @patch('patroni.ctl.get_dcs')
     @patch.object(etcd.Client, 'read', etcd_read)
-    def test_query(self):
-        with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)):
+    def test_query(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        # Mutually exclusive
+        result = self.runner.invoke(ctl, ['query', 'alpha', '--member', 'abc', '--role', 'master'])
+        assert result.exit_code == 1
+
+        with self.runner.isolated_filesystem():
+            with open('dummy', 'w') as dummy_file:
+                dummy_file.write('SELECT 1')
+
             # Mutually exclusive
-            result = self.runner.invoke(ctl, ['query', 'alpha', '--member', 'abc', '--role', 'master'])
+            result = self.runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy', '--command', 'dummy'])
             assert result.exit_code == 1
 
-            with self.runner.isolated_filesystem():
-                with open('dummy', 'w') as dummy_file:
-                    dummy_file.write('SELECT 1')
+            result = self.runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy'])
+            assert result.exit_code == 0
 
-                # Mutually exclusive
-                result = self.runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy', '--command', 'dummy'])
-                assert result.exit_code == 1
+            os.remove('dummy')
 
-                result = self.runner.invoke(ctl, ['query', 'alpha', '--file', 'dummy'])
-                assert result.exit_code == 0
+        result = self.runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1'])
+        assert 'mock column' in result.output
 
-                os.remove('dummy')
+        # --command or --file is mandatory
+        result = self.runner.invoke(ctl, ['query', 'alpha'])
+        assert result.exit_code == 1
 
-            result = self.runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1'])
-            assert 'mock column' in result.output
-
-            # --command or --file is mandatory
-            result = self.runner.invoke(ctl, ['query', 'alpha'])
-            assert result.exit_code == 1
-
-            result = self.runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1', '--username', 'root',
-                                              '--password', '--dbname', 'postgres'], input='ab\nab')
-            assert 'mock column' in result.output
+        result = self.runner.invoke(ctl, ['query', 'alpha', '--command', 'SELECT 1', '--username', 'root',
+                                          '--password', '--dbname', 'postgres'], input='ab\nab')
+        assert 'mock column' in result.output
 
     def test_query_member(self):
         with patch('patroni.ctl.get_cursor', Mock(return_value=MockConnect().cursor())):
-            rows = query_member(None, None, None, 'master', 'SELECT pg_is_in_recovery()')
+            rows = query_member(None, None, None, 'master', 'SELECT pg_is_in_recovery()', {})
             self.assertTrue('False' in str(rows))
 
-            rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
+            rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()', {})
             self.assertEquals(rows, (None, None))
 
             with patch('test_postgresql.MockCursor.execute', Mock(side_effect=OperationalError('bla'))):
-                rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
+                rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()', {})
 
         with patch('patroni.ctl.get_cursor', Mock(return_value=None)):
-            rows = query_member(None, None, None, None, 'SELECT pg_is_in_recovery()')
+            rows = query_member(None, None, None, None, 'SELECT pg_is_in_recovery()', {})
             self.assertTrue('No connection to' in str(rows))
 
-            rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
+            rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()', {})
             self.assertTrue('No connection to' in str(rows))
 
         with patch('patroni.ctl.get_cursor', Mock(side_effect=OperationalError('bla'))):
-            rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()')
+            rows = query_member(None, None, None, 'replica', 'SELECT pg_is_in_recovery()', {})
 
-    @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
-    def test_dsn(self):
-        with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)):
-            result = self.runner.invoke(ctl, ['dsn', 'alpha'])
-            assert 'host=127.0.0.1 port=5435' in result.output
+    @patch('patroni.ctl.get_dcs')
+    def test_dsn(self, mock_get_dcs):
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        result = self.runner.invoke(ctl, ['dsn', 'alpha'])
+        assert 'host=127.0.0.1 port=5435' in result.output
 
-            # Mutually exclusive options
-            result = self.runner.invoke(ctl, ['dsn', 'alpha', '--role', 'master', '--member', 'dummy'])
-            assert result.exit_code == 1
+        # Mutually exclusive options
+        result = self.runner.invoke(ctl, ['dsn', 'alpha', '--role', 'master', '--member', 'dummy'])
+        assert result.exit_code == 1
 
-            # Non-existing member
-            result = self.runner.invoke(ctl, ['dsn', 'alpha', '--member', 'dummy'])
-            assert result.exit_code == 1
+        # Non-existing member
+        result = self.runner.invoke(ctl, ['dsn', 'alpha', '--member', 'dummy'])
+        assert result.exit_code == 1
 
-    @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
-    @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
     @patch('requests.post', requests_get)
-    def test_restart_reinit(self):
-        result = self.runner.invoke(ctl, ['restart', 'alpha'], input='y')
-        assert 'restart failed for' in result.output
+    @patch('patroni.ctl.get_dcs')
+    def test_restart_reinit(self, mock_get_dcs):
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        result = self.runner.invoke(ctl, ['restart', 'alpha'], input='y\n\nnow')
+        assert 'Failed: restart for' in result.output
         assert result.exit_code == 0
 
         result = self.runner.invoke(ctl, ['reinit', 'alpha'], input='y')
         assert result.exit_code == 1
 
+        # successful reinit
+        result = self.runner.invoke(ctl, ['reinit', 'alpha', 'other'], input='y')
+        assert result.exit_code == 0
+
         # Aborted restart
         result = self.runner.invoke(ctl, ['restart', 'alpha'], input='N')
         assert result.exit_code == 1
 
+        result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force'])
+        assert result.exit_code == 0
+
         # Not a member
         result = self.runner.invoke(ctl, ['restart', 'alpha', 'dummy', '--any'], input='y')
         assert result.exit_code == 1
 
+        # Wrong pg version
+        result = self.runner.invoke(ctl, ['restart', 'alpha', '--any', '--pg-version', '9.1'], input='y')
+        assert 'Error: PostgreSQL version' in result.output
+        assert result.exit_code == 1
+
+        with patch('requests.delete', Mock(return_value=MockResponse(500))):
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force',
+                                              '--scheduled', '2300-10-01T14:30'])
+            assert 'Failed: flush scheduled restart' in result.output
+
+        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+            result = self.runner.invoke(ctl,
+                                        ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
+            assert result.exit_code == 1
+
         with patch('requests.post', Mock(return_value=MockResponse())):
-            result = self.runner.invoke(ctl, ['restart', 'alpha'], input='y')
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0.0',
+                                        '--scheduled', '2300-10-01T14:30'], input='y')
             assert result.exit_code == 0
 
-    @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
-    @patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
-    def test_remove(self):
-        with patch('patroni.ctl.get_dcs', Mock(return_value=self.e)):
-            result = self.runner.invoke(ctl, ['remove', 'alpha'], input='alpha\nslave')
-            assert 'Please confirm' in result.output
-            assert 'You are about to remove all' in result.output
-            # Not typing an exact confirmation
-            assert result.exit_code == 1
-
-            # master specified does not match master of cluster
-            result = self.runner.invoke(ctl, ['remove', 'alpha'], input='''alpha\nYes I am aware\nslave''')
-            assert result.exit_code == 1
-
-            # cluster specified on cmdline does not match verification prompt
-            result = self.runner.invoke(ctl, ['remove', 'alpha'], input='beta\nleader')
-            assert result.exit_code == 1
-
-            result = self.runner.invoke(ctl, ['remove', 'alpha'], input='''alpha\nYes I am aware\nleader''')
+        with patch('requests.post', Mock(return_value=MockResponse(204))):
+            # get restart with the non-200 return code
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0.0',
+                                        '--scheduled', '2300-10-01T14:30'], input='y')
             assert result.exit_code == 0
 
-    @patch('patroni.etcd.Etcd.watch', Mock(return_value=None))
-    @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
+        # force restart with restart already present
+        with patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse(204))):
+            result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force',
+                                              '--scheduled', '2300-10-01T14:30'])
+            assert result.exit_code == 0
+
+        with patch('requests.post', Mock(return_value=MockResponse(202))):
+            # get restart with the non-200 return code
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(
+                ctl, ['restart', 'alpha', '--pg-version', '99.0.0', '--scheduled', '2300-10-01T14:30'], input='y'
+            )
+            assert 'Success: restart scheduled' in result.output
+            assert result.exit_code == 0
+
+        with patch('requests.post', Mock(return_value=MockResponse(409))):
+            # get restart with the non-200 return code
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(
+                ctl, ['restart', 'alpha', '--pg-version', '99.0.0', '--scheduled', '2300-10-01T14:30'], input='y'
+            )
+            assert 'Failed: another restart is already' in result.output
+            assert result.exit_code == 0
+
+    @patch('patroni.ctl.get_dcs')
+    def test_remove(self, mock_get_dcs):
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        result = self.runner.invoke(ctl, ['remove', 'alpha'], input='alpha\nslave')
+        assert 'Please confirm' in result.output
+        assert 'You are about to remove all' in result.output
+        # Not typing an exact confirmation
+        assert result.exit_code == 1
+
+        # master specified does not match master of cluster
+        result = self.runner.invoke(ctl, ['remove', 'alpha'], input='alpha\nYes I am aware\nslave')
+        assert result.exit_code == 1
+
+        # cluster specified on cmdline does not match verification prompt
+        result = self.runner.invoke(ctl, ['remove', 'alpha'], input='beta\nleader')
+        assert result.exit_code == 1
+
+        result = self.runner.invoke(ctl, ['remove', 'alpha'], input='alpha\nYes I am aware\nleader')
+        assert result.exit_code == 0
+
+    @patch('patroni.dcs.AbstractDCS.watch', Mock(return_value=None))
+    @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
     def test_wait_for_leader(self):
         self.assertRaises(PatroniCtlException, wait_for_leader, self.e, 0)
 
@@ -270,9 +317,9 @@ class TestCtl(unittest.TestCase):
         assert cluster.leader.member.name == 'leader'
 
     @patch('requests.post', Mock(side_effect=requests.exceptions.ConnectionError('foo')))
-    def test_post_patroni(self):
+    def test_request_patroni(self):
         member = get_cluster_initialized_with_leader().leader.member
-        self.assertRaises(requests.exceptions.ConnectionError, post_patroni, member, 'dummy', {})
+        self.assertRaises(requests.exceptions.ConnectionError, request_patroni, member, 'post', 'dummy', {})
 
     def test_ctl(self):
         self.runner.invoke(ctl, ['list'])
@@ -299,9 +346,9 @@ class TestCtl(unittest.TestCase):
 
         self.assertEquals(len(list(get_all_members(get_cluster_initialized_without_leader(), role='replica'))), 2)
 
-    @patch('patroni.etcd.Etcd.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
-    @patch('patroni.etcd.Etcd.get_etcd_client', Mock(return_value=None))
-    def test_members(self):
+    @patch('patroni.ctl.get_dcs')
+    def test_members(self, mock_get_dcs):
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
         result = self.runner.invoke(members, ['alpha'])
         assert '127.0.0.1' in result.output
         assert result.exit_code == 0
@@ -309,3 +356,87 @@ class TestCtl(unittest.TestCase):
     def test_configure(self):
         result = self.runner.invoke(configure, ['--dcs', 'abc', '-c', 'dummy', '-n', 'bla'])
         assert result.exit_code == 0
+
+    @patch('patroni.ctl.get_dcs')
+    def test_scaffold(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_not_initialized_without_leader
+        mock_get_dcs.return_value.initialize = Mock(return_value=True)
+        mock_get_dcs.return_value.touch_member = Mock(return_value=True)
+        mock_get_dcs.return_value.attempt_to_acquire_leader = Mock(return_value=True)
+
+        with patch.object(self.e, 'initialize', return_value=False):
+            result = self.runner.invoke(ctl, ['scaffold', 'alpha'])
+            assert result.exception
+
+        with patch.object(mock_get_dcs.return_value, 'touch_member', Mock(return_value=False)):
+            result = self.runner.invoke(ctl, ['scaffold', 'alpha'])
+            assert result.exception
+
+        result = self.runner.invoke(ctl, ['scaffold', 'alpha'])
+        assert result.exit_code == 0
+
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        result = self.runner.invoke(ctl, ['scaffold', 'alpha'])
+        assert result.exception
+
+    @patch('patroni.ctl.get_dcs')
+    def test_list_extended(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+
+        result = self.runner.invoke(ctl, ['list', 'dummy', '--extended'])
+        assert '2100' in result.output
+        assert 'Scheduled restart' in result.output
+
+    @patch('patroni.ctl.get_dcs')
+    @patch('requests.delete', Mock(return_value=MockResponse()))
+    def test_flush(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+
+        result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '-r', 'master'], input='y')
+        assert 'No scheduled restart' in result.output
+
+        result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '--force'])
+        assert 'Success: flush scheduled restart' in result.output
+        with patch.object(requests, 'delete', return_value=MockResponse(404)):
+            result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '--force'])
+            assert 'Failed: flush scheduled restart' in result.output
+
+    @patch('patroni.ctl.get_dcs')
+    def test_pause_cluster(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+
+        with patch('requests.patch', Mock(return_value=MockResponse(200))):
+            result = self.runner.invoke(ctl, ['pause', 'dummy'])
+            assert 'Success' in result.output
+
+        with patch('requests.patch', Mock(return_value=MockResponse(500))):
+            result = self.runner.invoke(ctl, ['pause', 'dummy'])
+            assert 'Failed' in result.output
+
+        with patch('requests.patch', Mock(return_value=MockResponse(200))),\
+                patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+            result = self.runner.invoke(ctl, ['pause', 'dummy'])
+            assert 'Cluster is already paused' in result.output
+
+    @patch('patroni.ctl.get_dcs')
+    def test_resume_cluster(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+
+        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+            with patch('requests.patch', Mock(return_value=MockResponse(200))):
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Success' in result.output
+
+            with patch('requests.patch', Mock(return_value=MockResponse(500))):
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Failed' in result.output
+
+            with patch('requests.patch', Mock(return_value=MockResponse(200))),\
+                    patch('patroni.dcs.Cluster.is_paused', Mock(return_value=False)):
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Cluster is not paused' in result.output

tests/test_etcd.py

@@ -6,16 +6,15 @@ import unittest
 
 from dns.exception import DNSException
 from mock import Mock, patch
-from patroni.dcs import Cluster, AbstractDCS
-from patroni.etcd import Client, Etcd, EtcdError
+from patroni.dcs.etcd import AbstractDCS, Client, Cluster, Etcd, EtcdError
 from patroni.exceptions import DCSError
 from urllib3.exceptions import ReadTimeoutError
 
 
 class MockResponse(object):
 
-    def __init__(self):
-        self.status_code = 200
+    def __init__(self, status_code=200):
+        self.status_code = status_code
         self.content = '{}'
         self.ok = True
         self.text = ''
@@ -79,6 +78,8 @@ def etcd_read(self, key, **kwargs):
         raise etcd.EtcdKeyNotFound
 
     response = {"action": "get", "node": {"key": "/service/batman5", "dir": True, "nodes": [
+                {"key": "/service/batman5/config", "value": '{"foo": "bar"}',
+                 "modifiedIndex": 1582, "createdIndex": 1582},
                 {"key": "/service/batman5/failover", "value": "",
                  "modifiedIndex": 1582, "createdIndex": 1582},
                 {"key": "/service/batman5/initialize", "value": "postgresql0",
@@ -102,7 +103,9 @@ def etcd_read(self, key, **kwargs):
                      "expiration": "2015-05-15T09:11:09.611860899Z", "ttl": 30,
                      "modifiedIndex": 20730, "createdIndex": 20730}],
                  "modifiedIndex": 1581, "createdIndex": 1581}], "modifiedIndex": 1581, "createdIndex": 1581}}
-    return etcd.EtcdResult(**response)
+    result = etcd.EtcdResult(**response)
+    result.etcd_index = 0
+    return result
 
 
 class SleepException(Exception):
@@ -131,6 +134,10 @@ def socket_getaddrinfo(*args):
 def http_request(method, url, **kwargs):
     if url == 'http://localhost:2379/timeout':
         raise ReadTimeoutError(None, None, None)
+    if url == 'http://localhost:2379/v2/machines':
+        ret = MockResponse()
+        ret.content = 'http://localhost:2379,http://localhost:4001'
+        return ret
     if url == 'http://localhost:2379/':
         return MockResponse()
     raise socket.error
@@ -144,26 +151,39 @@ class TestClient(unittest.TestCase):
     @patch('dns.resolver.query', dns_query)
     @patch('requests.get', requests_get)
     def setUp(self):
-        with patch.object(etcd.Client, 'machines') as mock_machines:
+        with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
-            self.client = Client({'discovery_srv': 'test'})
+            self.client = Client({'discovery_srv': 'test', 'retry_timeout': 3})
             self.client.http.request = http_request
             self.client.http.request_encode_body = http_request
 
-    def test_api_execute(self):
+    def test_machines(self):
         self.client._base_uri = 'http://localhost:4001'
         self.client._machines_cache = ['http://localhost:2379']
-        self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
-        self.client._update_machines_cache = False
-        self.client.api_execute('/', 'POST', timeout=0)
-        self.client._update_machines_cache = False
+        self.assertIsNotNone(self.client.machines)
         self.client._base_uri = 'http://localhost:4001'
         self.client._machines_cache = []
-        self.assertRaises(etcd.EtcdConnectionFailed, self.client.api_execute, '/', 'GET')
-        self.assertTrue(self.client._update_machines_cache)
-        self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'GET')
-        self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '')
+        self.assertIsNotNone(self.client.machines)
+        self.client._update_machines_cache = True
+        machines = None
+        try:
+            machines = self.client.machines
+            self.assertFail()
+        except Exception:
+            self.assertIsNone(machines)
+
+    @patch.object(Client, 'machines')
+    def test_api_execute(self, mock_machines):
+        mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
         self.assertRaises(ValueError, self.client.api_execute, '', '')
+        self.client._base_uri = 'http://localhost:4001'
+        self.client._machines_cache = ['http://localhost:2379']
+        self.client.api_execute('/', 'POST', timeout=0)
+        self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
+        self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '')
+        self.client._update_machines_cache = True
+        with patch.object(Client, '_load_machines_cache', Mock(side_effect=etcd.EtcdException)):
+            self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'GET')
 
     def test_get_srv_record(self):
         self.assertEquals(self.client.get_srv_record('blabla'), [])
@@ -176,7 +196,9 @@ class TestClient(unittest.TestCase):
     def test__get_machines_cache_from_dns(self):
         self.client._get_machines_cache_from_dns('error:2379')
 
-    def test__load_machines_cache(self):
+    @patch.object(Client, 'machines')
+    def test__load_machines_cache(self, mock_machines):
+        mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
         self.client._config = {}
         self.assertRaises(Exception, self.client._load_machines_cache)
         self.client._config = {'discovery_srv': 'blabla'}
@@ -192,17 +214,19 @@ class TestEtcd(unittest.TestCase):
     def setUp(self):
         with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
-            self.etcd = Etcd('foo', {'namespace': '/patroni/', 'ttl': 30, 'host': 'localhost:2379', 'scope': 'test'})
+            self.etcd = Etcd({'namespace': '/patroni/', 'ttl': 30, 'retry_timeout': 10,
+                              'host': 'localhost:2379', 'scope': 'test', 'name': 'foo'})
 
     def test_base_path(self):
         self.assertEquals(self.etcd._base_path, '/patroni/test')
 
     @patch('dns.resolver.query', dns_query)
     def test_get_etcd_client(self):
-        with patch.object(etcd.Client, 'machines') as mock_machines:
+        with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(side_effect=etcd.EtcdException)
-            with patch('time.sleep', Mock(side_effect=SleepException())):
-                self.assertRaises(SleepException, self.etcd.get_etcd_client, {'discovery_srv': 'test'})
+            with patch('time.sleep', Mock(side_effect=SleepException)):
+                self.assertRaises(SleepException, self.etcd.get_etcd_client,
+                                  {'discovery_srv': 'test', 'retry_timeout': 10})
 
     def test_get_cluster(self):
         self.assertIsInstance(self.etcd.get_cluster(), Cluster)
@@ -240,6 +264,9 @@ class TestEtcd(unittest.TestCase):
     def test_delete_leader(self):
         self.assertFalse(self.etcd.delete_leader())
 
+    def test_delete_cluster(self):
+        self.assertFalse(self.etcd.delete_cluster())
+
     @patch.object(etcd.Client, 'watch', etcd_watch)
     def test_watch(self):
         self.etcd.watch(0)
@@ -249,6 +276,10 @@ class TestEtcd(unittest.TestCase):
         with patch.object(AbstractDCS, 'watch', Mock()):
             self.etcd.watch(9.5)
 
-    @patch('patroni.etcd.Etcd.retry', Mock(side_effect=AttributeError("foo")))
     def test_other_exceptions(self):
+        self.etcd.retry = Mock(side_effect=AttributeError('foo'))
         self.assertRaises(EtcdError, self.etcd.cancel_initialization)
+
+    def test_set_ttl(self):
+        self.etcd.set_ttl(20)
+        self.assertTrue(self.etcd.watch(1))

tests/test_exhibitor.py

@@ -0,0 +1,31 @@
+import unittest
+
+from mock import Mock, patch
+from patroni.dcs.exhibitor import ExhibitorEnsembleProvider, Exhibitor
+from patroni.dcs.zookeeper import ZooKeeperError
+from test_etcd import SleepException, requests_get
+from test_zookeeper import MockKazooClient
+
+
+@patch('requests.get', requests_get)
+@patch('time.sleep', Mock(side_effect=SleepException))
+class TestExhibitorEnsembleProvider(unittest.TestCase):
+
+    def test_init(self):
+        self.assertRaises(SleepException, ExhibitorEnsembleProvider, ['localhost'], 8181)
+
+    def test_poll(self):
+        self.assertFalse(ExhibitorEnsembleProvider(['exhibitor'], 8181).poll())
+
+
+class TestExhibitor(unittest.TestCase):
+
+    @patch('requests.get', requests_get)
+    @patch('patroni.dcs.zookeeper.KazooClient', MockKazooClient)
+    def setUp(self):
+        self.e = Exhibitor({'hosts': ['localhost', 'exhibitor'], 'port': 8181, 'scope': 'test',
+                            'name': 'foo', 'ttl': 30, 'retry_timeout': 10})
+
+    @patch.object(ExhibitorEnsembleProvider, 'poll', Mock(return_value=True))
+    def test_get_cluster(self):
+        self.assertRaises(ZooKeeperError, self.e.get_cluster)

tests/test_ha.py

@@ -1,11 +1,13 @@
-import etcd
-import unittest
 import datetime
+import etcd
+import os
 import pytz
+import unittest
 
-from mock import Mock, MagicMock, patch
-from patroni.dcs import Cluster, Failover, Leader, Member
-from patroni.etcd import Client, Etcd
+from mock import Mock, MagicMock, PropertyMock, patch
+from patroni.config import Config
+from patroni.dcs import Cluster, Failover, Leader, Member, get_dcs
+from patroni.dcs.etcd import Client
 from patroni.exceptions import DCSError, PostgresException
 from patroni.ha import Ha
 from patroni.postgresql import Postgresql
@@ -21,7 +23,7 @@ def false(*args, **kwargs):
 
 
 def get_cluster(initialize, leader, members, failover):
-    return Cluster(initialize, leader, 10, members, failover)
+    return Cluster(initialize, None, leader, 10, members, failover)
 
 
 def get_cluster_not_initialized_without_leader():
@@ -33,7 +35,10 @@ def get_cluster_initialized_without_leader(leader=False, failover=None):
                                   'api_url': 'http://127.0.0.1:8008/patroni', 'xlog_location': 4})
     l = Leader(0, 0, m1) if leader else None
     m2 = Member(0, 'other', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
-                                 'api_url': 'http://127.0.0.1:8011/patroni', 'tags': {'clonefrom': True}})
+                                 'api_url': 'http://127.0.0.1:8011/patroni',
+                                 'tags': {'clonefrom': True},
+                                 'scheduled_restart': {'schedule': "2100-01-01 10:53:07.560445+00:00",
+                                                       'postgres_version': '99.0.0'}})
     return get_cluster(True, l, [m1, m2], failover)
 
 
@@ -45,22 +50,47 @@ def get_cluster_initialized_with_only_leader(failover=None):
     l = get_cluster_initialized_without_leader(leader=True, failover=failover).leader
     return get_cluster(True, l, [l], failover)
 
+future_restart_time = datetime.datetime.now(pytz.utc) + datetime.timedelta(days=5)
+postmaster_start_time = datetime.datetime.now(pytz.utc)
+
 
 class MockPatroni(object):
 
     def __init__(self, p, d):
+        os.environ[Config.PATRONI_CONFIG_VARIABLE] = """
+restapi:
+  listen: 0.0.0.0:8008
+bootstrap:
+  users:
+    replicator:
+      password: rep-pass
+      options:
+        - replication
+postgresql:
+  name: foo
+  data_dir: data/postgresql0
+  pg_rewind:
+    username: postgres
+    password: postgres
+zookeeper:
+  exhibitor:
+    hosts: [localhost]
+    port: 8181
+"""
+        self.config = Config()
         self.postgresql = p
         self.dcs = d
         self.api = Mock()
         self.tags = {'foo': 'bar'}
         self.nofailover = None
-        self.nap_time = 10
         self.replicatefrom = None
         self.api.connection_string = 'http://127.0.0.1:8008'
         self.clonefrom = None
+        self.scheduled_restart = {'schedule': future_restart_time,
+                                  'postmaster_start_time': str(postmaster_start_time)}
 
 
-def run_async(func, args=()):
+def run_async(self, func, args=()):
     return func(*args) if args else func()
 
 
@@ -79,6 +109,8 @@ def run_async(func, args=()):
 @patch.object(etcd.Client, 'write', etcd_write)
 @patch.object(etcd.Client, 'read', etcd_read)
 @patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
+@patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=False))
+@patch('patroni.async_executor.AsyncExecutor.run_async', run_async)
 @patch('subprocess.call', Mock(return_value=0))
 class TestHa(unittest.TestCase):
 
@@ -88,15 +120,19 @@ class TestHa(unittest.TestCase):
         with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
             self.p = Postgresql({'name': 'postgresql0', 'scope': 'dummy', 'listen': '127.0.0.1:5432',
-                                 'data_dir': 'data/postgresql0', 'superuser': {}, 'admin': {},
-                                 'replication': {'username': '', 'password': '', 'network': ''}})
+                                 'data_dir': 'data/postgresql0', 'retry_timeout': 10,
+                                 'authentication': {'superuser': {'username': 'foo', 'password': 'bar'},
+                                                    'replication': {'username': '', 'password': ''}},
+                                 'parameters': {'wal_level': 'hot_standby', 'max_replication_slots': 5, 'foo': 'bar',
+                                                'hot_standby': 'on', 'max_wal_senders': 5, 'wal_keep_segments': 8}})
             self.p.set_state('running')
             self.p.set_role('replica')
+            self.p.postmaster_start_time = MagicMock(return_value=str(postmaster_start_time))
             self.p.check_replication_lag = true
             self.p.can_create_replica_without_replication_connection = MagicMock(return_value=False)
-            self.e = Etcd('foo', {'ttl': 30, 'host': 'ok:2379', 'scope': 'test'})
+            self.e = get_dcs({'etcd': {'ttl': 30, 'host': 'ok:2379', 'scope': 'test',
+                                       'name': 'foo', 'retry_timeout': 10}})
             self.ha = Ha(MockPatroni(self.p, self.e))
-            self.ha._async_executor.run_async = run_async
             self.ha.old_cluster = self.e.get_cluster()
             self.ha.cluster = get_cluster_not_initialized_without_leader()
             self.ha.load_cluster_from_dcs = Mock()
@@ -131,6 +167,9 @@ class TestHa(unittest.TestCase):
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.assertEquals(self.ha.run_cycle(), 'starting as readonly because i had the session lock')
 
+    def test_do_not_recover_in_pause(self):
+        pass
+
     @patch('sys.exit', return_value=1)
     @patch('patroni.ha.Ha.sysid_valid', MagicMock(return_value=True))
     def test_sysid_no_match(self, exit_mock):
@@ -199,6 +238,13 @@ class TestHa(unittest.TestCase):
         self.ha.patroni.replicatefrom = "foo"
         self.assertEquals(self.ha.run_cycle(), 'no action.  i am a secondary and i am following a leader')
 
+    def test_follow_in_pause(self):
+        self.ha.cluster.is_unlocked = false
+        self.ha.is_paused = true
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: continue to run as master without lock')
+        self.p.is_leader = false
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: no action')
+
     def test_no_etcd_connection_master_demote(self):
         self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
         self.assertEquals(self.ha.run_cycle(), 'demoted self because DCS is not accessible and i was a leader')
@@ -232,41 +278,40 @@ class TestHa(unittest.TestCase):
         self.assertRaises(PostgresException, self.ha.bootstrap)
 
     def test_reinitialize(self):
-        self.ha.schedule_reinitialize()
-        self.ha.schedule_reinitialize()
-        self.ha.run_cycle()
-        self.assertIsNone(self.ha._async_executor.scheduled_action)
+        self.assertIsNotNone(self.ha.reinitialize())
 
         self.ha.cluster = get_cluster_initialized_with_leader()
-        self.ha.has_lock = true
-        self.ha.schedule_reinitialize()
-        self.ha.run_cycle()
-        self.assertIsNone(self.ha._async_executor.scheduled_action)
+        self.assertIsNone(self.ha.reinitialize())
 
-        self.ha.has_lock = false
-        self.ha.schedule_reinitialize()
-        self.ha.run_cycle()
+        self.assertIsNotNone(self.ha.reinitialize())
+
+        self.ha.state_handler.name = self.ha.cluster.leader.name
+        self.assertIsNotNone(self.ha.reinitialize())
 
     def test_restart(self):
         self.assertEquals(self.ha.restart(), (True, 'restarted successfully'))
         self.p.restart = false
         self.assertEquals(self.ha.restart(), (False, 'restart failed'))
-        self.ha.schedule_reinitialize()
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.ha.reinitialize()
         self.assertEquals(self.ha.restart(), (False, 'reinitialize already in progress'))
+        with patch.object(self.ha, "restart_matches", return_value=False):
+            self.assertEquals(self.ha.restart({'foo': 'bar'}), (False, "restart conditions are not satisfied"))
 
     def test_restart_in_progress(self):
-        self.ha._async_executor.schedule('restart', True)
-        self.assertTrue(self.ha.restart_scheduled())
-        self.assertEquals(self.ha.run_cycle(), 'not healthy enough for leader race')
+        with patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=True)):
+            self.ha.restart(run_async=True)
+            self.assertTrue(self.ha.restart_scheduled())
+            self.assertEquals(self.ha.run_cycle(), 'not healthy enough for leader race')
 
-        self.ha.cluster = get_cluster_initialized_with_leader()
-        self.assertEquals(self.ha.run_cycle(), 'restart in progress')
+            self.ha.cluster = get_cluster_initialized_with_leader()
+            self.assertEquals(self.ha.run_cycle(), 'restart in progress')
 
-        self.ha.has_lock = true
-        self.assertEquals(self.ha.run_cycle(), 'updated leader lock during restart')
+            self.ha.has_lock = true
+            self.assertEquals(self.ha.run_cycle(), 'updated leader lock during restart')
 
-        self.ha.update_lock = false
-        self.assertEquals(self.ha.run_cycle(), 'failed to update leader lock during restart')
+            self.ha.update_lock = false
+            self.assertEquals(self.ha.run_cycle(), 'failed to update leader lock during restart')
 
     @patch('requests.get', requests_get)
     @patch('time.sleep', Mock())
@@ -309,6 +354,17 @@ class TestHa(unittest.TestCase):
         self.assertEquals('no action.  i am the leader with the lock', self.ha.run_cycle())
 
     @patch('requests.get', requests_get)
+    def test_manual_failover_from_leader_in_pause(self):
+        self.ha.has_lock = true
+        self.ha.is_paused = true
+        scheduled = datetime.datetime.now()
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
+        self.assertEquals('PAUSE: no action.  i am the leader with the lock', self.ha.run_cycle())
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, '', None))
+        self.assertEquals('PAUSE: no action.  i am the leader with the lock', self.ha.run_cycle())
+
+    @patch('requests.get', requests_get)
+    @patch('time.sleep', Mock())
     def test_manual_failover_process_no_leader(self):
         self.p.is_leader = false
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', self.p.name, None))
@@ -333,11 +389,27 @@ class TestHa(unittest.TestCase):
         self.ha.patroni.nofailover = True
         self.assertEquals(self.ha.run_cycle(), 'following a different leader because I am not allowed to promote')
 
+    @patch('time.sleep', Mock())
+    def test_manual_failover_process_no_leader_in_pause(self):
+        self.ha.is_paused = true
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', 'other', None))
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: continue to run as master without lock')
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', '', None))
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: continue to run as master without lock')
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', 'blabla', None))
+        self.assertEquals('PAUSE: acquired session lock as a leader', self.ha.run_cycle())
+        self.p.is_leader = false
+        self.p.set_role('replica')
+        self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, 'leader', self.p.name, None))
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: promoted self to leader by acquiring session lock')
+
     def test_is_healthiest_node(self):
         self.ha.state_handler.is_leader = false
         self.ha.patroni.nofailover = False
         self.ha.fetch_node_status = lambda e: (e, True, True, 0, {})
         self.assertTrue(self.ha.is_healthiest_node())
+        self.ha.is_paused = true
+        self.assertFalse(self.ha.is_healthiest_node())
 
     def test__is_healthiest_node(self):
         self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
@@ -369,3 +441,69 @@ class TestHa(unittest.TestCase):
         self.assertEqual(self.ha.post_recover(), 'failed to start postgres')
         self.p.is_running = true
         self.assertIsNone(self.ha.post_recover())
+
+    def test_schedule_future_restart(self):
+        self.ha.patroni.scheduled_restart = {}
+        # do the restart 2 times. The first one should succeed, the second one should fail
+        self.assertTrue(self.ha.schedule_future_restart({'schedule': future_restart_time}))
+        self.assertFalse(self.ha.schedule_future_restart({'schedule': future_restart_time}))
+
+    def test_delete_future_restarts(self):
+        self.ha.delete_future_restart()
+
+    def test_evaluate_scheduled_restart(self):
+        self.p.postmaster_start_time = Mock(return_value=str(postmaster_start_time))
+        # restart while the postmaster has been already restarted, fails
+        with patch.object(self.ha,
+                          'future_restart_scheduled',
+                          Mock(return_value={'postmaster_start_time':
+                                             str(postmaster_start_time - datetime.timedelta(days=1)),
+                                             'schedule': str(future_restart_time)})):
+            self.assertIsNone(self.ha.evaluate_scheduled_restart())
+        with patch.object(self.ha,
+                          'future_restart_scheduled',
+                          Mock(return_value={'postmaster_start_time': str(postmaster_start_time),
+                                             'schedule': str(future_restart_time)})):
+            with patch.object(self.ha,
+                              'should_run_scheduled_action', Mock(return_value=True)):
+                # restart in the future, ok
+                self.assertIsNotNone(self.ha.evaluate_scheduled_restart())
+                with patch.object(self.ha, 'restart', Mock(return_value=(False, "Test"))):
+                    # restart in the future, bit the actual restart failed
+                    self.assertIsNone(self.ha.evaluate_scheduled_restart())
+
+    def test_scheduled_restart(self):
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        with patch.object(self.ha, "evaluate_scheduled_restart", Mock(return_value="restart scheduled")):
+            self.assertEquals(self.ha.run_cycle(), "restart scheduled")
+
+    def test_restart_matches(self):
+        self.p._role = 'replica'
+        self.p.server_version = 90500
+        self.p._pending_restart = True
+        self.assertFalse(self.ha.restart_matches("master", "9.5.0", True))
+        self.assertFalse(self.ha.restart_matches("replica", "9.4.3", True))
+        self.p._pending_restart = False
+        self.assertFalse(self.ha.restart_matches("replica", "9.5.2", True))
+        self.assertTrue(self.ha.restart_matches("replica", "9.5.2", False))
+
+    def test_process_healthy_cluster_in_pause(self):
+        self.p.is_leader = false
+        self.ha.is_paused = true
+        self.p.name = 'leader'
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: removed leader lock because postgres is not running as master')
+        self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, '', self.p.name, None))
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: waiting to become master after promote...')
+
+    def test_postgres_unhealthy_in_pause(self):
+        self.ha.is_paused = true
+        self.p.is_healthy = false
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: postgres is not running')
+        self.ha.has_lock = true
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: removed leader lock because postgres is not running')
+
+    def test_no_etcd_connection_in_pause(self):
+        self.ha.is_paused = true
+        self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
+        self.assertEquals(self.ha.run_cycle(), 'PAUSE: DCS is not accessible')

tests/test_patroni.py

@@ -1,83 +1,89 @@
 import etcd
-import os
 import sys
 import time
 import unittest
-import yaml
 
 from mock import Mock, patch
 from patroni.api import RestApiServer
 from patroni.async_executor import AsyncExecutor
-from patroni.consul import Consul
-from patroni.etcd import Etcd
-from patroni import Patroni, PatroniException, main as _main
-from patroni.zookeeper import ZooKeeper
+from patroni.dcs.etcd import Client
+from patroni.exceptions import DCSError
+from patroni import Patroni, main as _main
 from six.moves import BaseHTTPServer
-from test_etcd import Client, SleepException, etcd_read, etcd_write
+from test_etcd import SleepException, etcd_read, etcd_write
 from test_postgresql import Postgresql, psycopg2_connect
-from test_zookeeper import MockKazooClient
+
+
+class MockFrozenImporter(object):
+
+    toc = set(['patroni.dcs.etcd'])
 
 
 @patch('time.sleep', Mock())
 @patch('subprocess.call', Mock(return_value=0))
 @patch('psycopg2.connect', psycopg2_connect)
 @patch.object(Postgresql, 'write_pg_hba', Mock())
+@patch.object(Postgresql, '_write_postgresql_conf', Mock())
 @patch.object(Postgresql, 'write_recovery_conf', Mock())
+@patch.object(Postgresql, 'is_running', Mock(return_value=True))
 @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
 @patch.object(AsyncExecutor, 'run', Mock())
 @patch.object(etcd.Client, 'write', etcd_write)
 @patch.object(etcd.Client, 'read', etcd_read)
 class TestPatroni(unittest.TestCase):
 
+    @patch('pkgutil.get_importer', Mock(return_value=MockFrozenImporter()))
+    @patch('sys.frozen', Mock(return_value=True), create=True)
+    @patch.object(etcd.Client, 'read', etcd_read)
     def setUp(self):
+        RestApiServer._BaseServer__is_shut_down = Mock()
+        RestApiServer._BaseServer__shutdown_request = True
+        RestApiServer.socket = 0
         with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
-            self.touched = False
-            self.init_cancelled = False
-            RestApiServer._BaseServer__is_shut_down = Mock()
-            RestApiServer._BaseServer__shutdown_request = True
-            RestApiServer.socket = 0
-            with open('postgres0.yml', 'r') as f:
-                config = yaml.load(f)
-                self.p = Patroni(config)
+            sys.argv = ['patroni.py', 'postgres0.yml']
+            self.p = Patroni()
 
-    @patch('patroni.zookeeper.KazooClient', MockKazooClient())
-    @patch.object(Consul, 'create_or_restore_session', Mock())
-    def test_get_dcs(self):
-        self.assertIsInstance(self.p.get_dcs('', {'zookeeper': {'scope': '', 'hosts': ''}}), ZooKeeper)
-        self.assertIsInstance(self.p.get_dcs('', {'consul': {'scope': '', 'hosts': '127.0.0.1:1'}}), Consul)
-        self.assertRaises(PatroniException, self.p.get_dcs, '', {})
+    @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(side_effect=[None, DCSError('foo'), None]))
+    def test_load_dynamic_configuration(self):
+        self.p.config._dynamic_configuration = {}
+        self.p.load_dynamic_configuration()
+        self.p.load_dynamic_configuration()
 
     @patch('time.sleep', Mock(side_effect=SleepException))
-    @patch.object(Etcd, 'delete_leader', Mock())
+    @patch.object(etcd.Client, 'delete', Mock())
     @patch.object(Client, 'machines')
     def test_patroni_main(self, mock_machines):
-        _main()
-        sys.argv = ['patroni.py', 'postgres0.yml']
+        with patch('subprocess.call', Mock(return_value=1)):
+            sys.argv = ['patroni.py', 'postgres0.yml']
 
-        mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
-        with patch.object(Patroni, 'run', Mock(side_effect=SleepException)):
-            self.assertRaises(SleepException, _main)
-        with patch.object(Patroni, 'run', Mock(side_effect=KeyboardInterrupt())):
-            _main()
-        sys.argv = ['patroni.py']
-        # read the content of the yaml configuration file into the environment variable
-        # in order to test how does patroni handle the configuration passed from the environment.
-        with open('postgres0.yml', 'r') as f:
-            os.environ[Patroni.PATRONI_CONFIG_VARIABLE] = f.read()
-        with patch.object(Patroni, 'run', Mock(side_effect=SleepException())):
-            self.assertRaises(SleepException, _main)
-        del os.environ[Patroni.PATRONI_CONFIG_VARIABLE]
+            mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
+            with patch.object(Patroni, 'run', Mock(side_effect=SleepException)):
+                self.assertRaises(SleepException, _main)
+            with patch.object(Patroni, 'run', Mock(side_effect=KeyboardInterrupt())):
+                with patch('patroni.ha.Ha.is_paused', Mock(return_value=True)):
+                    _main()
 
+    @patch('patroni.config.Config.save_cache', Mock())
+    @patch('patroni.config.Config.reload_local_configuration', Mock(return_value=True))
     def test_run(self):
+        self.p.sighup_handler()
         self.p.ha.dcs.watch = Mock(side_effect=SleepException)
         self.p.api.start = Mock()
+        self.p.config._dynamic_configuration = {}
         self.assertRaises(SleepException, self.p.run)
+        with patch('patroni.config.Config.set_dynamic_configuration', Mock(return_value=True)):
+            self.assertRaises(SleepException, self.p.run)
+        with patch('patroni.postgresql.Postgresql.data_directory_empty', Mock(return_value=False)):
+            self.assertRaises(SleepException, self.p.run)
+
+    def test_sigterm_handler(self):
+        self.assertRaises(SystemExit, self.p.sigterm_handler)
 
     def test_schedule_next_run(self):
         self.p.ha.dcs.watch = Mock(return_value=True)
         self.p.schedule_next_run()
-        self.p.next_run = time.time() - self.p.nap_time - 1
+        self.p.next_run = time.time() - self.p.dcs.loop_wait - 1
         self.p.schedule_next_run()
 
     def test_noloadbalance(self):
@@ -94,3 +100,8 @@ class TestPatroni(unittest.TestCase):
         self.assertIsNone(self.p.replicatefrom)
         self.p.tags['replicatefrom'] = 'foo'
         self.assertEqual(self.p.replicatefrom, 'foo')
+
+    def test_reload_config(self):
+        self.p.reload_config()
+        self.p.get_tags = Mock(side_effect=Exception)
+        self.p.reload_config()

tests/test_postgresql.py

@@ -32,21 +32,18 @@ class MockCursor(object):
             self.results = [(0,)]
         elif sql == 'SELECT pg_is_in_recovery()':
             self.results = [(False, )]
-        elif sql.startswith('SELECT to_char(pg_postmaster_start_time'):
-            self.results = [('', True, '', '', '', '', False)]
+        elif sql.startswith('WITH replication_info AS ('):
+            replication_info = '[{"application_name":"walreceiver","client_addr":"1.2.3.4",' +\
+                               '"state":"streaming","sync_state":"async","sync_priority":0}]'
+            self.results = [('', True, '', '', '', '', False, replication_info)]
+        elif sql.startswith('SELECT name, setting'):
+            self.results = [('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
+                            ('search_path', 'public', None, 'string', 'user'),
+                            ('port', '5433', None, 'integer', 'postmaster'),
+                            ('listen_addresses', '*', None, 'string', 'postmaster'),
+                            ('autovacuum', 'on', None, 'bool', 'sighup')]
         else:
-            self.results = [(
-                None,
-                None,
-                None,
-                None,
-                None,
-                None,
-                None,
-                None,
-                None,
-                None,
-            )]
+            self.results = [(None, None, None, None, None, None, None, None, None, None)]
 
     def fetchone(self):
         return self.results[0]
@@ -158,80 +155,77 @@ def fake_listdir(path):
 @patch('subprocess.call', Mock(return_value=0))
 @patch('psycopg2.connect', psycopg2_connect)
 class TestPostgresql(unittest.TestCase):
+    _PARAMETERS = {'wal_level': 'hot_standby', 'max_replication_slots': 5, 'f.oo': 'bar',
+                   'search_path': 'public', 'hot_standby': 'on', 'max_wal_senders': 5,
+                   'wal_keep_segments': 8, 'wal_log_hints': 'on', 'max_locks_per_transaction': 64,
+                   'max_worker_processes': 8, 'max_connections': 100, 'max_prepared_transactions': 0}
 
     @patch('subprocess.call', Mock(return_value=0))
     @patch('psycopg2.connect', psycopg2_connect)
+    @patch('os.rename', Mock())
+    @patch.object(Postgresql, 'get_major_version', Mock(return_value=9.4))
+    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def setUp(self):
-        self.p = Postgresql({'name': 'test0', 'scope': 'batman', 'data_dir': 'data/test0',
+        self.data_dir = 'data/test0'
+        if not os.path.exists(self.data_dir):
+            os.makedirs(self.data_dir)
+        self.p = Postgresql({'name': 'test0', 'scope': 'batman', 'data_dir': self.data_dir, 'retry_timeout': 10,
                              'listen': '127.0.0.1, *:5432', 'connect_address': '127.0.0.2:5432',
-                             'pg_hba': ['host replication replicator 127.0.0.1/32 md5',
-                                        'hostssl all all 0.0.0.0/0 md5',
-                                        'host all all 0.0.0.0/0 md5'],
-                             'superuser': {'username': 'test', 'password': 'test'},
-                             'admin': {'username': 'admin', 'password': 'admin'},
-                             'pg_rewind': {'username': 'admin', 'password': 'admin'},
-                             'replication': {'username': 'replicator',
-                                             'password': 'rep-pass'},
-                             'parameters': {'foo': 'bar'}, 'recovery_conf': {'foo': 'bar'},
+                             'authentication': {'superuser': {'username': 'test', 'password': 'test'},
+                                                'replication': {'username': 'replicator', 'password': 'rep-pass'}},
+                             'remove_data_directory_on_rewind_failure': True,
+                             'use_pg_rewind': True, 'pg_ctl_timeout': 'bla',
+                             'parameters': self._PARAMETERS,
+                             'recovery_conf': {'foo': 'bar'},
                              'callbacks': {'on_start': 'true', 'on_stop': 'true',
                                            'on_restart': 'true', 'on_role_change': 'true',
                                            'on_reload': 'true'
                                            },
                              'restore': 'true'})
-        if not os.path.exists(self.p.data_dir):
-            os.makedirs(self.p.data_dir)
         self.leadermem = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres'})
         self.leader = Leader(-1, 28, self.leadermem)
-        self.other = Member(0, 'test1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres',
+        self.other = Member(0, 'test-1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres',
                             'tags': {'replicatefrom': 'leader'}})
         self.me = Member(0, 'test0', 28, {'conn_url': 'postgres://replicator:[email protected]:5434/postgres'})
 
     def tearDown(self):
         shutil.rmtree('data')
 
-    def test_data_directory_empty(self):
-        self.assertTrue(self.p.data_directory_empty())
-
     def test_get_initdb_options(self):
-        self.p.initdb_options = [{'encoding': 'UTF8'}, 'data-checksums']
-        self.assertEquals(self.p.get_initdb_options(), ['--encoding=UTF8', '--data-checksums'])
-        self.p.initdb_options = [{'pgdata': 'bar'}]
-        self.assertRaises(Exception, self.p.get_initdb_options)
-        self.p.initdb_options = [{'foo': 'bar', 1: 2}]
-        self.assertRaises(Exception, self.p.get_initdb_options)
-        self.p.initdb_options = [1]
-        self.assertRaises(Exception, self.p.get_initdb_options)
-
-    def test_initialize(self):
-        self.assertTrue(self.p.initialize())
-        self.assertTrue(os.path.exists(os.path.join(self.p.data_dir, 'pg_hba.conf')))
-
-        with open(os.path.join(self.p.data_dir, 'pg_hba.conf')) as f:
-            lines = f.readlines()
-            assert 'host replication replicator 127.0.0.1/32 md5\n' in lines
-            assert 'host all all 0.0.0.0/0 md5\n' in lines
+        self.assertEquals(self.p.get_initdb_options([{'encoding': 'UTF8'}, 'data-checksums']),
+                          ['--encoding=UTF8', '--data-checksums'])
+        self.assertRaises(Exception, self.p.get_initdb_options, [{'pgdata': 'bar'}])
+        self.assertRaises(Exception, self.p.get_initdb_options, [{'foo': 'bar', 1: 2}])
+        self.assertRaises(Exception, self.p.get_initdb_options, [1])
 
     @patch('os.path.exists', Mock(return_value=True))
     @patch('os.unlink', Mock())
     def test_delete_trigger_file(self):
         self.p.delete_trigger_file()
 
-    def test_start(self):
+    @patch.object(Postgresql, 'is_running')
+    def test_start(self, mock_is_running):
+        mock_is_running.return_value = True
         self.assertTrue(self.p.start())
-        self.p.is_running = false
-        open(os.path.join(self.p.data_dir, 'postmaster.pid'), 'w').close()
+        mock_is_running.return_value = False
+        open(os.path.join(self.data_dir, 'postmaster.pid'), 'w').close()
+        pg_conf = os.path.join(self.data_dir, 'postgresql.conf')
+        open(pg_conf, 'w').close()
         self.assertTrue(self.p.start())
+        with open(pg_conf) as f:
+            lines = f.readlines()
+            self.assertTrue("f.oo = 'bar'\n" in lines)
 
-    def test_stop(self):
+    @patch.object(Postgresql, 'is_running')
+    def test_stop(self, mock_is_running):
+        mock_is_running.return_value = True
         self.assertTrue(self.p.stop())
         with patch('subprocess.call', Mock(return_value=1)):
+            mock_is_running.return_value = False
             self.assertTrue(self.p.stop())
-            self.p.is_running = Mock(return_value=True)
-            self.assertFalse(self.p.stop())
 
     def test_restart(self):
         self.p.start = false
-        self.p.is_running = false
         self.assertFalse(self.p.restart())
         self.assertEquals(self.p.state, 'restart failed (restarting)')
 
@@ -239,85 +233,98 @@ class TestPostgresql(unittest.TestCase):
     def test_write_pgpass(self):
         self.p.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo', 'password': 'bar'})
 
-    @patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
-    def test_sync_replica(self):
-        self.assertTrue(self.p.sync_replica(self.leader))
-        self.p.create_replica = Mock(return_value=1)
-        self.assertFalse(self.p.sync_replica(self.leader))
+    def test_checkpoint(self):
+        with patch.object(MockCursor, 'fetchone', Mock(return_value=(True, ))):
+            self.assertEquals(self.p.checkpoint({'user': 'postgres'}), 'is_in_recovery=true')
+        with patch.object(MockCursor, 'execute', Mock(return_value=None)):
+            self.assertIsNone(self.p.checkpoint())
+        self.assertEquals(self.p.checkpoint(), 'not accessible or not healty')
 
     @patch('subprocess.call', side_effect=OSError)
     @patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
     def test_pg_rewind(self, mock_call):
-        self.assertTrue(self.p.rewind(self.leader))
+        r = {'user': '', 'host': '', 'port': '', 'database': '', 'password': ''}
+        self.assertTrue(self.p.rewind(r))
         subprocess.call = mock_call
-        self.assertFalse(self.p.rewind(self.leader))
+        self.assertFalse(self.p.rewind(r))
 
-    @patch('patroni.postgresql.Postgresql.rewind', return_value=False)
-    @patch('patroni.postgresql.Postgresql.remove_data_directory', MagicMock(return_value=True))
-    @patch('patroni.postgresql.Postgresql.single_user_mode', MagicMock(return_value=1))
-    @patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
+    @patch('os.unlink', Mock(return_value=True))
     @patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string))
+    @patch.object(Postgresql, 'remove_data_directory', Mock(return_value=True))
+    @patch.object(Postgresql, 'single_user_mode', Mock(return_value=1))
+    @patch.object(Postgresql, 'write_pgpass', Mock(return_value={}))
+    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
+    @patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
+    @patch.object(Postgresql, 'rewind', return_value=False)
     def test_follow(self, mock_pg_rewind):
-        self.p.follow(None)
-        self.p.follow(self.leader)
-        self.p.follow(Leader(-1, 28, self.other))
-        self.p.rewind = mock_pg_rewind
-        self.p.follow(self.leader)
-        with mock.patch('os.path.islink', MagicMock(return_value=True)):
-            with mock.patch('patroni.postgresql.Postgresql.can_rewind', new_callable=PropertyMock(return_value=True)):
-                with mock.patch('os.unlink', MagicMock(return_value=True)):
-                    self.p.follow(self.leader, recovery=True)
-        with mock.patch('patroni.postgresql.Postgresql.can_rewind', new_callable=PropertyMock(return_value=True)):
-            self.p.rewind.return_value = True
-            self.p.follow(self.leader, recovery=True)
-            self.p.rewind.return_value = False
-            self.p.follow(self.leader, recovery=True)
-        with mock.patch('patroni.postgresql.Postgresql.check_recovery_conf', MagicMock(return_value=True)):
-            self.assertTrue(self.p.follow(None))
+        with patch.object(Postgresql, 'check_recovery_conf', Mock(return_value=True)):
+            self.assertTrue(self.p.follow(None, None))  # nothing to do, recovery.conf has good primary_conninfo
+
+        self.p.follow(self.me, self.me)  # follow is called when the node is holding leader lock
+
+        with patch.object(Postgresql, 'restart', Mock(return_value=False)):
+            self.p.set_role('replica')
+            self.p.follow(None, None)  # restart without rewind
+
+        with patch.object(Postgresql, 'stop', Mock(return_value=False)):
+            self.p.follow(self.leader, self.leader, need_rewind=True)  # failed to stop postgres
+
+        self.p.follow(self.leader, self.leader)  # "leader" is not accessible or is_in_recovery
+
+        with patch.object(Postgresql, 'checkpoint', Mock(return_value=None)):
+            self.p.follow(self.leader, self.leader)
+            mock_pg_rewind.return_value = True
+            self.p.follow(self.leader, self.leader, need_rewind=True)
+
+        self.p.follow(None, None)  # check_recovery_conf...
 
     @patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string))
     def test_can_rewind(self):
-        tmp = self.p.pg_rewind
-        self.p.pg_rewind = None
+        with patch('subprocess.call', MagicMock(return_value=1)):
+            self.assertFalse(self.p.can_rewind)
+        with patch('subprocess.call', side_effect=OSError):
+            self.assertFalse(self.p.can_rewind)
+        with patch.object(Postgresql, 'controldata', Mock(return_value={'wal_log_hints setting': 'on'})):
+            self.assertTrue(self.p.can_rewind)
+        self.p.config['use_pg_rewind'] = False
         self.assertFalse(self.p.can_rewind)
-        self.p.pg_rewind = tmp
-        with mock.patch('subprocess.call', MagicMock(return_value=1)):
-            self.assertFalse(self.p.can_rewind)
-        with mock.patch('subprocess.call', side_effect=OSError):
-            self.assertFalse(self.p.can_rewind)
-        tmp = self.p.controldata
-        self.p.controldata = lambda: {'wal_log_hints setting': 'on'}
-        self.assertTrue(self.p.can_rewind)
-        self.p.controldata = tmp
 
     @patch('time.sleep', Mock())
     def test_create_replica(self):
         self.p.delete_trigger_file = Mock(side_effect=OSError)
         with patch('subprocess.call', Mock(side_effect=[1, 0])):
-            self.assertEquals(self.p.create_replica(self.leader, ''), 0)
+            self.assertEquals(self.p.create_replica(self.leader), 0)
         with patch('subprocess.call', Mock(side_effect=[Exception(), 0])):
-            self.assertEquals(self.p.create_replica(self.leader, ''), 0)
+            self.assertEquals(self.p.create_replica(self.leader), 0)
 
         self.p.config['create_replica_method'] = ['wale', 'basebackup']
         self.p.config['wale'] = {'command': 'foo'}
         with patch('subprocess.call', Mock(return_value=0)):
-            self.assertEquals(self.p.create_replica(self.leader, ''), 0)
+            self.assertEquals(self.p.create_replica(self.leader), 0)
             del self.p.config['wale']
-            self.assertEquals(self.p.create_replica(self.leader, ''), 0)
+            self.assertEquals(self.p.create_replica(self.leader), 0)
 
         with patch('subprocess.call', Mock(side_effect=Exception("foo"))):
-            self.assertEquals(self.p.create_replica(self.leader, ''), 1)
+            self.assertEquals(self.p.create_replica(self.leader), 1)
 
+    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def test_sync_replication_slots(self):
         self.p.start()
-        cluster = Cluster(True, self.leader, 0, [self.me, self.other, self.leadermem], None)
+        cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None)
+        with mock.patch('patroni.postgresql.Postgresql._query', Mock(side_effect=psycopg2.OperationalError)):
+            self.p.sync_replication_slots(cluster)
         self.p.sync_replication_slots(cluster)
-        self.p.query = Mock(side_effect=psycopg2.OperationalError)
-        self.p.schedule_load_slots = True
-        self.p.sync_replication_slots(cluster)
-        self.p.schedule_load_slots = False
         with mock.patch('patroni.postgresql.Postgresql.role', new_callable=PropertyMock(return_value='replica')):
             self.p.sync_replication_slots(cluster)
+        with mock.patch('patroni.postgresql.logger.error', new_callable=Mock()) as errorlog_mock:
+            self.p.query = Mock()
+            alias1 = Member(0, 'test-3', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres'})
+            alias2 = Member(0, 'test.3', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres'})
+            cluster.members.extend([alias1, alias2])
+            self.p.sync_replication_slots(cluster)
+            errorlog_mock.assert_called_once()
+            assert "test-3" in errorlog_mock.call_args[0][1]
+            assert "test.3" in errorlog_mock.call_args[0][1]
 
     @patch.object(MockConnect, 'closed', 2)
     def test__query(self):
@@ -336,9 +343,11 @@ class TestPostgresql(unittest.TestCase):
     def test_reload(self):
         self.assertTrue(self.p.reload())
 
-    def test_is_healthy(self):
+    @patch.object(Postgresql, 'is_running')
+    def test_is_healthy(self, mock_is_running):
+        mock_is_running.return_value = True
         self.assertTrue(self.p.is_healthy())
-        self.p.is_running = false
+        mock_is_running.return_value = False
         self.assertFalse(self.p.is_healthy())
 
     def test_promote(self):
@@ -349,6 +358,15 @@ class TestPostgresql(unittest.TestCase):
     def test_last_operation(self):
         self.assertEquals(self.p.last_operation(), '0')
 
+    @patch('os.path.isfile', Mock(return_value=True))
+    @patch('os.kill', Mock(side_effect=Exception))
+    @patch('os.getpid', Mock(return_value=2))
+    @patch('os.getppid', Mock(return_value=2))
+    @patch.object(builtins, 'open', mock_open(read_data='-1'))
+    @patch.object(Postgresql, '_version_file_exists', Mock(return_value=True))
+    def test_is_running(self):
+        self.assertFalse(self.p.is_running())
+
     @patch('subprocess.Popen', Mock(side_effect=OSError))
     def test_call_nowait(self):
         self.assertFalse(self.p.call_nowait('on_start'))
@@ -356,6 +374,7 @@ class TestPostgresql(unittest.TestCase):
     def test_non_existing_callback(self):
         self.assertFalse(self.p.call_nowait('foobar'))
 
+    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def test_is_leader_exception(self):
         self.p.start()
         self.p.query = Mock(side_effect=psycopg2.OperationalError("not supported"))
@@ -367,31 +386,43 @@ class TestPostgresql(unittest.TestCase):
     @patch('os.rename', Mock())
     @patch('os.path.isdir', Mock(return_value=True))
     def test_move_data_directory(self):
-        self.p.is_running = false
         self.p.move_data_directory()
         with patch('os.rename', Mock(side_effect=OSError)):
             self.p.move_data_directory()
 
-    @patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
+    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def test_bootstrap(self):
         with patch('subprocess.call', Mock(return_value=1)):
-            self.assertRaises(PostgresException, self.p.bootstrap)
-        self.p.bootstrap()
-        with patch('patroni.postgresql.Postgresql.sync_replica', MagicMock(return_value=True)):
-            self.p.bootstrap(self.leader)
+            self.assertRaises(PostgresException, self.p.bootstrap, {})
+
+        self.p.bootstrap({'users': {'replicator': {'password': 'rep-pass', 'options': ['replication']}},
+                          'pg_hba': ['host replication replicator 127.0.0.1/32 md5',
+                                     'hostssl all all 0.0.0.0/0 md5',
+                                     'host all all 0.0.0.0/0 md5']})
+        with open(os.path.join(self.data_dir, 'pg_hba.conf')) as f:
+            lines = f.readlines()
+            assert 'host replication replicator 127.0.0.1/32 md5\n' in lines
+            assert 'host all all 0.0.0.0/0 md5\n' in lines
+
+    @patch('patroni.postgresql.Postgresql.create_replica', Mock(return_value=0))
+    def test_clone(self):
+        self.p.clone(self.leader)
+
+    @patch('os.listdir', Mock(return_value=['recovery.conf']))
+    @patch('os.path.exists', Mock(return_value=True))
+    def test_get_postgres_role_from_data_directory(self):
+        self.assertEquals(self.p.get_postgres_role_from_data_directory(), 'replica')
 
     def test_remove_data_directory(self):
-        self.p.data_dir = 'data_dir'
         self.p.remove_data_directory()
-        os.mkdir(self.p.data_dir)
+        open(self.data_dir, 'w').close()
         self.p.remove_data_directory()
-        open(self.p.data_dir, 'w').close()
-        self.p.remove_data_directory()
-        os.symlink('unexisting', self.p.data_dir)
+        os.symlink('unexisting', self.data_dir)
         with patch('os.unlink', Mock(side_effect=OSError)):
             self.p.remove_data_directory()
         self.p.remove_data_directory()
 
+    @patch('patroni.postgresql.Postgresql._version_file_exists', Mock(return_value=True))
     def test_controldata(self):
         with patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string)):
             data = self.p.controldata()
@@ -420,14 +451,14 @@ class TestPostgresql(unittest.TestCase):
     def test_single_user_mode(self, subprocess_popen_mock):
         subprocess_popen_mock.return_value.wait.return_value = 0
         self.assertEquals(self.p.single_user_mode(options=dict(archive_mode='on', archive_command='false')), 0)
-        subprocess_popen_mock.assert_called_once_with(['postgres', '--single', '-D', self.p.data_dir,
+        subprocess_popen_mock.assert_called_once_with(['postgres', '--single', '-D', self.data_dir,
                                                       '-c', 'archive_command=false', '-c', 'archive_mode=on',
                                                        'postgres'], stdin=subprocess.PIPE,
                                                       stdout=42,
                                                       stderr=subprocess.STDOUT)
         subprocess_popen_mock.reset_mock()
         self.assertEquals(self.p.single_user_mode(command="CHECKPOINT"), 0)
-        subprocess_popen_mock.assert_called_once_with(['postgres', '--single', '-D', self.p.data_dir,
+        subprocess_popen_mock.assert_called_once_with(['postgres', '--single', '-D', self.data_dir,
                                                       'postgres'], stdin=subprocess.PIPE,
                                                       stdout=42,
                                                       stderr=subprocess.STDOUT)
@@ -440,9 +471,9 @@ class TestPostgresql(unittest.TestCase):
     @patch('os.path.islink', return_value=False)
     @patch('os.path.isfile', return_value=True)
     def test_cleanup_archive_status(self, mock_file, mock_link, mock_remove, mock_unlink):
-        ap = os.path.join(self.p.data_dir, 'pg_xlog', 'archive_status/')
+        ap = os.path.join(self.data_dir, 'pg_xlog', 'archive_status/')
         self.p.cleanup_archive_status()
-        mock_remove.assert_has_calls([mock.call(ap+'a'), mock.call(ap+'b'), mock.call(ap+'c')])
+        mock_remove.assert_has_calls([mock.call(ap + 'a'), mock.call(ap + 'b'), mock.call(ap + 'c')])
         mock_unlink.assert_not_called()
 
         mock_remove.reset_mock()
@@ -450,7 +481,7 @@ class TestPostgresql(unittest.TestCase):
         mock_file.return_value = False
         mock_link.return_value = True
         self.p.cleanup_archive_status()
-        mock_unlink.assert_has_calls([mock.call(ap+'a'), mock.call(ap+'b'), mock.call(ap+'c')])
+        mock_unlink.assert_has_calls([mock.call(ap + 'a'), mock.call(ap + 'b'), mock.call(ap + 'c')])
         mock_remove.assert_not_called()
 
         mock_unlink.reset_mock()
@@ -462,6 +493,7 @@ class TestPostgresql(unittest.TestCase):
         mock_unlink.assert_not_called()
         mock_remove.assert_not_called()
 
+    @patch('patroni.postgresql.Postgresql._version_file_exists', Mock(return_value=True))
     @patch('subprocess.check_output', MagicMock(return_value=0, side_effect=pg_controldata_string))
     def test_sysid(self):
         self.assertEqual(self.p.sysid, "6200971513092291716")
@@ -490,3 +522,29 @@ class TestPostgresql(unittest.TestCase):
         self.assertTrue(self.p.replica_method_can_work_without_replication_connection('foo'))
         self.p.config['foo'] = {'command': 'bar'}
         self.assertFalse(self.p.replica_method_can_work_without_replication_connection('foo'))
+
+    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
+    def test_reload_config(self):
+        parameters = self._PARAMETERS.copy()
+        parameters.pop('f.oo')
+        self.p.reload_config({'retry_timeout': 10, 'listen': '*', 'parameters': parameters})
+        parameters['b.ar'] = 'bar'
+        self.p.reload_config({'retry_timeout': 10, 'listen': '*', 'parameters': parameters})
+        parameters['autovacuum'] = 'on'
+        self.p.reload_config({'retry_timeout': 10, 'listen': '*', 'parameters': parameters})
+        parameters['autovacuum'] = 'off'
+        parameters.pop('search_path')
+        self.p.reload_config({'retry_timeout': 10, 'listen': '*:5433', 'parameters': parameters})
+
+    @patch.object(Postgresql, '_version_file_exists', Mock(return_value=True))
+    def test_get_major_version(self):
+        with patch.object(builtins, 'open', mock_open(read_data='9.4')):
+            self.assertEquals(self.p.get_major_version(), 9.4)
+        with patch.object(builtins, 'open', Mock(side_effect=Exception)):
+            self.assertEquals(self.p.get_major_version(), 0.0)
+
+    def test_postmaster_start_time(self):
+        with patch.object(MockCursor, "fetchone", Mock(return_value=('foo', True, '', '', '', '', False))):
+            self.assertEqual(self.p.postmaster_start_time(), 'foo')
+        with patch.object(MockCursor, "execute", side_effect=psycopg2.Error):
+            self.assertIsNone(self.p.postmaster_start_time())

tests/test_utils.py

@@ -2,8 +2,7 @@ import unittest
 
 from mock import Mock, patch
 from patroni.exceptions import PatroniException
-from patroni.utils import reap_children, Retry, RetryFailedError, set_ignore_sigterm,\
-    sigchld_handler, sigterm_handler, sleep
+from patroni.utils import reap_children, Retry, RetryFailedError, sigchld_handler, sleep
 
 
 def time_sleep(_):
@@ -12,10 +11,6 @@ def time_sleep(_):
 
 class TestUtils(unittest.TestCase):
 
-    def test_sigterm_handler(self):
-        set_ignore_sigterm(False)
-        self.assertRaises(SystemExit, sigterm_handler, None, None)
-
     @patch('time.sleep', Mock())
     def test_reap_children(self):
         self.assertIsNone(reap_children())

tests/test_wale_restore.py

@@ -6,30 +6,10 @@ from mock import MagicMock, patch, PropertyMock
 from patroni.scripts.wale_restore import WALERestore, main as _main
 
 
-def fake_backup_data(self, *args, **kwargs):
-    """ return the fake result of WAL-E backup-list"""
-    return """name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop
-base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z 167772160   00000001000000000000007F    00000040 00000001000000000000007F    00000240
-"""
-
-
-def fake_backup_data_2(self, *args, **kwargs):
-    """ return the fake result of WAL-E backup-list"""
-    return """name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop """
-
-
-def fake_backup_data_3(self, *args, **kwargs):
-    """ return the fake result of WAL-E backup-list"""
-    return """name    last_modified   expanded_size_bytes wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop
-base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z 167772160   00000001000000000000007F    00000040 00000001000000000000007F    00000240
-"""
-
-
-def fake_backup_data_4(self, *args, **kwargs):
-    """ return the fake result of WAL-E backup-list"""
-    return """name    last_modified   expanded_size_foo wal_segment_backup_start    wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop
-base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z 167772160   00000001000000000000007F    00000040 00000001000000000000007F    00000240
-"""
+wale_output = b'name last_modified expanded_size_bytes wal_segment_backup_start ' +\
+              b'wal_segment_offset_backup_start wal_segment_backup_stop wal_segment_offset_backup_stop\n' +\
+              b'base_00000001000000000000007F_00000040 2015-05-18T10:13:25.000Z 167772160 ' +\
+              b'00000001000000000000007F 00000040 00000001000000000000007F 00000240\n'
 
 
 @patch('os.access', MagicMock(return_value=True))
@@ -39,7 +19,7 @@ base_00000001000000000000007F_00000040  2015-05-18T10:13:25.000Z 167772160   000
 @patch('psycopg2.extensions.cursor', MagicMock(autospec=True))
 @patch('psycopg2.extensions.connection', MagicMock(autospec=True))
 @patch('psycopg2.connect', MagicMock(autospec=True))
-@patch('subprocess.check_output', MagicMock(side_effect=fake_backup_data))
+@patch('subprocess.check_output', MagicMock(return_value=wale_output))
 class TestWALERestore(unittest.TestCase):
 
     def setUp(self):
@@ -50,11 +30,13 @@ class TestWALERestore(unittest.TestCase):
             self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
         with patch('subprocess.check_output', MagicMock(side_effect=subprocess.CalledProcessError(1, "cmd", "foo"))):
             self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output', MagicMock(side_effect=fake_backup_data_2)):
+        with patch('subprocess.check_output', MagicMock(return_value=wale_output.split(b'\n')[0])):
             self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output', MagicMock(side_effect=fake_backup_data_3)):
+        with patch('subprocess.check_output',
+                   MagicMock(return_value=wale_output.replace(b' wal_segment_offset_backup_stop', b''))):
             self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
-        with patch('subprocess.check_output', MagicMock(side_effect=fake_backup_data_4)):
+        with patch('subprocess.check_output',
+                   MagicMock(return_value=wale_output.replace(b'expanded_size_bytes', b'expanded_size_foo'))):
             self.assertFalse(self.wale_restore.should_use_s3_to_create_replica())
 
         self.wale_restore.should_use_s3_to_create_replica()

tests/test_zookeeper.py

@@ -1,13 +1,12 @@
 import six
 import unittest
 
-from mock import Mock, patch
-from patroni.dcs import Leader
-from patroni.zookeeper import ExhibitorEnsembleProvider, ZooKeeper, ZooKeeperError
 from kazoo.client import KazooState
 from kazoo.exceptions import NoNodeError, NodeExistsError
+from kazoo.handlers.threading import SequentialThreadingHandler
 from kazoo.protocol.states import ZnodeStat
-from test_etcd import SleepException, requests_get
+from mock import Mock, patch
+from patroni.dcs.zookeeper import Leader, PatroniSequentialThreadingHandler, ZooKeeper, ZooKeeperError
 
 
 class MockKazooClient(Mock):
@@ -15,6 +14,9 @@ class MockKazooClient(Mock):
     leader = False
     exists = True
 
+    def __init__(self, *args, **kwargs):
+        super(MockKazooClient, self).__init__()
+
     @property
     def client_id(self):
         return (-1, '')
@@ -56,11 +58,16 @@ class MockKazooClient(Mock):
             raise TypeError("Invalid type for 'path' (string expected)")
         if not isinstance(value, (six.binary_type,)):
             raise TypeError("Invalid type for 'value' (must be a byte string)")
+        if value == b'Exception':
+            raise Exception
         if path.endswith('/initialize') or path == '/service/test/optime/leader':
             raise Exception
         elif value == b'retry' or (value == b'exists' and self.exists):
             raise NodeExistsError
 
+    def create_async(self, path, value=b"", acl=None, ephemeral=False, sequence=False, makepath=False):
+        return self.create(path, value, acl, ephemeral, sequence, makepath) or Mock()
+
     @staticmethod
     def set(path, value, version=-1):
         if not isinstance(path, six.string_types):
@@ -71,13 +78,16 @@ class MockKazooClient(Mock):
             raise Exception
         if path == '/service/test/members/bar' and value == b'retry':
             return
-        if path == '/service/test/failover':
+        if path in ('/service/test/failover', '/service/test/config'):
             if value == b'Exception':
                 raise Exception
             elif value == b'ok':
                 return
         raise NoNodeError
 
+    def set_async(self, path, value, version=-1):
+        return self.set(path, value, version) or Mock()
+
     def delete(self, path, version=-1, recursive=False):
         if not isinstance(path, six.string_types):
             raise TypeError("Invalid type for 'path' (string expected)")
@@ -90,25 +100,35 @@ class MockKazooClient(Mock):
         elif path.endswith('/') or path.endswith('/initialize') or path == '/service/test/members/bar':
             raise NoNodeError
 
+    def delete_async(self, path, version=-1, recursive=False):
+        return self.delete(path, version, recursive) or Mock()
 
-@patch('requests.get', requests_get)
-@patch('patroni.zookeeper.sleep', Mock(side_effect=SleepException()))
-class TestExhibitorEnsembleProvider(unittest.TestCase):
 
-    def test_init(self):
-        self.assertRaises(SleepException, ExhibitorEnsembleProvider, ['localhost'], 8181)
+class TestPatroniSequentialThreadingHandler(unittest.TestCase):
+
+    def setUp(self):
+        self.handler = PatroniSequentialThreadingHandler(10)
+
+    @patch.object(SequentialThreadingHandler, 'create_connection', Mock())
+    def test_create_connection(self):
+        self.assertIsNotNone(self.handler.create_connection(()))
+        self.assertIsNotNone(self.handler.create_connection((), 40))
 
 
 class TestZooKeeper(unittest.TestCase):
 
-    @patch('requests.get', requests_get)
-    @patch('patroni.zookeeper.KazooClient', MockKazooClient)
+    @patch('patroni.dcs.zookeeper.KazooClient', MockKazooClient)
     def setUp(self):
-        self.zk = ZooKeeper('foo', {'exhibitor': {'hosts': ['localhost', 'exhibitor'], 'port': 8181}, 'scope': 'test'})
+        self.zk = ZooKeeper({'hosts': ['localhost:2181'], 'scope': 'test',
+                             'name': 'foo', 'ttl': 30, 'retry_timeout': 10, 'loop_wait': 10})
 
     def test_session_listener(self):
         self.zk.session_listener(KazooState.SUSPENDED)
 
+    def test_reload_config(self):
+        self.zk.reload_config({'ttl': 20, 'retry_timeout': 10, 'loop_wait': 10})
+        self.zk.reload_config({'ttl': 20, 'retry_timeout': 10, 'loop_wait': 5})
+
     def test_get_node(self):
         self.assertIsNone(self.zk.get_node('/no_node'))
 
@@ -123,17 +143,23 @@ class TestZooKeeper(unittest.TestCase):
 
     def test_get_cluster(self):
         self.assertRaises(ZooKeeperError, self.zk.get_cluster)
-        self.zk.exhibitor.poll = lambda: True
         cluster = self.zk.get_cluster()
         self.assertIsInstance(cluster.leader, Leader)
         self.zk.touch_member('foo')
-        self.zk.delete_leader()
+
+    def test_delete_leader(self):
+        self.assertTrue(self.zk.delete_leader())
 
     def test_set_failover_value(self):
         self.zk.set_failover_value('')
         self.zk.set_failover_value('ok')
         self.zk.set_failover_value('Exception')
 
+    def test_set_config_value(self):
+        self.zk.set_config_value('')
+        self.zk.set_config_value('ok')
+        self.zk.set_config_value('Exception')
+
     def test_initialize(self):
         self.assertFalse(self.zk.initialize())
 
@@ -148,7 +174,7 @@ class TestZooKeeper(unittest.TestCase):
         self.zk.touch_member('new')
         self.zk._name = 'na'
         self.zk._client.exists = 1
-        self.zk.touch_member('exists')
+        self.zk.touch_member('Exception')
         self.zk._name = 'bar'
         self.zk.touch_member('retry')
         self.zk._fetch_cluster = True
@@ -166,8 +192,12 @@ class TestZooKeeper(unittest.TestCase):
     def test_write_leader_optime(self):
         self.zk.last_leader_operation = '0'
         self.zk.write_leader_optime('1')
+        with patch.object(MockKazooClient, 'create_async', Mock()):
+            self.zk.write_leader_optime('1')
+        with patch.object(MockKazooClient, 'set_async', Mock()):
+            self.zk.write_leader_optime('2')
         self.zk._base_path = self.zk._base_path.replace('test', 'bla')
-        self.zk.write_leader_optime('2')
+        self.zk.write_leader_optime('3')
 
     def test_delete_cluster(self):
         self.assertTrue(self.zk.delete_cluster())
@@ -176,3 +206,8 @@ class TestZooKeeper(unittest.TestCase):
         self.zk.watch(0)
         self.zk.event.isSet = lambda: True
         self.zk.watch(0)
+
+    def test__kazoo_connect(self):
+        self.zk._client._retry.deadline = 1
+        self.zk._orig_kazoo_connect = Mock(return_value=(0, 0))
+        self.zk._kazoo_connect(None, None)