BUGFIX: don't leak password when running pg_rewind (#1321)

In addition to that:
* enforce security settings from `postgresql.authention`
* update release notes
* bump version
* close https://github.com/zalando/patroni/issues/1320

.travis.yml

@@ -119,7 +119,7 @@ install:
             fi
             source ~/virtualenv/python${pv}/bin/activate
             # explicitly install all needed python modules to cache them
-            for p in '-r requirements.txt' 'behave codacy-coverage coverage coveralls flake8 mock pytest-cov pytest setuptools'; do
+            for p in '-r requirements.txt' 'psycopg2-binary behave codacy-coverage coverage coveralls flake8 mock pytest-cov pytest setuptools'; do
                 pip install $p --upgrade
             done
         fi

Dockerfile

@@ -25,6 +25,7 @@ RUN set -ex \
             | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \
             | xargs apt-get install -y vim curl less jq locales haproxy sudo \
                             python3-etcd python3-kazoo python3-pip busybox \
+                            net-tools iputils-ping --fix-missing \
     && pip3 install dumb-init \
 \
     # Cleanup all locales but en_US.UTF-8
@@ -43,7 +44,7 @@ RUN set -ex \
 \
     # Prepare postgres/patroni/haproxy environment
     && mkdir -p $PGHOME/.config/patroni /patroni /run/haproxy \
-    && ln -s ../../patroni.yaml $PGHOME/.config/patroni/patronictl.yaml \
+    && ln -s ../../postgres0.yml $PGHOME/.config/patroni/patronictl.yaml \
     && ln -s /patronictl.py /usr/local/bin/patronictl \
     && sed -i "s|/var/lib/postgresql.*|$PGHOME:/bin/bash|" /etc/passwd \
     && chown -R postgres:postgres /var/log \
@@ -146,9 +147,10 @@ RUN sed -i 's/env python/&3/' /patroni*.py \
     && sed -i "s|^#\(  bin_dir: \).*|\1$PGBIN|" postgres?.yml \
     && sed -i 's/^  - encoding: UTF8/  - locale: en_US.UTF-8\n&/' postgres?.yml \
     && sed -i 's/^\(scope\|name\|etcd\|  host\|  authentication\|  pg_hba\|  parameters\):/#&/' postgres?.yml \
-    && sed -i 's/^    \(replication\|superuser\|unix_socket_directories\|\(\(  \)\{0,1\}\(username\|password\)\)\):/#&/' postgres?.yml \
+    && sed -i 's/^    \(replication\|superuser\|rewind\|unix_socket_directories\|\(\(  \)\{0,1\}\(username\|password\)\)\):/#&/' postgres?.yml \
     && sed -i 's/^      parameters:/      pg_hba:\n      - local all all trust\n      - host replication all all md5\n      - host all all all md5\n&\n        max_connections: 100/'  postgres?.yml \
     && if [ "$COMPRESS" = "true" ]; then chmod u+s /usr/bin/sudo; fi \
+    && chmod +s /bin/ping \
     && chown -R postgres:postgres $PGHOME /run /etc/haproxy
 
 USER postgres

README.rst

@@ -59,6 +59,33 @@ To install requirements on a Mac, run the following:
 
     brew install postgresql etcd haproxy libyaml python
 
+**Psycopg2**
+
+Starting from `psycopg2-2.8 <http://initd.org/psycopg/articles/2019/04/04/psycopg-28-released/>`__ the binary version of psycopg2 will no longer be installed by default. Installing it from the source code requires C compiler and postgres+python dev packages.
+Since in the python world it is not possible to specify dependency as ``psycopg2 OR psycopg2-binary`` you will have to decide how to install it.
+
+There are a few options available:
+
+1. Use the package manager from your distro
+
+::
+
+    sudo apt-get install python-psycopg2   # install python2 psycopg2 module on Debian/Ubuntu
+    sudo apt-get install python3-psycopg2  # install python3 psycopg2 module on Debian/Ubuntu
+    sudo yum install python-psycopg2       # install python2 psycopg2 on RedHat/Fedora/CentOS
+
+2. Install psycopg2 from the binary package
+
+::
+
+    pip install psycopg2-binary
+
+3. Install psycopg2 from source
+
+::
+
+    pip install psycopg2>=2.5.4
+
 **General installation for pip**
 
 Patroni can be installed with pip:

docker/README.md

@@ -67,24 +67,24 @@ Example session:
 
     $ docker exec -ti demo-patroni1 bash
     postgres@patroni1:~$ patronictl list
-    +-------------+----------+------------+--------+---------+----+-----------+
-    |   Cluster   |  Member  |    Host    |  Role  |  State  | TL | Lag in MB |
-    +-------------+----------+------------+--------+---------+----+-----------+
-    | testcluster | patroni1 | 172.21.0.3 | Leader | running |  1 |         0 |
-    | testcluster | patroni2 | 172.21.0.4 |        | running |  1 |         0 |
-    | testcluster | patroni3 | 172.21.0.5 |        | running |  1 |         0 |
-    +-------------+----------+------------+--------+---------+----+-----------+
+    +---------+----------+------------+--------+---------+----+-----------+
+    | Cluster |  Member  |    Host    |  Role  |  State  | TL | Lag in MB |
+    +---------+----------+------------+--------+---------+----+-----------+
+    |   demo  | patroni1 | 172.22.0.3 | Leader | running |  1 |         0 |
+    |   demo  | patroni2 | 172.22.0.7 |        | running |  1 |         0 |
+    |   demo  | patroni3 | 172.22.0.4 |        | running |  1 |         0 |
+    +---------+----------+------------+--------+---------+----+-----------+
 
-    postgres@patroni1:~$ etcdctl ls --recursive --sort -p /service/testcluster
-    /service/testcluster/config
-    /service/testcluster/initialize
-    /service/testcluster/leader
-    /service/testcluster/members/
-    /service/testcluster/members/patroni1
-    /service/testcluster/members/patroni2
-    /service/testcluster/members/patroni3
-    /service/testcluster/optime/
-    /service/testcluster/optime/leader
+    postgres@patroni1:~$ etcdctl ls --recursive --sort -p /service/demo
+    /service/demo/config
+    /service/demo/initialize
+    /service/demo/leader
+    /service/demo/members/
+    /service/demo/members/patroni1
+    /service/demo/members/patroni2
+    /service/demo/members/patroni3
+    /service/demo/optime/
+    /service/demo/optime/leader
 
     postgres@patroni1:~$ etcdctl member list
     1bab629f01fa9065: name=etcd3 peerURLs=http://etcd3:2380 clientURLs=http://etcd3:2379 isLeader=false

docs/ENVIRONMENT.rst

@@ -11,9 +11,14 @@ Global/Universal
 -  **PATRONI\_NAME**: name of the node where the current instance of Patroni is running. Must be unique for the cluster.
 -  **PATRONI\_NAMESPACE**: path within the configuration store where Patroni will keep information about the cluster. Default value: "/service"
 -  **PATRONI\_SCOPE**: cluster name
+
+Log
+---
 -  **PATRONI\_LOG\_LEVEL**: sets the general logging level. Default value is **INFO** (see `the docs for Python logging <https://docs.python.org/3.6/library/logging.html#levels>`_)
+-  **PATRONI\_LOG\_TRACEBACK\_LEVEL**: sets the level where tracebacks will be visible. Default value is **ERROR**. Set it to **DEBUG** if you want to see tracebacks only if you enable **PATRONI\_LOG\_LEVEL=DEBUG**.
 -  **PATRONI\_LOG\_FORMAT**: sets the log formatting string. Default value is **%(asctime)s %(levelname)s: %(message)s** (see `the LogRecord attributes <https://docs.python.org/3.6/library/logging.html#logrecord-attributes>`_)
 -  **PATRONI\_LOG\_DATEFORMAT**: sets the datetime formatting string. (see the `formatTime() documentation <https://docs.python.org/3.6/library/logging.html#logging.Formatter.formatTime>`_)
+-  **PATRONI\_LOG\_MAX\_QUEUE\_SIZE**: Patroni is using two-step logging. Log records are written into the in-memory queue and there is a separate thread which pulls them from the queue and writes to stderr or file. The maximum size of the internal queue is limited by default by **1000** records, which is enough to keep logs for the past 1h20m.
 -  **PATRONI\_LOG\_DIR**: Directory to write application logs to. The directory must exist and be writable by the user executing Patroni. If you set this env variable, the application will retain 4 25MB logs by default. You can tune those retention values with `PATRONI_LOG_FILE_NUM` and `PATRONI_LOG_FILE_SIZE` (see below).
 -  **PATRONI\_LOG\_FILE\_NUM**: The number of application logs to retain.
 -  **PATRONI\_LOG\_FILE\_SIZE**: Size of patroni.log file (in bytes) that triggers a log rolling.
@@ -40,6 +45,7 @@ Consul
 -  **PATRONI\_CONSUL\_CERT**: (optional) File with the client certificate
 -  **PATRONI\_CONSUL\_KEY**: (optional) File with the client key. Can be empty if the key is part of certificate.
 -  **PATRONI\_CONSUL\_DC**: (optional) Datacenter to communicate with. By default the datacenter of the host is used.
+-  **PATRONI\_CONSUL\_CONSISTENCY**: (optional) Select consul consistency mode. Possible values are ``default``, ``consistent``, or ``stale`` (more details in `consul API reference <https://www.consul.io/api/features/consistency.html/>`__)
 -  **PATRONI\_CONSUL\_CHECKS**: (optional) list of Consul health checks used for the session. If not specified Consul will use "serfHealth" in additional to the TTL based check created by Patroni. Additional checks, in particular the "serfHealth", may cause the leader lock to expire faster than in `ttl` seconds when the leader instance becomes unavailable.
 -  **PATRONI\_CONSUL\_REGISTER\_SERVICE**: (optional) whether or not to register a service with the name defined by the scope parameter and the tag master, replica or standby-leader depending on the node's role. Defaults to **false**
 -  **PATRONI\_CONSUL\_SERVICE\_CHECK\_INTERVAL**: (optional) how often to perform health check against registered url
@@ -60,6 +66,10 @@ Etcd
 -  **PATRONI\_ETCD\_CERT**: File with the client certificate.
 -  **PATRONI\_ETCD\_KEY**: File with the client key. Can be empty if the key is part of certificate.
 
+ZooKeeper
+---------
+-  **PATRONI\_ZOOKEEPER\_HOSTS**: comma separated list of ZooKeeper cluster members: "'host1:port1','host2:port2','etc...'". It is important to quote every single entity!
+
 Exhibitor
 ---------
 -  **PATRONI\_EXHIBITOR\_HOSTS**: initial list of Exhibitor (ZooKeeper) nodes in format: 'host1,host2,etc...'. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes.
@@ -75,7 +85,7 @@ Kubernetes
 -  **PATRONI\_KUBERNETES\_ROLE\_LABEL**: (optional) name of the label containing Postgres role (`master` or `replica`). Patroni will set this label on the pod it is running in. Default value is `role`.
 -  **PATRONI\_KUBERNETES\_USE\_ENDPOINTS**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
 -  **PATRONI\_KUBERNETES\_POD\_IP**: (optional) IP address of the pod Patroni is running in. This value is required when `PATRONI_KUBERNETES_USE_ENDPOINTS` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
--  **PATRONI\_KUBERNETES\_PORTS**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``PATRONI_KUBERNETES_PORTS='{[{"name": "postgresql", "port": 5432}]}'`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `PATRONI_KUBERNETES_USE_ENDPOINTS` is set.
+-  **PATRONI\_KUBERNETES\_PORTS**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``PATRONI_KUBERNETES_PORTS='[{"name": "postgresql", "port": 5432}]'`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `PATRONI_KUBERNETES_USE_ENDPOINTS` is set.
 
 PostgreSQL
 ----------
@@ -87,8 +97,25 @@ PostgreSQL
 -  **PATRONI\_POSTGRESQL\_PGPASS**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni.
 -  **PATRONI\_REPLICATION\_USERNAME**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication
 -  **PATRONI\_REPLICATION\_PASSWORD**: replication password; the user will be created during initialization.
+-  **PATRONI\_REPLICATION\_SSLMODE**: (optional) maps to the `sslmode <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE>`__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation <https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS>`__. The default mode is ``prefer``.
+-  **PATRONI\_REPLICATION\_SSLKEY**: (optional) maps to the `sslkey <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY>`__ connection parameter, which specifies the location of the secret key used with the client's certificate.
+-  **PATRONI\_REPLICATION\_SSLCERT**: (optional) maps to the `sslcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCERT>`__ connection parameter, which specifies the location of the client certificate.
+-  **PATRONI\_REPLICATION\_SSLROOTCERT**: (optional) maps to the `sslrootcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLROOTCERT>`__ connection parameter, which specifies the location of a file containing one ore more certificate authorities (CA) certificates that the client will use to verify a server's certificate.
+-  **PATRONI\_REPLICATION\_SSLCRL**: (optional) maps to the `sslcrl <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCRL>`__ connection parameter, which specifies the location of a file containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list.
 -  **PATRONI\_SUPERUSER\_USERNAME**: name for the superuser, set during initialization (initdb) and later used by Patroni to connect to the postgres. Also this user is used by pg_rewind.
 -  **PATRONI\_SUPERUSER\_PASSWORD**: password for the superuser, set during initialization (initdb).
+-  **PATRONI\_SUPERUSER\_SSLMODE**: (optional) maps to the `sslmode <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE>`__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation <https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS>`__. The default mode is ``prefer``.
+-  **PATRONI\_SUPERUSER\_SSLKEY**: (optional) maps to the `sslkey <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY>`__ connection parameter, which specifies the location of the secret key used with the client's certificate.
+-  **PATRONI\_SUPERUSER\_SSLCERT**: (optional) maps to the `sslcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCERT>`__ connection parameter, which specifies the location of the client certificate.
+-  **PATRONI\_SUPERUSER\_SSLROOTCERT**: (optional) maps to the `sslrootcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLROOTCERT>`__ connection parameter, which specifies the location of a file containing one ore more certificate authorities (CA) certificates that the client will use to verify a server's certificate.
+-  **PATRONI\_SUPERUSER\_SSLCRL**: (optional) maps to the `sslcrl <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCRL>`__ connection parameter, which specifies the location of a file containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list.
+-  **PATRONI\_REWIND\_USERNAME**: name for the user for ``pg_rewind``; the user will be created during initialization of postgres 11+ and all necessary `permissions <https://www.postgresql.org/docs/11/app-pgrewind.html#id-1.9.5.8.8>`__ will be granted.
+-  **PATRONI\_REWIND\_PASSWORD**: password for the user for ``pg_rewind``; the user will be created during initialization.
+-  **PATRONI\_REWIND\_SSLMODE**: (optional) maps to the `sslmode <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE>`__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation <https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS>`__. The default mode is ``prefer``.
+-  **PATRONI\_REWIND\_SSLKEY**: (optional) maps to the `sslkey <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY>`__ connection parameter, which specifies the location of the secret key used with the client's certificate.
+-  **PATRONI\_REWIND\_SSLCERT**: (optional) maps to the `sslcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCERT>`__ connection parameter, which specifies the location of the client certificate.
+-  **PATRONI\_REWIND\_SSLROOTCERT**: (optional) maps to the `sslrootcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLROOTCERT>`__ connection parameter, which specifies the location of a file containing one ore more certificate authorities (CA) certificates that the client will use to verify a server's certificate.
+-  **PATRONI\_REWIND\_SSLCRL**: (optional) maps to the `sslcrl <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCRL>`__ connection parameter, which specifies the location of a file containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list.
 
 REST API
 --------
@@ -98,7 +125,12 @@ REST API
 -  **PATRONI\_RESTAPI\_PASSWORD**: Basic-auth password to protect unsafe REST API endpoints.
 -  **PATRONI\_RESTAPI\_CERTFILE**: Specifies the file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL.
 -  **PATRONI\_RESTAPI\_KEYFILE**: Specifies the file with the secret key in the PEM format.
+-  **PATRONI\_RESTAPI\_CAFILE**: Specifies the file with the CA_BUNDLE with certificates of trusted CAs to use while verifying client certs.
+-  **PATRONI\_RESTAPI\_VERIFY\_CLIENT**: ``none``, ``optional`` or ``required``. When ``none`` REST API will not check client certificates. When ``required`` client certificates are required for all REST API calls. When ``optional`` client certificates are required for all unsafe REST API endpoints. If ``verify_client`` is set to ``optional`` or ``required`` basic-auth is not checked.
 
-ZooKeeper
----------
--  **PATRONI\_ZOOKEEPER\_HOSTS**: comma separated list of ZooKeeper cluster members: "'host1:port1','host2:port2','etc...'". It is important to quote every single entity!
+CTL
+---
+-  **PATRONI\_CTL\_INSECURE**: Allow connections to REST API without verifying SSL certs.
+-  **PATRONI\_CTL\_CACERT**: Specifies the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs. If not provided patronictl will use the value provided for REST API "cafile" parameter.
+-  **PATRONI\_CTL\_CERTFILE**: Specifies the file with the client certificate in the PEM format. If not provided patronictl will use the value provided for REST API "certfile" parameter.
+-  **PATRONI\_CTL\_KEYFILE**: Specifies the file with the client secret key in the PEM format. If not provided patronictl will use the value provided for REST API "keyfile" parameter.

docs/README.rst

@@ -33,6 +33,35 @@ To install requirements on a Mac, run the following:
 
     brew install postgresql etcd haproxy libyaml python
 
+.. _psycopg2_install_options:
+
+**Psycopg2**
+
+Starting from `psycopg2-2.8 <http://initd.org/psycopg/articles/2019/04/04/psycopg-28-released/>`__ the binary version of psycopg2 will no longer be installed by default. Installing it from the source code requires C compiler and postgres+python dev packages.
+Since in the python world it is not possible to specify dependency as ``psycopg2 OR psycopg2-binary`` you will have to decide how to install it.
+
+There are a few options available:
+
+1. Use the package manager from your distro
+
+::
+
+    sudo apt-get install python-psycopg2   # install python2 psycopg2 module on Debian/Ubuntu
+    sudo apt-get install python3-psycopg2  # install python3 psycopg2 module on Debian/Ubuntu
+    sudo yum install python-psycopg2       # install python2 psycopg2 on RedHat/Fedora/CentOS
+
+2. Install psycopg2 from the binary package
+
+::
+
+    pip install psycopg2-binary
+
+3. Install psycopg2 from source
+
+::
+
+    pip install psycopg2>=2.5.4
+
 **General installation for pip**
 
 Patroni can be installed with pip:

docs/SETTINGS.rst

@@ -4,6 +4,39 @@
 YAML Configuration Settings
 ===========================
 
+.. _dynamic_configuration_settings:
+
+Dynamic configuration settings
+------------------------------
+
+Dynamic configuration is stored in the DCS (Distributed Configuration Store) and applied on all cluster nodes. Some parameters, like **loop_wait**, **ttl**, **postgresql.parameters.max_connections**, **postgresql.parameters.max_worker_processes** and so on could be set only in the dynamic configuration. Some other parameters like **postgresql.listen**, **postgresql.data_dir** could be set only locally, i.e. in the Patroni config file or via :ref:`configuration <environment>` variable. In most cases the local configuration will override the dynamic configuration. In order to change the dynamic configuration you can use either ``patronictl edit-config`` tool or Patroni :ref:`REST API <rest_api>`.
+
+-  **loop\_wait**: the number of seconds the loop will sleep. Default value: 10
+-  **ttl**: the TTL to acquire the leader lock (in seconds). Think of it as the length of time before initiation of the automatic failover process. Default value: 30
+-  **retry\_timeout**: timeout for DCS and PostgreSQL operation retries (in seconds). DCS or network issues shorter than this will not cause Patroni to demote the leader. Default value: 10
+-  **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election.
+-  **master\_start\_timeout**: the amount of time a master is allowed to recover from failures before failover is triggered (in seconds). Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Worst case failover time for master failure is: loop\_wait + master\_start\_timeout + loop\_wait, unless master\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff.
+-  **synchronous\_mode**: turns on synchronous replication mode. In this mode a replica will be chosen as synchronous and only the latest leader and synchronous replica are able to participate in leader election. Synchronous mode makes sure that successfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See :ref:`replication modes documentation <replication_modes>` for details.
+-  **synchronous\_mode\_strict**: prevents disabling synchronous replication if no synchronous replicas are available, blocking all client writes to the master. See :ref:`replication modes documentation <replication_modes>` for details.
+-  **postgresql**:
+    -  **use\_pg\_rewind**: whether or not to use pg_rewind. Defaults to `false`.
+    -  **use\_slots**: whether or not to use replication_slots. Defaults to `true` on PostgreSQL 9.4+.
+    -  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower. There is no recovery.conf anymore in PostgreSQL 12, but you may continue using this section, because Patroni handles it transparently.
+    -  **parameters**: list of configuration settings for Postgres.
+-  **standby\_cluster**: if this section is defined, we want to bootstrap a standby cluster.
+    -  **host**: an address of remote master
+    -  **port**: a port of remote master
+    -  **primary\_slot\_name**: which slot on the remote master to use for replication. This parameter is optional, the default value is derived from the instance name (see function `slot_name_from_member_name`).
+    -  **create\_replica\_methods**: an ordered list of methods that can be used to bootstrap standby leader from the remote master, can be different from the list defined in :ref:`postgresql_settings`
+    -  **restore\_command**: command to restore WAL records from the remote master to standby leader, can be different from the list defined in :ref:`postgresql_settings`
+    -  **archive\_cleanup\_command**: cleanup command for standby leader
+    -  **recovery\_min\_apply\_delay**: how long to wait before actually apply WAL records on a standby leader
+-  **slots**: define permanent replication slots. These slots will be preserved during switchover/failover. Patroni will try to create slots before opening connections to the cluster.
+    -  **my_slot_name**: the name of replication slot. It is the responsibility of the operator to make sure that there are no clashes in names between replication slots automatically created by Patroni for members and permanent replication slots.
+        -  **type**: slot type. Could be ``physical`` or ``logical``. If the slot is logical, you have to additionally define ``database`` and ``plugin``.
+        -  **database**: the database name where logical slots should be created.
+        -  **plugin**: the plugin name for the logical slot.
+
 Global/Universal
 ----------------
 -  **name**: the name of the host. Must be unique for the cluster.
@@ -13,8 +46,10 @@ Global/Universal
 Log
 ---
 -  **level**: sets the general logging level. Default value is **INFO** (see `the docs for Python logging <https://docs.python.org/3.6/library/logging.html#levels>`_)
+-  **traceback\_level**: sets the level where tracebacks will be visible. Default value is **ERROR**. Set it to **DEBUG** if you want to see tracebacks only if you enable **log.level=DEBUG**.
 -  **format**: sets the log formatting string. Default value is **%(asctime)s %(levelname)s: %(message)s** (see `the LogRecord attributes <https://docs.python.org/3.6/library/logging.html#logrecord-attributes>`_)
 -  **dateformat**: sets the datetime formatting string. (see the `formatTime() documentation <https://docs.python.org/3.6/library/logging.html#logging.Formatter.formatTime>`_)
+-  **max\_queue\_size**: Patroni is using two-step logging. Log records are written into the in-memory queue and there is a separate thread which pulls them from the queue and writes to stderr or file. The maximum size of the internal queue is limited by default by **1000** records, which is enough to keep logs for the past 1h20m.
 -  **dir**: Directory to write application logs to. The directory must exist and be writable by the user executing Patroni. If you set this value, the application will retain 4 25MB logs by default. You can tune those retention values with `file_num` and `file_size` (see below).
 -  **file\_num**: The number of application logs to retain.
 -  **file\_size**: Size of patroni.log file (in bytes) that triggers a log rolling.
@@ -26,32 +61,7 @@ Log
 
 Bootstrap configuration
 -----------------------
--  **dcs**: This section will be written into `/<namespace>/<scope>/config` of a given configuration store after initializing of new cluster. This is the global configuration for the cluster. If you want to change some parameters for all cluster nodes - just do it in DCS (or via Patroni API) and all nodes will apply this configuration.
-    -  **loop\_wait**: the number of seconds the loop will sleep. Default value: 10
-    -  **ttl**: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process. Default value: 30
-    -  **retry\_timeout**: timeout for DCS and PostgreSQL operation retries. DCS or network issues shorter than this will not cause Patroni to demote the leader. Default value: 10
-    -  **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election.
-    -  **master\_start\_timeout**: the amount of time a master is allowed to recover from failures before failover is triggered. Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Best worst case failover time for master failure is: loop\_wait + master\_start\_timeout + loop\_wait, unless master\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff.
-    -  **synchronous\_mode**: turns on synchronous replication mode. In this mode a replica will be chosen as synchronous and only the latest leader and synchronous replica are able to participate in leader election. Synchronous mode makes sure that successfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See :ref:`replication modes documentation <replication_modes>` for details.
-    -  **synchronous\_mode\_strict**: prevents disabling synchronous replication if no synchronous replicas are available, blocking all client writes to the master. See :ref:`replication modes documentation <replication_modes>` for details.
-    -  **postgresql**:
-        -  **use\_pg\_rewind**: whether or not to use pg_rewind
-        -  **use\_slots**: whether or not to use replication_slots. Must be False for PostgreSQL 9.3. You should comment out max_replication_slots before it becomes ineligible for leader status.
-        -  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower.
-        -  **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work.
-    -  **standby\_cluster**: if this section is defined, we want to bootstrap a standby cluster.
-        -  **host**: an address of remote master
-        -  **port**: a port of remote master
-        -  **primary\_slot\_name**: which slot on the remote master to use for replication. This parameter is optional, the default value is derived from the instance name (see function `slot_name_from_member_name`).
-        -  **create\_replica\_methods**: an ordered list of methods that can be used to bootstrap standby leader from the remote master, can be different from the list defined in :ref:`postgresql_settings`
-        -  **restore\_command**: command to restore WAL records from the remote master to standby leader, can be different from the list defined in :ref:`postgresql_settings`
-        -  **archive\_cleanup\_command**: cleanup command for standby leader
-        -  **recovery\_min\_apply\_delay**: how long to wait before actually apply WAL records on a standby leader
-    -  **slots**: define permanent replication slots. These slots will be preserved during switchover/failover. Patroni will try to create slots before opening connections to the cluster.
-        -  **my_slot_name**: the name of replication slot. It is the responsibility of the operator to make sure that there are no clashes in names between replication slots automatically created by Patroni for members and permanent replication slots.
-            -  **type**: slot type. Could be ``physical`` or ``logical``. If the slot is logical, you have to additionally define ``database`` and ``plugin``.
-            -  **database**: the database name where logical slots should be created.
-            -  **plugin**: the plugin name for the logical slot.
+-  **dcs**: This section will be written into `/<namespace>/<scope>/config` of the given configuration store after initializing of new cluster. The global dynamic configuration for the cluster. Under the ``bootstrap.dcs`` you can put any of the parameters described in the :ref:`Dynamic Configuration settings <dynamic_configuration_settings>` and after Patroni initialized (bootstrapped) the new cluster, it will write this section into `/<namespace>/<scope>/config` of the configuration store. All later changes of ``bootstrap.dcs`` will not take any effect! If you want to change them please use either ``patronictl edit-config`` or Patroni :ref:`REST API <rest_api>`.
 -  **method**: custom script to use for bootstrapping this cluster.
    See :ref:`custom bootstrap methods documentation <custom_bootstrap>` for details.
    When ``initdb`` is specified revert to the default ``initdb`` command. ``initdb`` is also triggered when no ``method``
@@ -87,6 +97,7 @@ Most of the parameters are optional, but you have to specify one of the **host**
 -  **cert**: (optional) file with the client certificate
 -  **key**: (optional) file with the client key. Can be empty if the key is part of **cert**.
 -  **dc**: (optional) Datacenter to communicate with. By default the datacenter of the host is used.
+-  **consistency**: (optional) Select consul consistency mode. Possible values are ``default``, ``consistent``, or ``stale`` (more details in `consul API reference <https://www.consul.io/api/features/consistency.html/>`__)
 -  **checks**: (optional) list of Consul health checks used for the session. If not specified Consul will use "serfHealth" in additional to the TTL based check created by Patroni. Additional checks, in particular the "serfHealth", may cause the leader lock to expire faster than in `ttl` seconds when the leader instance becomes unavailable
 -  **register\_service**: (optional) whether or not to register a service with the name defined by the scope parameter and the tag master, replica or standby-leader depending on the node's role. Defaults to **false**
 -  **service\_check\_interval**: (optional) how often to perform health check against registered url
@@ -108,6 +119,10 @@ Most of the parameters are optional, but you have to specify one of the **host**
 -  **cert**: (optional) file with the client certificate.
 -  **key**: (optional) file with the client key. Can be empty if the key is part of **cert**.
 
+ZooKeeper
+----------
+-  **hosts**: list of ZooKeeper cluster members in format: ['host1:port1', 'host2:port2', 'etc...'].
+
 Exhibitor
 ---------
 -  **hosts**: initial list of Exhibitor (ZooKeeper) nodes in format: 'host1,host2,etc...'. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes.
@@ -124,7 +139,7 @@ Kubernetes
 -  **role\_label**: (optional) name of the label containing role (master or replica). Patroni will set this label on the pod it runs in. Default value is ``role``.
 -  **use\_endpoints**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
 -  **pod\_ip**: (optional) IP address of the pod Patroni is running in. This value is required when `use_endpoints` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
--  **ports**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``kubernetes.ports: {[{"name": "postgresql", "port": 5432}]}`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `kubernetes.use_endpoints` is set.
+-  **ports**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``kubernetes.ports: [{"name": "postgresql", "port": 5432}]`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `kubernetes.use_endpoints` is set.
 
 .. _postgresql_settings:
 
@@ -134,9 +149,27 @@ PostgreSQL
     -  **superuser**:
         -  **username**: name for the superuser, set during initialization (initdb) and later used by Patroni to connect to the postgres.
         -  **password**: password for the superuser, set during initialization (initdb).
+        -  **sslmode**: (optional) maps to the `sslmode <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE>`__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation <https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS>`__. The default mode is ``prefer``.
+        -  **sslkey**: (optional) maps to the `sslkey <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY>`__ connection parameter, which specifies the location of the secret key used with the client's certificate.
+        -  **sslcert**: (optional) maps to the `sslcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCERT>`__ connection parameter, which specifies the location of the client certificate.
+        -  **sslrootcert**: (optional) maps to the `sslrootcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLROOTCERT>`__ connection parameter, which specifies the location of a file containing one ore more certificate authorities (CA) certificates that the client will use to verify a server's certificate.
+        -  **sslcrl**: (optional) maps to the `sslcrl <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCRL>`__ connection parameter, which specifies the location of a file containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list.
     -  **replication**:
         -  **username**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication
         -  **password**: replication password; the user will be created during initialization.
+        -  **sslmode**: (optional) maps to the `sslmode <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE>`__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation <https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS>`__. The default mode is ``prefer``.
+        -  **sslkey**: (optional) maps to the `sslkey <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY>`__ connection parameter, which specifies the location of the secret key used with the client's certificate.
+        -  **sslcert**: (optional) maps to the `sslcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCERT>`__ connection parameter, which specifies the location of the client certificate.
+        -  **sslrootcert**: (optional) maps to the `sslrootcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLROOTCERT>`__ connection parameter, which specifies the location of a file containing one ore more certificate authorities (CA) certificates that the client will use to verify a server's certificate.
+        -  **sslcrl**: (optional) maps to the `sslcrl <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCRL>`__ connection parameter, which specifies the location of a file containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list.
+    -  **rewind**:
+        -  **username**: name for the user for ``pg_rewind``; the user will be created during initialization of postgres 11+ and all necessary `permissions <https://www.postgresql.org/docs/11/app-pgrewind.html#id-1.9.5.8.8>`__ will be granted.
+        -  **password**: password for the user for ``pg_rewind``; the user will be created during initialization.
+        -  **sslmode**: (optional) maps to the `sslmode <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE>`__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation <https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS>`__. The default mode is ``prefer``.
+        -  **sslkey**: (optional) maps to the `sslkey <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY>`__ connection parameter, which specifies the location of the secret key used with the client's certificate.
+        -  **sslcert**: (optional) maps to the `sslcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCERT>`__ connection parameter, which specifies the location of the client certificate.
+        -  **sslrootcert**: (optional) maps to the `sslrootcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLROOTCERT>`__ connection parameter, which specifies the location of a file containing one ore more certificate authorities (CA) certificates that the client will use to verify a server's certificate.
+        -  **sslcrl**: (optional) maps to the `sslcrl <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCRL>`__ connection parameter, which specifies the location of a file containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list.
 -  **callbacks**: callback scripts to run on certain actions. Patroni will pass the action, role and cluster name. (See scripts/aws.py as an example of how to write them.)
         -  **on\_reload**: run this script when configuration reload is triggered.
         -  **on\_restart**: run this script when the postgres restarts (without changing role).
@@ -151,7 +184,7 @@ PostgreSQL
 -  **config\_dir**: The location of the Postgres configuration directory, defaults to the data directory. Must be writable by Patroni.
 -  **bin\_dir**: Path to PostgreSQL binaries (pg_ctl, pg_rewind, pg_basebackup, postgres). The default value is an empty string meaning that PATH environment variable will be used to find the executables.
 -  **listen**: IP address + port that Postgres listens to; must be accessible from other nodes in the cluster, if you're using streaming replication. Multiple comma-separated addresses are permitted, as long as the port component is appended after to the last one with a colon, i.e. ``listen: 127.0.0.1,127.0.0.2:5432``. Patroni will use the first address from this list to establish local connections to the PostgreSQL node.
--  **use\_unix\_socket**: specifies that Patroni should prefer to use unix sockets to connect to the cluster. Default value is ``false``. If ``unix_socket_directories`` is definded, Patroni will use first suitable value from it to connect to the cluster and fallback to tcp if nothing is suitable. If ``unix_socket_directories`` is not specified in ``postgresql.parameters``, Patroni will assume that default value should be used and omit ``host`` from connection parameters.
+-  **use\_unix\_socket**: specifies that Patroni should prefer to use unix sockets to connect to the cluster. Default value is ``false``. If ``unix_socket_directories`` is defined, Patroni will use the first suitable value from it to connect to the cluster and fallback to tcp if nothing is suitable. If ``unix_socket_directories`` is not specified in ``postgresql.parameters``, Patroni will assume that the default value should be used and omit ``host`` from the connection parameters.
 -  **pgpass**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup, the post_init script and under some other circumstances. The location must be writable by Patroni.
 -  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower.
 -  **custom\_conf** : path to an optional custom ``postgresql.conf`` file, that will be used in place of ``postgresql.base.conf``. The file must exist on all cluster nodes, be readable by PostgreSQL and will be included from its location on the real ``postgresql.conf``. Note that Patroni will not monitor this file for changes, nor backup it. However, its settings can still be overridden by Patroni's own configuration facilities - see :ref:`dynamic configuration <dynamic_configuration>` for details.
@@ -159,15 +192,18 @@ PostgreSQL
 -  **pg\_hba**: list of lines that Patroni will use to generate ``pg_hba.conf``. This parameter has higher priority than ``bootstrap.pg_hba``. Together with :ref:`dynamic configuration <dynamic_configuration>` it simplifies management of ``pg_hba.conf``.
         -  **- host all all 0.0.0.0/0 md5**.
         -  **- host replication replicator 127.0.0.1/32 md5**: A line like this is required for replication.
+-  **pg\_ident**: list of lines that Patroni will use to generate ``pg_ident.conf``. Together with :ref:`dynamic configuration <dynamic_configuration>` it simplifies management of ``pg_ident.conf``.
+        -  **- mapname1 systemname1 pguser1**.
+        -  **- mapname1 systemname2 pguser2**.
 -  **pg\_ctl\_timeout**: How long should pg_ctl wait when doing ``start``, ``stop`` or ``restart``. Default value is 60 seconds.
 -  **use\_pg\_rewind**: try to use pg\_rewind on the former leader when it joins cluster as a replica.
--  **remove\_data\_directory\_on\_rewind\_failure**: If this option is enabled, Patroni will remove postgres data directory and recreate replica. Otherwise it will try to follow the new leader. Default value is **false**.
--  **remove\_data\_directory\_on\_diverged\_timelines**: Patroni will remove postgres data directory and recreate replica if it notices that timelines are diverging and the former master can not start streaming from the new master. This option is useful when ``pg_rewind`` can not be used. Default value is **false**.
+-  **remove\_data\_directory\_on\_rewind\_failure**: If this option is enabled, Patroni will remove the PostgreSQL data directory and recreate the replica. Otherwise it will try to follow the new leader. Default value is **false**.
+-  **remove\_data\_directory\_on\_diverged\_timelines**: Patroni will remove the PostgreSQL data directory and recreate the replica if it notices that timelines are diverging and the former master can not start streaming from the new master. This option is useful when ``pg_rewind`` can not be used. Default value is **false**.
 -  **replica\_method**: for each create_replica_methods other than basebackup, you would add a configuration section of the same name. At a minimum, this should include "command" with a full path to the actual script to be executed. Other configuration parameters will be passed along to the script in the form "parameter=value".
 
 REST API
 --------
--  **connect\_address**: IP address (or hostname) and port, to access the Patroni's REST API. All the members of the cluster must be able to connect to this address, so unless the Patroni setup is intended for a demo inside the localhost, this address must be a non "localhost" or loopback addres (ie: "localhost" or "127.0.0.1"). It can serve as a endpoint for HTTP health checks (read below about the "listen" REST API parameter), and also for user queries (either directly or via the REST API), as well as for the health checks done by the cluster members during leader elections (for example, to determine whether the master is still running, or if there is a node which has a WAL position that is ahead of the one doing the query; etc.) The connect_address is put in the member key in DCS, making it possible to translate the member name into the address to connect to its REST API.
+-  **connect\_address**: IP address (or hostname) and port, to access the Patroni's :ref:`REST API <rest_api>`. All the members of the cluster must be able to connect to this address, so unless the Patroni setup is intended for a demo inside the localhost, this address must be a non "localhost" or loopback address (ie: "localhost" or "127.0.0.1"). It can serve as an endpoint for HTTP health checks (read below about the "listen" REST API parameter), and also for user queries (either directly or via the REST API), as well as for the health checks done by the cluster members during leader elections (for example, to determine whether the master is still running, or if there is a node which has a WAL position that is ahead of the one doing the query; etc.) The connect_address is put in the member key in DCS, making it possible to translate the member name into the address to connect to its REST API.
 
 -  **listen**: IP address (or hostname) and port that Patroni will listen to for the REST API - to provide also the same health checks and cluster messaging between the participating nodes, as described above. to provide health-check information for HAProxy (or any other load balancer capable of doing a HTTP "OPTION" or "GET" checks).
 
@@ -178,6 +214,8 @@ REST API
 
         -  **certfile**: Specifies the file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL.
         -  **keyfile**: Specifies the file with the secret key in the PEM format.
+        -  **cafile**: Specifies the file with the CA_BUNDLE with certificates of trusted CAs to use while verifying client certs.
+        -  **verify\_client**: ``none``, ``optional`` or ``required``. When ``none`` REST API will not check client certificates. When ``required`` client certificates are required for all REST API calls. When ``optional`` client certificates are required for all unsafe REST API endpoints. If ``verify_client`` is set to ``optional`` or ``required`` basic-auth is not checked.
 
 .. _patronictl_settings:
 
@@ -185,15 +223,20 @@ CTL
 ---
 -  **Optional**:
     -  **insecure**: Allow connections to REST API without verifying SSL certs.
-    -  **cacert**: Specifices the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs.
-    -  **certfile**: Specifies the file with the certificate in the PEM format to use while verifying REST API SSL certs. If not provided patronictl will use the value provided for REST API "certfile" parameter.
-
-ZooKeeper
-----------
--  **hosts**: list of ZooKeeper cluster members in format: ['host1:port1', 'host2:port2', 'etc...'].
+    -  **cacert**: Specifies the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs. If not provided patronictl will use the value provided for REST API "cafile" parameter.
+    -  **certfile**: Specifies the file with the client certificate in the PEM format. If not provided patronictl will use the value provided for REST API "certfile" parameter.
+    -  **keyfile**: Specifies the file with the client secret key in the PEM format. If not provided patronictl will use the value provided for REST API "keyfile" parameter.
 
 Watchdog
 --------
 - **mode**: ``off``, ``automatic`` or ``required``. When ``off`` watchdog is disabled. When ``automatic`` watchdog will be used if available, but ignored if it is not. When ``required`` the node will not become a leader unless watchdog can be successfully enabled.
 - **device**: Path to watchdog device. Defaults to ``/dev/watchdog``.
 - **safety_margin**: Number of seconds of safety margin between watchdog triggering and leader key expiration.
+
+Tags
+----
+- **nofailover**: ``true`` or ``false``, controls whether this node is allowed to participate in the leader race and become a leader. Defaults to ``false``
+- **clonefrom**: ``true`` or ``false``. If set to ``true`` other nodes might prefer to use this node for bootstrap (take ``pg_basebackup`` from). If there are several nodes with ``clonefrom`` tag set to ``true`` the node to bootstrap from will be chosen randomly. The default value is ``false``.
+- **noloadbalance**: ``true`` or ``false``. If set to ``true`` the node will return HTTP Status Code 503 for the ``GET /replica`` REST API health-check and therefore will be excluded from the load-balancing. Defaults to ``false``.
+- **replicatefrom**: The IP address/hostname of another replica. Used to support cascading replication.
+- **nosync**: ``true`` or ``false``. If set to ``true`` the node will never be selected as a synchronous replica.

docs/dynamic_configuration.rst

@@ -14,7 +14,7 @@ Patroni configuration is stored in the DCS (Distributed Configuration Store). Th
 
 - Local :ref:`configuration <settings>` (patroni.yml).
 	These options are defined in the configuration file and take precedence over dynamic configuration.
-	patroni.yml could be changed and reload in runtime (without restart of Patroni) by sending SIGHUP to the Patroni process or by performing ``POST /reload`` REST-API request.
+	patroni.yml could be changed and reloaded in runtime (without restart of Patroni) by sending SIGHUP to the Patroni process, performing ``POST /reload`` REST-API request or executing ``patronictl reload``.
 
 - Environment :ref:`configuration <environment>`.
 	It is possible to set/override some of the "Local" configuration parameters with environment variables.
@@ -83,147 +83,3 @@ Upon changing these options, Patroni will read the relevant section of the confi
 run-time values.
 
 Patroni nodes are dumping the state of the DCS options to disk upon for every change of the configuration into the file ``patroni.dynamic.json`` located in the Postgres data directory. Only the master is allowed to restore these options from the on-disk dump if these are completely absent from the DCS or if they are invalid.
-
-REST API
-========
-
-We provide a REST API endpoint for working with dynamic configuration.
-
-GET /config
------------
-Get current version of dynamic configuration.
-
-.. code-block:: bash
-
-	$ curl -s localhost:8008/config | jq .
-	{
-	  "ttl": 30,
-	  "loop_wait": 10,
-	  "retry_timeout": 10,
-	  "maximum_lag_on_failover": 1048576,
-	  "postgresql": {
-	    "use_slots": true,
-	    "use_pg_rewind": true,
-	    "parameters": {
-	      "hot_standby": "on",
-	      "wal_log_hints": "on",
-	      "wal_keep_segments": 8,
-	      "wal_level": "hot_standby",
-	      "max_wal_senders": 5,
-	      "max_replication_slots": 5,
-	      "max_connections": "100"
-	    }
-	  }
-	}
-
-PATCH /config
--------------
-Change existing configuration.
-
-.. code-block:: bash
-
-	$ curl -s -XPATCH -d \
-		'{"loop_wait":5,"ttl":20,"postgresql":{"parameters":{"max_connections":"101"}}}' \
-		http://localhost:8008/config | jq .
-	{
-	  "ttl": 20,
-	  "loop_wait": 5,
-	  "maximum_lag_on_failover": 1048576,
-	  "retry_timeout": 10,
-	  "postgresql": {
-	    "use_slots": true,
-	    "use_pg_rewind": true,
-	    "parameters": {
-	      "hot_standby": "on",
-	      "wal_log_hints": "on",
-	      "wal_keep_segments": 8,
-	      "wal_level": "hot_standby",
-	      "max_wal_senders": 5,
-	      "max_replication_slots": 5,
-	      "max_connections": "101"
-	    }
-	  }
-	}
-
-The above REST API call patches the existing configuration and returns the new configuration.
-
-Let's check that the node processed this configuration. First of all it should start printing log lines every 5 seconds (loop_wait=5). The change of "max_connections" requires a restart, so the "restart_pending" flag should be exposed:
-
-.. code-block:: bash
-
-	$ curl -s http://localhost:8008/patroni | jq .
-	{
-	  "pending_restart": true,
-	  "database_system_identifier": "6287881213849985952",
-	  "postmaster_start_time": "2016-06-13 13:13:05.211 CEST",
-	  "xlog": {
-	    "location": 2197818976
-	  },
-	  "patroni": {
-	    "scope": "batman",
-	    "version": "1.0"
-	  },
-	  "state": "running",
-	  "role": "master",
-	  "server_version": 90503
-	}
-
-Removing parameters:
-
-If you want to remove (reset) some setting just patch it with ``null``:
-
-.. code-block:: bash
-
-	$ curl -s -XPATCH -d \
-		'{"postgresql":{"parameters":{"max_connections":null}}}' \
-		http://localhost:8008/config | jq .
-	{
-	  "ttl": 20,
-	  "loop_wait": 5,
-	  "retry_timeout": 10,
-	  "maximum_lag_on_failover": 1048576,
-	  "postgresql": {
-	    "use_slots": true,
-	    "use_pg_rewind": true,
-	    "parameters": {
-	      "hot_standby": "on",
-	      "unix_socket_directories": ".",
-	      "wal_keep_segments": 8,
-	      "wal_level": "hot_standby",
-	      "wal_log_hints": "on",
-	      "max_wal_senders": 5,
-	      "max_replication_slots": 5
-	    }
-	  }
-	}
-
-Above call removes ``postgresql.parameters.max_connections`` from the dynamic configuration.
-
-PUT /config
------------
-
-It's also possible to perform the full rewrite of an existing dynamic configuration unconditionally:
-
-.. code-block:: bash
-
-	$ curl -s -XPUT -d \
-		'{"maximum_lag_on_failover":1048576,"retry_timeout":10,"postgresql":{"use_slots":true,"use_pg_rewind":true,"parameters":{"hot_standby":"on","wal_log_hints":"on","wal_keep_segments":8,"wal_level":"hot_standby","unix_socket_directories":".","max_wal_senders":5}},"loop_wait":3,"ttl":20}' \
-		http://localhost:8008/config | jq .
-	{
-	  "ttl": 20,
-	  "maximum_lag_on_failover": 1048576,
-	  "retry_timeout": 10,
-	  "postgresql": {
-	    "use_slots": true,
-	    "parameters": {
-	      "hot_standby": "on",
-	      "unix_socket_directories": ".",
-	      "wal_keep_segments": 8,
-	      "wal_level": "hot_standby",
-	      "wal_log_hints": "on",
-	      "max_wal_senders": 5
-	    },
-	    "use_pg_rewind": true
-	  },
-	  "loop_wait": 3
-	}

docs/existing_data.rst

@@ -12,24 +12,25 @@ Procedure
 
 A Patroni cluster can be started with a data directory from a single-node PostgreSQL database. This is achieved by following closely these steps:
 
-#. Manually start PostgreSQL daemon
-#. Create Patroni superuser and replication users as defined in the :ref:`authentication <postgresql_settings>` section of the Patroni configuration. If this user is created in SQL, the following queries achieve this:
+1. Manually start PostgreSQL daemon
+2. Create Patroni superuser and replication users as defined in the :ref:`authentication <postgresql_settings>` section of the Patroni configuration. If this user is created in SQL, the following queries achieve this:
 
 .. code-block:: sql
-  CREATE USER $PATRONI_SUPERUSER_USERNAME WITH SUPERUSER ENCRYPTED PASSWORD '$PATRONI_SUPERUSER_PASSWORD';
-  CREATE USER $PATRONI_REPLICATION_USERNAME WITH REPLICATION ENCRYPTED PASSWORD '$PATRONI_REPLICATION_PASSWORD';
 
-#. Start Patroni (e.g. ``patroni /etc/patroni/patroni.yml``). It automatically detects that PostgreSQL daemon is already running but its configuration might be out-of-date.
-#. Ask Patroni to restart the node with ``patronictl restart cluster-name node-name``.
+   CREATE USER $PATRONI_SUPERUSER_USERNAME WITH SUPERUSER ENCRYPTED PASSWORD '$PATRONI_SUPERUSER_PASSWORD';
+   CREATE USER $PATRONI_REPLICATION_USERNAME WITH REPLICATION ENCRYPTED PASSWORD '$PATRONI_REPLICATION_PASSWORD';
+
+3. Start Patroni (e.g. ``patroni /etc/patroni/patroni.yml``). It automatically detects that PostgreSQL daemon is already running but its configuration might be out-of-date.
+4. Ask Patroni to restart the node with ``patronictl restart cluster-name node-name``. This step is only required if PostgreSQL configuration is out-of-date.
 
 
 FAQ
 ---
 
-#. During Patroni startup, Patroni complains that it cannot bind to the PostgreSQL port.
+- During Patroni startup, Patroni complains that it cannot bind to the PostgreSQL port.
 
-You need to verify ``listen_addresses`` and ``port`` in ``postgresql.conf`` and ``postgresql.listen`` in ``patroni.yml``. Don't forget that ``pg_hba.conf`` should allow such access.
+  You need to verify ``listen_addresses`` and ``port`` in ``postgresql.conf`` and ``postgresql.listen`` in ``patroni.yml``. Don't forget that ``pg_hba.conf`` should allow such access.
 
-#. After asking Patroni to restart the node, PostgreSQL displays the error message ``could not open configuration file "/etc/postgresql/10/main/pg_hba.conf": No such file or directory``
+- After asking Patroni to restart the node, PostgreSQL displays the error message ``could not open configuration file "/etc/postgresql/10/main/pg_hba.conf": No such file or directory``
 
-It can mean various things depending on how you manage PostgreSQL configuration. If you specified `postgresql.config_dir`, Patroni generates the ``pg_hba.conf`` based on the settings in the :ref:`bootstrap <bootstrap_settings>` section only when it bootstraps a new cluster. In this scenario the ``PGDATA`` was not empty, therefore no bootstrap happened. This file must exist beforehand.
+  It can mean various things depending on how you manage PostgreSQL configuration. If you specified `postgresql.config_dir`, Patroni generates the ``pg_hba.conf`` based on the settings in the :ref:`bootstrap <bootstrap_settings>` section only when it bootstraps a new cluster. In this scenario the ``PGDATA`` was not empty, therefore no bootstrap happened. This file must exist beforehand.

docs/index.rst

@@ -19,6 +19,7 @@ We call Patroni a "template" because it is far from being a one-size-fits-all or
 
    README
    dynamic_configuration
+   rest_api
    ENVIRONMENT
    SETTINGS
    replica_bootstrap

docs/releases.rst

@@ -3,6 +3,316 @@
 Release notes
 =============
 
+Version 1.6.3
+-------------
+
+**Bugfixes**
+
+- Don't expose password when running ``pg_rewind`` (Alexander Kukushkin)
+
+  Bug was introduced in the `#1301 <https://github.com/zalando/patroni/pull/1301>`__
+
+- Apply connection parameters specified in the ``postgresql.authentication`` to ``pg_basebackup`` and custom replica creation methods (Alexander)
+
+  They were relying on url-like connection string and therefore parameters never applied.
+
+
+Version 1.6.2
+-------------
+
+**New features**
+
+- Implemented ``patroni --version`` (Igor Yanchenko)
+
+  It prints the current version of Patroni and exits.
+
+- Set the ``user-agent`` http header for all http requests (Alexander Kukushkin)
+
+  Patroni is communicating with Consul, Etcd, and Kubernetes API via the http protocol. Having a specifically crafted ``user-agent`` (example: ``Patroni/1.6.2 Python/3.6.8 Linux``) might be useful for debugging and monitoring.
+
+- Make it possible to configure log level for exception tracebacks (Igor)
+
+  If you set ``log.traceback_level=DEBUG`` the tracebacks will be visible only when ``log.level=DEBUG``. The default behavior remains the same.
+
+
+**Stability improvements**
+
+- Avoid importing all DCS modules when searching for the module required by the config file (Alexander)
+
+  There is no need to import modules for Etcd, Consul, and Kubernetes if we need only e.g. Zookeeper. It helps to reduce memory usage and solves the problem of having INFO messages ``Failed to import smth``.
+
+- Removed python ``requests`` module from explicit requirements (Alexander)
+
+  It wasn't used for anything critical, but causing a lot of problems when the new version of ``urllib3`` is released.
+
+- Improve handling of ``etcd.hosts`` written as a comma-separated string instead of YAML array (Igor)
+
+  Previously it was failing when written in format ``host1:port1, host2:port2`` (the space character after the comma).
+
+
+**Usability improvements**
+
+- Don't force users to choose members from an empty list in ``patronictl`` (Igor)
+
+  If the user provides a wrong cluster name, we will raise an exception rather than ask to choose a member from an empty list.
+
+- Make the error message more helpful if the REST API cannot bind (Igor)
+
+  For an inexperienced user it might be hard to figure out what is wrong from the Python stacktrace.
+
+
+**Bugfixes**
+
+- Fix calculation of ``wal_buffers`` (Alexander)
+
+  The base unit has been changed from 8 kB blocks to bytes in PostgreSQL 11.
+
+- Use ``passfile`` in ``primary_conninfo`` only on PostgreSQL 10+ (Alexander)
+
+  On older versions there is no guarantee that ``passfile`` will work, unless the latest version of ``libpq`` is installed.
+
+
+Version 1.6.1
+-------------
+
+**New features**
+
+- Added ``PATRONICTL_CONFIG_FILE`` environment variable (msvechla)
+
+  It allows configuring the ``--config-file`` argument for ``patronictl`` from the environment.
+
+- Implement ``patronictl history`` (Alexander Kukushkin)
+
+  It shows the history of failovers/switchovers.
+
+- Pass ``-c statement_timeout=0`` in ``PGOPTIONS`` when doing ``pg_rewind`` (Alexander Kukushkin)
+
+  It protects from the case when ``statement_timeout`` on the server is set to some small value and one of the statements executed by pg_rewind is canceled.
+
+- Allow lower values for PostgreSQL configuration (Soulou)
+
+  Patroni didn't allow some of the PostgreSQL configuration parameters be set smaller than some hardcoded values. Now the minimal allowed values are smaller, default values have not been changed.
+
+- Allow for certificate-based authentication (Jonathan S. Katz)
+
+  This feature enables certificate-based authentication for superuser, replication, rewind accounts and allows the user to specify the ``sslmode`` they wish to connect with.
+
+- Use the ``passfile`` in the ``primary_conninfo`` instead of password (Alexander Kukushkin)
+
+  It allows to avoid setting ``600`` permissions on postgresql.conf
+
+- Perform ``pg_ctl reload`` regardless of config changes (Alexander Kukushkin)
+
+  It is possible that some config files are not controlled by Patroni. When somebody is doing a reload via the REST API or by sending SIGHUP to the Patroni process, the usual expectation is that Postgres will also be reloaded. Previously it didn't happen when there were no changes in the ``postgresql`` section of Patroni config.
+
+- Compare all recovery parameters, not only ``primary_conninfo`` (Alexander Kukushkin)
+
+  Previously the ``check_recovery_conf()`` method was only checking whether ``primary_conninfo`` has changed, never taking into account all other recovery parameters.
+
+- Make it possible to apply some recovery parameters without restart (Alexander Kukushkin)
+
+  Starting from PostgreSQL 12 the following recovery parameters could be changed without restart: ``archive_cleanup_command``, ``promote_trigger_file``, ``recovery_end_command``, and ``recovery_min_apply_delay``. In future Postgres releases this list will be extended and Patroni will support it automatically.
+
+- Make it possible to change ``use_slots`` online (Alexander Kukushkin)
+
+  Previously it required restarting Patroni and removing slots manually.
+
+- Remove only ``PATRONI_`` prefixed environment variables when starting up Postgres (Cody Coons)
+
+  It will solve a lot of problems with running different Foreign Data Wrappers.
+
+
+**Stability improvements**
+
+- Use LIST + WATCH when working with K8s API (Alexander Kukushkin)
+
+  It allows to efficiently receive object changes (pods, endpoints/configmaps) and makes less stress on K8s master nodes.
+
+- Improve the workflow when PGDATA is not empty during bootstrap (Alexander Kukushkin)
+
+  According to the ``initdb`` source code it might consider a PGDATA empty when there are only ``lost+found`` and ``.dotfiles`` in it. Now Patroni does the same. If ``PGDATA`` happens to be non-empty, and at the same time not valid from the ``pg_controldata`` point of view, Patroni will complain and exit.
+
+- Avoid calling expensive ``os.listdir()`` on every HA loop (Alexander Kukushkin)
+
+  When the system is under IO stress, ``os.listdir()`` could take a few seconds (or even minutes) to execute, badly affecting the HA loop of Patroni. This could even cause the leader key to disappear from DCS due to the lack of updates. There is a better and less expensive way to check that the PGDATA is not empty. Now we check the presence of the ``global/pg_control`` file in the PGDATA.
+
+- Some improvements in logging infrastructure (Alexander Kukushkin)
+
+  Previously threre was a possibility to loose the last few log lines on shutdown because the logging thread was a ``daemon`` thread.
+
+- Use ``spawn`` multiprocessing start method on python 3.4+ (Maciej Kowalczyk)
+
+  It is a known `issue <https://bugs.python.org/issue6721>`__ in Python that threading and multiprocessing do not mix well. Switching from the default method ``fork`` to the ``spawn`` is a recommended workaround. Not doing so might result in the Postmaster starting process hanging and Patroni indefinitely reporting ``INFO: restarting after failure in progress``, while  Postgres is actually up and running.
+
+**Improvements in REST API**
+
+- Make it possible to check client certificates in the REST API (Alexander Kukushkin)
+
+  If the ``verify_client`` is set to ``required``, Patroni will check client certificates for all REST API calls. When it is set to ``optional``, client certificates are checked for all unsafe REST API endpoints.
+
+- Return the response code 503 for the ``GET /replica`` health check request if Postgres is not running (Alexander Anikin)
+
+  Postgres might spend significant time in recovery before it starts accepting client connections.
+
+- Implement ``/history`` and ``/cluster`` endpoints (Alexander Kukushkin)
+
+  The ``/history`` endpoint shows the content of the ``history`` key in DCS. The ``/cluster`` endpoint shows all cluster members and some service info like pending and scheduled restarts or switchovers.
+
+
+**Improvements in Etcd support**
+
+- Retry on Etcd RAFT internal error (Alexander Kukushkin)
+
+  When the Etcd node is being shut down, it sends ``response code=300, data='etcdserver: server stopped'``, which was causing Patroni to demote the primary.
+
+- Don't give up on Etcd request retry too early (Alexander Kukushkin)
+
+  When there were some network problems, Patroni was quickly exhausting the list of Etcd nodes and giving up without using the whole ``retry_timeout``, potentially resulting in demoting the primary.
+
+
+**Bugfixes**
+
+- Disable ``synchronous_commit`` when granting execute permissions to the ``pg_rewind`` user (kremius)
+
+  If the bootstrap is done with ``synchronous_mode_strict: true`` the `GRANT EXECUTE` statement was waiting indefinitely due to the non-synchronous nodes being available.
+
+- Fix memory leak on python 3.7 (Alexander Kukushkin)
+
+  Patroni is using ``ThreadingMixIn`` to process REST API requests and python 3.7 made threads spawn for every request non-daemon by default.
+
+- Fix race conditions in asynchronous actions (Alexander Kukushkin)
+
+  There was a chance that ``patronictl reinit --force`` could be overwritten by the attempt to recover stopped Postgres. This ended up in a situation when Patroni was trying to start Postgres while basebackup was running.
+
+- Fix race condition in ``postmaster_start_time()`` method (Alexander Kukushkin)
+
+  If the method is executed from the REST API thread, it requires a separate cursor object to be created.
+
+- Fix the problem of not promoting the sync standby that had a name contaning upper case letters (Alexander Kukushkin)
+
+  We converted the name to the lower case because Postgres was doing the same while comparing the ``application_name`` with the value in ``synchronous_standby_names``.
+
+- Kill all children along with the callback process before starting the new one (Alexander Kukushkin)
+
+  Not doing so makes it hard to implement callbacks in bash and eventually can lead to the situation when two callbacks are running at the same time. 
+
+- Fix 'start failed' issue (Alexander Kukushkin)
+
+  Under certain conditions the Postgres state might be set to 'start failed' despite Postgres being up and running.
+
+
+Version 1.6.0
+-------------
+
+This version adds compatibility with PostgreSQL 12, makes is possible to run pg_rewind without superuser on PostgreSQL 11 and newer, and enables IPv6 support.
+
+
+**New features**
+
+- Psycopg2 was removed from requirements and must be installed independently (Alexander Kukushkin)
+
+  Starting from 2.8.0 ``psycopg2`` was split into two different packages, ``psycopg2``, and ``psycopg2-binary``, which could be installed at the same time into the same place on the filesystem. In order to decrease dependency hell problem, we let a user choose how to install it. There are a few options available, please consult the :ref:`documentation <psycopg2_install_options>`.
+
+- Compatibility with PostgreSQL 12 (Alexander Kukushkin)
+
+  Starting from PostgreSQL 12 there is no ``recovery.conf`` anymore and all former recovery parameters are converted into `GUC <https://www.enterprisedb.com/blog/what-is-a-guc-variable>`_. In order to protect from ``ALTER SYSTEM SET primary_conninfo`` or similar, Patroni will parse ``postgresql.auto.conf`` and remove all standby and recovery parameters from there. Patroni config remains backward compatible. For example despite ``restore_command`` being a GUC, one can still specify it in the ``postgresql.recovery_conf.restore_command`` section and Patroni will write it into ``postgresql.conf`` for PostgreSQL 12.
+
+- Make it possible to use ``pg_rewind`` without superuser on PostgreSQL 11 and newer (Alexander Kukushkin)
+
+  If you want to use this feature please define ``username`` and ``password`` in the ``postgresql.authentication.rewind`` section of Patroni configuration file. For an already existing cluster you will have to create the user manually and ``GRANT EXECUTE`` permission on a few functions. You can find more details in the PostgreSQL `documentation <https://www.postgresql.org/docs/11/app-pgrewind.html#id-1.9.5.8.8>`__.
+
+- Do a smart comparison of actual and desired ``primary_conninfo`` values on replicas (Alexander Kukushkin)
+
+  It might help to avoid replica restart when you are converting an already existing primary-standby cluster to one managed by Patroni
+
+- IPv6 support (Alexander Kukushkin)
+
+  There were two major issues. Patroni REST API service was listening only on ``0.0.0.0`` and IPv6 IP addresses used in the ``api_url`` and ``conn_url`` were not properly quoted.
+
+- Kerberos support (Ajith Vilas, Alexander Kukushkin)
+
+  It makes possible using Kerberos authentication between Postgres nodes instead of defining passwords in Patroni configuration file
+
+- Manage ``pg_ident.conf`` (Alexander Kukushkin)
+
+  This functionality works similarly to ``pg_hba.conf``: if the ``postgresql.pg_ident`` is defined in the config file or DCS, Patroni will write its value to ``pg_ident.conf``, however, if ``postgresql.parameters.ident_file`` is defined, Patroni will assume that ``pg_ident`` is managed from outside and not update the file.
+
+
+**Improvements in REST API**
+
+- Added ``/health`` endpoint (Wilfried Roset)
+
+  It will return an HTTP status code only if PostgreSQL is running
+
+- Added ``/read-only`` and ``/read-write`` endpoints (Julien Riou)
+
+  The ``/read-only`` endpoint enables reads balanced across replicas and the primary. The ``/read-write`` endpoint is an alias for ``/primary``, ``/leader`` and ``/master``.
+
+- Use ``SSLContext`` to wrap the REST API socket (Julien Riou)
+
+  Usage of ``ssl.wrap_socket()`` is deprecated and was still allowing soon-to-be-deprecated protocols like TLS 1.1.
+
+
+**Logging improvements**
+
+- Two-step logging (Alexander Kukushkin)
+
+  All log messages are first written into the in-memory queue and later they are asynchronously flushed into the stderr or file from a separate thread. The maximum queue size is limited (configurable). If the limit is reached, Patroni will start losing logs, which is still better than blocking the HA loop.
+
+- Enable debug logging for GET/OPTIONS API calls together with latency (Jan Tomsa)
+
+  It will help with debugging of health-checks performed by HAProxy, Consul or other tooling that decides which node is the primary/replica.
+
+- Log exceptions caught in Retry (Daniel Kucera)
+
+  Log the final exception when either the number of attempts or the timeout were reached. It will hopefully help to debug some issues when communication to DCS fails.
+
+
+**Improvements in patronictl**
+
+- Enhance dialogues for scheduled switchover and restart (Rafia Sabih)
+
+  Previously dialogues did not take into account scheduled actions and therefore were misleading.
+
+- Check if config file exists (Wilfried Roset)
+
+  Be verbose about configuration file when the given filename does not exists, instead of ignoring silently (which can lead to misunderstanding).
+
+- Add fallback value for ``EDITOR`` (Wilfried Roset)
+
+  When the ``EDITOR`` environment variable was not defined, ``patronictl edit-config`` was failing with `PatroniCtlException`. The new strategy is to try ``editor`` and than ``vi``, which should be available on most systems.
+
+
+**Improvements in Consul support**
+
+- Allow to specify Consul consistency mode (Jan Tomsa)
+
+  You can read more about consistency mode `here <https://www.consul.io/api/features/consistency.html>`__.
+
+- Reload Consul config on SIGHUP (Cameron Daniel, Alexander Kukushkin)
+
+  It is especially useful when somebody is changing the value of ``token``.
+
+
+**Bugfixes**
+
+- Fix corner case in switchover/failover (Sharoon Thomas)
+
+  The variable ``scheduled_at`` may be undefined if REST API is not accessible and we are using DCS as a fallback.
+
+- Open trust to localhost in ``pg_hba.conf`` during custom bootstrap (Alexander Kukushkin)
+
+  Previously it was open only to unix_socket, which was causing a lot of errors: ``FATAL:  no pg_hba.conf entry for replication connection from host "127.0.0.1", user "replicator"``
+
+- Consider synchronous node as healthy even when the former leader is ahead (Alexander Kukushkin)
+
+  If the primary loses access to the DCS, it restarts Postgres in read-only, but it might happen that other nodes can still access the old primary via the REST API. Such a situation was causing the synchronous standby not to promote because the old primary was reporting WAL position ahead of the synchronous standby.
+
+- Standby cluster bugfixes (Alexander Kukushkin)
+
+  Make it possible to bootstrap a replica in a standby cluster when the standby_leader is not accessible and a few other minor fixes.
+
+
 Version 1.5.6
 -------------
 

docs/replica_bootstrap.rst

@@ -127,7 +127,7 @@ A special ``no_master`` parameter, if defined, allows Patroni to call the replic
 running master or replicas. In that case, an empty string will be passed in a connection string. This is useful for
 restoring the formerly running cluster from the binary backup.
 
-A special ``keep_data`` parameter, if defined, will instuct Patroni to  not clean PGDATA folder before calling restore.
+A special ``keep_data`` parameter, if defined, will instruct Patroni to  not clean PGDATA folder before calling restore.
 
 A special ``no_params`` parameter, if defined, restricts passing parameters to custom command.
 
@@ -204,4 +204,4 @@ in a patroni configuration:
 Note, that these options will be applied only once during cluster bootstrap,
 and the only way to change them afterwards is through DCS.
 
-If you use replication slots on the standby cluster, you must also create the corresponding replication slot on the primary cluster.  It will not be done automatically by the standby cluster implementation.  You can use Patroni's permenant replication slots feature on the primary cluster to maintain a replication slot with the same name as ``primary_slot_name``, or its default value if ``primary_slot_name`` is not provided.
+If you use replication slots on the standby cluster, you must also create the corresponding replication slot on the primary cluster.  It will not be done automatically by the standby cluster implementation.  You can use Patroni's permanent replication slots feature on the primary cluster to maintain a replication slot with the same name as ``primary_slot_name``, or its default value if ``primary_slot_name`` is not provided.

docs/replication_modes.rst

@@ -46,7 +46,7 @@ When ``synchronous_mode`` is on and a standby crashes, commits will block until
 
 When it is absolutely necessary to guarantee that each write is stored durably
 on at least two nodes, enable ``synchronous_mode_strict`` in addition to the
-``synchronous_node``. This parameter prevents Patroni from switching off the
+``synchronous_mode``. This parameter prevents Patroni from switching off the
 synchronous replication on the primary when no synchronous standby candidates
 are available. As a downside, the primary is not be available for writes
 (unless the Postgres transaction explicitly turns of ``synchronous_mode``),

docs/rest_api.rst

@@ -0,0 +1,336 @@
+.. _rest_api:
+
+Patroni REST API
+================
+
+Patroni has a rich REST API, which is used by Patroni itself during the leader race, by the ``patronictl`` tool in order to perform failovers/switchovers/reinitialize/restarts/reloads, by HAProxy or any other kind of load balancer to perform HTTP health checks, and of course could also be used for monitoring. Below you will find the list of Patroni REST API endpoints.
+
+Health check endpoints
+----------------------
+For all health check ``GET`` requests Patroni returns a JSON document with the status of the node, along with the HTTP status code. If you don't want or don't need the JSON document, you might consider using the ``OPTIONS`` method instead of ``GET``.
+
+- The following requests to Patroni REST API will return HTTP status code **200** only when the Patroni node is running as the leader:
+
+  - ``GET /``
+  - ``GET /master``
+  - ``GET /leader``
+  - ``GET /primary``
+  - ``GET /read-write``
+
+- ``GET /replica``: replica health check endpoint. It returns HTTP status code **200** only when the Patroni node is in the state ``running``, the role is ``replica`` and ``noloadbalance`` tag is not set.
+
+- ``GET /read-only``: like the above endpoint, but also includes the primary.
+
+- ``GET /standby-leader``: returns HTTP status code **200** only when the Patroni node is running as the leader in a :ref:`standby cluster <standby_cluster>`.
+
+- ``GET /synchronous`` or ``GET /sync``: returns HTTP status code **200** only when the Patroni node is running as a synchronous standby.
+
+- ``GET /asynchronous`` or ``GET /async``: returns HTTP status code **200** only when the Patroni node is running as an asynchronous standby.
+
+- ``GET /health``: returns HTTP status code **200** only when PostgreSQL is up and running.
+
+Monitoring endpoint
+-------------------
+
+The ``GET /patroni`` is used by Patroni during the leader race. It also could be used by your monitoring system. The JSON document produced by this endpoint has the same structure as the JSON produced by the health check endpoints.
+
+.. code-block:: bash
+
+    $ curl -s http://localhost:8008/patroni | jq .
+    {
+      "state": "running",
+      "postmaster_start_time": "2019-09-24 09:22:32.555 CEST",
+      "role": "master",
+      "server_version": 110005,
+      "cluster_unlocked": false,
+      "xlog": {
+        "location": 25624640
+      },
+      "timeline": 3,
+      "database_system_identifier": "6739877027151648096",
+      "patroni": {
+        "version": "1.6.0",
+        "scope": "batman"
+      }
+    }
+
+Cluster status endpoints
+------------------------
+
+- The ``GET /cluster`` endpoint generates a JSON document describing the current cluster topology and state:
+
+.. code-block:: bash
+
+    $ curl -s http://localhost:8008/cluster | jq .
+    {
+      "members": [
+        {
+          "name": "postgresql0",
+          "host": "127.0.0.1",
+          "port": 5432,
+          "role": "leader",
+          "state": "running",
+          "api_url": "http://127.0.0.1:8008/patroni",
+          "timeline": 5,
+          "tags": {
+            "clonefrom": true
+          }
+        },
+        {
+          "name": "postgresql1",
+          "host": "127.0.0.1",
+          "port": 5433,
+          "role": "replica",
+          "state": "running",
+          "api_url": "http://127.0.0.1:8009/patroni",
+          "timeline": 5,
+          "tags": {
+            "clonefrom": true
+          },
+          "lag": 0
+        }
+      ],
+      "scheduled_switchover": {
+        "at": "2019-09-24T10:36:00+02:00",
+        "from": "postgresql0"
+      }
+    }
+
+
+- The ``GET /history`` endpoint provides a view on the history of cluster switchovers/failovers. The format is very similar to the content of history files in the ``pg_wal`` directory. The only difference is the timestamp field showing when the new timeline was created.
+
+.. code-block:: bash
+
+    $ curl -s http://localhost:8008/history | jq .
+    [
+      [
+        1,
+        25623960,
+        "no recovery target specified",
+        "2019-09-23T16:57:57+02:00"
+      ],
+      [
+        2,
+        25624344,
+        "no recovery target specified",
+        "2019-09-24T09:22:33+02:00"
+      ],
+      [
+        3,
+        25624752,
+        "no recovery target specified",
+        "2019-09-24T09:26:15+02:00"
+      ],
+      [
+        4,
+        50331856,
+        "no recovery target specified",
+        "2019-09-24T09:35:52+02:00"
+      ]
+    ]
+
+
+Config endpoint
+---------------
+
+``GET /config``: Get the current version of the dynamic configuration:
+
+.. code-block:: bash
+
+	$ curl -s localhost:8008/config | jq .
+	{
+	  "ttl": 30,
+	  "loop_wait": 10,
+	  "retry_timeout": 10,
+	  "maximum_lag_on_failover": 1048576,
+	  "postgresql": {
+	    "use_slots": true,
+	    "use_pg_rewind": true,
+	    "parameters": {
+	      "hot_standby": "on",
+	      "wal_log_hints": "on",
+	      "wal_keep_segments": 8,
+	      "wal_level": "hot_standby",
+	      "max_wal_senders": 5,
+	      "max_replication_slots": 5,
+	      "max_connections": "100"
+	    }
+	  }
+	}
+
+
+``PATCH /config``: Change the existing configuration.
+
+.. code-block:: bash
+
+	$ curl -s -XPATCH -d \
+		'{"loop_wait":5,"ttl":20,"postgresql":{"parameters":{"max_connections":"101"}}}' \
+		http://localhost:8008/config | jq .
+	{
+	  "ttl": 20,
+	  "loop_wait": 5,
+	  "maximum_lag_on_failover": 1048576,
+	  "retry_timeout": 10,
+	  "postgresql": {
+	    "use_slots": true,
+	    "use_pg_rewind": true,
+	    "parameters": {
+	      "hot_standby": "on",
+	      "wal_log_hints": "on",
+	      "wal_keep_segments": 8,
+	      "wal_level": "hot_standby",
+	      "max_wal_senders": 5,
+	      "max_replication_slots": 5,
+	      "max_connections": "101"
+	    }
+	  }
+	}
+
+The above REST API call patches the existing configuration and returns the new configuration.
+
+Let's check that the node processed this configuration. First of all it should start printing log lines every 5 seconds (loop_wait=5). The change of "max_connections" requires a restart, so the "pending_restart" flag should be exposed:
+
+.. code-block:: bash
+
+	$ curl -s http://localhost:8008/patroni | jq .
+	{
+	  "pending_restart": true,
+	  "database_system_identifier": "6287881213849985952",
+	  "postmaster_start_time": "2016-06-13 13:13:05.211 CEST",
+	  "xlog": {
+	    "location": 2197818976
+	  },
+	  "patroni": {
+	    "scope": "batman",
+	    "version": "1.0"
+	  },
+	  "state": "running",
+	  "role": "master",
+	  "server_version": 90503
+	}
+
+Removing parameters:
+
+If you want to remove (reset) some setting just patch it with ``null``:
+
+.. code-block:: bash
+
+	$ curl -s -XPATCH -d \
+		'{"postgresql":{"parameters":{"max_connections":null}}}' \
+		http://localhost:8008/config | jq .
+	{
+	  "ttl": 20,
+	  "loop_wait": 5,
+	  "retry_timeout": 10,
+	  "maximum_lag_on_failover": 1048576,
+	  "postgresql": {
+	    "use_slots": true,
+	    "use_pg_rewind": true,
+	    "parameters": {
+	      "hot_standby": "on",
+	      "unix_socket_directories": ".",
+	      "wal_keep_segments": 8,
+	      "wal_level": "hot_standby",
+	      "wal_log_hints": "on",
+	      "max_wal_senders": 5,
+	      "max_replication_slots": 5
+	    }
+	  }
+	}
+
+The above call removes ``postgresql.parameters.max_connections`` from the dynamic configuration.
+
+``PUT /config``: It's also possible to perform the full rewrite of an existing dynamic configuration unconditionally:
+
+.. code-block:: bash
+
+	$ curl -s -XPUT -d \
+		'{"maximum_lag_on_failover":1048576,"retry_timeout":10,"postgresql":{"use_slots":true,"use_pg_rewind":true,"parameters":{"hot_standby":"on","wal_log_hints":"on","wal_keep_segments":8,"wal_level":"hot_standby","unix_socket_directories":".","max_wal_senders":5}},"loop_wait":3,"ttl":20}' \
+		http://localhost:8008/config | jq .
+	{
+	  "ttl": 20,
+	  "maximum_lag_on_failover": 1048576,
+	  "retry_timeout": 10,
+	  "postgresql": {
+	    "use_slots": true,
+	    "parameters": {
+	      "hot_standby": "on",
+	      "unix_socket_directories": ".",
+	      "wal_keep_segments": 8,
+	      "wal_level": "hot_standby",
+	      "wal_log_hints": "on",
+	      "max_wal_senders": 5
+	    },
+	    "use_pg_rewind": true
+	  },
+	  "loop_wait": 3
+	}
+
+
+Switchover and failover endpoints
+---------------------------------
+
+``POST /switchover`` or ``POST /failover``. These endpoints are very similar to each other. There are a couple of minor differences though:
+
+1. The failover endpoint allows to perform a manual failover when there are no healthy nodes, but at the same time it will not allow you to schedule a switchover.
+
+2. The switchover endpoint is the opposite. It works only when the cluster is healthy (there is a leader) and allows to schedule a switchover at a given time.
+
+
+In the JSON body of the ``POST`` request you must specify at least the ``leader`` or ``candidate`` fields and optionally the ``scheduled_at`` field if you want to schedule a switchover at a specific time.
+
+
+Example: perform a failover to the specific node:
+
+.. code-block:: bash
+
+    $ curl -s http://localhost:8009/failover -XPOST -d '{"candidate":"postgresql1"}'
+    Successfully failed over to "postgresql1"
+
+
+Example: schedule a switchover from the leader to any other healthy replica in the cluster at a specific time:
+
+.. code-block:: bash
+
+    $ curl -s http://localhost:8008/switchover -XPOST -d \
+	    '{"leader":"postgresql0","scheduled_at":"2019-09-24T12:00+00"}'
+    Switchover scheduled
+
+
+Depending on the situation the request might finish with a different HTTP status code and body. The status code **200** is returned when the switchover or failover successfully completed. If the switchover was successfully scheduled, Patroni will return HTTP status code **202**. In case something went wrong, the error status code (one of **400**, **412** or **503**) will be returned with some details in the response body. For more information please check the source code of ``patroni/api.py:do_POST_failover()`` method.
+
+The switchover and failover endpoints are used by ``patronictl switchover`` and ``patronictl failover``, respectively.
+
+
+Restart endpoint
+----------------
+
+- ``POST /restart``: You can restart Postgres on the specific node by performing the ``POST /restart`` call. In the JSON body of ``POST`` request it is possible to optionally specify some restart conditions:
+
+  - **restart_pending**: boolean, if set to ``true`` Patroni will restart PostgreSQL only when restart is pending in order to apply some changes in the PostgreSQL config.
+  - **role**: perform restart only if the current role of the node matches with the role from the POST request.
+  - **postgres_version**: perform restart only if the current version of postgres is smaller than specified in the POST request.
+  - **timeout**: how long we should wait before PostgreSQL starts accepting connections. Overrides ``master_start_timeout``.
+  - **schedule**: timestamp with time zone, schedule the restart somewhere in the future.
+
+- ``DELETE /restart``: delete the scheduled restart
+
+``POST /restart`` and ``DELETE /restart`` endpoints are used by ``patronictl restart`` and ``patronictl flush`` respectively.
+
+
+Reload endpoint
+---------------
+
+The ``POST /reload`` call will order Patroni to re-read and apply the configuration file. This is the equivalent of sending the ``SIGHUP`` signal to the Patroni process. In case you changed some of the Postgres parameters which require a restart (like **shared_buffers**), you still have to explicitly do the restart of Postgres by either calling the ``POST /restart`` endpoint or with the help of ``patronictl restart``.
+
+The reload endpoint is used by ``patronictl reload``.
+
+
+Reinitialize endpoint
+---------------------
+
+``POST /reinitialize``: reinitialize the PostgreSQL data directory on the specified node. It is allowed to be executed only on replicas. Once called, it will remove the data directory and start ``pg_basebackup`` or some alternative :ref:`replica creation method <custom_replica_creation>`.
+
+The call might fail if Patroni is in a loop trying to recover (restart) a failed Postgres. In order to overcome this problem one can specify ``{"force":true}`` in the request body.
+
+The reinitialize endpoint is used by ``patronictl reinit``.

features/basic_replication.feature

@@ -4,6 +4,7 @@ Feature: basic replication
   Scenario: check replication of a single table
     Given I start postgres0
     Then postgres0 is a leader after 10 seconds
+    And there is a non empty initialize key in DCS after 15 seconds
     When I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "loop_wait": 2, "synchronous_mode": true}
     Then I receive a response code 200
     When I start postgres1
@@ -35,10 +36,12 @@ Feature: basic replication
     And I run patronictl.py resume batman
     Then I receive a response returncode 0
     And postgres2 role is the primary after 24 seconds
+    And Response on GET http://127.0.0.1:8010/history contains recovery after 10 seconds
     When I issue a PATCH request to http://127.0.0.1:8010/config with {"synchronous_mode": null, "master_start_timeout": 0}
     Then I receive a response code 200
     When I add the table bar to postgres2
     Then table bar is present on postgres1 after 20 seconds
+    And Response on GET http://127.0.0.1:8010/config contains master_start_timeout after 10 seconds
 
   Scenario: check immediate failover when master_start_timeout=0
     Given I kill postmaster on postgres2

features/custom_bootstrap.feature

@@ -13,5 +13,5 @@ Scenario: make a backup and do a restore into a new cluster
     Given I add the table bar to postgres1
     And I do a backup of postgres1
     When I start postgres2 in a cluster batman2 from backup
-    Then postgres2 is a leader of batman2 after 10 seconds
+    Then postgres2 is a leader of batman2 after 30 seconds
     And table bar is present on postgres2 after 10 seconds

features/environment.py

@@ -1,11 +1,6 @@
 import abc
-import consul
 import datetime
-import etcd
-import kazoo.client
-import kazoo.exceptions
 import os
-import psutil
 import psycopg2
 import json
 import shutil
@@ -249,58 +244,23 @@ class PatroniController(AbstractController):
         except Exception:
             return None
 
-    def database_is_running(self):
-        pid = self._get_pid()
-        if not pid:
-            return False
-        try:
-            os.kill(pid, 0)
-        except OSError:
-            return False
-        return True
-
     def patroni_hang(self, timeout):
         hang = ProcessHang(self._handle.pid, timeout)
         self._closables.append(hang)
         hang.start()
 
-    def checkpoint_hang(self, timeout):
-        pid = self._get_pid()
-        if not pid:
-            return False
-        proc = psutil.Process(pid)
-        for child in proc.children():
-            if 'checkpoint' in child.cmdline()[0]:
-                checkpointer = child
-                break
-        else:
-            return False
-        hang = ProcessHang(checkpointer.pid, timeout)
-        self._closables.append(hang)
-        hang.start()
-        return True
-
     def cancel_background(self):
         for obj in self._closables:
             obj.close()
         self._closables = []
 
-    def terminate_backends(self):
-        pid = self._get_pid()
-        if not pid:
-            return False
-        proc = psutil.Process(pid)
-        for p in proc.children():
-            if 'process' not in p.cmdline()[0]:
-                p.terminate()
-
     @property
     def backup_source(self):
         return 'postgres://{username}:{password}@{host}:{port}/{database}'.format(**self._replication)
 
-    def backup(self, dest='basebackup'):
+    def backup(self, dest='data/basebackup'):
         subprocess.call([PatroniPoolController.BACKUP_SCRIPT, '--walmethod=none',
-                         '--datadir=' + os.path.join(self._output_dir, dest),
+                         '--datadir=' + os.path.join(self._work_directory, dest),
                          '--dbname=' + self.backup_source])
 
 
@@ -375,9 +335,11 @@ class ConsulController(AbstractDcsController):
         super(ConsulController, self).__init__(context)
         os.environ['PATRONI_CONSUL_HOST'] = 'localhost:8500'
         os.environ['PATRONI_CONSUL_REGISTER_SERVICE'] = 'on'
-        self._client = consul.Consul()
         self._config_file = None
 
+        import consul
+        self._client = consul.Consul()
+
     def _start(self):
         self._config_file = self._work_directory + '.json'
         with open(self._config_file, 'wb') as f:
@@ -417,6 +379,8 @@ class EtcdController(AbstractDcsController):
     def __init__(self, context):
         super(EtcdController, self).__init__(context)
         os.environ['PATRONI_ETCD_HOST'] = 'localhost:2379'
+
+        import etcd
         self._client = etcd.Client(port=2379)
 
     def _start(self):
@@ -424,12 +388,14 @@ class EtcdController(AbstractDcsController):
                                 stdout=self._log, stderr=subprocess.STDOUT)
 
     def query(self, key, scope='batman'):
+        import etcd
         try:
             return self._client.get(self.path(key, scope)).value
         except etcd.EtcdKeyNotFound:
             return None
 
     def cleanup_service_tree(self):
+        import etcd
         try:
             self._client.delete(self.path(scope=''), recursive=True)
         except (etcd.EtcdKeyNotFound, etcd.EtcdConnectionFailed):
@@ -473,13 +439,13 @@ class KubernetesController(AbstractDcsController):
 
     def delete_pod(self, name):
         try:
-            self._api.delete_namespaced_pod(name, self._namespace, self._client.V1DeleteOptions())
-        except:
+            self._api.delete_namespaced_pod(name, self._namespace, body=self._client.V1DeleteOptions())
+        except Exception:
             pass
         while True:
             try:
                 self._api.read_namespaced_pod(name, self._namespace)
-            except:
+            except Exception:
                 break
 
     def query(self, key, scope='batman'):
@@ -488,22 +454,23 @@ class KubernetesController(AbstractDcsController):
             return (pod.metadata.annotations or {}).get('status', '')
         else:
             try:
-                e = self._api.read_namespaced_endpoints(scope + ('' if key == 'leader' else '-' + key), self._namespace)
-                if key == 'leader':
+                ep = scope + {'leader': '', 'history': '-config', 'initialize': '-config'}.get(key, '-' + key)
+                e = self._api.read_namespaced_endpoints(ep, self._namespace)
+                if key != 'sync':
                     return e.metadata.annotations[key]
                 else:
                     return json.dumps(e.metadata.annotations)
-            except:
+            except Exception:
                 return None
 
     def cleanup_service_tree(self):
         try:
             self._api.delete_collection_namespaced_pod(self._namespace, label_selector=self._label_selector)
-        except:
+        except Exception:
             pass
         try:
             self._api.delete_collection_namespaced_endpoints(self._namespace, label_selector=self._label_selector)
-        except:
+        except Exception:
             pass
 
         while True:
@@ -523,18 +490,22 @@ class ZooKeeperController(AbstractDcsController):
         super(ZooKeeperController, self).__init__(context, False)
         if export_env:
             os.environ['PATRONI_ZOOKEEPER_HOSTS'] = "'localhost:2181'"
+
+        import kazoo.client
         self._client = kazoo.client.KazooClient()
 
     def _start(self):
         pass  # TODO: implement later
 
     def query(self, key, scope='batman'):
+        import kazoo.exceptions
         try:
             return self._client.get(self.path(key, scope))[0].decode('utf-8')
         except kazoo.exceptions.NoNodeError:
             return None
 
     def cleanup_service_tree(self):
+        import kazoo.exceptions
         try:
             self._client.delete(self.path(scope=''), recursive=True)
         except (kazoo.exceptions.NoNodeError):
@@ -594,9 +565,8 @@ class PatroniPoolController(object):
         self._processes[name].start(max_wait_limit)
 
     def __getattr__(self, func):
-        if func not in ['stop', 'query', 'write_label', 'read_label', 'check_role_has_changed_to', 'add_tag_to_config',
-                        'get_watchdog', 'database_is_running', 'checkpoint_hang', 'patroni_hang',
-                        'terminate_backends', 'backup']:
+        if func not in ['stop', 'query', 'write_label', 'read_label', 'check_role_has_changed_to',
+                        'add_tag_to_config', 'get_watchdog', 'patroni_hang', 'backup']:
             raise AttributeError("PatroniPoolController instance has no attribute '{0}'".format(func))
 
         def wrapper(name, *args, **kwargs):
@@ -610,7 +580,7 @@ class PatroniPoolController(object):
         self._processes.clear()
 
     def create_and_set_output_directory(self, feature_name):
-        feature_dir = os.path.join(self.patroni_path, 'features/output', feature_name.replace(' ', '_'))
+        feature_dir = os.path.join(self.patroni_path, 'features', 'output', feature_name.replace(' ', '_'))
         if os.path.exists(feature_dir):
             shutil.rmtree(feature_dir)
         os.makedirs(feature_dir)
@@ -637,7 +607,7 @@ class PatroniPoolController(object):
                 'parameters': {
                     'archive_mode': 'on',
                     'archive_command': 'mkdir -p {0} && test ! -f {0}/%f && cp %p {0}/%f'.format(
-                            os.path.join(self._output_dir, 'wal_archive'))
+                            os.path.join(self.patroni_path, 'data', 'wal_archive'))
                 },
                 'authentication': {
                     'superuser': {'password': 'zalando1'},
@@ -653,11 +623,12 @@ class PatroniPoolController(object):
             'bootstrap': {
                 'method': 'backup_restore',
                 'backup_restore': {
-                    'command': 'features/backup_restore.sh --sourcedir=' + os.path.join(self._output_dir, 'basebackup'),
+                    'command': 'features/backup_restore.sh --sourcedir=' + os.path.join(self.patroni_path,
+                                                                                        'data', 'basebackup'),
                     'recovery_conf': {
                         'recovery_target_action': 'promote',
                         'recovery_target_timeline': 'latest',
-                        'restore_command': 'cp {0}/wal_archive/%f %p'.format(self._output_dir)
+                        'restore_command': 'cp {0}/data/wal_archive/%f %p'.format(self.patroni_path)
                     }
                 }
             },

features/patroni_api.feature

@@ -8,6 +8,8 @@ Scenario: check API requests on a stand-alone server
 	Then I receive a response code 200
 	And I receive a response state running
 	And I receive a response role master
+	When I issue a GET request to http://127.0.0.1:8008/health
+	Then I receive a response code 200
 	When I issue a GET request to http://127.0.0.1:8008/replica
 	Then I receive a response code 503
 	When I run patronictl.py reinit batman postgres0 --force
@@ -26,10 +28,7 @@ Scenario: check API requests on a stand-alone server
 	And I receive a response text "Failover could be performed only to a specific candidate"
 
 Scenario: check local configuration reload
-	Given I issue an empty POST request to http://127.0.0.1:8008/reload
-	Then I receive a response code 200
-	And I receive a response text nothing changed
-	When I add tag new_tag new_value to postgres0 config
+	Given I add tag new_tag new_value to postgres0 config
 	And I issue an empty POST request to http://127.0.0.1:8008/reload
 	Then I receive a response code 202
 
@@ -44,6 +43,20 @@ Scenario: check dynamic configuration change via DCS
 	When I issue a GET request to http://127.0.0.1:8008/patroni
 	Then I receive a response code 200
 	And I receive a response tags {'new_tag': 'new_value'}
+	And I sleep for 4 seconds
+
+Scenario: check the scheduled restart
+	Given I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"superuser_reserved_connections": "6"}}}
+	Then I receive a response code 200
+		And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 5 seconds
+	Given I issue a scheduled restart at http://127.0.0.1:8008 in 3 seconds with {"role": "replica"}
+	Then I receive a response code 202
+		And I sleep for 4 seconds
+		And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 10 seconds
+	Given I issue a scheduled restart at http://127.0.0.1:8008 in 3 seconds with {"restart_pending": "True"}
+	Then I receive a response code 202
+		And Response on GET http://127.0.0.1:8008/patroni does not contain pending_restart after 10 seconds
+		And postgres0 role is the primary after 10 seconds
 
 Scenario: check API requests for the primary-replica pair in the pause mode
 	Given I run patronictl.py pause batman
@@ -71,6 +84,7 @@ Scenario: check the switchover via the API in the pause mode
 	And postgres1 role is the primary after 10 seconds
 	And postgres0 role is the secondary after 10 seconds
 	And replication works from postgres1 to postgres0 after 20 seconds
+	And "members/postgres0" key in DCS has state=running after 10 seconds
 	When I issue a GET request to http://127.0.0.1:8008/master
 	Then I receive a response code 503
 	When I issue a GET request to http://127.0.0.1:8008/replica
@@ -92,6 +106,7 @@ Scenario: check the scheduled switchover
 	And postgres0 role is the primary after 10 seconds
 	And postgres1 role is the secondary after 10 seconds
 	And replication works from postgres0 to postgres1 after 25 seconds
+	And "members/postgres1" key in DCS has state=running after 10 seconds
 	When I issue a GET request to http://127.0.0.1:8008/master
 	Then I receive a response code 200
 	When I issue a GET request to http://127.0.0.1:8008/replica
@@ -100,16 +115,3 @@ Scenario: check the scheduled switchover
 	Then I receive a response code 503
 	When I issue a GET request to http://127.0.0.1:8009/replica
 	Then I receive a response code 200
-
-Scenario: check the scheduled restart
-	Given I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"superuser_reserved_connections": "6"}}}
-	Then I receive a response code 200
-		And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 5 seconds
-	Given I issue a scheduled restart at http://127.0.0.1:8008 in 3 seconds with {"role": "replica"}
-	Then I receive a response code 202
-		And I sleep for 4 seconds
-		And Response on GET http://127.0.0.1:8008/patroni contains pending_restart after 10 seconds
-	Given I issue a scheduled restart at http://127.0.0.1:8008 in 3 seconds with {"restart_pending": "True"}
-	Then I receive a response code 202
-		And Response on GET http://127.0.0.1:8008/patroni does not contain pending_restart after 10 seconds
-

features/standby_cluster.feature

@@ -2,13 +2,14 @@ Feature: standby cluster
   Scenario: check permanent logical slots are preserved on failover/switchover
     Given I start postgres1
     Then postgres1 is a leader after 10 seconds
-    And I sleep for 3 seconds
+    And there is a non empty initialize key in DCS after 15 seconds
     When I issue a PATCH request to http://127.0.0.1:8009/config with {"loop_wait": 2, "slots": {"pm_1": {"type": "physical"}}, "postgresql": {"parameters": {"wal_level": "logical"}}}
     Then I receive a response code 200
     And Response on GET http://127.0.0.1:8009/config contains slots after 10 seconds
     And I sleep for 3 seconds
     When I issue a PATCH request to http://127.0.0.1:8009/config with {"slots": {"test_logical": {"type": "logical", "database": "postgres", "plugin": "test_decoding"}}}
     Then I receive a response code 200
+    And I do a backup of postgres1
     When I start postgres0 with callback configured
     Then "members/postgres0" key in DCS has state=running after 10 seconds
     And replication works from postgres1 to postgres0 after 15 seconds

features/steps/cascading_replication.py

@@ -34,3 +34,17 @@ def check_member(context, name, key, value, time_limit):
             pass
         time.sleep(1)
     assert False, "{0} does not have {1}={2} in dcs after {3} seconds".format(name, key, value, time_limit)
+
+
+@step('there is a non empty {key:w} key in DCS after {time_limit:d} seconds')
+def check_initialize(context, key, time_limit):
+    time_limit *= context.timeout_multiplier
+    max_time = time.time() + int(time_limit)
+    while time.time() < max_time:
+        try:
+            if context.dcs_ctl.query(key):
+                return
+        except Exception:
+            pass
+        time.sleep(1)
+    assert False, "There is no {0} in dcs after {1} seconds".format(key, time_limit)

features/steps/patroni_api.py

@@ -1,8 +1,6 @@
-import base64
 import json
 import os
 import parse
-import requests
 import shlex
 import subprocess
 import sys
@@ -12,8 +10,10 @@ import yaml
 from behave import register_type, step, then
 from dateutil import tz
 from datetime import datetime, timedelta
+from patroni.request import PatroniRequest
 
 tzutc = tz.tzutc()
+request_executor = PatroniRequest({'ctl': {'auth': 'username:password'}})
 
 
 @parse.with_pattern(r'https?://(?:\w|\.|:|/)+')
@@ -45,9 +45,9 @@ def sleep_for_n_seconds(context, value):
 
 
 def _set_response(context, response):
-    context.status_code = response.status_code
-    data = response.content.decode('utf-8')
-    ct = response.headers.get('content-type', '')
+    context.status_code = response.status
+    data = response.data.decode('utf-8')
+    ct = response.getheader('content-type', '')
     if ct.startswith('application/json') or\
             ct.startswith('text/yaml') or\
             ct.startswith('text/x-yaml') or\
@@ -63,13 +63,7 @@ def _set_response(context, response):
 
 @step('I issue a GET request to {url:url}')
 def do_get(context, url):
-    try:
-        r = requests.get(url)
-    except requests.exceptions.RequestException:
-        context.status_code = None
-        context.response = None
-    else:
-        _set_response(context, r)
+    do_request(context, 'GET', url, None)
 
 
 @step('I issue an empty POST request to {url:url}')
@@ -79,17 +73,11 @@ def do_post_empty(context, url):
 
 @step('I issue a {request_method:w} request to {url:url} with {data}')
 def do_request(context, request_method, url, data):
-    data = data and json.loads(data) or {}
-    headers = {'Authorization': 'Basic ' + base64.b64encode('username:password'.encode('utf-8')).decode('utf-8'),
-               'Content-Type': 'application/json'}
+    data = data and json.loads(data)
     try:
-        if request_method == 'PATCH':
-            r = requests.patch(url, headers=headers, json=data)
-        else:
-            r = requests.post(url, headers=headers, json=data)
-    except requests.exceptions.RequestException:
-        context.status_code = None
-        context.response = None
+        r = request_executor.request(request_method, url, data)
+    except Exception:
+        context.status_code = context.response = None
     else:
         _set_response(context, r)
 
@@ -149,8 +137,8 @@ def add_tag_to_config(context, tag, value, pg_name):
 def check_http_response(context, url, value, timeout, negate=False):
     timeout *= context.timeout_multiplier
     for _ in range(int(timeout)):
-        r = requests.get(url)
-        if (value in r.content.decode('utf-8')) != negate:
+        r = request_executor.request('GET', url)
+        if (value in r.data.decode('utf-8')) != negate:
             break
         time.sleep(1)
     else:

features/steps/standby_cluster.py

@@ -28,7 +28,10 @@ def start_patroni(context, name, cluster_name):
     return context.pctl.start(name, custom_config={
         "scope": cluster_name,
         "postgresql": {
-            "callbacks": {c: callback + name for c in ('on_start', 'on_stop', 'on_restart', 'on_role_change')}
+            "callbacks": {c: callback + name for c in ('on_start', 'on_stop', 'on_restart', 'on_role_change')},
+            "backup_restore": {
+                "command": "features/backup_restore.sh --sourcedir=" + os.path.join(context.pctl.patroni_path,
+                                                                                    'data', 'basebackup')}
         }
     })
 
@@ -49,6 +52,7 @@ def start_patroni_standby_cluster(context, name, cluster_name, name2):
                     "host": "localhost",
                     "port": port,
                     "primary_slot_name": "pm_1",
+                    "create_replica_methods": ["backup_restore", "basebackup"]
                 }
             }
         },

features/steps/watchdog.py

@@ -44,31 +44,6 @@ def watchdog_was_triggered(context, name, timeout):
     assert False
 
 
-@then('{name:w} watchdog was not triggered')
-def watchdog_was_not_triggered(context, name):
-    assert not context.pctl.get_watchdog(name).was_triggered
-
-
-@step('{name:w} checkpoint takes {timeout:d} seconds')
-def checkpoint_hang(context, name, timeout):
-    assert context.pctl.checkpoint_hang(name, timeout)
-
-
 @step('{name:w} hangs for {timeout:d} seconds')
 def patroni_hang(context, name, timeout):
     return context.pctl.patroni_hang(name, timeout)
-
-
-@step('I terminate {name:w} user processes')
-def terminate_backends(context, name):
-    return context.pctl.terminate_backends(name)
-
-
-@step('Sleep for {timeout:d} seconds')
-def dcs_connection_lost(context, timeout):
-    time.sleep(timeout)
-
-
-@then('{name:w} database is running')
-def database_is_running(context, name):
-    assert context.pctl.database_is_running(name)

kubernetes/patroni_k8s.yaml

@@ -159,8 +159,9 @@ rules:
   - patch
   - update
   - watch
-  # delete is required only for 'patronictl remove'
+  # delete and deletecollection are required only for 'patronictl remove'
   - delete
+  - deletecollection
 - apiGroups:
   - ""
   resources:
@@ -173,8 +174,9 @@ rules:
   - create
   - list
   - watch
-  # delete is required only for for 'patronictl remove'
+  # delete and deletecollection are required only for for 'patronictl remove'
   - delete
+  - deletecollection
 - apiGroups:
   - ""
   resources:

patroni/__init__.py

@@ -4,26 +4,29 @@ import signal
 import sys
 import time
 
+from patroni.version import __version__
+
 logger = logging.getLogger(__name__)
 
+PATRONI_ENV_PREFIX = 'PATRONI_'
+
 
 class Patroni(object):
 
-    def __init__(self):
+    def __init__(self, conf):
         from patroni.api import RestApiServer
-        from patroni.config import Config
         from patroni.dcs import get_dcs
         from patroni.ha import Ha
         from patroni.log import PatroniLogger
         from patroni.postgresql import Postgresql
-        from patroni.version import __version__
+        from patroni.request import PatroniRequest
         from patroni.watchdog import Watchdog
 
         self.setup_signal_handlers()
 
         self.version = __version__
         self.logger = PatroniLogger()
-        self.config = Config()
+        self.config = conf
         self.logger.reload_config(self.config.get('log', {}))
         self.dcs = get_dcs(self.config)
         self.watchdog = Watchdog(self.config)
@@ -31,6 +34,7 @@ class Patroni(object):
 
         self.postgresql = Postgresql(self.config['postgresql'])
         self.api = RestApiServer(self, self.config['restapi'])
+        self.request = PatroniRequest(self.config, True)
         self.ha = Ha(self)
 
         self.tags = self.get_tags()
@@ -66,14 +70,16 @@ class Patroni(object):
     def nosync(self):
         return bool(self.tags.get('nosync', False))
 
-    def reload_config(self):
+    def reload_config(self, sighup=False):
         try:
             self.tags = self.get_tags()
             self.logger.reload_config(self.config.get('log', {}))
-            self.dcs.reload_config(self.config)
             self.watchdog.reload_config(self.config)
+            if sighup:
+                self.request.reload_config(self.config)
             self.api.reload_config(self.config['restapi'])
-            self.postgresql.reload_config(self.config['postgresql'])
+            self.postgresql.reload_config(self.config['postgresql'], sighup)
+            self.dcs.reload_config(self.config)
         except Exception:
             logger.exception('Failed to reload config_file=%s', self.config.config_file)
 
@@ -114,13 +120,16 @@ class Patroni(object):
 
     def run(self):
         self.api.start()
+        self.logger.start()
         self.next_run = time.time()
 
         while not self.received_sigterm:
             if self._received_sighup:
                 self._received_sighup = False
                 if self.config.reload_local_configuration():
-                    self.reload_config()
+                    self.reload_config(True)
+                else:
+                    self.postgresql.config.reload_config(self.config['postgresql'], True)
 
             logger.info(self.ha.run_cycle())
 
@@ -151,21 +160,64 @@ class Patroni(object):
         except Exception:
             logger.exception('Exception during RestApi.shutdown')
         self.ha.shutdown()
+        self.logger.shutdown()
 
 
 def patroni_main():
-    patroni = Patroni()
+    import argparse
+    from patroni.config import Config, ConfigParseError
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--version', action='version', version='%(prog)s {0}'.format(__version__))
+    parser.add_argument('configfile', nargs='?', default='',
+                        help='Patroni may also read the configuration from the {0} environment variable'
+                        .format(Config.PATRONI_CONFIG_VARIABLE))
+    args = parser.parse_args()
+    try:
+        conf = Config(args.configfile)
+    except ConfigParseError as e:
+        if e.value:
+            print(e.value)
+        parser.print_help()
+        sys.exit(1)
+    patroni = Patroni(conf)
     try:
         patroni.run()
     except KeyboardInterrupt:
         pass
     finally:
         patroni.shutdown()
-        logging.shutdown()
+
+
+def fatal(string, *args):
+    sys.stderr.write('FATAL: ' + string.format(*args) + '\n')
+    sys.exit(1)
+
+
+def check_psycopg2():
+    min_psycopg2 = (2, 5, 4)
+    min_psycopg2_str = '.'.join(map(str, min_psycopg2))
+
+    def parse_version(version):
+        for e in version.split('.'):
+            try:
+                yield int(e)
+            except ValueError:
+                break
+
+    try:
+        import psycopg2
+        version_str = psycopg2.__version__.split(' ')[0]
+        version = tuple(parse_version(version_str))
+        if version < min_psycopg2:
+            fatal('Patroni requires psycopg2>={0}, but only {1} is available', min_psycopg2_str, version_str)
+    except ImportError:
+        fatal('Patroni requires psycopg2>={0} or psycopg2-binary', min_psycopg2_str)
 
 
 def main():
     if os.getpid() != 1:
+        check_psycopg2()
         return patroni_main()
 
     # Patroni started with PID=1, it looks like we are in the container

patroni/api.py

@@ -7,10 +7,13 @@ import traceback
 import dateutil.parser
 import datetime
 import os
+import six
+import socket
 
-from patroni.postgresql import PostgresConnectionException, PostgresException, Postgresql
+from patroni.exceptions import PostgresConnectionException, PostgresException
+from patroni.postgresql.misc import postgres_version_to_int
 from patroni.utils import deep_compare, parse_bool, patch_config, Retry, \
-    RetryFailedError, parse_int, split_host_port, tzutc
+    RetryFailedError, parse_int, split_host_port, tzutc, uri, cluster_as_json
 from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 from six.moves.socketserver import ThreadingMixIn
 from threading import Thread
@@ -18,20 +21,6 @@ from threading import Thread
 logger = logging.getLogger(__name__)
 
 
-def check_auth(func):
-    """Decorator function to check authorization header.
-
-    Usage example:
-    @check_auth
-    def do_PUT_foo():
-        pass
-    """
-    def wrapper(handler, *args, **kwargs):
-        if handler.check_auth_header():
-            return func(handler, *args, **kwargs)
-    return wrapper
-
-
 class RestApiHandler(BaseHTTPRequestHandler):
 
     def _write_response(self, status_code, body, content_type='text/html', headers=None):
@@ -47,14 +36,20 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def _write_json_response(self, status_code, response):
         self._write_response(status_code, json.dumps(response), content_type='application/json')
 
-    def send_auth_request(self, body):
-        headers = {'WWW-Authenticate': 'Basic realm="' + self.server.patroni.__class__.__name__ + '"'}
-        self._write_response(401, body, headers=headers)
+    def check_auth(func):
+        """Decorator function to check authorization header or client certificates
 
-    def check_auth_header(self):
-        auth_header = self.headers.get('Authorization')
-        status = self.server.check_auth_header(auth_header)
-        return not status or self.send_auth_request(status)
+        Usage example:
+        @check_auth
+        def do_PUT_foo():
+            pass
+        """
+
+        def wrapper(self, *args, **kwargs):
+            if self.server.check_auth(self):
+                return func(self, *args, **kwargs)
+
+        return wrapper
 
     def _write_status_response(self, status_code, response):
         patroni = self.server.patroni
@@ -74,30 +69,45 @@ class RestApiHandler(BaseHTTPRequestHandler):
             response['watchdog_failed'] = True
         if patroni.ha.is_paused():
             response['pause'] = True
+        qsize = patroni.logger.queue_size
+        if qsize > patroni.logger.NORMAL_LOG_QUEUE_SIZE:
+            response['logger_queue_size'] = qsize
+            lost = patroni.logger.records_lost
+            if lost:
+                response['logger_records_lost'] = lost
         self._write_json_response(status_code, response)
 
     def do_GET(self, write_status_code_only=False):
         """Default method for processing all GET requests which can not be routed to other methods"""
 
+        time_start = time.time()
+        request_type = 'OPTIONS' if write_status_code_only else 'GET'
+
         path = '/master' if self.path == '/' else self.path
         response = self.get_postgresql_status()
 
         patroni = self.server.patroni
         cluster = patroni.dcs.cluster
 
-        replica_status_code = 200 if not patroni.noloadbalance and response.get('role') == 'replica' else 503
+        if not cluster and patroni.ha.is_paused():
+            primary_status_code = 200 if response['role'] == 'master' else 503
+        else:
+            primary_status_code = 200 if patroni.ha.is_leader() else 503
+
+        replica_status_code = 200 if not patroni.noloadbalance and \
+            response.get('role') == 'replica' and response.get('state') == 'running' else 503
         status_code = 503
 
         if patroni.ha.is_standby_cluster() and ('standby_leader' in path or 'standby-leader' in path):
             status_code = 200 if patroni.ha.is_leader() else 503
-        elif 'master' in path or 'leader' in path or 'primary' in path:
-            # Round-robing across all masters in pause mode if DCS is not accessible
-            if not cluster and patroni.ha.is_paused():
-                status_code = 200 if response['role'] == 'master' else 503
-            else:
-                status_code = 200 if patroni.ha.is_leader() else 503
+        elif 'master' in path or 'leader' in path or 'primary' in path or 'read-write' in path:
+            status_code = primary_status_code
         elif 'replica' in path:
             status_code = replica_status_code
+        elif 'read-only' in path:
+            status_code = 200 if primary_status_code == 200 else replica_status_code
+        elif 'health' in path:
+            status_code = 200 if response.get('state') == 'running' else 503
         elif cluster:  # dcs is available
             is_synchronous = cluster.is_synchronous_mode() and cluster.sync \
                     and cluster.sync.sync_standby == patroni.postgresql.name
@@ -112,6 +122,10 @@ class RestApiHandler(BaseHTTPRequestHandler):
         else:
             self._write_status_response(status_code, response)
 
+        time_end = time.time()
+        self.log_message('%s %s %s latency: %s ms', request_type, path,
+                         status_code, (time_end - time_start) * 1000)
+
     def do_OPTIONS(self):
         self.do_GET(write_status_code_only=True)
 
@@ -119,6 +133,14 @@ class RestApiHandler(BaseHTTPRequestHandler):
         response = self.get_postgresql_status(True)
         self._write_status_response(200, response)
 
+    def do_GET_cluster(self):
+        cluster = self.server.patroni.dcs.cluster or self.server.patroni.dcs.get_cluster()
+        self._write_json_response(200, cluster_as_json(cluster))
+
+    def do_GET_history(self):
+        cluster = self.server.patroni.dcs.cluster or self.server.patroni.dcs.get_cluster()
+        self._write_json_response(200, cluster.history and cluster.history.lines or [])
+
     def do_GET_config(self):
         cluster = self.server.patroni.dcs.cluster or self.server.patroni.dcs.get_cluster()
         if cluster.config:
@@ -166,18 +188,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
     @check_auth
     def do_POST_reload(self):
-        try:
-            if self.server.patroni.config.reload_local_configuration(True):
-                status_code = 202
-                response = 'reload scheduled'
-                self.server.patroni.sighup_handler()
-            else:
-                status_code = 200
-                response = 'nothing changed'
-        except Exception as e:
-            status_code = 500
-            response = str(e)
-        self._write_response(status_code, response)
+        self.server.patroni.sighup_handler()
+        self._write_response(202, 'reload scheduled')
 
     @staticmethod
     def parse_schedule(schedule, action):
@@ -229,7 +241,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     break
             elif k == 'postgres_version':
                 try:
-                    Postgresql.postgres_version_to_int(request[k])
+                    postgres_version_to_int(request[k])
                 except PostgresException as e:
                     status_code = 400
                     data = e.value
@@ -411,10 +423,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
             if self.server.patroni.postgresql.state not in ('running', 'restarting', 'starting'):
                 raise RetryFailedError('')
-            stmt = ("WITH replication_info AS ("
-                    "SELECT usename, application_name, client_addr, state, sync_state, sync_priority"
-                    " FROM pg_catalog.pg_stat_replication) SELECT"
-                    " pg_catalog.to_char(pg_catalog.pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),"
+            stmt = ("SELECT pg_catalog.to_char(pg_catalog.pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),"
                     " CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0"
                     " ELSE ('x' || pg_catalog.substr(pg_catalog.pg_{0}file_name("
                     "pg_catalog.pg_current_{0}_{1}()), 1, 8))::bit(32)::int END,"
@@ -425,8 +434,10 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     " pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), '0/0')::bigint,"
                     " pg_catalog.to_char(pg_catalog.pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),"
                     " pg_catalog.pg_is_in_recovery() AND pg_catalog.pg_is_{0}_replay_paused(), "
-                    "(SELECT pg_catalog.array_to_json(pg_catalog.array_agg("
-                    "pg_catalog.row_to_json(ri))) FROM replication_info ri)")
+                    " pg_catalog.array_to_json(pg_catalog.array_agg(pg_catalog.row_to_json(ri))) "
+                    "FROM (SELECT (SELECT rolname FROM pg_authid WHERE oid = usesysid) AS usename,"
+                    " application_name, client_addr, w.state, sync_state, sync_priority"
+                    " FROM pg_catalog.pg_stat_get_wal_senders() w, pg_catalog.pg_stat_get_activity(pid)) AS ri")
 
             row = self.query(stmt.format(self.server.patroni.postgresql.wal_name,
                                          self.server.patroni.postgresql.lsn_name), retry=retry)[0]
@@ -471,12 +482,14 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
 
 class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
+    # On 3.7+ the `ThreadingMixIn` gathers all non-daemon worker threads in order to join on them at server close.
+    daemon_threads = True  # Make worker threads "fire and forget" to prevent a memory leak.
 
     def __init__(self, patroni, config):
         self.patroni = patroni
         self.__listen = None
-        self.__initialize(config)
-        self.__set_config_parameters(config)
+        self.__ssl_options = None
+        self.reload_config(config)
         self.daemon = True
 
     def query(self, sql, *params):
@@ -507,50 +520,99 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
             if not auth_header.startswith('Basic ') or not self.check_basic_auth_key(auth_header[6:]):
                 return 'not authenticated'
 
+    def check_auth(self, rh):
+        if not hasattr(rh.request, 'getpeercert') or not rh.request.getpeercert():  # valid client cert isn't present
+            if self.__protocol == 'https' and self.__ssl_options.get('verify_client') in ('required', 'optional'):
+                return rh._write_response(403, 'client certificate required')
+
+            reason = self.check_auth_header(rh.headers.get('Authorization'))
+            if reason:
+                headers = {'WWW-Authenticate': 'Basic realm="' + self.patroni.__class__.__name__ + '"'}
+                return rh._write_response(401, reason, headers=headers)
+        return True
+
     @staticmethod
-    def __get_ssl_options(config):
-        return {option: config[option] for option in ['certfile', 'keyfile'] if option in config}
+    def __has_dual_stack():
+        if hasattr(socket, 'AF_INET6') and hasattr(socket, 'IPPROTO_IPV6') and hasattr(socket, 'IPV6_V6ONLY'):
+            sock = None
+            try:
+                sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+                sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, False)
+                return True
+            except socket.error as e:
+                logger.debug('Error when working with ipv6 socket: %s', e)
+            finally:
+                if sock:
+                    sock.close()
+        return False
 
-    def __set_config_parameters(self, config):
-        self.__auth_key = base64.b64encode(config['auth'].encode('utf-8')).decode('utf-8') if 'auth' in config else None
-        self.connection_string = '{0}://{1}/patroni'.format(self.__protocol,
-                                                            config.get('connect_address') or self.__listen)
+    def __httpserver_init(self, host, port):
+        dual_stack = self.__has_dual_stack()
+        if host in ('', '*'):
+            host = None
 
-    def __initialize(self, config):
+        info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE)
+        # in case dual stack is not supported we want IPv4 to be preferred over IPv6
+        info.sort(key=lambda x: x[0] == socket.AF_INET, reverse=not dual_stack)
+
+        self.address_family = info[0][0]
         try:
-            host, port = split_host_port(config['listen'], None)
+            HTTPServer.__init__(self, info[0][-1][:2], RestApiHandler)
+        except socket.error:
+            logger.error(
+                    "Couldn't start a service on '%s:%s', please check your `restapi.listen` configuration", host, port)
+            raise
+
+    def __initialize(self, listen, ssl_options):
+        try:
+            host, port = split_host_port(listen, None)
         except Exception:
             raise ValueError('Invalid "restapi" config: expected <HOST>:<PORT> for "listen", but got "{0}"'
-                             .format(config['listen']))
+                             .format(listen))
 
-        if self.__listen is not None:  # changing config in runtime
+        reloading_config = self.__listen is not None  # changing config in runtime
+        if reloading_config:
             self.shutdown()
 
-        self.__listen = config['listen']
-        self.__ssl_options = self.__get_ssl_options(config)
+        self.__listen = listen
+        self.__ssl_options = ssl_options
 
-        HTTPServer.__init__(self, (host, port), RestApiHandler)
+        self.__httpserver_init(host, port)
         Thread.__init__(self, target=self.serve_forever)
         self._set_fd_cloexec(self.socket)
 
-        self.__protocol = 'http'
-
         # wrap socket with ssl if 'certfile' is defined in a config.yaml
         # Sometime it's also needed to pass reference to a 'keyfile'.
-        if self.__ssl_options.get('certfile'):
+        self.__protocol = 'https' if ssl_options.get('certfile') else 'http'
+        if self.__protocol == 'https':
             import ssl
-            self.socket = ssl.wrap_socket(self.socket, server_side=True, **self.__ssl_options)
-            self.__protocol = 'https'
-        return True
+            ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH, cafile=ssl_options.get('cafile'))
+            ctx.load_cert_chain(certfile=ssl_options['certfile'], keyfile=ssl_options.get('keyfile'))
+            verify_client = ssl_options.get('verify_client')
+            if verify_client:
+                modes = {'none': ssl.CERT_NONE, 'optional': ssl.CERT_OPTIONAL, 'required': ssl.CERT_REQUIRED}
+                if verify_client in modes:
+                    ctx.verify_mode = modes[verify_client]
+                else:
+                    logger.error('Bad value in the "restapi.verify_client": %s', verify_client)
+            self.socket = ctx.wrap_socket(self.socket, server_side=True)
+        if reloading_config:
+            self.start()
 
     def reload_config(self, config):
         if 'listen' not in config:  # changing config in runtime
             raise ValueError('Can not find "restapi.listen" config')
 
-        elif (self.__listen != config['listen'] or self.__ssl_options != self.__get_ssl_options(config)) \
-                and self.__initialize(config):
-            self.start()
-        self.__set_config_parameters(config)
+        ssl_options = {n: config[n] for n in ('certfile', 'keyfile', 'cafile') if n in config}
+
+        if isinstance(config.get('verify_client'), six.string_types):
+            ssl_options['verify_client'] = config['verify_client'].lower()
+
+        if self.__listen != config['listen'] or self.__ssl_options != ssl_options:
+            self.__initialize(config['listen'], ssl_options)
+
+        self.__auth_key = base64.b64encode(config['auth'].encode('utf-8')).decode('utf-8') if 'auth' in config else None
+        self.connection_string = uri(self.__protocol, config.get('connect_address') or self.__listen, 'patroni')
 
     @staticmethod
     def handle_error(request, client_address):

patroni/async_executor.py

@@ -52,8 +52,8 @@ class CriticalTask(object):
 
 class AsyncExecutor(object):
 
-    def __init__(self, state_handler, ha_wakeup):
-        self.state_handler = state_handler
+    def __init__(self, cancellable, ha_wakeup):
+        self._cancellable = cancellable
         self._ha_wakeup = ha_wakeup
         self._thread_lock = RLock()
         self._scheduled_action = None
@@ -92,7 +92,7 @@ class AsyncExecutor(object):
                     return
                 self._finish_event.clear()
 
-            self.state_handler.reset_is_cancelled()
+            self._cancellable.reset_is_cancelled()
             # if the func returned something (not None) - wake up main HA loop
             wakeup = func(*args) if args else func()
             return wakeup
@@ -110,6 +110,12 @@ class AsyncExecutor(object):
     def run_async(self, func, args=()):
         Thread(target=self.run, args=(func, args)).start()
 
+    def try_run_async(self, action, func, args=()):
+        prev = self.schedule(action)
+        if prev is None:
+            return self.run_async(func, args)
+        return 'Failed to run {0}, {1} is already in progress'.format(action, prev)
+
     def cancel(self):
         with self:
             with self._scheduled_action_lock:
@@ -118,7 +124,7 @@ class AsyncExecutor(object):
                 logger.warning('Cancelling long running task %s', self._scheduled_action)
             self._is_cancelled = True
 
-        self.state_handler.cancel()
+        self._cancellable.cancel()
         self._finish_event.wait()
 
         with self:

patroni/callback_executor.py

@@ -1,42 +0,0 @@
-import logging
-import subprocess
-from threading import Event, Lock, Thread
-
-logger = logging.getLogger(__name__)
-
-
-class CallbackExecutor(Thread):
-
-    def __init__(self):
-        super(CallbackExecutor, self).__init__()
-        self.daemon = True
-        self._lock = Lock()
-        self._cmd = None
-        self._process = None
-        self._callback_event = Event()
-        self.start()
-
-    def call(self, cmd):
-        with self._lock:
-            if self._process and self._process.poll() is None:
-                try:
-                    self._process.kill()
-                    logger.warning('Killed the old callback process because it was still running: %s', self._cmd)
-                except OSError:
-                    logger.exception('Failed to kill the old callback')
-                    logger.warning('Wait until callback end')
-                    self._process.wait()
-        self._cmd = cmd
-        self._callback_event.set()
-
-    def run(self):
-        while True:
-            self._callback_event.wait()
-            self._callback_event.clear()
-            with self._lock:
-                try:
-                    self._process = subprocess.Popen(self._cmd, close_fds=True)
-                except Exception:
-                    logger.exception('Failed to execute %s',  self._cmd)
-                    continue
-            self._process.wait()

patroni/config.py

@@ -2,19 +2,34 @@ import json
 import logging
 import os
 import shutil
-import sys
 import tempfile
 import yaml
 
 from collections import defaultdict
 from copy import deepcopy
+from patroni import PATRONI_ENV_PREFIX
+from patroni.exceptions import ConfigParseError
 from patroni.dcs import ClusterConfig
-from patroni.postgresql import Postgresql
+from patroni.postgresql.config import CaseInsensitiveDict, ConfigHandler
 from patroni.utils import deep_compare, parse_bool, parse_int, patch_config
-from requests.structures import CaseInsensitiveDict
 
 logger = logging.getLogger(__name__)
 
+_AUTH_ALLOWED_PARAMETERS = (
+    'username',
+    'password',
+    'sslmode',
+    'sslcert',
+    'sslkey',
+    'sslrootcert',
+    'sslcrl'
+)
+
+
+def default_validator(conf):
+    if not conf:
+        return "Config is empty."
+
 
 class Config(object):
     """
@@ -36,7 +51,6 @@ class Config(object):
          to work with it as with the old `config` object.
     """
 
-    PATRONI_ENV_PREFIX = 'PATRONI_'
     PATRONI_CONFIG_VARIABLE = PATRONI_ENV_PREFIX + 'CONFIGURATION'
 
     __CACHE_FILENAME = 'patroni.dynamic.json'
@@ -59,34 +73,33 @@ class Config(object):
         'postgresql': {
             'bin_dir': '',
             'use_slots': True,
-            'parameters': CaseInsensitiveDict({p: v[0] for p, v in Postgresql.CMDLINE_OPTIONS.items()})
+            'parameters': CaseInsensitiveDict({p: v[0] for p, v in ConfigHandler.CMDLINE_OPTIONS.items()})
         },
         'watchdog': {
             'mode': 'automatic',
         }
     }
 
-    def __init__(self):
+    def __init__(self, configfile, validator=default_validator):
         self._modify_index = -1
         self._dynamic_configuration = {}
 
         self.__environment_configuration = self._build_environment_configuration()
 
         # Patroni reads the configuration from the command-line argument if it exists, otherwise from the environment
-        self._config_file = len(sys.argv) >= 2 and os.path.isfile(sys.argv[1]) and sys.argv[1]
+        self._config_file = configfile and os.path.isfile(configfile) and configfile
         if self._config_file:
             self._local_configuration = self._load_config_file()
         else:
             config_env = os.environ.pop(self.PATRONI_CONFIG_VARIABLE, None)
             self._local_configuration = config_env and yaml.safe_load(config_env) or self.__environment_configuration
-            if not self._local_configuration:
-                print('Usage: {0} config.yml'.format(sys.argv[0]))
-                print('\tPatroni may also read the configuration from the {0} environment variable'.
-                      format(self.PATRONI_CONFIG_VARIABLE))
-                sys.exit(1)
+        if validator:
+            error = validator(self._local_configuration)
+            if error:
+                raise ConfigParseError(error)
 
         self.__effective_configuration = self._build_effective_configuration({}, self._local_configuration)
-        self._data_dir = self.__effective_configuration['postgresql']['data_dir']
+        self._data_dir = self.__effective_configuration.get('postgresql', {}).get('data_dir', "")
         self._cache_file = os.path.join(self._data_dir, self.__CACHE_FILENAME)
         self._load_cache()
         self._cache_needs_saving = False
@@ -158,31 +171,25 @@ class Config(object):
             except Exception:
                 logger.exception('Exception when setting dynamic_configuration')
 
-    def reload_local_configuration(self, dry_run=False):
+    def reload_local_configuration(self):
         if self.config_file:
             try:
                 configuration = self._load_config_file()
                 if not deep_compare(self._local_configuration, configuration):
                     new_configuration = self._build_effective_configuration(self._dynamic_configuration, configuration)
-                    if dry_run:
-                        return not deep_compare(new_configuration, self.__effective_configuration)
                     self._local_configuration = configuration
                     self.__effective_configuration = new_configuration
                     return True
                 else:
-                    logger.info('No configuration items changed, nothing to reload.')
+                    logger.info('No local configuration items changed.')
             except Exception:
                 logger.exception('Exception when reloading local configuration from %s', self.config_file)
-                if dry_run:
-                    raise
 
     @staticmethod
     def _process_postgresql_parameters(parameters, is_local=False):
-        ret = {}
-        for name, value in (parameters or {}).items():
-            if name not in Postgresql.CMDLINE_OPTIONS or not is_local and Postgresql.CMDLINE_OPTIONS[name][1](value):
-                ret[name] = value
-        return ret
+        return {name: value for name, value in (parameters or {}).items()
+                if name not in ConfigHandler.CMDLINE_OPTIONS or
+                not is_local and ConfigHandler.CMDLINE_OPTIONS[name][1](value)}
 
     def _safe_copy_dynamic_configuration(self, dynamic_configuration):
         config = deepcopy(self.__DEFAULT_CONFIG)
@@ -210,7 +217,7 @@ class Config(object):
         ret = defaultdict(dict)
 
         def _popenv(name):
-            return os.environ.pop(Config.PATRONI_ENV_PREFIX + name.upper(), None)
+            return os.environ.pop(PATRONI_ENV_PREFIX + name.upper(), None)
 
         for param in ('name', 'namespace', 'scope'):
             value = _popenv(param)
@@ -219,7 +226,7 @@ class Config(object):
 
         def _fix_log_env(name, oldname):
             value = _popenv(oldname)
-            name = Config.PATRONI_ENV_PREFIX + 'LOG_' + name.upper()
+            name = PATRONI_ENV_PREFIX + 'LOG_' + name.upper()
             if value and name not in os.environ:
                 os.environ[name] = value
 
@@ -232,9 +239,11 @@ class Config(object):
                 if value:
                     ret[section][param] = value
 
-        _set_section_values('restapi', ['listen', 'connect_address', 'certfile', 'keyfile'])
+        _set_section_values('restapi', ['listen', 'connect_address', 'certfile', 'keyfile', 'cafile', 'verify_client'])
+        _set_section_values('ctl', ['insecure', 'cacert', 'certfile', 'keyfile'])
         _set_section_values('postgresql', ['listen', 'connect_address', 'config_dir', 'data_dir', 'pgpass', 'bin_dir'])
-        _set_section_values('log', ['level', 'format', 'dateformat', 'dir', 'file_size', 'file_num', 'loggers'])
+        _set_section_values('log', ['level', 'traceback_level', 'format', 'dateformat', 'max_queue_size',
+                                    'dir', 'file_size', 'file_num', 'loggers'])
 
         def _parse_dict(value):
             if not value.strip().startswith('{'):
@@ -251,9 +260,9 @@ class Config(object):
             if value:
                 ret['log']['loggers'] = value
 
-        def _get_auth(name):
+        def _get_auth(name, params=None):
             ret = {}
-            for param in ('username', 'password'):
+            for param in params or _AUTH_ALLOWED_PARAMETERS[:2]:
                 value = _popenv(name + '_' + param)
                 if value:
                     ret[param] = value
@@ -264,8 +273,8 @@ class Config(object):
             ret['restapi']['authentication'] = restapi_auth
 
         authentication = {}
-        for user_type in ('replication', 'superuser'):
-            entry = _get_auth(user_type)
+        for user_type in ('replication', 'superuser', 'rewind'):
+            entry = _get_auth(user_type, _AUTH_ALLOWED_PARAMETERS)
             if entry:
                 authentication[user_type] = entry
 
@@ -282,13 +291,13 @@ class Config(object):
                 return None
 
         for param in list(os.environ.keys()):
-            if param.startswith(Config.PATRONI_ENV_PREFIX):
+            if param.startswith(PATRONI_ENV_PREFIX):
                 # PATRONI_(ETCD|CONSUL|ZOOKEEPER|EXHIBITOR|...)_(HOSTS?|PORT|..)
                 name, suffix = (param[8:].split('_', 1) + [''])[:2]
                 if suffix in ('HOST', 'HOSTS', 'PORT', 'USE_PROXIES', 'PROTOCOL', 'SRV', 'URL', 'PROXY',
-                              'CACERT', 'CERT', 'KEY', 'VERIFY', 'TOKEN', 'CHECKS', 'DC', 'REGISTER_SERVICE',
-                              'SERVICE_CHECK_INTERVAL', 'NAMESPACE', 'CONTEXT', 'USE_ENDPOINTS', 'SCOPE_LABEL',
-                              'ROLE_LABEL', 'POD_IP', 'PORTS', 'LABELS') and name:
+                              'CACERT', 'CERT', 'KEY', 'VERIFY', 'TOKEN', 'CHECKS', 'DC', 'CONSISTENCY',
+                              'REGISTER_SERVICE', 'SERVICE_CHECK_INTERVAL', 'NAMESPACE', 'CONTEXT',
+                              'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL', 'POD_IP', 'PORTS', 'LABELS') and name:
                     value = os.environ.pop(param)
                     if suffix == 'PORT':
                         value = value and parse_int(value)
@@ -305,7 +314,7 @@ class Config(object):
 
         users = {}
         for param in list(os.environ.keys()):
-            if param.startswith(Config.PATRONI_ENV_PREFIX):
+            if param.startswith(PATRONI_ENV_PREFIX):
                 name, suffix = (param[8:].rsplit('_', 1) + [''])[:2]
                 # PATRONI_<username>_PASSWORD=<password>, PATRONI_<username>_OPTIONS=<option1,option2,...>
                 # CREATE USER "<username>" WITH <OPTIONS> PASSWORD '<password>'
@@ -335,7 +344,7 @@ class Config(object):
                 config[name] = deepcopy(value) if value else {}
 
         # restapi server expects to get restapi.auth = 'username:password'
-        if 'authentication' in config['restapi']:
+        if 'restapi' in config and 'authentication' in config['restapi']:
             config['restapi']['auth'] = '{username}:{password}'.format(**config['restapi']['authentication'])
 
         # special treatment for old config
@@ -354,6 +363,11 @@ class Config(object):
         if 'superuser' not in pg_config['authentication'] and 'pg_rewind' in pg_config:
             pg_config['authentication']['superuser'] = pg_config['pg_rewind']
 
+        # handle setting additional connection parameters that may be available
+        # in the configuration file, such as SSL connection parameters
+        for name, value in pg_config['authentication'].items():
+            pg_config['authentication'][name] = {n: v for n, v in value.items() if n in _AUTH_ALLOWED_PARAMETERS}
+
         # no 'name' in config
         if 'name' not in config and 'name' in pg_config:
             config['name'] = pg_config['name']

patroni/ctl.py

@@ -2,7 +2,6 @@
 Patroni Control
 '''
 
-import base64
 import click
 import codecs
 import datetime
@@ -14,9 +13,8 @@ import io
 import json
 import logging
 import os
-import psycopg2
 import random
-import requests
+import six
 import subprocess
 import sys
 import tempfile
@@ -26,15 +24,15 @@ import yaml
 
 from click import ClickException
 from contextlib import contextmanager
-from patroni.config import Config
 from patroni.dcs import get_dcs as _get_dcs
 from patroni.exceptions import PatroniException
 from patroni.postgresql import Postgresql
-from patroni.utils import patch_config, polling_loop
+from patroni.postgresql.misc import postgres_version_to_int
+from patroni.utils import cluster_as_json, patch_config, polling_loop
+from patroni.request import PatroniRequest
 from patroni.version import __version__
 from prettytable import PrettyTable
 from six.moves.urllib_parse import urlparse
-from six import text_type
 
 CONFIG_DIR_PATH = click.get_app_dir('patroni')
 CONFIG_FILE_PATH = os.path.join(CONFIG_DIR_PATH, 'patronictl.yaml')
@@ -68,18 +66,13 @@ def parse_dcs(dcs):
 
 
 def load_config(path, dcs):
-    logging.debug('Loading configuration from file %s', path)
-    config = {}
-    old_argv = list(sys.argv)
-    try:
-        sys.argv[1] = path
-        if Config.PATRONI_CONFIG_VARIABLE not in os.environ:
-            for p in ('PATRONI_RESTAPI_LISTEN', 'PATRONI_POSTGRESQL_DATA_DIR'):
-                if p not in os.environ:
-                    os.environ[p] = '.'
-        config = Config().copy()
-    finally:
-        sys.argv = old_argv
+    from patroni.config import Config
+
+    if not (os.path.exists(path) and os.access(path, os.R_OK)):
+        logging.debug('Ignoring configuration file "%s". It does not exists or is not readable.', path)
+    else:
+        logging.debug('Loading configuration from file %s', path)
+    config = Config(path, validator=None).copy()
 
     dcs = parse_dcs(dcs) or parse_dcs(config.get('dcs_api')) or {}
     if dcs:
@@ -107,12 +100,16 @@ option_insecure = click.option('-k', '--insecure', is_flag=True, help='Allow con
 
 
 @click.group()
-@click.option('--config-file', '-c', help='Configuration file', default=CONFIG_FILE_PATH)
+@click.option('--config-file', '-c', help='Configuration file',
+              envvar='PATRONICTL_CONFIG_FILE', default=CONFIG_FILE_PATH)
 @click.option('--dcs', '-d', help='Use this DCS', envvar='DCS')
 @option_insecure
 @click.pass_context
 def ctl(ctx, config_file, dcs, insecure):
-    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=os.environ.get('LOGLEVEL', 'WARNING'))
+    level = 'WARNING'
+    for name in ('LOGLEVEL', 'PATRONI_LOGLEVEL', 'PATRONI_LOG_LEVEL'):
+        level = os.environ.get(name, level)
+    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=level)
     logging.captureWarnings(True)  # Capture eventual SSL warning
     ctx.obj = load_config(config_file, dcs)
     # backward compatibility for configuration file where ctl section is not define
@@ -128,36 +125,12 @@ def get_dcs(config, scope):
         raise PatroniCtlException(str(e))
 
 
-def auth_header(config):
-    if config.get('restapi', {}).get('auth', ''):
-        return {'Authorization': 'Basic ' + base64.b64encode(config['restapi']['auth'].encode('utf-8')).decode('utf-8')}
-
-
-def request_patroni(member, request_type, endpoint, content=None, headers=None):
+def request_patroni(member, method='GET', endpoint=None, data=None):
     ctx = click.get_current_context()  # the current click context
-    headers = headers or {}
-    url_parts = urlparse(member.api_url)
-    logging.debug(url_parts)
-    if 'Content-Type' not in headers:
-        headers['Content-Type'] = 'application/json'
-
-    url = '{0}://{1}/{2}'.format(url_parts.scheme, url_parts.netloc, endpoint)
-
-    insecure = ctx.obj.get('ctl', {}).get('insecure', False)
-    # Get certfile if any from several configuration namespace
-    cert = ctx.obj.get('ctl', {}).get('cacert') or \
-        ctx.obj.get('restapi', {}).get('cacert') or \
-        ctx.obj.get('restapi', {}).get('certfile')
-    # In the case we specificaly disable SSL cert verification we don't want to have the warning
-    if insecure:
-        verify = False
-    elif cert:
-        verify = cert
-    else:
-        verify = True
-    return getattr(requests, request_type)(url, headers=headers,
-                                           data=json.dumps(content) if content else None, timeout=60,
-                                           verify=verify)
+    request_executor = ctx.obj.get('__request_patroni')
+    if not request_executor:
+        request_executor = ctx.obj['__request_patroni'] = PatroniRequest(ctx.obj)
+    return request_executor(member, method, endpoint, data)
 
 
 def print_output(columns, rows=None, alignment=None, fmt='pretty', header=True, delimiter='\t'):
@@ -246,6 +219,7 @@ def get_cursor(cluster, connect_parameters, role='master', member=None):
     else:
         params.pop('database')
 
+    import psycopg2
     conn = psycopg2.connect(**params)
     conn.autocommit = True
     cursor = conn.cursor()
@@ -263,10 +237,12 @@ def get_cursor(cluster, connect_parameters, role='master', member=None):
     return None
 
 
-def get_members(cluster, cluster_name, member_names, role, force, action):
+def get_members(cluster, cluster_name, member_names, role, force, action, ask_confirmation=True):
     candidates = {m.name: m for m in cluster.members}
 
     if not force or role:
+        if not member_names and not candidates:
+            raise PatroniCtlException('{0} cluster doesn\'t have any members'.format(cluster_name))
         output_members(cluster, cluster_name)
 
     if role:
@@ -286,12 +262,25 @@ def get_members(cluster, cluster_name, member_names, role, force, action):
         if member_name not in candidates:
             raise PatroniCtlException('{0} is not a member of cluster'.format(member_name))
 
-    if not force:
-        confirm = click.confirm('Are you sure you want to {0} members {1}?'.format(action, ', '.join(member_names)))
-        if not confirm:
-            raise PatroniCtlException('Aborted {0}'.format(action))
+    members = [candidates[n] for n in member_names]
+    if ask_confirmation:
+        confirm_members_action(members, force, action)
+    return members
 
-    return [candidates[n] for n in member_names]
+
+def confirm_members_action(members, force, action, scheduled_at=None):
+    if scheduled_at:
+        if not force:
+            confirm = click.confirm('Are you sure you want to schedule {0} of members {1} at {2}?'
+                                    .format(action, ', '.join([m.name for m in members]), scheduled_at))
+            if not confirm:
+                raise PatroniCtlException('Aborted scheduled {0}'.format(action))
+    else:
+        if not force:
+            confirm = click.confirm('Are you sure you want to {0} members {1}?'
+                                    .format(action, ', '.join([m.name for m in members])))
+            if not confirm:
+                raise PatroniCtlException('Aborted {0}'.format(action))
 
 
 @ctl.command('dsn', help='Generate a dsn for the provided member, defaults to a dsn of the master')
@@ -380,6 +369,7 @@ def query(
 
 
 def query_member(cluster, cursor, member, role, command, connect_parameters):
+    import psycopg2
     try:
         if cursor is None:
             cursor = get_cursor(cluster, connect_parameters, role=role, member=member)
@@ -440,9 +430,9 @@ def remove(obj, cluster_name, fmt):
 
 
 def check_response(response, member_name, action_name, silent_success=False):
-    if response.status_code >= 400:
+    if response.status >= 400:
         click.echo('Failed: {0} for member {1}, status code={2}, ({3})'.format(
-            action_name, member_name, response.status_code, response.text
+            action_name, member_name, response.status, response.data.decode('utf-8')
         ))
         return False
     elif not silent_success:
@@ -476,18 +466,17 @@ def reload(obj, cluster_name, member_names, force, role):
 
     members = get_members(cluster, cluster_name, member_names, role, force, 'reload')
 
-    content = {}
     for member in members:
-        r = request_patroni(member, 'post', 'reload', content, auth_header(obj))
-        if r.status_code == 200:
+        r = request_patroni(member, 'post', 'reload')
+        if r.status == 200:
             click.echo('No changes to apply on member {0}'.format(member.name))
-        elif r.status_code == 202:
+        elif r.status == 202:
             click.echo('Reload request received for member {0} and will be processed within {1} seconds'.format(
                 member.name, cluster.config.data.get('loop_wait'))
             )
         else:
             click.echo('Failed: reload for member {0}, status code={1}, ({2})'.format(
-                member.name, r.status_code, r.text)
+                member.name, r.status, r.data.decode('utf-8'))
             )
 
 
@@ -509,7 +498,15 @@ def reload(obj, cluster_name, member_names, force, role):
 def restart(obj, cluster_name, member_names, force, role, p_any, scheduled, version, pending, timeout):
     cluster = get_dcs(obj, cluster_name).get_cluster()
 
-    members = get_members(cluster, cluster_name, member_names, role, force, 'restart')
+    members = get_members(cluster, cluster_name, member_names, role, force, 'restart', False)
+    if scheduled is None and not force:
+        next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M')
+        scheduled = click.prompt('When should the restart take place (e.g. ' + next_hour + ') ',
+                                 type=str, default='now')
+
+    scheduled_at = parse_scheduled(scheduled)
+    confirm_members_action(members, force, 'restart', scheduled_at)
+
     if p_any:
         random.shuffle(members)
         members = members[:1]
@@ -524,16 +521,12 @@ def restart(obj, cluster_name, member_names, force, role, p_any, scheduled, vers
 
     if version:
         try:
-            Postgresql.postgres_version_to_int(version)
+            postgres_version_to_int(version)
         except PatroniException as e:
             raise PatroniCtlException(e.value)
 
         content['postgres_version'] = version
 
-    if scheduled is None and not force:
-        scheduled = click.prompt('When should the restart take place (e.g. 2015-10-01T14:30) ', type=str, default='now')
-
-    scheduled_at = parse_scheduled(scheduled)
     if scheduled_at:
         if cluster.is_paused():
             raise PatroniCtlException("Can't schedule restart in the paused state")
@@ -545,19 +538,19 @@ def restart(obj, cluster_name, member_names, force, role, p_any, scheduled, vers
     for member in members:
         if 'schedule' in content:
             if force and member.data.get('scheduled_restart'):
-                r = request_patroni(member, 'delete', 'restart', headers=auth_header(obj))
+                r = request_patroni(member, 'delete', 'restart')
                 check_response(r, member.name, 'flush scheduled restart', True)
 
-        r = request_patroni(member, 'post', 'restart', content, auth_header(obj))
-        if r.status_code == 200:
+        r = request_patroni(member, 'post', 'restart', content)
+        if r.status == 200:
             click.echo('Success: restart on member {0}'.format(member.name))
-        elif r.status_code == 202:
+        elif r.status == 202:
             click.echo('Success: restart scheduled on member {0}'.format(member.name))
-        elif r.status_code == 409:
+        elif r.status == 409:
             click.echo('Failed: another restart is already scheduled on member {0}'.format(member.name))
         else:
             click.echo('Failed: restart for member {0}, status code={1}, ({2})'.format(
-                member.name, r.status_code, r.text)
+                member.name, r.status, r.data.decode('utf-8'))
             )
 
 
@@ -573,8 +566,8 @@ def reinit(obj, cluster_name, member_names, force):
     for member in members:
         body = {'force': force}
         while True:
-            r = request_patroni(member, 'post', 'reinitialize', body, auth_header(obj))
-            if not check_response(r, member.name, 'reinitialize') and r.text.endswith(' already in progress') \
+            r = request_patroni(member, 'post', 'reinitialize', body)
+            if not check_response(r, member.name, 'reinitialize') and r.data.endswith(b' already in progress') \
                     and not force and click.confirm('Do you want to cancel it and reinitialize anyway?'):
                 body['force'] = True
                 continue
@@ -625,10 +618,12 @@ def _do_failover_or_switchover(obj, action, cluster_name, master, candidate, for
         raise PatroniCtlException('Member {0} does not exist in cluster {1}'.format(candidate, cluster_name))
 
     scheduled_at_str = None
+    scheduled_at = None
 
     if action == 'switchover':
         if scheduled is None and not force:
-            scheduled = click.prompt('When should the switchover take place (e.g. 2015-10-01T14:30) ',
+            next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M')
+            scheduled = click.prompt('When should the switchover take place (e.g. ' + next_hour + ' ) ',
                                      type=str, default='now')
 
         scheduled_at = parse_scheduled(scheduled)
@@ -647,27 +642,32 @@ def _do_failover_or_switchover(obj, action, cluster_name, master, candidate, for
 
     if not force:
         demote_msg = ', demoting current master ' + master if master else ''
-
-        if not click.confirm('Are you sure you want to {0} cluster {1}{2}?'.format(action, cluster_name, demote_msg)):
-            raise PatroniCtlException('Aborting ' + action)
+        if scheduled_at_str:
+            if not click.confirm('Are you sure you want to schedule {0} of cluster {1} at {2}{3}?'
+                                 .format(action, cluster_name, scheduled_at_str, demote_msg)):
+                raise PatroniCtlException('Aborting scheduled ' + action)
+        else:
+            if not click.confirm('Are you sure you want to {0} cluster {1}{2}?'
+                                 .format(action, cluster_name, demote_msg)):
+                raise PatroniCtlException('Aborting ' + action)
 
     r = None
     try:
         member = cluster.leader.member if cluster.leader else cluster.get_member(candidate, False)
 
-        r = request_patroni(member, 'post', action, failover_value, auth_header(obj))
+        r = request_patroni(member, 'post', action, failover_value)
 
         # probably old patroni, which doesn't support switchover yet
-        if r.status_code == 501 and action == 'switchover' and 'Server does not support this operation' in r.text:
-            r = request_patroni(member, 'post', 'failover', failover_value, auth_header(obj))
+        if r.status == 501 and action == 'switchover' and b'Server does not support this operation' in r.data:
+            r = request_patroni(member, 'post', 'failover', failover_value)
 
-        if r.status_code in (200, 202):
+        if r.status in (200, 202):
             logging.debug(r)
             cluster = dcs.get_cluster()
             logging.debug(cluster)
-            click.echo('{0} {1}'.format(timestamp(), r.text))
+            click.echo('{0} {1}'.format(timestamp(), r.data.decode('utf-8')))
         else:
-            click.echo('{0} failed, details: {1}, {2}'.format(action.title(), r.status_code, r.text))
+            click.echo('{0} failed, details: {1}, {2}'.format(action.title(), r.status, r.data.decode('utf-8')))
             return
     except Exception:
         logging.exception(r)
@@ -704,80 +704,51 @@ def switchover(obj, cluster_name, master, candidate, force, scheduled):
 def output_members(cluster, name, extended=False, fmt='pretty'):
     rows = []
     logging.debug(cluster)
-    leader_name = None
-    if cluster.leader:
-        leader_name = cluster.leader.name
-
-    xlog_location_cluster = cluster.last_leader_operation or 0
-
-    # Mainly for consistent pretty printing and watching we sort the output
-    cluster.members.sort(key=lambda x: x.name)
-
-    has_scheduled_restarts = any(m.data.get('scheduled_restart') for m in cluster.members)
-    has_pending_restarts = any(m.data.get('pending_restart') for m in cluster.members)
-
-    # Show Host as 'host:port' if somebody is running on non-standard port or two nodes are running on the same host
-    append_port = any(str(m.conn_kwargs()['port']) != '5432' for m in cluster.members) or\
-        len(set(m.conn_kwargs()['host'] for m in cluster.members)) < len(cluster.members)
-
-    for m in cluster.members:
-        logging.debug(m)
-
-        role = ''
-        if m.name == leader_name:
-            role = 'Leader'
-        elif m.name == cluster.sync.sync_standby:
-            role = 'Sync standby'
-
-        xlog_location = m.data.get('xlog_location')
-        lag = ''
-        if xlog_location is None:
-            lag = 'unknown'
-        elif xlog_location_cluster >= xlog_location:
-            lag = round((xlog_location_cluster - xlog_location)/1024/1024)
-
-        host = m.conn_kwargs()['host']
-        if append_port:
-            host += ':{0}'.format(m.conn_kwargs()['port'])
-
-        row = [name, m.name, host, role, m.data.get('state', ''), m.data.get('timeline', ''), lag]
-
-        if extended or has_pending_restarts:
-            row.append('*' if m.data.get('pending_restart') else '')
-
-        if extended or has_scheduled_restarts:
-            value = ''
-            scheduled_restart = m.data.get('scheduled_restart')
-            if scheduled_restart:
-                value = scheduled_restart['schedule']
-                if 'postgres_version' in scheduled_restart:
-                    value += ' if version < {0}'.format(scheduled_restart['postgres_version'])
-
-            row.append(value)
-
-        rows.append(row)
+    cluster = cluster_as_json(cluster)
 
     columns = ['Cluster', 'Member', 'Host', 'Role', 'State', 'TL', 'Lag in MB']
-    alignment = {'Lag in MB': 'r', 'TL': 'r'}
+    for c in ('Pending restart', 'Scheduled restart'):
+        if extended or any(m.get(c.lower().replace(' ', '_')) for m in cluster['members']):
+            columns.append(c)
 
-    if extended or has_pending_restarts:
-        columns.append('Pending restart')
+    # Show Host as 'host:port' if somebody is running on non-standard port or two nodes are running on the same host
+    append_port = any(m['port'] != 5432 for m in cluster['members']) or\
+        len(set(m['host'] for m in cluster['members'])) < len(cluster['members'])
 
-    if extended or has_scheduled_restarts:
-        columns.append('Scheduled restart')
+    for m in cluster['members']:
+        logging.debug(m)
 
-    print_output(columns, rows, alignment, fmt)
+        lag = m.get('lag', '')
+        m.update(cluster=name, member=m['name'], tl=m.get('timeline', ''),
+                 role='' if m['role'] == 'replica' else m['role'].replace('_', ' ').title(),
+                 lag_in_mb=round(lag/1024/1024) if isinstance(lag, six.integer_types) else lag,
+                 pending_restart='*' if m.get('pending_restart') else '')
+
+        if append_port:
+            m['host'] = ':'.join([m['host'], str(m['port'])])
+
+        if 'scheduled_restart' in m:
+            value = m['scheduled_restart']['schedule']
+            if 'postgres_version' in m['scheduled_restart']:
+                value += ' if version < {0}'.format(m['scheduled_restart']['postgres_version'])
+            m['scheduled_restart'] = value
+
+        rows.append([m.get(n.lower().replace(' ', '_'), '') for n in columns])
+
+    print_output(columns, rows, {'Lag in MB': 'r', 'TL': 'r'}, fmt)
+
+    if fmt != 'pretty':  # Omit service info when using machine-readable formats
+        return
 
     service_info = []
-    if cluster.is_paused():
+    if cluster.get('pause'):
         service_info.append('Maintenance mode: on')
 
-    if cluster.failover and cluster.failover.scheduled_at:
-        info = 'Switchover scheduled at: ' + cluster.failover.scheduled_at.isoformat()
-        if cluster.failover.leader:
-            info += '\n                    from: ' + cluster.failover.leader
-        if cluster.failover.candidate:
-            info += '\n                      to: ' + cluster.failover.candidate
+    if 'scheduled_switchover' in cluster:
+        info = 'Switchover scheduled at: ' + cluster['scheduled_switchover']['at']
+        for name in ('from', 'to'):
+            if name in cluster['scheduled_switchover']:
+                info += '\n{0:>24}: {1}'.format(name, cluster['scheduled_switchover'][name])
         service_info.append(info)
 
     if service_info:
@@ -892,7 +863,7 @@ def flush(obj, cluster_name, member_names, force, role, target):
     for member in members:
         if target == 'restart':
             if member.data.get('scheduled_restart'):
-                r = request_patroni(member, 'delete', 'restart', None, auth_header(obj))
+                r = request_patroni(member, 'delete', 'restart')
                 check_response(r, member.name, 'flush scheduled restart')
             else:
                 click.echo('No scheduled restart for member {0}'.format(member.name))
@@ -929,20 +900,20 @@ def toggle_pause(config, cluster_name, paused, wait):
 
     for member in members:
         try:
-            r = request_patroni(member, 'patch', 'config', {'pause': paused or None}, auth_header(config))
+            r = request_patroni(member, 'patch', 'config', {'pause': paused or None})
         except Exception as err:
             logging.warning(str(err))
             logging.warning('Member %s is not accessible', member.name)
             continue
 
-        if r.status_code == 200:
+        if r.status == 200:
             if wait:
                 wait_until_pause_is_applied(dcs, paused, cluster)
             else:
                 click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
         else:
             click.echo('Failed: {0} cluster management status code={1}, ({2})'.format(
-                       paused and 'pause' or 'resume', r.status_code, r.text))
+                       paused and 'pause' or 'resume', r.status, r.data.decode('utf-8')))
         break
     else:
         raise PatroniCtlException('Can not find accessible cluster member')
@@ -997,7 +968,7 @@ def show_diff(before_editing, after_editing):
         buf = io.StringIO()
         for line in unified_diff:
             # Force cast to unicode as difflib on Python 2.7 returns a mix of unicode and str.
-            buf.write(text_type(line))
+            buf.write(six.text_type(line))
         buf.seek(0)
 
         class opts:
@@ -1079,15 +1050,40 @@ def apply_yaml_file(data, filename):
     return format_config_for_editing(changed_data), changed_data
 
 
+def find_executable(executable, path=None):
+    _, ext = os.path.splitext(executable)
+
+    if (sys.platform == 'win32') and (ext != '.exe'):
+        executable = executable + '.exe'
+
+    if os.path.isfile(executable):
+        return executable
+
+    if path is None:
+        path = os.environ.get('PATH', os.defpath)
+
+    for p in path.split(os.pathsep):
+        f = os.path.join(p, executable)
+        if os.path.isfile(f):
+            return f
+
+
 def invoke_editor(before_editing, cluster_name):
     """Starts editor command to edit configuration in human readable format
 
     :param before_editing: human representation before editing
     :returns tuple of human readable and parsed datastructure after changes
     """
+
     editor_cmd = os.environ.get('EDITOR')
     if not editor_cmd:
-        raise PatroniCtlException('EDITOR environment variable is not set')
+        for editor in ('editor', 'vi'):
+            editor_cmd = find_executable(editor)
+            if editor_cmd:
+                logging.debug('Setting fallback editor_cmd=%s', editor)
+                break
+    if not editor_cmd:
+        raise PatroniCtlException('EDITOR environment variable is not set. editor or vi are not available')
 
     with temporary_file(contents=before_editing.encode('utf-8'),
                         suffix='.yaml',
@@ -1181,8 +1177,8 @@ def version(obj, cluster_name, member_names):
         if m.api_url:
             if not member_names or m.name in member_names:
                 try:
-                    response = request_patroni(m, 'get', 'patroni')
-                    data = response.json()
+                    response = request_patroni(m)
+                    data = json.loads(response.data.decode('utf-8'))
                     version = data.get('patroni', {}).get('version')
                     pg_version = data.get('server_version')
                     pg_version_str = " PostgreSQL {0}".format(format_pg_version(pg_version)) if pg_version else ""
@@ -1191,6 +1187,19 @@ def version(obj, cluster_name, member_names):
                     click.echo("{0}: failed to get version: {1}".format(m.name, e))
 
 
+@ctl.command('history', help="Show the history of failovers/switchovers")
+@arg_cluster_name
+@option_format
+@click.pass_obj
+def history(obj, cluster_name, fmt):
+    cluster = get_dcs(obj, cluster_name).get_cluster()
+    history = cluster.history and cluster.history.lines or []
+    for line in history:
+        if len(line) < 4:
+            line.append('')
+    print_output(['TL', 'LSN', 'Reason', 'Timestamp'], history, {'TL': 'r', 'LSN': 'r'}, fmt)
+
+
 def format_pg_version(version):
     if version < 100000:
         return "{0}.{1}.{2}".format(version // 10000, version // 100 % 100, version % 100)

patroni/dcs/__init__.py

@@ -14,7 +14,7 @@ import time
 from collections import defaultdict, namedtuple
 from copy import deepcopy
 from patroni.exceptions import PatroniException
-from patroni.utils import parse_bool
+from patroni.utils import parse_bool, uri
 from random import randint
 from six.moves.urllib_parse import urlparse, urlunparse, parse_qsl
 from threading import Event, Lock
@@ -73,26 +73,34 @@ def dcs_modules():
 
 
 def get_dcs(config):
-    available_implementations = set()
-    for module_name in dcs_modules():
-        try:
-            module = importlib.import_module(module_name)
-            for name in filter(lambda name: not name.startswith('__'), dir(module)):  # iterate through module content
-                item = getattr(module, name)
-                name = name.lower()
-                # try to find implementation of AbstractDCS interface, class name must match with module_name
-                if inspect.isclass(item) and issubclass(item, AbstractDCS) and __package__ + '.' + name == module_name:
-                    available_implementations.add(name)
-                    if name in config:  # which has configuration section in the config file
+    modules = dcs_modules()
+
+    for module_name in modules:
+        name = module_name.split('.')[-1]
+        if name in config:  # we will try to import only modules which have configuration section in the config file
+            try:
+                module = importlib.import_module(module_name)
+                for key, item in module.__dict__.items():  # iterate through the module content
+                    # try to find implementation of AbstractDCS interface, class name must match with module_name
+                    if key.lower() == name and inspect.isclass(item) and issubclass(item, AbstractDCS):
                         # propagate some parameters
                         config[name].update({p: config[p] for p in ('namespace', 'name', 'scope', 'loop_wait',
                                              'patronictl', 'ttl', 'retry_timeout') if p in config})
                         return item(config[name])
+            except ImportError:
+                logger.debug('Failed to import %s', module_name)
+
+    available_implementations = []
+    for module_name in modules:
+        name = module_name.split('.')[-1]
+        try:
+            module = importlib.import_module(module_name)
+            available_implementations.extend(name for key, item in module.__dict__.items() if key.lower() == name
+                                             and inspect.isclass(item) and issubclass(item, AbstractDCS))
         except ImportError:
-            if not config.get('patronictl'):
-                logger.info('Failed to import %s', module_name)
+            logger.info('Failed to import %s', module_name)
     raise PatroniException("""Can not find suitable configuration of distributed configuration store
-Available implementations: """ + ', '.join(available_implementations))
+Available implementations: """ + ', '.join(sorted(set(available_implementations))))
 
 
 class Member(namedtuple('Member', 'index,name,session,data')):
@@ -134,10 +142,7 @@ class Member(namedtuple('Member', 'index,name,session,data')):
             return conn_url
 
         if conn_kwargs:
-            conn_url = 'postgresql://{host}:{port}'.format(
-                host=conn_kwargs.get('host'),
-                port=conn_kwargs.get('port'),
-            )
+            conn_url = uri('postgresql', (conn_kwargs.get('host'), conn_kwargs.get('port', 5432)))
             self.data['conn_url'] = conn_url
             return conn_url
 
@@ -160,11 +165,11 @@ class Member(namedtuple('Member', 'index,name,session,data')):
             }
             self.data['conn_kwargs'] = ret.copy()
 
+        # apply any remaining authentication parameters
         if auth and isinstance(auth, dict):
+            ret.update(auth)
             if 'username' in auth:
-                ret['user'] = auth['username']
-            if 'password' in auth:
-                ret['password'] = auth['password']
+                ret['user'] = ret.pop('username')
         return ret
 
     @property
@@ -239,6 +244,20 @@ class Leader(namedtuple('Leader', 'index,session,member')):
     def timeline(self):
         return self.member.data.get('timeline')
 
+    @property
+    def checkpoint_after_promote(self):
+        """
+        >>> Leader(1, '', Member.from_node(1, '', '', '{"version":"z"}')).checkpoint_after_promote
+        """
+        version = self.member.data.get('version')
+        if version:
+            try:
+                # 1.5.6 is the last version which doesn't expose checkpoint_after_promote: false
+                if tuple(map(int, version.split('.'))) > (1, 5, 6):
+                    return self.member.data['role'] == 'master' and 'checkpoint_after_promote' not in self.member.data
+            except Exception:
+                logger.debug('Failed to parse Patroni version %s', version)
+
 
 class Failover(namedtuple('Failover', 'index,leader,candidate,scheduled_at')):
 
@@ -431,14 +450,16 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operat
         # the current master, because that member would replicate from elsewhere. We still create the slot if
         # the replicatefrom destination member is currently not a member of the cluster (fallback to the
         # master), or if replicatefrom destination member happens to be the current master
+        use_slots = self.config and self.config.data.get('postgresql', {}).get('use_slots', True)
         if role in ('master', 'standby_leader'):
-            slot_members = [m.name for m in self.members if m.name != name and
+            slot_members = [m.name for m in self.members if use_slots and m.name != name and
                             (m.replicatefrom is None or m.replicatefrom == name or
                              not self.has_member(m.replicatefrom))]
             permanent_slots = (self.config and self.config.permanent_slots or {}).copy()
         else:
             # only manage slots for replicas that replicate from this one, except for the leader among them
-            slot_members = [m.name for m in self.members if m.replicatefrom == name and m.name != self.leader.name]
+            slot_members = [m.name for m in self.members if use_slots and
+                            m.replicatefrom == name and m.name != self.leader.name]
             permanent_slots = {}
 
         slots = {slot_name_from_member_name(name): {'type': 'physical'} for name in slot_members}

patroni/dcs/consul.py

@@ -9,13 +9,14 @@ import time
 import urllib3
 
 from consul import ConsulException, NotFound, base
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
-from patroni.exceptions import DCSError
-from patroni.utils import deep_compare, parse_bool, Retry, RetryFailedError, split_host_port
 from urllib3.exceptions import HTTPError
 from six.moves.urllib.parse import urlencode, urlparse, quote
 from six.moves.http_client import HTTPException
 
+from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState, TimelineHistory
+from ..exceptions import DCSError
+from ..utils import deep_compare, parse_bool, Retry, RetryFailedError, split_host_port, uri, USER_AGENT
+
 logger = logging.getLogger(__name__)
 
 
@@ -40,7 +41,7 @@ class HTTPClient(object):
     def __init__(self, host='127.0.0.1', port=8500, token=None, scheme='http', verify=True, cert=None, ca_cert=None):
         self.token = token
         self._read_timeout = 10
-        self.base_uri = '{0}://{1}:{2}'.format(scheme, host, port)
+        self.base_uri = uri(scheme, (host, port))
         kwargs = {}
         if cert:
             if isinstance(cert, tuple):
@@ -111,8 +112,9 @@ class HTTPClient(object):
             else:
                 kwargs['timeout'] = self._read_timeout
             token = params.pop('token', self.token) if isinstance(params, dict) else self.token
+            kwargs['headers'] = urllib3.make_headers(user_agent=USER_AGENT)
             if token:
-                kwargs['headers'] = {'X-Consul-Token': token}
+                kwargs['headers']['X-Consul-Token'] = token
             return callback(self.response(self.http.request(method.upper(), self.uri(path, params), **kwargs)))
         return wrapper
 
@@ -122,7 +124,7 @@ class ConsulClient(base.Consul):
     def __init__(self, *args, **kwargs):
         self._cert = kwargs.pop('cert', None)
         self._ca_cert = kwargs.pop('ca_cert', None)
-        self._token = kwargs.get('token')
+        self.token = kwargs.get('token')
         super(ConsulClient, self).__init__(*args, **kwargs)
 
     def connect(self, *args, **kwargs):
@@ -131,10 +133,15 @@ class ConsulClient(base.Consul):
             kwargs['cert'] = self._cert
         if self._ca_cert:
             kwargs['ca_cert'] = self._ca_cert
-        if self._token:
-            kwargs['token'] = self._token
+        if self.token:
+            kwargs['token'] = self.token
         return HTTPClient(**kwargs)
 
+    def reload_config(self, config):
+        self.http.token = self.token = config.get('token')
+        self.consistency = config.get('consistency', 'default')
+        self.dc = config.get('dc')
+
 
 def catch_consul_errors(func):
     def wrapper(*args, **kwargs):
@@ -197,7 +204,7 @@ class Consul(AbstractDCS):
         if config.get('key') and config.get('cert'):
             config['cert'] = (config['cert'], config['key'])
 
-        config_keys = ('host', 'port', 'token', 'scheme', 'cert', 'ca_cert', 'dc')
+        config_keys = ('host', 'port', 'token', 'scheme', 'cert', 'ca_cert', 'dc', 'consistency')
         kwargs = {p: config.get(p) for p in config_keys if config.get(p)}
 
         verify = config.get('verify')
@@ -232,6 +239,10 @@ class Consul(AbstractDCS):
                 logger.info('waiting on consul')
                 time.sleep(5)
 
+    def reload_config(self, config):
+        super(Consul, self).reload_config(config)
+        self._client.reload_config(config.get('consul', {}))
+
     def set_ttl(self, ttl):
         if self._client.http.set_ttl(ttl/2.0):  # Consul multiplies the TTL by 2x
             self._session = None

patroni/dcs/etcd.py

@@ -5,28 +5,28 @@ import logging
 import os
 import urllib3.util.connection
 import random
-import requests
 import six
 import socket
 import time
 
 from dns.exception import DNSException
 from dns import resolver
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
-from patroni.exceptions import DCSError
-from patroni.utils import Retry, RetryFailedError, split_host_port
-from urllib3.exceptions import HTTPError, ReadTimeoutError
-from requests.exceptions import RequestException
+from urllib3.exceptions import HTTPError, ReadTimeoutError, ProtocolError
 from six.moves.queue import Queue
 from six.moves.http_client import HTTPException
 from six.moves.urllib_parse import urlparse
 from threading import Thread
 
+from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState, TimelineHistory
+from ..exceptions import DCSError
+from ..request import get as requests_get
+from ..utils import Retry, RetryFailedError, split_host_port, uri, USER_AGENT
+
 logger = logging.getLogger(__name__)
 
 
-def uri(protocol, host, port, endpoint=''):
-    return '{0}://{1}:{2}{3}'.format(protocol, host, port, endpoint)
+class EtcdRaftInternal(etcd.EtcdException):
+    """Raft Internal Error"""
 
 
 class EtcdError(DCSError):
@@ -93,22 +93,51 @@ class Client(etcd.Client):
         self._config = config
         self._load_machines_cache()
         self._allow_reconnect = True
+        # allow passing retry argument to api_execute in params
+        self._comparison_conditions.add('retry')
+        self._read_options.add('retry')
+        self._del_conditions.add('retry')
 
-    def _build_request_parameters(self):
+    def _calculate_timeouts(self, etcd_nodes=None, timeout=None):
+        """Calculate a request timeout and number of retries per single etcd node.
+        In case if the timeout per node is too small (less than one second) we will reduce the number of nodes.
+        For the cluster with only one node we will try to do 2 retries.
+        For clusters with 2 nodes we will try to do 1 retry for every node.
+        No retries for clusters with 3 or more nodes. We better rely on switching to a different node."""
+
+        etcd_nodes = etcd_nodes or len(self._machines_cache) + 1
+        per_node_timeout = timeout = float(timeout or self.read_timeout)
+
+        max_retries = 4 - min(etcd_nodes, 3)
+        per_node_retries = 1
+        min_timeout = 1.0
+
+        while etcd_nodes > 0:
+            per_node_timeout = float(timeout) / etcd_nodes
+            if per_node_timeout >= min_timeout:
+                # for small clusters we will try to do more than on try on every node
+                while per_node_retries < max_retries and per_node_timeout / (per_node_retries + 1) >= min_timeout:
+                    per_node_retries += 1
+                per_node_timeout /= per_node_retries
+                break
+            # if the timeout per one node is to small try to reduce number of nodes
+            etcd_nodes -= 1
+            max_retries = 1
+
+        return etcd_nodes, per_node_timeout, per_node_retries - 1
+
+    def _get_headers(self):
+        basic_auth = ':'.join((self.username, self.password)) if self.username and self.password else None
+        return urllib3.make_headers(basic_auth=basic_auth, user_agent=USER_AGENT)
+
+    def _build_request_parameters(self, timeout=None):
         kwargs = {'headers': self._get_headers(), 'redirect': self.allow_redirect}
 
-        # calculate the number of retries and timeout *per node*
-        # actual number of retries depends on the number of nodes
-        etcd_nodes = len(self._machines_cache) + 1
-        kwargs['retries'] = 0 if etcd_nodes > 3 else (1 if etcd_nodes > 1 else 2)
-
-        # if etcd_nodes > 3:
-        #     kwargs.update({'retries': 0, 'timeout': float(self.read_timeout)/etcd_nodes})
-        # elif etcd_nodes > 1:
-        #     kwargs.update({'retries': 1, 'timeout': self.read_timeout/2.0/etcd_nodes})
-        # else:
-        #     kwargs.update({'retries': 2, 'timeout': self.read_timeout/3.0})
-        kwargs['timeout'] = self.read_timeout/float(kwargs['retries'] + 1)/etcd_nodes
+        if timeout is not None:
+            kwargs.update(retries=0, timeout=timeout)
+        else:
+            _, per_node_timeout, per_node_retries = self._calculate_timeouts()
+            kwargs.update(timeout=per_node_timeout, retries=per_node_retries)
         return kwargs
 
     def set_machines_cache_ttl(self, cache_ttl):
@@ -166,8 +195,12 @@ class Client(etcd.Client):
             self._check_cluster_id(response)
         except (HTTPError, HTTPException, socket.error, socket.timeout) as e:
             if (isinstance(fields, dict) and fields.get("wait") == "true" and
-                    isinstance(e, ReadTimeoutError)):
+                    isinstance(e, (ReadTimeoutError, ProtocolError))):
                 logger.debug("Watch timed out.")
+                # switch to the next etcd node because we don't know exactly what happened,
+                # whether the key didn't received an update or there is a network problem.
+                self._machines_cache.insert(0, self._base_uri)
+                self._base_uri = self._next_server()
                 raise etcd.EtcdWatchTimedOut("Watch timed out: {0}".format(e), cause=e)
             logger.error("Request to server %s failed: %r", self._base_uri, e)
             logger.info("Reconnection allowed, looking for another server.")
@@ -179,6 +212,7 @@ class Client(etcd.Client):
         if not path.startswith('/'):
             raise ValueError('Path does not start with /')
 
+        retry = params.pop('retry', None) if isinstance(params, dict) else None
         kwargs = {'fields': params, 'preload_content': False}
 
         if method in [self._MGET, self._MDELETE]:
@@ -194,31 +228,44 @@ class Client(etcd.Client):
             self._load_machines_cache()
         elif not self._use_proxies and time.time() - self._machines_cache_updated > self._machines_cache_ttl:
             self._refresh_machines_cache()
-            self._machines_cache_updated = time.time()
 
-        kwargs.update(self._build_request_parameters())
+        kwargs.update(self._build_request_parameters(timeout))
 
-        if timeout is not None:
-            kwargs.update({'retries': 0, 'timeout': timeout})
+        if retry:
+            machines_cache = [self._base_uri] + self._machines_cache
 
         response = False
 
-        try:
-            some_request_failed = False
-            while not response:
-                response = self._do_http_request(request_executor, method, self._base_uri + path, **kwargs)
+        while True:
+            try:
+                some_request_failed = False
+                while not response:
+                    response = self._do_http_request(request_executor, method, self._base_uri + path, **kwargs)
+
+                    if response is False:
+                        if not retry:
+                            raise etcd.EtcdException('{0} {1} request failed'.format(method, path))
+                        some_request_failed = True
+                if some_request_failed:
+                    self._refresh_machines_cache()
+                if response:
+                    break
+            except etcd.EtcdConnectionFailed:
+                if not retry:
+                    raise
+                sleeptime = retry.sleeptime
+                remaining_time = retry.stoptime - sleeptime - time.time()
+                nodes, timeout, retries = self._calculate_timeouts(len(machines_cache), remaining_time)
+                if nodes == 0:
+                    self._update_machines_cache = True
+                    raise
+                retry.sleep_func(sleeptime)
+                retry.update_delay()
+                # We still have some time left. Partially restore `_machines_cache` and retry request
+                kwargs.update(timeout=timeout, retries=retries)
+                self._base_uri = machines_cache[0]
+                self._machines_cache = machines_cache[1:nodes]
 
-                if response is False:
-                    some_request_failed = True
-            if some_request_failed:
-                self._refresh_machines_cache()
-        except etcd.EtcdConnectionFailed as e:
-            if isinstance(e, etcd.EtcdWatchTimedOut) and self._machines_cache:
-                self._base_uri = self._next_server()
-            else:
-                self._update_machines_cache = True
-            if not response:
-                raise
         return self._handle_server_response(response)
 
     @staticmethod
@@ -238,15 +285,15 @@ class Client(etcd.Client):
             protocol = 'https' if '-ssl' in r else 'http'
             endpoint = '/members' if '-server' in r else ''
             for host, port in self.get_srv_record('_etcd{0}._tcp.{1}'.format(r, srv)):
-                url = uri(protocol, host, port, endpoint)
+                url = uri(protocol, (host, port), endpoint)
                 if endpoint:
                     try:
-                        response = requests.get(url, timeout=self.read_timeout, verify=False)
-                        if response.ok:
-                            for member in response.json():
+                        response = requests_get(url, timeout=self.read_timeout, verify=False)
+                        if response.status < 400:
+                            for member in json.loads(response.data.decode('utf-8')):
                                 ret.extend(member['clientURLs'])
                             break
-                    except RequestException:
+                    except Exception:
                         logger.exception('GET %s', url)
                 else:
                     ret.append(url)
@@ -260,19 +307,14 @@ class Client(etcd.Client):
     def _get_machines_cache_from_dns(self, host, port):
         """One host might be resolved into multiple ip addresses. We will make list out of it"""
         if self.protocol == 'http':
-            ret = []
-            for af, _, _, _, sa in self._dns_resolver.resolve(host, port):
-                host, port = sa[:2]
-                if af == socket.AF_INET6:
-                    host = '[{0}]'.format(host)
-                ret.append(uri(self.protocol, host, port))
+            ret = map(lambda res: uri(self.protocol, res[-1][:2]), self._dns_resolver.resolve(host, port))
             if ret:
                 return list(set(ret))
-        return [uri(self.protocol, host, port)]
+        return [uri(self.protocol, (host, port))]
 
     def _get_machines_cache_from_config(self):
         if 'proxy' in self._config:
-            return [uri(self.protocol, self._config['host'], self._config['port'])]
+            return [uri(self.protocol, (self._config['host'], self._config['port']))]
 
         machines_cache = []
         if 'srv' in self._config:
@@ -305,14 +347,15 @@ class Client(etcd.Client):
         # After filling up initial list of machines_cache we should ask etcd-cluster about actual list
         self._base_uri = self._next_server()
         self._refresh_machines_cache()
-
         self._update_machines_cache = False
-        self._machines_cache_updated = time.time()
 
     def _refresh_machines_cache(self):
         self._machines_cache = self._get_machines_cache_from_config() if self._use_proxies else self.machines
         if self._base_uri in self._machines_cache:
             self._machines_cache.remove(self._base_uri)
+        elif self._machines_cache:
+            self._base_uri = self._next_server()
+        self._machines_cache_updated = time.time()
 
 
 class Etcd(AbstractDCS):
@@ -321,15 +364,15 @@ class Etcd(AbstractDCS):
         super(Etcd, self).__init__(config)
         self._ttl = int(config.get('ttl') or 30)
         self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
-                            retry_exceptions=(etcd.EtcdLeaderElectionInProgress,
-                                              etcd.EtcdWatcherCleared,
-                                              etcd.EtcdEventIndexCleared))
+                            retry_exceptions=(etcd.EtcdLeaderElectionInProgress, EtcdRaftInternal))
         self._client = self.get_etcd_client(config)
         self.__do_not_watch = False
         self._has_failed = False
 
     def retry(self, *args, **kwargs):
-        return self._retry.copy()(*args, **kwargs)
+        retry = self._retry.copy()
+        kwargs['retry'] = retry
+        return retry(*args, **kwargs)
 
     def _handle_exception(self, e, name='', do_sleep=False, raise_ex=None):
         if not self._has_failed:
@@ -377,7 +420,7 @@ class Etcd(AbstractDCS):
             config['hosts'] = []
             for value in hosts:
                 if isinstance(value, six.string_types):
-                    config['hosts'].append(uri(protocol, *split_host_port(value, default_port)))
+                    config['hosts'].append(uri(protocol, split_host_port(value.strip(), default_port)))
         elif 'host' in config:
             host, port = split_host_port(config['host'], 2379)
             config['host'] = host
@@ -515,7 +558,7 @@ class Etcd(AbstractDCS):
 
     @catch_etcd_errors
     def take_leader(self):
-        return self.retry(self._client.set, self.leader_path, self._name, self._ttl)
+        return self.retry(self._client.write, self.leader_path, self._name, ttl=self._ttl)
 
     def attempt_to_acquire_leader(self, permanent=False):
         try:
@@ -544,7 +587,7 @@ class Etcd(AbstractDCS):
 
     @catch_etcd_errors
     def _update_leader(self):
-        return self.retry(self._client.test_and_set, self.leader_path, self._name, self._name, self._ttl)
+        return self.retry(self._client.write, self.leader_path, self._name, prevValue=self._name, ttl=self._ttl)
 
     @catch_etcd_errors
     def initialize(self, create_new=True, sysid=""):
@@ -605,3 +648,6 @@ class Etcd(AbstractDCS):
             return super(Etcd, self).watch(None, timeout)
         finally:
             self.event.clear()
+
+
+etcd.EtcdError.error_exceptions[300] = EtcdRaftInternal

patroni/dcs/exhibitor.py

@@ -1,10 +1,11 @@
+import json
 import logging
 import random
-import requests
 import time
 
 from patroni.dcs.zookeeper import ZooKeeper
-from requests.exceptions import RequestException
+from patroni.request import get as requests_get
+from patroni.utils import uri
 
 logger = logging.getLogger(__name__)
 
@@ -46,12 +47,11 @@ class ExhibitorEnsembleProvider(object):
     def _query_exhibitors(self, exhibitors):
         random.shuffle(exhibitors)
         for host in exhibitors:
-            uri = 'http://{0}:{1}{2}'.format(host, self._exhibitor_port, self._uri_path)
             try:
-                response = requests.get(uri, timeout=self.TIMEOUT)
-                return response.json()
-            except RequestException:
-                pass
+                response = requests_get(uri('http', (host, self._exhibitor_port), self._uri_path), timeout=self.TIMEOUT)
+                return json.loads(response.data.decode('utf-8'))
+            except Exception:
+                logging.debug('Request to %s failed', host)
         return None
 
     @property

patroni/dcs/kubernetes.py

@@ -8,11 +8,14 @@ import sys
 import time
 
 from kubernetes import client as k8s_client, config as k8s_config, watch as k8s_watch
-from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
-from patroni.exceptions import DCSError
-from patroni.utils import deep_compare, tzutc, Retry, RetryFailedError
+from urllib3 import Timeout
 from urllib3.exceptions import HTTPError
 from six.moves.http_client import HTTPException
+from threading import Condition, Lock, Thread
+
+from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, SyncState, TimelineHistory
+from ..exceptions import DCSError
+from ..utils import deep_compare, Retry, RetryFailedError, tzutc, USER_AGENT
 
 logger = logging.getLogger(__name__)
 
@@ -33,6 +36,8 @@ class CoreV1ApiProxy(object):
 
     def __init__(self, use_endpoints=False):
         self._api = k8s_client.CoreV1Api()
+        self._api.api_client.user_agent = USER_AGENT
+        self._api.api_client.rest_client.pool_manager.connection_pool_kw['maxsize'] = 10
         self._request_timeout = None
         self._use_endpoints = use_endpoints
 
@@ -70,6 +75,110 @@ def catch_kubernetes_errors(func):
     return wrapper
 
 
+class ObjectCache(Thread):
+
+    def __init__(self, dcs, func, retry, condition):
+        Thread.__init__(self)
+        self.daemon = True
+        self._api_client = k8s_client.ApiClient()
+        self._dcs = dcs
+        self._func = func
+        self._retry = retry
+        self._condition = condition
+        self._is_ready = False
+        self._object_cache = {}
+        self._object_cache_lock = Lock()
+        self._annotations_map = {self._dcs.leader_path: self._dcs._LEADER, self._dcs.config_path: self._dcs._CONFIG}
+        self.start()
+
+    def _list(self):
+        return self._func(_request_timeout=(self._retry.deadline, Timeout.DEFAULT_TIMEOUT))
+
+    def _watch(self, resource_version):
+        return self._func(_request_timeout=(self._retry.deadline, Timeout.DEFAULT_TIMEOUT),
+                          _preload_content=False, watch=True, resource_version=resource_version)
+
+    def set(self, name, value):
+        with self._object_cache_lock:
+            old_value = self._object_cache.get(name)
+            ret = not old_value or int(old_value.metadata.resource_version) < int(value.metadata.resource_version)
+            if ret:
+                self._object_cache[name] = value
+        return ret, old_value
+
+    def delete(self, name, resource_version):
+        with self._object_cache_lock:
+            old_value = self._object_cache.get(name)
+            ret = old_value and int(old_value.metadata.resource_version) < int(resource_version)
+            if ret:
+                del self._object_cache[name]
+        return not old_value or ret, old_value
+
+    def copy(self):
+        with self._object_cache_lock:
+            return self._object_cache.copy()
+
+    def _build_cache(self):
+        objects = self._list()
+        return_type = 'V1' + objects.kind[:-4]
+        with self._object_cache_lock:
+            self._object_cache = {item.metadata.name: item for item in objects.items}
+        with self._condition:
+            self._is_ready = True
+            self._condition.notify()
+
+        response = self._watch(objects.metadata.resource_version)
+        try:
+            for line in k8s_watch.watch.iter_resp_lines(response):
+                event = json.loads(line)
+                obj = event['object']
+                if obj.get('code') == 410:
+                    break
+
+                ev_type = event['type']
+                name = obj['metadata']['name']
+
+                if ev_type in ('ADDED', 'MODIFIED'):
+                    obj = k8s_watch.watch.SimpleNamespace(data=json.dumps(obj))
+                    obj = self._api_client.deserialize(obj, return_type)
+                    success, old_value = self.set(name, obj)
+                    if success:
+                        new_value = (obj.metadata.annotations or {}).get(self._annotations_map.get(name))
+                elif ev_type == 'DELETED':
+                    success, old_value = self.delete(name, obj['metadata']['resourceVersion'])
+                    new_value = None
+                else:
+                    logger.warning('Unexpected event type: %s', ev_type)
+                    continue
+
+                if success and return_type != 'V1Pod':
+                    if old_value:
+                        old_value = (old_value.metadata.annotations or {}).get(self._annotations_map.get(name))
+
+                    if old_value != new_value and \
+                            (name != self._dcs.config_path or old_value is not None and new_value is not None):
+                        logger.debug('%s changed from %s to %s', name, old_value, new_value)
+                        self._dcs.event.set()
+        finally:
+            with self._condition:
+                self._is_ready = False
+            response.close()
+            response.release_conn()
+
+    def run(self):
+        while True:
+            try:
+                self._build_cache()
+            except Exception as e:
+                with self._condition:
+                    self._is_ready = False
+                logger.error('ObjectCache.run %r', e)
+
+    def is_ready(self):
+        """Must be called only when holding the lock on `_condition`"""
+        return self._is_ready
+
+
 class Kubernetes(AbstractDCS):
 
     def __init__(self, config):
@@ -110,6 +219,16 @@ class Kubernetes(AbstractDCS):
         self._config_resource_version = None
         self.__do_not_watch = False
 
+        self._condition = Condition()
+
+        pods_func = functools.partial(self._api.list_namespaced_pod, self._namespace,
+                                      label_selector=self._label_selector)
+        self._pods = ObjectCache(self, pods_func, self._retry, self._condition)
+
+        kinds_func = functools.partial(self._api.list_namespaced_kind, self._namespace,
+                                       label_selector=self._label_selector)
+        self._kinds = ObjectCache(self, kinds_func, self._retry, self._condition)
+
     def retry(self, *args, **kwargs):
         return self._retry.copy()(*args, **kwargs)
 
@@ -140,14 +259,21 @@ class Kubernetes(AbstractDCS):
         member.data['pod_labels'] = pod.metadata.labels
         return member
 
+    def _wait_caches(self):
+        stop_time = time.time() + self._retry.deadline
+        while not (self._pods.is_ready() and self._kinds.is_ready()):
+            timeout = stop_time - time.time()
+            if timeout <= 0:
+                raise RetryFailedError('Exceeded retry deadline')
+            self._condition.wait(timeout)
+
     def _load_cluster(self):
         try:
-            # get list of members
-            response = self.retry(self._api.list_namespaced_pod, self._namespace, label_selector=self._label_selector)
-            members = [self.member(pod) for pod in response.items]
+            with self._condition:
+                self._wait_caches()
 
-            response = self.retry(self._api.list_namespaced_kind, self._namespace, label_selector=self._label_selector)
-            nodes = {item.metadata.name: item for item in response.items}
+                members = [self.member(pod) for pod in self._pods.copy().values()]
+                nodes = self._kinds.copy()
 
             config = nodes.get(self.config_path)
             metadata = config and config.metadata
@@ -195,12 +321,13 @@ class Kubernetes(AbstractDCS):
             if metadata:
                 member = Member(-1, leader, None, {})
                 member = ([m for m in members if m.name == leader] or [member])[0]
-                leader = Leader(response.metadata.resource_version, None, member)
+                leader = Leader(metadata.resource_version, None, member)
 
             # failover key
             failover = nodes.get(self.failover_path)
             metadata = failover and failover.metadata
-            failover = Failover.from_node(metadata and metadata.resource_version, metadata and metadata.annotations)
+            failover = Failover.from_node(metadata and metadata.resource_version,
+                                          metadata and (metadata.annotations or {}).copy())
 
             # get synchronization state
             sync = nodes.get(self.sync_path)
@@ -279,7 +406,10 @@ class Kubernetes(AbstractDCS):
             body = k8s_client.V1Endpoints(**endpoints)
         else:
             body = k8s_client.V1ConfigMap(metadata=metadata)
-        return self.retry(func, self._namespace, body) if retry else func(self._namespace, body)
+        ret = self.retry(func, self._namespace, body) if retry else func(self._namespace, body)
+        if ret:
+            self._kinds.set(name, ret)
+        return ret
 
     def patch_or_create_config(self, annotations, resource_version=None, patch=False, retry=True):
         # SCOPE-config endpoint requires corresponding service otherwise it might be "cleaned" by k8s master
@@ -353,7 +483,8 @@ class Kubernetes(AbstractDCS):
         """Unused"""
 
     def manual_failover(self, leader, candidate, scheduled_at=None, index=None):
-        annotations = {'leader': leader or None, 'member': candidate or None, 'scheduled_at': scheduled_at}
+        annotations = {'leader': leader or None, 'member': candidate or None,
+                       'scheduled_at': scheduled_at and scheduled_at.isoformat()}
         patch = bool(self.cluster and isinstance(self.cluster.failover, Failover) and self.cluster.failover.index)
         return self.patch_or_create(self.failover_path, annotations, index, bool(index or patch), False)
 
@@ -417,24 +548,6 @@ class Kubernetes(AbstractDCS):
             self.__do_not_watch = False
             return True
 
-        if leader_index:
-            end_time = time.time() + timeout
-            w = k8s_watch.Watch()
-            while timeout >= 1:
-                try:
-                    for event in w.stream(self._api.list_namespaced_kind, self._namespace,
-                                          resource_version=leader_index, timeout_seconds=int(timeout + 0.5),
-                                          field_selector='metadata.name=' + self.leader_path,
-                                          _request_timeout=(1, timeout + 1)):
-                        return event['raw_object'].get('metadata', {}).get('resourceVersion') != leader_index
-                    return False
-                except KeyboardInterrupt:
-                    raise
-                except Exception:
-                    logger.exception('watch')
-
-                timeout = end_time - time.time()
-
         try:
             return super(Kubernetes, self).watch(None, timeout)
         finally:

patroni/dcs/zookeeper.py

@@ -274,7 +274,10 @@ class ZooKeeper(AbstractDCS):
         cluster = self.cluster
         member = cluster and cluster.get_member(self._name, fallback_to_leader=False)
         encoded_data = json.dumps(data, separators=(',', ':')).encode('utf-8')
-        if member and self._client.client_id is not None and member.session != self._client.client_id[0]:
+        if member and (self._client.client_id is not None and member.session != self._client.client_id[0] or
+                       not (deep_compare(member.data.get('tags', {}), data.get('tags', {})) and
+                            member.data.get('version') == data.get('version') and
+                            member.data.get('checkpoint_after_promote') == data.get('checkpoint_after_promote'))):
             try:
                 self._client.delete_async(self.member_path).get(timeout=1)
             except NoNodeError:

patroni/exceptions.py

@@ -27,3 +27,7 @@ class PostgresConnectionException(PostgresException):
 
 class WatchdogError(PatroniException):
     pass
+
+
+class ConfigParseError(PatroniException):
+    pass

patroni/ha.py

@@ -3,7 +3,6 @@ import functools
 import json
 import logging
 import psycopg2
-import requests
 import sys
 import time
 import uuid
@@ -13,7 +12,9 @@ from multiprocessing.pool import ThreadPool
 from patroni.async_executor import AsyncExecutor, CriticalTask
 from patroni.exceptions import DCSError, PostgresConnectionException, PatroniException
 from patroni.postgresql import ACTION_ON_START, ACTION_ON_ROLE_CHANGE
-from patroni.utils import polling_loop, tzutc
+from patroni.postgresql.misc import postgres_version_to_int
+from patroni.postgresql.rewind import Rewind
+from patroni.utils import polling_loop, tzutc, is_standby_cluster as _is_standby_cluster
 from patroni.dcs import RemoteMember
 from threading import RLock
 
@@ -59,6 +60,7 @@ class Ha(object):
     def __init__(self, patroni):
         self.patroni = patroni
         self.state_handler = patroni.postgresql
+        self._rewind = Rewind(self.state_handler)
         self.dcs = patroni.dcs
         self.cluster = None
         self.old_cluster = None
@@ -71,7 +73,7 @@ class Ha(object):
         self._post_bootstrap_task = None
         self._crash_recovery_executed = False
         self._start_timeout = None
-        self._async_executor = AsyncExecutor(self.state_handler, self.wakeup)
+        self._async_executor = AsyncExecutor(self.state_handler.cancellable, self.wakeup)
         self.watchdog = patroni.watchdog
 
         # Each member publishes various pieces of information to the DCS using touch_member. This lock protects
@@ -106,9 +108,7 @@ class Ha(object):
         return config.get('standby_cluster')
 
     def is_standby_cluster(self):
-        config = self.get_standby_cluster_config()
-        # Check whether or not provided configuration describes a standby cluster
-        return isinstance(config, dict) and (config.get('host') or config.get('port') or config.get('restore_command'))
+        return _is_standby_cluster(self.get_standby_cluster_config())
 
     def is_leader(self):
         with self._is_leader_lock:
@@ -174,12 +174,15 @@ class Ha(object):
                 'conn_url': self.state_handler.connection_string,
                 'api_url': self.patroni.api.connection_string,
                 'state': self.state_handler.state,
-                'role': self.state_handler.role
+                'role': self.state_handler.role,
+                'version': self.patroni.version
             }
 
             # following two lines are mainly necessary for consul, to avoid creation of master service
             if data['role'] == 'master' and not self.is_leader():
                 data['role'] = 'promoted'
+            if self.is_leader() and not self._rewind.checkpoint_after_promote():
+                data['checkpoint_after_promote'] = False
             tags = self.get_effective_tags()
             if tags:
                 data['tags'] = tags
@@ -210,7 +213,8 @@ class Ha(object):
         if self.is_standby_cluster() and not isinstance(clone_member, RemoteMember):
             clone_member = self.get_remote_member(clone_member)
 
-        if self.state_handler.clone(clone_member):
+        self._rewind.reset_state()
+        if self.state_handler.bootstrap.clone(clone_member):
             logger.info('bootstrapped %s', msg)
             cluster = self.dcs.get_cluster()
             node_to_follow = self._get_node_to_follow(cluster)
@@ -224,9 +228,8 @@ class Ha(object):
             clone_member = self.cluster.get_clone_member(self.state_handler.name)
             member_role = 'leader' if clone_member == self.cluster.leader else 'replica'
             msg = "from {0} '{1}'".format(member_role, clone_member.name)
-            self._async_executor.schedule('bootstrap {0}'.format(msg))
-            self._async_executor.run_async(self.clone, args=(clone_member, msg))
-            return 'trying to bootstrap {0}'.format(msg)
+            ret = self._async_executor.try_run_async('bootstrap {0}'.format(msg), self.clone, args=(clone_member, msg))
+            return ret or 'trying to bootstrap {0}'.format(msg)
 
         # no initialize key and node is allowed to be master and has 'bootstrap' section in a configuration file
         elif self.cluster.initialize is None and not self.patroni.nofailover and 'bootstrap' in self.patroni.config:
@@ -235,25 +238,21 @@ class Ha(object):
                 self._post_bootstrap_task = CriticalTask()
 
                 if self.is_standby_cluster():
-                    self._async_executor.schedule('bootstrap_standby_leader')
-                    self._async_executor.run_async(self.bootstrap_standby_leader)
-                    return 'trying to bootstrap a new standby leader'
+                    ret = self._async_executor.try_run_async('bootstrap_standby_leader', self.bootstrap_standby_leader)
+                    return ret or 'trying to bootstrap a new standby leader'
                 else:
-                    self._async_executor.schedule('bootstrap')
-                    self._async_executor.run_async(
-                        self.state_handler.bootstrap,
-                        args=(self.patroni.config['bootstrap'],)
-                    )
-                    return 'trying to bootstrap a new cluster'
+                    ret = self._async_executor.try_run_async('bootstrap', self.state_handler.bootstrap.bootstrap,
+                                                             args=(self.patroni.config['bootstrap'],))
+                    return ret or 'trying to bootstrap a new cluster'
             else:
                 return 'failed to acquire initialize lock'
         else:
-            if self.state_handler.can_create_replica_without_replication_connection():
+            create_replica_methods = self.get_standby_cluster_config().get('create_replica_methods', []) \
+                                     if self.is_standby_cluster() else None
+            if self.state_handler.can_create_replica_without_replication_connection(create_replica_methods):
                 msg = 'bootstrap (without leader)'
-                self._async_executor.schedule(msg)
-                self._async_executor.run_async(self.clone)
-                return 'trying to ' + msg
-            return 'waiting for leader to bootstrap'
+                return self._async_executor.try_run_async(msg, self.clone) or 'trying to ' + msg
+            return 'waiting for {0}leader to bootstrap'.format('standby_' if self.is_standby_cluster() else '')
 
     def bootstrap_standby_leader(self):
         """ If we found 'standby' key in the configuration, we need to bootstrap
@@ -271,19 +270,17 @@ class Ha(object):
 
     def _handle_rewind_or_reinitialize(self):
         leader = self.get_remote_master() if self.is_standby_cluster() else self.cluster.leader
-        if not self.state_handler.rewind_or_reinitialize_needed_and_possible(leader):
+        if not self._rewind.rewind_or_reinitialize_needed_and_possible(leader):
             return None
 
-        if self.state_handler.can_rewind:
-            self._async_executor.schedule('running pg_rewind from ' + leader.name)
-            self._async_executor.run_async(self.state_handler.rewind, (leader,))
-            return True
+        if self._rewind.can_rewind:
+            msg = 'running pg_rewind from ' + leader.name
+            return self._async_executor.try_run_async(msg, self._rewind.execute, args=(leader,)) or msg
 
         # remove_data_directory_on_diverged_timelines is set
         if not self.is_standby_cluster():
-            self._async_executor.schedule('reinitializing due to diverged timelines')
-            self._async_executor.run_async(self._do_reinitialize, args=(self.cluster, ))
-            return True
+            msg = 'reinitializing due to diverged timelines'
+            return self._async_executor.try_run_async(msg, self._do_reinitialize, args=(self.cluster,)) or msg
 
     def recover(self):
         # Postgres is not running and we will restart in standby mode. Watchdog is not needed until we promote.
@@ -306,21 +303,22 @@ class Ha(object):
 
         data = self.state_handler.controldata()
         logger.info('pg_controldata:\n%s\n', '\n'.join('  {0}: {1}'.format(k, v) for k, v in data.items()))
-        if data.get('Database cluster state') in ('in production', 'shutting down', 'in crash recovery') and \
-                not self._crash_recovery_executed and (self.cluster.is_unlocked() or self.state_handler.can_rewind):
+        if data.get('Database cluster state') in ('in production', 'shutting down', 'in crash recovery') \
+                and not self._crash_recovery_executed and \
+                (self.cluster.is_unlocked() or self._rewind.can_rewind):
             self._crash_recovery_executed = True
-            self._async_executor.schedule('doing crash recovery in a single user mode')
-            self._async_executor.run_async(self.state_handler.fix_cluster_state)
-            return self._async_executor.scheduled_action
+            msg = 'doing crash recovery in a single user mode'
+            return self._async_executor.try_run_async(msg, self.state_handler.fix_cluster_state) or msg
 
         self.load_cluster_from_dcs()
 
         role = 'replica'
         if self.is_standby_cluster() or not self.has_lock():
-            if not self.state_handler.rewind_executed:
-                self.state_handler.trigger_check_diverged_lsn()
-            if self._handle_rewind_or_reinitialize():
-                return self._async_executor.scheduled_action
+            if not self._rewind.executed:
+                self._rewind.trigger_check_diverged_lsn()
+            msg = self._handle_rewind_or_reinitialize()
+            if msg:
+                return msg
 
             if self.has_lock():  # in standby cluster
                 msg = "starting as a standby leader because i had the session lock"
@@ -336,10 +334,9 @@ class Ha(object):
             msg = "starting as readonly because i had the session lock"
             node_to_follow = None
 
-        self.recovering = True
-
-        self._async_executor.schedule('restarting after failure')
-        self._async_executor.run_async(self.state_handler.follow, (node_to_follow, role, timeout))
+        if self._async_executor.try_run_async('restarting after failure', self.state_handler.follow,
+                                              args=(node_to_follow, role, timeout)) is None:
+            self.recovering = True
         return msg
 
     def _get_node_to_follow(self, cluster):
@@ -363,7 +360,7 @@ class Ha(object):
         node_to_follow = self._get_node_to_follow(self.cluster)
 
         if self.is_paused():
-            if not (self.state_handler.need_rewind and self.state_handler.can_rewind_or_reinitialize_allowed)\
+            if not (self._rewind.is_needed and self._rewind.can_rewind_or_reinitialize_allowed)\
                     or self.cluster.is_unlocked():
                 self.state_handler.set_role('master' if is_leader else 'replica')
                 if is_leader:
@@ -374,20 +371,26 @@ class Ha(object):
             self.demote('immediate-nolock')
             return demote_reason
 
-        if self._handle_rewind_or_reinitialize():
-            return self._async_executor.scheduled_action
+        msg = self._handle_rewind_or_reinitialize()
+        if msg:
+            return msg
 
         role = 'standby_leader' if isinstance(node_to_follow, RemoteMember) and self.has_lock(False) else 'replica'
         # It might happen that leader key in the standby cluster references non-exiting member.
         # In this case it is safe to continue running without changing recovery.conf
         if self.is_standby_cluster() and role == 'replica' and not (node_to_follow and node_to_follow.conn_url):
             return 'continue following the old known standby leader'
-        elif not self.state_handler.check_recovery_conf(node_to_follow):
-            self._async_executor.schedule('changing primary_conninfo and restarting')
-            self._async_executor.run_async(self.state_handler.follow, (node_to_follow, role))
-        elif role == 'standby_leader' and self.state_handler.role != role:
-            self.state_handler.set_role(role)
-            self.state_handler.call_nowait(ACTION_ON_ROLE_CHANGE)
+        else:
+            change_required, restart_required = self.state_handler.config.check_recovery_conf(node_to_follow)
+            if change_required:
+                if restart_required:
+                    self._async_executor.try_run_async('changing primary_conninfo and restarting',
+                                                       self.state_handler.follow, args=(node_to_follow, role))
+                else:
+                    self.state_handler.follow(node_to_follow, role, do_reload=True)
+            elif role == 'standby_leader' and self.state_handler.role != role:
+                self.state_handler.set_role(role)
+                self.state_handler.call_nowait(ACTION_ON_ROLE_CHANGE)
 
         return follow_reason
 
@@ -422,7 +425,7 @@ class Ha(object):
                     logger.warning("No standbys available!")
 
                 logger.info("Assigning synchronous standby status to %s", picked)
-                self.state_handler.set_synchronous_standby(picked)
+                self.state_handler.config.set_synchronous_standby(picked)
 
                 if picked and picked != '*' and not allow_promote:
                     # Wait for PostgreSQL to enable synchronous mode and see if we can immediately set sync_standby
@@ -443,7 +446,7 @@ class Ha(object):
         else:
             if self.cluster.sync.leader and self.dcs.delete_sync_state(index=self.cluster.sync.index):
                 logger.info("Disabled synchronous replication")
-            self.state_handler.set_synchronous_standby(None)
+            self.state_handler.config.set_synchronous_standby(None)
 
     def is_sync_standby(self, cluster):
         return cluster.leader and cluster.sync.leader == cluster.leader.name \
@@ -535,24 +538,29 @@ class Ha(object):
                     # Somebody else updated sync state, it may be due to us losing the lock. To be safe, postpone
                     # promotion until next cycle. TODO: trigger immediate retry of run_cycle
                     return 'Postponing promotion because synchronous replication state was updated by somebody else'
-                self.state_handler.set_synchronous_standby('*' if self.is_synchronous_mode_strict() else None)
+                self.state_handler.config.set_synchronous_standby('*' if self.is_synchronous_mode_strict() else None)
             if self.state_handler.role != 'master':
                 self.set_leader_access_is_restricted(self.cluster.has_permanent_logical_slots(self.state_handler.name))
-                self._async_executor.schedule('promote')
-                self._async_executor.run_async(self.state_handler.promote,
-                                               args=(self.dcs.loop_wait, self._leader_access_is_restricted))
+
+                def on_success():
+                    self._rewind.reset_state()
+                    logger.info("cleared rewind state after becoming the leader")
+
+                self._async_executor.try_run_async('promote', self.state_handler.promote,
+                                                   args=(self.dcs.loop_wait, on_success,
+                                                         self._leader_access_is_restricted))
             return promote_message
 
-    @staticmethod
-    def fetch_node_status(member):
+    def fetch_node_status(self, member):
         """This function perform http get request on member.api_url and fetches its status
         :returns: `_MemberStatus` object
         """
 
         try:
-            response = requests.get(member.api_url, timeout=2, verify=False)
-            logger.info('Got response from %s %s: %s', member.name, member.api_url, response.content)
-            return _MemberStatus.from_api_response(member, response.json())
+            response = self.patroni.request(member, timeout=2, retries=0)
+            data = response.data.decode('utf-8')
+            logger.info('Got response from %s %s: %s', member.name, member.api_url, data)
+            return _MemberStatus.from_api_response(member, json.loads(data))
         except Exception as e:
             logger.warning("Request failed to %s: GET %s (%s)", member.name, member.api_url, e)
         return _MemberStatus.unknown(member)
@@ -576,6 +584,7 @@ class Ha(object):
     def _is_healthiest_node(self, members, check_replication_lag=True):
         """This method tries to determine whether I am healthy enough to became a new leader candidate or not."""
 
+        # We don't call `last_operation()` here because it returns a string
         _, my_wal_position = self.state_handler.timeline_wal_position()
         if check_replication_lag and self.is_lagging(my_wal_position):
             logger.info('My wal position exceeds maximum replication lag')
@@ -599,7 +608,11 @@ class Ha(object):
                         return False
                     if my_wal_position < st.wal_position:
                         logger.info('Wal position of %s is ahead of my wal position', st.member.name)
-                        return False
+                        # In synchronous mode the former leader might be still accessible and even be ahead of us.
+                        # We should not disqualify himself from the leader race in such a situation.
+                        if not self.is_synchronous_mode() or st.member.name != self.cluster.sync.leader:
+                            return False
+                        logger.info('Ignoring the former leader being ahead of us')
         return True
 
     def is_failover_possible(self, members):
@@ -735,7 +748,7 @@ class Ha(object):
             'immediate-nolock': dict(stop='immediate', checkpoint=False, release=False, offline=False, async_req=True),
         }[mode]
 
-        self.state_handler.trigger_check_diverged_lsn()
+        self._rewind.trigger_check_diverged_lsn()
         self.state_handler.stop(mode_control['stop'], checkpoint=mode_control['checkpoint'],
                                 on_safepoint=self.watchdog.disable if self.watchdog.is_running else None)
         self.state_handler.set_role('demoted')
@@ -755,12 +768,11 @@ class Ha(object):
         # there could be an async action already running, calling follow from here will lead
         # to racy state handler state updates.
         if mode_control['async_req']:
-            self._async_executor.schedule('starting after demotion')
-            self._async_executor.run_async(self.state_handler.follow, (node_to_follow,))
+            self._async_executor.try_run_async('starting after demotion', self.state_handler.follow, (node_to_follow,))
         else:
             if self.is_synchronous_mode():
-                self.state_handler.set_synchronous_standby(None)
-            if self.state_handler.rewind_or_reinitialize_needed_and_possible(leader):
+                self.state_handler.config.set_synchronous_standby(None)
+            if self._rewind.rewind_or_reinitialize_needed_and_possible(leader):
                 return False  # do not start postgres, but run pg_rewind on the next iteration
             self.state_handler.follow(node_to_follow)
 
@@ -829,9 +841,8 @@ class Ha(object):
                         members = [m for m in self.cluster.members
                                    if not failover.candidate or m.name == failover.candidate]
                     if self.is_failover_possible(members):  # check that there are healthy members
-                        self._async_executor.schedule('manual failover: demote')
-                        self._async_executor.run_async(self.demote, ('graceful',))
-                        return 'manual failover: demoting myself'
+                        ret = self._async_executor.try_run_async('manual failover: demote', self.demote, ('graceful',))
+                        return ret or 'manual failover: demoting myself'
                     else:
                         logger.warning('manual failover: no healthy members found, failover is not possible')
             else:
@@ -876,7 +887,7 @@ class Ha(object):
             # when we are doing manual failover there is no guaranty that new leader is ahead of any other node
             # node tagged as nofailover can be ahead of the new leader either, but it is always excluded from elections
             if bool(self.cluster.failover) or self.patroni.nofailover:
-                self.state_handler.trigger_check_diverged_lsn()
+                self._rewind.trigger_check_diverged_lsn()
                 time.sleep(2)  # Give a time to somebody to take the leader lock
 
             if self.patroni.nofailover:
@@ -895,7 +906,7 @@ class Ha(object):
                 return 'removed leader lock because postgres is not running as master'
 
             if self.state_handler.is_leader() and self._leader_access_is_restricted:
-                self.state_handler.sync_replication_slots(self.cluster)
+                self.state_handler.slots_handler.sync_replication_slots(self.cluster)
                 self.state_handler.call_nowait(ACTION_ON_ROLE_CHANGE)
                 self.set_leader_access_is_restricted(False)
 
@@ -904,6 +915,9 @@ class Ha(object):
                 if msg is not None:
                     return msg
 
+                # check if the node is ready to be used by pg_rewind
+                self._rewind.check_for_checkpoint_after_promote()
+
                 if self.is_standby_cluster():
                     # in case of standby cluster we don't really need to
                     # enforce anything, since the leader is not a master.
@@ -968,8 +982,7 @@ class Ha(object):
         if role and role != self.state_handler.role:
             reason_to_cancel = "host role mismatch"
 
-        if (postgres_version and
-           self.state_handler.postgres_version_to_int(postgres_version) <= int(self.state_handler.server_version)):
+        if postgres_version and postgres_version_to_int(postgres_version) <= int(self.state_handler.server_version):
             reason_to_cancel = "postgres version mismatch"
 
         if pending_restart and not self.state_handler.pending_restart:
@@ -1132,17 +1145,16 @@ class Ha(object):
                 return 'waiting for end of recovery after bootstrap'
 
             self.state_handler.set_role('master')
-            self._async_executor.schedule('post_bootstrap')
-            self._async_executor.run_async(self.state_handler.post_bootstrap,
-                                           args=(self.patroni.config['bootstrap'], self._post_bootstrap_task))
-            return 'running post_bootstrap'
+            ret = self._async_executor.try_run_async('post_bootstrap', self.state_handler.bootstrap.post_bootstrap,
+                                                     args=(self.patroni.config['bootstrap'], self._post_bootstrap_task))
+            return ret or 'running post_bootstrap'
 
         self.state_handler.bootstrapping = False
         self.dcs.set_config_value(json.dumps(self.patroni.config.dynamic_configuration, separators=(',', ':')))
         if not self.watchdog.activate():
             logger.error('Cancelling bootstrap because watchdog activation failed')
             self.cancel_initialization()
-        self.state_handler.sync_replication_slots(self.cluster)
+        self.state_handler.slots_handler.sync_replication_slots(self.cluster)
         self.dcs.take_leader()
         self.set_is_leader(True)
         self.state_handler.call_nowait(ACTION_ON_START)
@@ -1231,7 +1243,7 @@ class Ha(object):
             if self.state_handler.bootstrapping:
                 return self.post_bootstrap()
 
-            if self.recovering and not self.state_handler.need_rewind:
+            if self.recovering and not self._rewind.is_needed:
                 self.recovering = False
                 # Check if we tried to recover and failed
                 msg = self.post_recover()
@@ -1251,15 +1263,6 @@ class Ha(object):
                     return 'released leader key voluntarily as data dir empty and currently leader'
 
                 return self.bootstrap()  # new node
-            # "bootstrap", but data directory is not empty
-            elif not self.sysid_valid(self.cluster.initialize) and self.cluster.is_unlocked() and not self.is_paused():
-                if not self.state_handler.cb_called and self.state_handler.is_running() \
-                        and not self.state_handler.is_leader():
-                    self._join_aborted = True
-                    logger.error('No initialize key in DCS and PostgreSQL is running as replica, aborting start')
-                    logger.error('Please first start Patroni on the node running as master')
-                    sys.exit(1)
-                self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=self.state_handler.sysid)
             else:
                 # check if we are allowed to join
                 data_sysid = self.state_handler.sysid
@@ -1267,10 +1270,20 @@ class Ha(object):
                     # data directory is not empty, but no valid sysid, cluster must be broken, suggest reinit
                     return "data dir for the cluster is not empty, but system ID is invalid; consider doing reinitalize"
 
-                if self.sysid_valid(self.cluster.initialize) and self.cluster.initialize != self.state_handler.sysid:
-                    logger.fatal("system ID mismatch, node %s belongs to a different cluster: %s != %s",
-                                 self.state_handler.name, self.cluster.initialize, self.state_handler.sysid)
-                    sys.exit(1)
+                if self.sysid_valid(self.cluster.initialize):
+                    if self.cluster.initialize != data_sysid:
+                        logger.fatal("system ID mismatch, node %s belongs to a different cluster: %s != %s",
+                                     self.state_handler.name, self.cluster.initialize, data_sysid)
+                        sys.exit(1)
+                elif self.cluster.is_unlocked() and not self.is_paused():
+                    # "bootstrap", but data directory is not empty
+                    if not self.state_handler.cb_called and self.state_handler.is_running() \
+                            and not self.state_handler.is_leader():
+                        self._join_aborted = True
+                        logger.error('No initialize key in DCS and PostgreSQL is running as replica, aborting start')
+                        logger.error('Please first start Patroni on the node running as master')
+                        sys.exit(1)
+                    self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=data_sysid)
 
             if not self.state_handler.is_healthy():
                 if self.is_paused():
@@ -1282,8 +1295,8 @@ class Ha(object):
                     # the demote code follows through to starting Postgres right away, however, in the rewind case
                     # it returns from demote and reaches this point to start PostgreSQL again after rewind. In that
                     # case it makes no sense to continue to recover() unless rewind has finished successfully.
-                    elif self.state_handler.rewind_failed or not self.state_handler.rewind_executed and not \
-                            (self.state_handler.need_rewind and self.state_handler.can_rewind_or_reinitialize_allowed):
+                    elif self._rewind.failed or not self._rewind.executed and not \
+                            (self._rewind.is_needed and self._rewind.can_rewind_or_reinitialize_allowed):
                         return 'postgres is not running'
 
                 # try to start dead postgres
@@ -1300,10 +1313,10 @@ class Ha(object):
                 # stops PostgreSQL, therefore, we only reload replication slots if no
                 # asynchronous processes are running (should be always the case for the master)
                 if not self._async_executor.busy and not self.state_handler.is_starting():
-                    self.state_handler.sync_replication_slots(self.cluster)
+                    self.state_handler.slots_handler.sync_replication_slots(self.cluster)
                     if not self.state_handler.cb_called:
                         if not self.state_handler.is_leader():
-                            self.state_handler.trigger_check_diverged_lsn()
+                            self._rewind.trigger_check_diverged_lsn()
                         self.state_handler.call_nowait(ACTION_ON_START)
         except DCSError:
             dcs_failed = True

patroni/log.py

@@ -1,66 +1,194 @@
 import logging
 import os
+import sys
 
 from copy import deepcopy
 from logging.handlers import RotatingFileHandler
 from patroni.utils import deep_compare
+from six.moves.queue import Queue, Full
+from threading import Lock, Thread
+
+_LOGGER = logging.getLogger(__name__)
 
 
-class PatroniLogger(object):
+def debug_exception(logger_obj, msg, *args, **kwargs):
+    kwargs.pop("exc_info", False)
+    if logger_obj.isEnabledFor(logging.DEBUG):
+        logger_obj.debug(msg, *args, exc_info=True, **kwargs)
+    else:
+        msg = "{0}, DETAIL: '{1}'".format(msg, sys.exc_info()[1])
+        logger_obj.error(msg, *args, exc_info=False, **kwargs)
 
-    DEFAULT_LEVEL = 'INFO'
-    DEFAULT_FORMAT = '%(asctime)s %(levelname)s: %(message)s'
+
+def error_exception(logger_obj, msg, *args, **kwargs):
+    exc_info = kwargs.pop("exc_info", True)
+    logger_obj.error(msg, *args, exc_info=exc_info, **kwargs)
+
+
+class QueueHandler(logging.Handler):
 
     def __init__(self):
-        self.root_logger = logging.getLogger()
-        self.config = None
-        self.handler = None
+        logging.Handler.__init__(self)
+        self.queue = Queue()
+        self._records_lost = 0
+
+    def _put_record(self, record):
+        self.format(record)
+        record.msg = record.message
+        record.args = None
+        record.exc_info = None
+        self.queue.put_nowait(record)
+
+    def _try_to_report_lost_records(self):
+        if self._records_lost:
+            try:
+                record = _LOGGER.makeRecord(_LOGGER.name, logging.WARNING, __file__, 0,
+                                            'QueueHandler has lost %s log records',
+                                            (self._records_lost,), None, 'emit')
+                self._put_record(record)
+                self._records_lost = 0
+            except Exception:
+                pass
+
+    def emit(self, record):
+        try:
+            self._put_record(record)
+            self._try_to_report_lost_records()
+        except Exception:
+            self._records_lost += 1
+
+    @property
+    def records_lost(self):
+        return self._records_lost
+
+
+class ProxyHandler(logging.Handler):
+
+    def __init__(self, patroni_logger):
+        logging.Handler.__init__(self)
+        self.patroni_logger = patroni_logger
+
+    def emit(self, record):
+        self.patroni_logger.log_handler.handle(record)
+
+
+class PatroniLogger(Thread):
+
+    DEFAULT_LEVEL = 'INFO'
+    DEFAULT_TRACEBACK_LEVEL = 'ERROR'
+    DEFAULT_FORMAT = '%(asctime)s %(levelname)s: %(message)s'
+
+    NORMAL_LOG_QUEUE_SIZE = 2  # When everything goes normal Patroni writes only 2 messages per HA loop
+    DEFAULT_MAX_QUEUE_SIZE = 1000
+    LOGGING_BROKEN_EXIT_CODE = 5
+
+    def __init__(self):
+        super(PatroniLogger, self).__init__()
+        self._queue_handler = QueueHandler()
+        self._root_logger = logging.getLogger()
+        self._config = None
+        self.log_handler = None
+        self.log_handler_lock = Lock()
+        self._old_handlers = []
         self.reload_config({'level': 'DEBUG'})
+        # We will switch to the QueueHandler only when thread was started.
+        # This is necessary to protect from the cases when Patroni constructor
+        # failed and PatroniLogger thread remain running and prevent shutdown.
+        self._proxy_handler = ProxyHandler(self)
+        self._root_logger.addHandler(self._proxy_handler)
 
     def update_loggers(self):
-        loggers = deepcopy(self.config.get('loggers') or {})
-        for name, logger in self.root_logger.manager.loggerDict.items():
+        loggers = deepcopy(self._config.get('loggers') or {})
+        for name, logger in self._root_logger.manager.loggerDict.items():
             if not isinstance(logger, logging.PlaceHolder):
                 level = loggers.pop(name, logging.NOTSET)
                 logger.setLevel(level)
 
         for name, level in loggers.items():
-            logger = self.root_logger.manager.getLogger(name)
+            logger = self._root_logger.manager.getLogger(name)
             logger.setLevel(level)
 
     def reload_config(self, config):
-        if self.config is None or not deep_compare(self.config, config):
-            self.root_logger.setLevel(config.get('level', PatroniLogger.DEFAULT_LEVEL))
+        if self._config is None or not deep_compare(self._config, config):
+            with self._queue_handler.queue.mutex:
+                self._queue_handler.queue.maxsize = config.get('max_queue_size', self.DEFAULT_MAX_QUEUE_SIZE)
 
-            add_handler = None
+            self._root_logger.setLevel(config.get('level', PatroniLogger.DEFAULT_LEVEL))
+            if config.get('traceback_level', PatroniLogger.DEFAULT_TRACEBACK_LEVEL).lower() == 'debug':
+                logging.Logger.exception = debug_exception
+            else:
+                logging.Logger.exception = error_exception
+
+            new_handler = None
             if 'dir' in config:
-                if not isinstance(self.handler, RotatingFileHandler):
-                    add_handler = RotatingFileHandler(os.path.join(config['dir'], __name__))
-                handler = add_handler or self.handler
+                if not isinstance(self.log_handler, RotatingFileHandler):
+                    new_handler = RotatingFileHandler(os.path.join(config['dir'], __name__))
+                handler = new_handler or self.log_handler
                 handler.maxBytes = int(config.get('file_size', 25000000))
                 handler.backupCount = int(config.get('file_num', 4))
             else:
-                if self.handler is None or isinstance(self.handler, RotatingFileHandler):
-                    add_handler = logging.StreamHandler()
-                handler = add_handler or self.handler
+                if self.log_handler is None or isinstance(self.log_handler, RotatingFileHandler):
+                    new_handler = logging.StreamHandler()
+                handler = new_handler or self.log_handler
 
-            oldlogformat = (self.config or {}).get('format', PatroniLogger.DEFAULT_FORMAT)
+            oldlogformat = (self._config or {}).get('format', PatroniLogger.DEFAULT_FORMAT)
             logformat = config.get('format', PatroniLogger.DEFAULT_FORMAT)
 
-            olddateformat = (self.config or {}).get('dateformat') or None
+            olddateformat = (self._config or {}).get('dateformat') or None
             dateformat = config.get('dateformat') or None  # Convert empty string to `None`
 
-            if oldlogformat != logformat or olddateformat != dateformat or add_handler:
+            if oldlogformat != logformat or olddateformat != dateformat or new_handler:
                 handler.setFormatter(logging.Formatter(logformat, dateformat))
 
-            if add_handler:
-                self.root_logger.addHandler(add_handler)
+            if new_handler:
+                with self.log_handler_lock:
+                    if self.log_handler:
+                        self._old_handlers.append(self.log_handler)
+                    self.log_handler = new_handler
 
-                if self.handler is not None:
-                    self.root_logger.removeHandler(self.handler)
-                    self.handler.close()
-
-                self.handler = add_handler
-
-            self.config = config.copy()
+            self._config = config.copy()
             self.update_loggers()
+
+    def _close_old_handlers(self):
+        while True:
+            with self.log_handler_lock:
+                if not self._old_handlers:
+                    break
+                handler = self._old_handlers.pop()
+            try:
+                handler.close()
+            except Exception:
+                _LOGGER.exception('Failed to close the old log handler %s', handler)
+
+    def run(self):
+        # switch to QueueHandler only when the thread was started
+        with self.log_handler_lock:
+            self._root_logger.addHandler(self._queue_handler)
+            self._root_logger.removeHandler(self._proxy_handler)
+
+        while True:
+            self._close_old_handlers()
+
+            record = self._queue_handler.queue.get(True)
+            if record is None:
+                break
+
+            self.log_handler.handle(record)
+            self._queue_handler.queue.task_done()
+
+    def shutdown(self):
+        try:
+            self._queue_handler.queue.put_nowait(None)
+        except Full:  # Queue is full.
+            # It seems that logging is not working, exiting with non-standard exit-code is the best we can do.
+            sys.exit(self.LOGGING_BROKEN_EXIT_CODE)
+        self.join()
+        logging.shutdown()
+
+    @property
+    def queue_size(self):
+        return self._queue_handler.queue.qsize()
+
+    @property
+    def records_lost(self):
+        return self._queue_handler.records_lost

patroni/postgresql/__init__.py

@@ -0,0 +1,883 @@
+import logging
+import os
+import psycopg2
+import shlex
+import shutil
+import subprocess
+import time
+
+from contextlib import contextmanager
+from copy import deepcopy
+from patroni.postgresql.callback_executor import CallbackExecutor
+from patroni.postgresql.bootstrap import Bootstrap
+from patroni.postgresql.cancellable import CancellableSubprocess
+from patroni.postgresql.config import ConfigHandler
+from patroni.postgresql.connection import Connection, get_connection_cursor
+from patroni.postgresql.misc import parse_history, postgres_major_version_to_int
+from patroni.postgresql.postmaster import PostmasterProcess
+from patroni.postgresql.slots import SlotsHandler
+from patroni.exceptions import PostgresConnectionException
+from patroni.utils import Retry, RetryFailedError, polling_loop
+from threading import current_thread, Lock
+
+
+logger = logging.getLogger(__name__)
+
+ACTION_ON_START = "on_start"
+ACTION_ON_STOP = "on_stop"
+ACTION_ON_RESTART = "on_restart"
+ACTION_ON_RELOAD = "on_reload"
+ACTION_ON_ROLE_CHANGE = "on_role_change"
+ACTION_NOOP = "noop"
+
+STATE_RUNNING = 'running'
+STATE_REJECT = 'rejecting connections'
+STATE_NO_RESPONSE = 'not responding'
+STATE_UNKNOWN = 'unknown'
+
+STOP_POLLING_INTERVAL = 1
+
+
+@contextmanager
+def null_context():
+    yield
+
+
+class Postgresql(object):
+
+    def __init__(self, config):
+        self.name = config['name']
+        self.scope = config['scope']
+        self._data_dir = config['data_dir']
+        self._database = config.get('database', 'postgres')
+        self._version_file = os.path.join(self._data_dir, 'PG_VERSION')
+        self._pg_control = os.path.join(self._data_dir, 'global', 'pg_control')
+        self._major_version = self.get_major_version()
+
+        self._state_lock = Lock()
+        self.set_state('stopped')
+
+        self._pending_restart = False
+        self._connection = Connection()
+        self.config = ConfigHandler(self, config)
+
+        self._bin_dir = config.get('bin_dir') or ''
+        self.bootstrap = Bootstrap(self)
+        self.bootstrapping = False
+        self.__thread_ident = current_thread().ident
+
+        self.slots_handler = SlotsHandler(self)
+
+        self._callback_executor = CallbackExecutor()
+        self.__cb_called = False
+        self.__cb_pending = None
+
+        self.cancellable = CancellableSubprocess()
+
+        self._sysid = None
+        self.retry = Retry(max_tries=-1, deadline=config['retry_timeout']/2.0, max_delay=1,
+                           retry_exceptions=PostgresConnectionException)
+
+        # Retry 'pg_is_in_recovery()' only once
+        self._is_leader_retry = Retry(max_tries=1, deadline=config['retry_timeout']/2.0, max_delay=1,
+                                      retry_exceptions=PostgresConnectionException)
+
+        self._role_lock = Lock()
+        self.set_role(self.get_postgres_role_from_data_directory())
+        self._state_entry_timestamp = None
+
+        self._cluster_info_state = {}
+        self._cached_replica_timeline = None
+
+        # Last known running process
+        self._postmaster_proc = None
+
+        if self.is_running():
+            self.set_state('running')
+            self.set_role('master' if self.is_leader() else 'replica')
+            self.config.write_postgresql_conf()  # we are "joining" already running postgres
+            hba_saved = self.config.replace_pg_hba()
+            ident_saved = self.config.replace_pg_ident()
+            if hba_saved or ident_saved:
+                self.reload()
+        elif self.role == 'master':
+            self.set_role('demoted')
+
+    @property
+    def create_replica_methods(self):
+        return self.config.get('create_replica_methods', []) or self.config.get('create_replica_method', [])
+
+    @property
+    def major_version(self):
+        return self._major_version
+
+    @property
+    def database(self):
+        return self._database
+
+    @property
+    def data_dir(self):
+        return self._data_dir
+
+    @property
+    def callback(self):
+        return self.config.get('callbacks') or {}
+
+    @property
+    def wal_name(self):
+        return 'wal' if self._major_version >= 100000 else 'xlog'
+
+    @property
+    def lsn_name(self):
+        return 'lsn' if self._major_version >= 100000 else 'location'
+
+    @property
+    def cluster_info_query(self):
+        return ("SELECT CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0 "
+                "ELSE ('x' || pg_catalog.substr(pg_catalog.pg_{0}file_name("
+                "pg_catalog.pg_current_{0}_{1}()), 1, 8))::bit(32)::int END, "
+                "CASE WHEN pg_catalog.pg_is_in_recovery() THEN GREATEST("
+                " pg_catalog.pg_{0}_{1}_diff(COALESCE("
+                "pg_catalog.pg_last_{0}_receive_{1}(), '0/0'), '0/0')::bigint,"
+                " pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), '0/0')::bigint)"
+                "ELSE pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}_{1}(), '0/0')::bigint "
+                "END").format(self.wal_name, self.lsn_name)
+
+    def _version_file_exists(self):
+        return not self.data_directory_empty() and os.path.isfile(self._version_file)
+
+    def get_major_version(self):
+        if self._version_file_exists():
+            try:
+                with open(self._version_file) as f:
+                    return postgres_major_version_to_int(f.read().strip())
+            except Exception:
+                logger.exception('Failed to read PG_VERSION from %s', self._data_dir)
+        return 0
+
+    def pgcommand(self, cmd):
+        """Returns path to the specified PostgreSQL command"""
+        return os.path.join(self._bin_dir, cmd)
+
+    def pg_ctl(self, cmd, *args, **kwargs):
+        """Builds and executes pg_ctl command
+
+        :returns: `!True` when return_code == 0, otherwise `!False`"""
+
+        pg_ctl = [self.pgcommand('pg_ctl'), cmd]
+        return subprocess.call(pg_ctl + ['-D', self._data_dir] + list(args), **kwargs) == 0
+
+    def pg_isready(self):
+        """Runs pg_isready to see if PostgreSQL is accepting connections.
+
+        :returns: 'ok' if PostgreSQL is up, 'reject' if starting up, 'no_resopnse' if not up."""
+
+        r = self.config.local_connect_kwargs
+        cmd = [self.pgcommand('pg_isready'), '-p', r['port'], '-d', self._database]
+
+        # Host is not set if we are connecting via default unix socket
+        if 'host' in r:
+            cmd.extend(['-h', r['host']])
+
+        # We only need the username because pg_isready does not try to authenticate
+        if 'user' in r:
+            cmd.extend(['-U', r['user']])
+
+        ret = subprocess.call(cmd)
+        return_codes = {0: STATE_RUNNING,
+                        1: STATE_REJECT,
+                        2: STATE_NO_RESPONSE,
+                        3: STATE_UNKNOWN}
+        return return_codes.get(ret, STATE_UNKNOWN)
+
+    def reload_config(self, config, sighup=False):
+        self.config.reload_config(config, sighup)
+        self._is_leader_retry.deadline = self.retry.deadline = config['retry_timeout']/2.0
+
+    @property
+    def pending_restart(self):
+        return self._pending_restart
+
+    def set_pending_restart(self, value):
+        self._pending_restart = value
+
+    @property
+    def sysid(self):
+        if not self._sysid and not self.bootstrapping:
+            data = self.controldata()
+            self._sysid = data.get('Database system identifier', "")
+        return self._sysid
+
+    def get_postgres_role_from_data_directory(self):
+        if self.data_directory_empty() or not self.controldata():
+            return 'uninitialized'
+        elif self.config.recovery_conf_exists():
+            return 'replica'
+        else:
+            return 'master'
+
+    @property
+    def server_version(self):
+        return self._connection.server_version
+
+    def connection(self):
+        return self._connection.get()
+
+    def set_connection_kwargs(self, kwargs):
+        self._connection.set_conn_kwargs(kwargs)
+
+    def _query(self, sql, *params):
+        """We are always using the same cursor, therefore this method is not thread-safe!!!
+        You can call it from different threads only if you are holding explicit `AsyncExecutor` lock,
+        because the main thread is always holding this lock when running HA cycle."""
+        cursor = None
+        try:
+            cursor = self._connection.cursor()
+            cursor.execute(sql, params)
+            return cursor
+        except psycopg2.Error as e:
+            if cursor and cursor.connection.closed == 0:
+                # When connected via unix socket, psycopg2 can't recoginze 'connection lost'
+                # and leaves `_cursor_holder.connection.closed == 0`, but psycopg2.OperationalError
+                # is still raised (what is correct). It doesn't make sense to continiue with existing
+                # connection and we will close it, to avoid its reuse by the `cursor` method.
+                if isinstance(e, psycopg2.OperationalError):
+                    self._connection.close()
+                else:
+                    raise e
+            if self.state == 'restarting':
+                raise RetryFailedError('cluster is being restarted')
+            raise PostgresConnectionException('connection problems')
+
+    def query(self, sql, *args, **kwargs):
+        if not kwargs.get('retry', True):
+            return self._query(sql, *args)
+        try:
+            return self.retry(self._query, sql, *args)
+        except RetryFailedError as e:
+            raise PostgresConnectionException(str(e))
+
+    def pg_control_exists(self):
+        return os.path.isfile(self._pg_control)
+
+    def data_directory_empty(self):
+        if self.pg_control_exists():
+            return False
+        if not os.path.exists(self._data_dir):
+            return True
+        return all(os.name != 'nt' and (n.startswith('.') or n == 'lost+found') for n in os.listdir(self._data_dir))
+
+    def replica_method_options(self, method):
+        return deepcopy(self.config.get(method, {}))
+
+    def replica_method_can_work_without_replication_connection(self, method):
+        return method != 'basebackup' and self.replica_method_options(method).get('no_master')
+
+    def can_create_replica_without_replication_connection(self, replica_methods=None):
+        """ go through the replication methods to see if there are ones
+            that does not require a working replication connection.
+        """
+        if replica_methods is None:
+            replica_methods = self.create_replica_methods
+        return any(self.replica_method_can_work_without_replication_connection(m) for m in replica_methods)
+
+    def reset_cluster_info_state(self):
+        self._cluster_info_state = {}
+
+    def _cluster_info_state_get(self, name):
+        if not self._cluster_info_state:
+            try:
+                result = self._is_leader_retry(self._query, self.cluster_info_query).fetchone()
+                self._cluster_info_state = dict(zip(['timeline', 'wal_position'], result))
+            except RetryFailedError as e:  # SELECT failed two times
+                self._cluster_info_state = {'error': str(e)}
+                if not self.is_starting() and self.pg_isready() == STATE_REJECT:
+                    self.set_state('starting')
+
+        if 'error' in self._cluster_info_state:
+            raise PostgresConnectionException(self._cluster_info_state['error'])
+
+        return self._cluster_info_state.get(name)
+
+    def is_leader(self):
+        return bool(self._cluster_info_state_get('timeline'))
+
+    def is_running(self):
+        """Returns PostmasterProcess if one is running on the data directory or None. If most recently seen process
+        is running updates the cached process based on pid file."""
+        if self._postmaster_proc:
+            if self._postmaster_proc.is_running():
+                return self._postmaster_proc
+            self._postmaster_proc = None
+
+        # we noticed that postgres was restarted, force syncing of replication
+        self.slots_handler.schedule()
+
+        self._postmaster_proc = PostmasterProcess.from_pidfile(self._data_dir)
+        return self._postmaster_proc
+
+    @property
+    def cb_called(self):
+        return self.__cb_called
+
+    def call_nowait(self, cb_name):
+        """ pick a callback command and call it without waiting for it to finish """
+        if self.bootstrapping:
+            return
+        if cb_name in (ACTION_ON_START, ACTION_ON_STOP, ACTION_ON_RESTART, ACTION_ON_ROLE_CHANGE):
+            self.__cb_called = True
+
+        if self.callback and cb_name in self.callback:
+            cmd = self.callback[cb_name]
+            try:
+                cmd = shlex.split(self.callback[cb_name]) + [cb_name, self.role, self.scope]
+                self._callback_executor.call(cmd)
+            except Exception:
+                logger.exception('callback %s %s %s %s failed', cmd, cb_name, self.role, self.scope)
+
+    @property
+    def role(self):
+        with self._role_lock:
+            return self._role
+
+    def set_role(self, value):
+        with self._role_lock:
+            self._role = value
+
+    @property
+    def state(self):
+        with self._state_lock:
+            return self._state
+
+    def set_state(self, value):
+        with self._state_lock:
+            self._state = value
+            self._state_entry_timestamp = time.time()
+
+    def time_in_state(self):
+        return time.time() - self._state_entry_timestamp
+
+    def is_starting(self):
+        return self.state == 'starting'
+
+    def wait_for_port_open(self, postmaster, timeout):
+        """Waits until PostgreSQL opens ports."""
+        for _ in polling_loop(timeout):
+            if self.cancellable.is_cancelled:
+                return False
+
+            if not postmaster.is_running():
+                logger.error('postmaster is not running')
+                self.set_state('start failed')
+                return False
+
+            isready = self.pg_isready()
+            if isready != STATE_NO_RESPONSE:
+                if isready not in [STATE_REJECT, STATE_RUNNING]:
+                    logger.warning("Can't determine PostgreSQL startup status, assuming running")
+                return True
+
+        logger.warning("Timed out waiting for PostgreSQL to start")
+        return False
+
+    def start(self, timeout=None, task=None, block_callbacks=False, role=None):
+        """Start PostgreSQL
+
+        Waits for postmaster to open ports or terminate so pg_isready can be used to check startup completion
+        or failure.
+
+        :returns: True if start was initiated and postmaster ports are open, False if start failed"""
+        # make sure we close all connections established against
+        # the former node, otherwise, we might get a stalled one
+        # after kill -9, which would report incorrect data to
+        # patroni.
+        self._connection.close()
+
+        if self.is_running():
+            logger.error('Cannot start PostgreSQL because one is already running.')
+            self.set_state('starting')
+            return True
+
+        if not block_callbacks:
+            self.__cb_pending = ACTION_ON_START
+
+        self.set_role(role or self.get_postgres_role_from_data_directory())
+
+        self.set_state('starting')
+        self._pending_restart = False
+
+        configuration = self.config.effective_configuration
+        self.config.write_postgresql_conf(configuration)
+        self.config.resolve_connection_addresses()
+        self.config.replace_pg_hba()
+        self.config.replace_pg_ident()
+
+        options = ['--{0}={1}'.format(p, configuration[p]) for p in self.config.CMDLINE_OPTIONS
+                   if p in configuration and p != 'wal_keep_segments']
+
+        if self.cancellable.is_cancelled:
+            return False
+
+        with task or null_context():
+            if task and task.is_cancelled:
+                logger.info("PostgreSQL start cancelled.")
+                return False
+
+            self._postmaster_proc = PostmasterProcess.start(self.pgcommand('postgres'),
+                                                            self._data_dir,
+                                                            self.config.postgresql_conf,
+                                                            options)
+
+            if task:
+                task.complete(self._postmaster_proc)
+
+        start_timeout = timeout
+        if not start_timeout:
+            try:
+                start_timeout = float(self.config.get('pg_ctl_timeout', 60))
+            except ValueError:
+                start_timeout = 60
+
+        # We want postmaster to open ports before we continue
+        if not self._postmaster_proc or not self.wait_for_port_open(self._postmaster_proc, start_timeout):
+            return False
+
+        ret = self.wait_for_startup(start_timeout)
+        if ret is not None:
+            return ret
+        elif timeout is not None:
+            return False
+        else:
+            return None
+
+    def checkpoint(self, connect_kwargs=None):
+        check_not_is_in_recovery = connect_kwargs is not None
+        connect_kwargs = connect_kwargs or self.config.local_connect_kwargs
+        for p in ['connect_timeout', 'options']:
+            connect_kwargs.pop(p, None)
+        try:
+            with get_connection_cursor(**connect_kwargs) as cur:
+                cur.execute("SET statement_timeout = 0")
+                if check_not_is_in_recovery:
+                    cur.execute('SELECT pg_catalog.pg_is_in_recovery()')
+                    if cur.fetchone()[0]:
+                        return 'is_in_recovery=true'
+                return cur.execute('CHECKPOINT')
+        except psycopg2.Error:
+            logger.exception('Exception during CHECKPOINT')
+            return 'not accessible or not healty'
+
+    def stop(self, mode='fast', block_callbacks=False, checkpoint=None, on_safepoint=None):
+        """Stop PostgreSQL
+
+        Supports a callback when a safepoint is reached. A safepoint is when no user backend can return a successful
+        commit to users. Currently this means we wait for user backends to close. But in the future alternate mechanisms
+        could be added.
+
+        :param on_safepoint: This callback is called when no user backends are running.
+        """
+        if checkpoint is None:
+            checkpoint = False if mode == 'immediate' else True
+
+        success, pg_signaled = self._do_stop(mode, block_callbacks, checkpoint, on_safepoint)
+        if success:
+            # block_callbacks is used during restart to avoid
+            # running start/stop callbacks in addition to restart ones
+            if not block_callbacks:
+                self.set_state('stopped')
+                if pg_signaled:
+                    self.call_nowait(ACTION_ON_STOP)
+        else:
+            logger.warning('pg_ctl stop failed')
+            self.set_state('stop failed')
+        return success
+
+    def _do_stop(self, mode, block_callbacks, checkpoint, on_safepoint):
+        postmaster = self.is_running()
+        if not postmaster:
+            if on_safepoint:
+                on_safepoint()
+            return True, False
+
+        if checkpoint and not self.is_starting():
+            self.checkpoint()
+
+        if not block_callbacks:
+            self.set_state('stopping')
+
+        # Send signal to postmaster to stop
+        success = postmaster.signal_stop(mode)
+        if success is not None:
+            if success and on_safepoint:
+                on_safepoint()
+            return success, True
+
+        # We can skip safepoint detection if we don't have a callback
+        if on_safepoint:
+            # Wait for our connection to terminate so we can be sure that no new connections are being initiated
+            self._wait_for_connection_close(postmaster)
+            postmaster.wait_for_user_backends_to_close()
+            on_safepoint()
+
+        postmaster.wait()
+
+        return True, True
+
+    @staticmethod
+    def terminate_starting_postmaster(postmaster):
+        """Terminates a postmaster that has not yet opened ports or possibly even written a pid file. Blocks
+        until the process goes away."""
+        postmaster.signal_stop('immediate')
+        postmaster.wait()
+
+    def _wait_for_connection_close(self, postmaster):
+        try:
+            with self.connection().cursor() as cur:
+                while postmaster.is_running():  # Need a timeout here?
+                    cur.execute("SELECT 1")
+                    time.sleep(STOP_POLLING_INTERVAL)
+        except psycopg2.Error:
+            pass
+
+    def reload(self):
+        ret = self.pg_ctl('reload')
+        if ret:
+            self.call_nowait(ACTION_ON_RELOAD)
+        return ret
+
+    def check_for_startup(self):
+        """Checks PostgreSQL status and returns if PostgreSQL is in the middle of startup."""
+        return self.is_starting() and not self.check_startup_state_changed()
+
+    def check_startup_state_changed(self):
+        """Checks if PostgreSQL has completed starting up or failed or still starting.
+
+        Should only be called when state == 'starting'
+
+        :returns: True if state was changed from 'starting'
+        """
+        ready = self.pg_isready()
+
+        if ready == STATE_REJECT:
+            return False
+        elif ready == STATE_NO_RESPONSE:
+            ret = not self.is_running()
+            if ret:
+                self.set_state('start failed')
+                self.slots_handler.schedule(False)  # TODO: can remove this?
+                self.config.save_configuration_files(True)  # TODO: maybe remove this?
+            return ret
+        else:
+            if ready != STATE_RUNNING:
+                # Bad configuration or unexpected OS error. No idea of PostgreSQL status.
+                # Let the main loop of run cycle clean up the mess.
+                logger.warning("%s status returned from pg_isready",
+                               "Unknown" if ready == STATE_UNKNOWN else "Invalid")
+            self.set_state('running')
+            self.slots_handler.schedule()
+            self.config.save_configuration_files(True)
+            # TODO: __cb_pending can be None here after PostgreSQL restarts on its own. Do we want to call the callback?
+            # Previously we didn't even notice.
+            action = self.__cb_pending or ACTION_ON_START
+            self.call_nowait(action)
+            self.__cb_pending = None
+
+            return True
+
+    def wait_for_startup(self, timeout=None):
+        """Waits for PostgreSQL startup to complete or fail.
+
+        :returns: True if start was successful, False otherwise"""
+        if not self.is_starting():
+            # Should not happen
+            logger.warning("wait_for_startup() called when not in starting state")
+
+        while not self.check_startup_state_changed():
+            if self.cancellable.is_cancelled or timeout and self.time_in_state() > timeout:
+                return None
+            time.sleep(1)
+
+        return self.state == 'running'
+
+    def restart(self, timeout=None, task=None, block_callbacks=False, role=None):
+        """Restarts PostgreSQL.
+
+        When timeout parameter is set the call will block either until PostgreSQL has started, failed to start or
+        timeout arrives.
+
+        :returns: True when restart was successful and timeout did not expire when waiting.
+        """
+        self.set_state('restarting')
+        if not block_callbacks:
+            self.__cb_pending = ACTION_ON_RESTART
+        ret = self.stop(block_callbacks=True) and self.start(timeout, task, True, role)
+        if not ret and not self.is_starting():
+            self.set_state('restart failed ({0})'.format(self.state))
+        return ret
+
+    def is_healthy(self):
+        if not self.is_running():
+            logger.warning('Postgresql is not running.')
+            return False
+        return True
+
+    def controldata(self):
+        """ return the contents of pg_controldata, or non-True value if pg_controldata call failed """
+        result = {}
+        # Don't try to call pg_controldata during backup restore
+        if self._version_file_exists() and self.state != 'creating replica':
+            try:
+                env = os.environ.copy()
+                env.update(LANG='C', LC_ALL='C')
+                data = subprocess.check_output([self.pgcommand('pg_controldata'), self._data_dir], env=env)
+                if data:
+                    data = data.decode('utf-8').splitlines()
+                    # pg_controldata output depends on major verion. Some of parameters are prefixed by 'Current '
+                    result = {l.split(':')[0].replace('Current ', '', 1): l.split(':', 1)[1].strip() for l in data
+                              if l and ':' in l}
+            except subprocess.CalledProcessError:
+                logger.exception("Error when calling pg_controldata")
+        return result
+
+    @contextmanager
+    def get_replication_connection_cursor(self, host='localhost', port=5432, database=None, **kwargs):
+        conn_kwargs = self.config.replication.copy()
+        conn_kwargs.update(host=host, port=int(port), database=database or self._database, connect_timeout=3,
+                           user=conn_kwargs.pop('username'), replication=1, options='-c statement_timeout=2000')
+        with get_connection_cursor(**conn_kwargs) as cur:
+            yield cur
+
+    def get_local_timeline_lsn_from_replication_connection(self):
+        timeline = lsn = None
+        try:
+            with self.get_replication_connection_cursor(**self.config.local_replication_address) as cur:
+                cur.execute('IDENTIFY_SYSTEM')
+                timeline, lsn = cur.fetchone()[1:3]
+        except Exception:
+            logger.exception('Can not fetch local timeline and lsn from replication connection')
+        return timeline, lsn
+
+    def get_replica_timeline(self):
+        return self.get_local_timeline_lsn_from_replication_connection()[0]
+
+    def replica_cached_timeline(self, master_timeline):
+        if not self._cached_replica_timeline or not master_timeline or self._cached_replica_timeline != master_timeline:
+            self._cached_replica_timeline = self.get_replica_timeline()
+        return self._cached_replica_timeline
+
+    def get_master_timeline(self):
+        return self._cluster_info_state_get('timeline')
+
+    def get_history(self, timeline):
+        history_path = 'pg_{0}/{1:08X}.history'.format(self.wal_name, timeline)
+        try:
+            cursor = self._connection.cursor()
+            cursor.execute('SELECT isdir, modification FROM pg_catalog.pg_stat_file(%s)', (history_path,))
+            isdir, modification = cursor.fetchone()
+            if not isdir:
+                cursor.execute('SELECT pg_catalog.pg_read_file(%s)', (history_path,))
+                history = list(parse_history(cursor.fetchone()[0]))
+                if history[-1][0] == timeline - 1:
+                    history[-1].append(modification.isoformat())
+                return history
+        except Exception:
+            logger.exception('Failed to read and parse %s', (history_path,))
+
+    def follow(self, member, role='replica', timeout=None, do_reload=False):
+        recovery_params = self.config.build_recovery_params(member)
+        self.config.write_recovery_conf(recovery_params)
+
+        # When we demoting the master or standby_leader to replica or promoting replica to a standby_leader
+        # and we know for sure that postgres was already running before, we will only execute on_role_change
+        # callback and prevent execution of on_restart/on_start callback.
+        # If the role remains the same (replica or standby_leader), we will execute on_start or on_restart
+        change_role = self.cb_called and (self.role in ('master', 'demoted') or
+                                          not {'standby_leader', 'replica'} - {self.role, role})
+        if change_role:
+            self.__cb_pending = ACTION_NOOP
+
+        if self.is_running():
+            if do_reload:
+                self.config.write_postgresql_conf()
+                self.reload()
+            else:
+                self.restart(block_callbacks=change_role, role=role)
+        else:
+            self.start(timeout=timeout, block_callbacks=change_role, role=role)
+
+        if change_role:
+            # TODO: postpone this until start completes, or maybe do even earlier
+            self.call_nowait(ACTION_ON_ROLE_CHANGE)
+        return True
+
+    def _wait_promote(self, wait_seconds):
+        for _ in polling_loop(wait_seconds):
+            data = self.controldata()
+            if data.get('Database cluster state') == 'in production':
+                return True
+
+    def promote(self, wait_seconds, on_success=None, access_is_restricted=False):
+        if self.role == 'master':
+            return True
+        ret = self.pg_ctl('promote', '-W')
+        if ret:
+            self.set_role('master')
+            if on_success is not None:
+                on_success()
+            if not access_is_restricted:
+                self.call_nowait(ACTION_ON_ROLE_CHANGE)
+            ret = self._wait_promote(wait_seconds)
+        return ret
+
+    def timeline_wal_position(self):
+        # This method could be called from different threads (simultaneously with some other `_query` calls).
+        # If it is called not from main thread we will create a new cursor to execute statement.
+        if current_thread().ident == self.__thread_ident:
+            return self._cluster_info_state_get('timeline'), self._cluster_info_state_get('wal_position')
+
+        with self.connection().cursor() as cursor:
+            cursor.execute(self.cluster_info_query)
+            return cursor.fetchone()[:2]
+
+    def postmaster_start_time(self):
+        try:
+            query = "SELECT pg_catalog.to_char(pg_catalog.pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ')"
+            if current_thread().ident == self.__thread_ident:
+                return self.query(query).fetchone()[0]
+            with self.connection().cursor() as cursor:
+                cursor.execute(query)
+                return cursor.fetchone()[0]
+        except psycopg2.Error:
+            return None
+
+    def last_operation(self):
+        return str(self._cluster_info_state_get('wal_position'))
+
+    def configure_server_parameters(self):
+        self._major_version = self.get_major_version()
+        self.config.setup_server_parameters()
+        return True
+
+    def move_data_directory(self):
+        if os.path.isdir(self._data_dir) and not self.is_running():
+            try:
+                new_name = '{0}_{1}'.format(self._data_dir, time.strftime('%Y-%m-%d-%H-%M-%S'))
+                logger.info('renaming data directory to %s', new_name)
+                os.rename(self._data_dir, new_name)
+            except OSError:
+                logger.exception("Could not rename data directory %s", self._data_dir)
+
+    def remove_data_directory(self):
+        self.set_role('uninitialized')
+        logger.info('Removing data directory: %s', self._data_dir)
+        try:
+            if os.path.islink(self._data_dir):
+                os.unlink(self._data_dir)
+            elif not os.path.exists(self._data_dir):
+                return
+            elif os.path.isfile(self._data_dir):
+                os.remove(self._data_dir)
+            elif os.path.isdir(self._data_dir):
+
+                # let's see if pg_xlog|pg_wal is a symlink, in this case we
+                # should clean the target
+                for pg_wal_dir in ('pg_xlog', 'pg_wal'):
+                    pg_wal_path = os.path.join(self._data_dir, pg_wal_dir)
+                    if os.path.exists(pg_wal_path) and os.path.islink(pg_wal_path):
+                        pg_wal_realpath = os.path.realpath(pg_wal_path)
+                        logger.info('Removing WAL directory: %s', pg_wal_realpath)
+                        shutil.rmtree(pg_wal_realpath)
+
+                shutil.rmtree(self._data_dir)
+        except (IOError, OSError):
+            logger.exception('Could not remove data directory %s', self._data_dir)
+            self.move_data_directory()
+
+    def pick_synchronous_standby(self, cluster):
+        """Finds the best candidate to be the synchronous standby.
+
+        Current synchronous standby is always preferred, unless it has disconnected or does not want to be a
+        synchronous standby any longer.
+
+        :returns tuple of candidate name or None, and bool showing if the member is the active synchronous standby.
+        """
+        current = cluster.sync.sync_standby
+        current = current.lower() if current else current
+        members = {m.name.lower(): m for m in cluster.members}
+        candidates = []
+        # Pick candidates based on who has flushed WAL farthest.
+        # TODO: for synchronous_commit = remote_write we actually want to order on write_location
+        for app_name, state, sync_state in self.query(
+                "SELECT pg_catalog.lower(application_name), state, sync_state"
+                " FROM pg_catalog.pg_stat_replication"
+                " ORDER BY flush_{0} DESC".format(self.lsn_name)):
+            member = members.get(app_name)
+            if state != 'streaming' or not member or member.tags.get('nosync', False):
+                continue
+            if sync_state == 'sync':
+                return app_name, True
+            if sync_state == 'potential' and app_name == current:
+                # Prefer current even if not the best one any more to avoid indecisivness and spurious swaps.
+                return cluster.sync.sync_standby, False
+            if sync_state in ('async', 'potential'):
+                candidates.append(member.name)
+
+        if candidates:
+            return candidates[0], False
+        return None, False
+
+    def read_postmaster_opts(self):
+        """returns the list of option names/values from postgres.opts, Empty dict if read failed or no file"""
+        result = {}
+        try:
+            with open(os.path.join(self._data_dir, 'postmaster.opts')) as f:
+                data = f.read()
+                for opt in data.split('" "'):
+                    if '=' in opt and opt.startswith('--'):
+                        name, val = opt.split('=', 1)
+                        result[name.strip('-')] = val.rstrip('"\n')
+        except IOError:
+            logger.exception('Error when reading postmaster.opts')
+        return result
+
+    def single_user_mode(self, command=None, options=None):
+        """run a given command in a single-user mode. If the command is empty - then just start and stop"""
+        cmd = [self.pgcommand('postgres'), '--single', '-D', self._data_dir]
+        for opt, val in sorted((options or {}).items()):
+            cmd.extend(['-c', '{0}={1}'.format(opt, val)])
+        # need a database name to connect
+        cmd.append(self._database)
+        return self.cancellable.call(cmd, communicate_input=command)
+
+    def cleanup_archive_status(self):
+        status_dir = os.path.join(self._data_dir, 'pg_' + self.wal_name, 'archive_status')
+        try:
+            for f in os.listdir(status_dir):
+                path = os.path.join(status_dir, f)
+                try:
+                    if os.path.islink(path):
+                        os.unlink(path)
+                    elif os.path.isfile(path):
+                        os.remove(path)
+                except OSError:
+                    logger.exception('Unable to remove %s', path)
+        except OSError:
+            logger.exception('Unable to list %s', status_dir)
+
+    def fix_cluster_state(self):
+        self.cleanup_archive_status()
+
+        # Start in a single user mode and stop to produce a clean shutdown
+        opts = self.read_postmaster_opts()
+        opts.update({'archive_mode': 'on', 'archive_command': 'false'})
+        self.config.remove_recovery_conf()
+        return self.single_user_mode(options=opts) == 0 or None
+
+    def schedule_sanity_checks_after_pause(self):
+        """
+            After coming out of pause we have to:
+            1. sync replication slots, because it might happen that slots were removed
+            2. get new 'Database system identifier' to make sure that it wasn't changed
+        """
+        self.slots_handler.schedule()
+        self._sysid = None

patroni/postgresql/bootstrap.py

@@ -0,0 +1,370 @@
+import logging
+import os
+import shlex
+import tempfile
+import time
+
+from patroni.dcs import RemoteMember
+from patroni.utils import deep_compare
+from six import string_types
+
+logger = logging.getLogger(__name__)
+
+
+class Bootstrap(object):
+
+    def __init__(self, postgresql):
+        self._postgresql = postgresql
+        self._running_custom_bootstrap = False
+
+    @property
+    def running_custom_bootstrap(self):
+        return self._running_custom_bootstrap
+
+    @property
+    def keep_existing_recovery_conf(self):
+        return self._running_custom_bootstrap and self._keep_existing_recovery_conf
+
+    @staticmethod
+    def process_user_options(tool, options, not_allowed_options, error_handler):
+        user_options = []
+
+        def option_is_allowed(name):
+            ret = name not in not_allowed_options
+            if not ret:
+                error_handler('{0} option for {1} is not allowed'.format(name, tool))
+            return ret
+
+        if isinstance(options, dict):
+            for k, v in options.items():
+                if k and v:
+                    user_options.append('--{0}={1}'.format(k, v))
+        elif isinstance(options, list):
+            for opt in options:
+                if isinstance(opt, string_types) and option_is_allowed(opt):
+                    user_options.append('--{0}'.format(opt))
+                elif isinstance(opt, dict):
+                    keys = list(opt.keys())
+                    if len(keys) != 1 or not isinstance(opt[keys[0]], string_types) or not option_is_allowed(keys[0]):
+                        error_handler('Error when parsing {0} key-value option {1}: only one key-value is allowed'
+                                      ' and value should be a string'.format(tool, opt[keys[0]]))
+                    user_options.append('--{0}={1}'.format(keys[0], opt[keys[0]]))
+                else:
+                    error_handler('Error when parsing {0} option {1}: value should be string value'
+                                  ' or a single key-value pair'.format(tool, opt))
+        else:
+            error_handler('{0} options must be list ot dict'.format(tool))
+        return user_options
+
+    def _initdb(self, config):
+        self._postgresql.set_state('initalizing new cluster')
+        not_allowed_options = ('pgdata', 'nosync', 'pwfile', 'sync-only', 'version')
+
+        def error_handler(e):
+            raise Exception(e)
+
+        options = self.process_user_options('initdb', config or [], not_allowed_options, error_handler)
+        pwfile = None
+
+        if self._postgresql.config.superuser:
+            if 'username' in self._postgresql.config.superuser:
+                options.append('--username={0}'.format(self._postgresql.config.superuser['username']))
+            if 'password' in self._postgresql.config.superuser:
+                (fd, pwfile) = tempfile.mkstemp()
+                os.write(fd, self._postgresql.config.superuser['password'].encode('utf-8'))
+                os.close(fd)
+                options.append('--pwfile={0}'.format(pwfile))
+        options = ['-o', ' '.join(options)] if options else []
+
+        ret = self._postgresql.pg_ctl('initdb', *options)
+        if pwfile:
+            os.remove(pwfile)
+        if ret:
+            self._postgresql.configure_server_parameters()
+        else:
+            self._postgresql.set_state('initdb failed')
+        return ret
+
+    def _post_restore(self):
+        self._postgresql.config.restore_configuration_files()
+        self._postgresql.configure_server_parameters()
+
+        # make sure there is no trigger file or postgres will be automatically promoted
+        trigger_file = 'promote_trigger_file' if self._postgresql.major_version >= 120000 else 'trigger_file'
+        trigger_file = self._postgresql.config.get('recovery_conf', {}).get(trigger_file) or 'promote'
+        trigger_file = os.path.abspath(os.path.join(self._postgresql.data_dir, trigger_file))
+        if os.path.exists(trigger_file):
+            os.unlink(trigger_file)
+
+    def _custom_bootstrap(self, config):
+        self._postgresql.set_state('running custom bootstrap script')
+        params = ['--scope=' + self._postgresql.scope, '--datadir=' + self._postgresql.data_dir]
+        try:
+            logger.info('Running custom bootstrap script: %s', config['command'])
+            if self._postgresql.cancellable.call(shlex.split(config['command']) + params) != 0:
+                self._postgresql.set_state('custom bootstrap failed')
+                return False
+        except Exception:
+            logger.exception('Exception during custom bootstrap')
+            return False
+        self._post_restore()
+
+        if 'recovery_conf' in config:
+            self._postgresql.config.write_recovery_conf(config['recovery_conf'])
+        elif not self.keep_existing_recovery_conf:
+            self._postgresql.config.remove_recovery_conf()
+        return True
+
+    def call_post_bootstrap(self, config):
+        """
+        runs a script after initdb or custom bootstrap script is called and waits until completion.
+        """
+        cmd = config.get('post_bootstrap') or config.get('post_init')
+        if cmd:
+            r = self._postgresql.config.local_connect_kwargs
+            connstring = self._postgresql.config.format_dsn(r, True)
+            if 'host' not in r:
+                # https://www.postgresql.org/docs/current/static/libpq-pgpass.html
+                # A host name of localhost matches both TCP (host name localhost) and Unix domain socket
+                # (pghost empty or the default socket directory) connections coming from the local machine.
+                r['host'] = 'localhost'  # set it to localhost to write into pgpass
+
+            env = self._postgresql.config.write_pgpass(r) if 'password' in r else None
+
+            try:
+                ret = self._postgresql.cancellable.call(shlex.split(cmd) + [connstring], env=env)
+            except OSError:
+                logger.error('post_init script %s failed', cmd)
+                return False
+            if ret != 0:
+                logger.error('post_init script %s returned non-zero code %d', cmd, ret)
+                return False
+        return True
+
+    def create_replica(self, clone_member):
+        """
+            create the replica according to the replica_method
+            defined by the user.  this is a list, so we need to
+            loop through all methods the user supplies
+        """
+
+        self._postgresql.set_state('creating replica')
+        self._postgresql.schedule_sanity_checks_after_pause()
+
+        is_remote_master = isinstance(clone_member, RemoteMember)
+
+        # get list of replica methods either from clone member or from
+        # the config. If there is no configuration key, or no value is
+        # specified, use basebackup
+        replica_methods = (clone_member.create_replica_methods if is_remote_master
+                           else self._postgresql.create_replica_methods) or ['basebackup']
+
+        if clone_member and clone_member.conn_url:
+            r = clone_member.conn_kwargs(self._postgresql.config.replication)
+            # add the credentials to connect to the replica origin to pgpass.
+            env = self._postgresql.config.write_pgpass(r)
+            connstring = self._postgresql.config.format_dsn(r, True)
+        else:
+            connstring = ''
+            env = os.environ.copy()
+            # if we don't have any source, leave only replica methods that work without it
+            replica_methods = [r for r in replica_methods
+                               if self._postgresql.replica_method_can_work_without_replication_connection(r)]
+
+        # go through them in priority order
+        ret = 1
+        for replica_method in replica_methods:
+            if self._postgresql.cancellable.is_cancelled:
+                break
+
+            method_config = self._postgresql.replica_method_options(replica_method)
+
+            # if the method is basebackup, then use the built-in
+            if replica_method == "basebackup":
+                ret = self.basebackup(connstring, env, method_config)
+                if ret == 0:
+                    logger.info("replica has been created using basebackup")
+                    # if basebackup succeeds, exit with success
+                    break
+            else:
+                if not self._postgresql.data_directory_empty():
+                    if method_config.get('keep_data', False):
+                        logger.info('Leaving data directory uncleaned')
+                    else:
+                        self._postgresql.remove_data_directory()
+
+                cmd = replica_method
+                # user-defined method; check for configuration
+                # not required, actually
+                if method_config:
+                    # look to see if the user has supplied a full command path
+                    # if not, use the method name as the command
+                    cmd = method_config.pop('command', cmd)
+
+                # add the default parameters
+                if not method_config.get('no_params', False):
+                    method_config.update({"scope": self._postgresql.scope,
+                                          "role": "replica",
+                                          "datadir": self._postgresql.data_dir,
+                                          "connstring": connstring})
+                else:
+                    for param in ('no_params', 'no_master', 'keep_data'):
+                        method_config.pop(param, None)
+                params = ["--{0}={1}".format(arg, val) for arg, val in method_config.items()]
+                try:
+                    # call script with the full set of parameters
+                    ret = self._postgresql.cancellable.call(shlex.split(cmd) + params, env=env)
+                    # if we succeeded, stop
+                    if ret == 0:
+                        logger.info('replica has been created using %s', replica_method)
+                        break
+                    else:
+                        logger.error('Error creating replica using method %s: %s exited with code=%s',
+                                     replica_method, cmd, ret)
+                except Exception:
+                    logger.exception('Error creating replica using method %s', replica_method)
+                    ret = 1
+
+        self._postgresql.set_state('stopped')
+        return ret
+
+    def basebackup(self, conn_url, env, options):
+        # creates a replica data dir using pg_basebackup.
+        # this is the default, built-in create_replica_methods
+        # tries twice, then returns failure (as 1)
+        # uses "stream" as the xlog-method to avoid sync issues
+        # supports additional user-supplied options, those are not validated
+        maxfailures = 2
+        ret = 1
+        not_allowed_options = ('pgdata', 'format', 'wal-method', 'xlog-method', 'gzip',
+                               'version', 'compress', 'dbname', 'host', 'port', 'username', 'password')
+        user_options = self.process_user_options('basebackup', options, not_allowed_options, logger.error)
+
+        for bbfailures in range(0, maxfailures):
+            if self._postgresql.cancellable.is_cancelled:
+                break
+            if not self._postgresql.data_directory_empty():
+                self._postgresql.remove_data_directory()
+            try:
+                ret = self._postgresql.cancellable.call([self._postgresql.pgcommand('pg_basebackup'),
+                                                         '--pgdata=' + self._postgresql.data_dir, '-X', 'stream',
+                                                         '--dbname=' + conn_url] + user_options, env=env)
+                if ret == 0:
+                    break
+                else:
+                    logger.error('Error when fetching backup: pg_basebackup exited with code=%s', ret)
+
+            except Exception as e:
+                logger.error('Error when fetching backup with pg_basebackup: %s', e)
+
+            if bbfailures < maxfailures - 1:
+                logger.warning('Trying again in 5 seconds')
+                time.sleep(5)
+
+        return ret
+
+    def clone(self, clone_member):
+        """
+             - initialize the replica from an existing member (master or replica)
+             - initialize the replica using the replica creation method that
+               works without the replication connection (i.e. restore from on-disk
+               base backup)
+        """
+
+        ret = self.create_replica(clone_member) == 0
+        if ret:
+            self._post_restore()
+        return ret
+
+    def bootstrap(self, config):
+        """ Initialize a new node from scratch and start it. """
+        pg_hba = config.get('pg_hba', [])
+        method = config.get('method') or 'initdb'
+        if method != 'initdb' and method in config and 'command' in config[method]:
+            self._keep_existing_recovery_conf = config[method].get('keep_existing_recovery_conf')
+            self._running_custom_bootstrap = True
+            do_initialize = self._custom_bootstrap
+        else:
+            method = 'initdb'
+            do_initialize = self._initdb
+        return do_initialize(config.get(method)) and self._postgresql.config.append_pg_hba(pg_hba) \
+            and self._postgresql.config.save_configuration_files() and self._postgresql.start()
+
+    def create_or_update_role(self, name, password, options):
+        options = list(map(str.upper, options))
+        if 'NOLOGIN' not in options and 'LOGIN' not in options:
+            options.append('LOGIN')
+
+        params = [name]
+        if password:
+            options.extend(['PASSWORD', '%s'])
+            params.extend([password, password])
+
+        sql = """DO $$
+BEGIN
+    SET local synchronous_commit = 'local';
+    PERFORM * FROM pg_authid WHERE rolname = %s;
+    IF FOUND THEN
+        ALTER ROLE "{0}" WITH {1};
+    ELSE
+        CREATE ROLE "{0}" WITH {1};
+    END IF;
+END;$$""".format(name, ' '.join(options))
+        self._postgresql.query(sql, *params)
+
+    def post_bootstrap(self, config, task):
+        try:
+            postgresql = self._postgresql
+            superuser = postgresql.config.superuser
+            if 'username' in superuser and 'password' in superuser:
+                self.create_or_update_role(superuser['username'], superuser['password'], ['SUPERUSER'])
+
+            task.complete(self.call_post_bootstrap(config))
+            if task.result:
+                replication = postgresql.config.replication
+                self.create_or_update_role(replication['username'], replication.get('password'), ['REPLICATION'])
+
+                rewind = postgresql.config.rewind_credentials
+                if not deep_compare(rewind, superuser):
+                    self.create_or_update_role(rewind['username'], rewind.get('password'), [])
+                    for f in ('pg_ls_dir(text, boolean, boolean)', 'pg_stat_file(text, boolean)',
+                              'pg_read_binary_file(text)', 'pg_read_binary_file(text, bigint, bigint, boolean)'):
+                        sql = """DO $$
+BEGIN
+    SET local synchronous_commit = 'local';
+    GRANT EXECUTE ON function pg_catalog.{0} TO "{1}";
+END;$$""".format(f, rewind['username'])
+                        postgresql.query(sql)
+
+                for name, value in (config.get('users') or {}).items():
+                    if all(name != a.get('username') for a in (superuser, replication, rewind)):
+                        self.create_or_update_role(name, value.get('password'), value.get('options', []))
+
+                # We were doing a custom bootstrap instead of running initdb, therefore we opened trust
+                # access from certain addresses to be able to reach cluster and change password
+                if self._running_custom_bootstrap:
+                    self._running_custom_bootstrap = False
+                    # If we don't have custom configuration for pg_hba.conf we need to restore original file
+                    if not postgresql.config.get('pg_hba'):
+                        os.unlink(postgresql.config.pg_hba_conf)
+                        postgresql.config.restore_configuration_files()
+                    postgresql.config.write_postgresql_conf()
+                    postgresql.config.replace_pg_ident()
+
+                    # at this point there should be no recovery.conf
+                    postgresql.config.remove_recovery_conf()
+
+                    if postgresql.config.hba_file and postgresql.config.hba_file != postgresql.config.pg_hba_conf:
+                        postgresql.restart()
+                    else:
+                        postgresql.config.replace_pg_hba()
+                        if postgresql.pending_restart:
+                            postgresql.restart()
+                        else:
+                            postgresql.reload()
+                            time.sleep(1)  # give a time to postgres to "reload" configuration files
+                            postgresql.connection().close()  # close connection to reconnect with a new password
+        except Exception:
+            logger.exception('post_bootstrap')
+            task.complete(False)
+        return task.result

patroni/postgresql/callback_executor.py

@@ -0,0 +1,36 @@
+import logging
+
+from patroni.postgresql.cancellable import CancellableExecutor
+from threading import Condition, Thread
+
+logger = logging.getLogger(__name__)
+
+
+class CallbackExecutor(CancellableExecutor, Thread):
+
+    def __init__(self):
+        CancellableExecutor.__init__(self)
+        Thread.__init__(self)
+        self.daemon = True
+        self._cmd = None
+        self._condition = Condition()
+        self.start()
+
+    def call(self, cmd):
+        self._kill_process()
+        with self._condition:
+            self._cmd = cmd
+            self._condition.notify()
+
+    def run(self):
+        while True:
+            with self._condition:
+                if self._cmd is None:
+                    self._condition.wait()
+                cmd, self._cmd = self._cmd, None
+
+            with self._lock:
+                if not self._start_process(cmd, close_fds=True):
+                    continue
+            self._process.wait()
+            self._kill_children()

patroni/postgresql/cancellable.py

@@ -0,0 +1,129 @@
+import logging
+import os
+import psutil
+import subprocess
+
+from patroni.exceptions import PostgresException
+from patroni.utils import polling_loop
+from six import string_types
+from threading import Lock
+
+logger = logging.getLogger(__name__)
+
+
+class CancellableExecutor(object):
+
+    def __init__(self):
+        self._process = None
+        self._process_cmd = None
+        self._process_children = []
+        self._lock = Lock()
+
+    def _start_process(self, cmd, *args, **kwargs):
+        """This method must be executed only when the `_lock` is acquired"""
+
+        try:
+            self._process_children = []
+            self._process_cmd = cmd
+            self._process = psutil.Popen(cmd, *args, **kwargs)
+        except Exception:
+            return logger.exception('Failed to execute %s',  cmd)
+        return True
+
+    def _kill_process(self):
+        with self._lock:
+            if self._process is not None and self._process.is_running() and not self._process_children:
+                try:
+                    self._process.suspend()  # Suspend the process before getting list of childrens
+                except psutil.Error as e:
+                    logger.info('Failed to suspend the process: %s', e.msg)
+
+                try:
+                    self._process_children = self._process.children(recursive=True)
+                except psutil.Error:
+                    pass
+
+                try:
+                    self._process.kill()
+                    logger.warning('Killed %s because it was still running', self._process_cmd)
+                except psutil.NoSuchProcess:
+                    pass
+                except psutil.AccessDenied as e:
+                    logger.warning('Failed to kill the process: %s', e.msg)
+
+    def _kill_children(self):
+        waitlist = []
+        with self._lock:
+            for child in self._process_children:
+                try:
+                    child.kill()
+                except psutil.NoSuchProcess:
+                    continue
+                except psutil.AccessDenied as e:
+                    logger.info('Failed to kill child process: %s', e.msg)
+                waitlist.append(child)
+        psutil.wait_procs(waitlist)
+
+
+class CancellableSubprocess(CancellableExecutor):
+
+    def __init__(self):
+        super(CancellableSubprocess, self).__init__()
+        self._is_cancelled = False
+
+    def call(self, *args, **kwargs):
+        for s in ('stdin', 'stdout', 'stderr'):
+            kwargs.pop(s, None)
+
+        communicate_input = 'communicate_input' in kwargs
+        if communicate_input:
+            input_data = kwargs.pop('communicate_input', None)
+            if not isinstance(input_data, string_types):
+                input_data = ''
+            if input_data and input_data[-1] != '\n':
+                input_data += '\n'
+            kwargs['stdin'] = subprocess.PIPE
+            kwargs['stdout'] = open(os.devnull, 'w')
+            kwargs['stderr'] = subprocess.STDOUT
+
+        try:
+            with self._lock:
+                if self._is_cancelled:
+                    raise PostgresException('cancelled')
+
+                self._is_cancelled = False
+                started = self._start_process(*args, **kwargs)
+
+            if started:
+                if communicate_input:
+                    if input_data:
+                        self._process.communicate(input_data)
+                    self._process.stdin.close()
+                return self._process.wait()
+        finally:
+            with self._lock:
+                self._process = None
+            self._kill_children()
+
+    def reset_is_cancelled(self):
+        with self._lock:
+            self._is_cancelled = False
+
+    @property
+    def is_cancelled(self):
+        with self._lock:
+            return self._is_cancelled
+
+    def cancel(self):
+        with self._lock:
+            self._is_cancelled = True
+            if self._process is None or not self._process.is_running():
+                return
+            self._process.terminate()
+
+        for _ in polling_loop(10):
+            with self._lock:
+                if self._process is None or not self._process.is_running():
+                    return
+
+        self._kill_process()

patroni/postgresql/connection.py

@@ -0,0 +1,46 @@
+import logging
+import psycopg2
+
+from contextlib import contextmanager
+from threading import Lock
+
+logger = logging.getLogger(__name__)
+
+
+class Connection(object):
+
+    def __init__(self):
+        self._lock = Lock()
+        self._connection = None
+        self._cursor_holder = None
+
+    def set_conn_kwargs(self, conn_kwargs):
+        self._conn_kwargs = conn_kwargs
+
+    def get(self):
+        with self._lock:
+            if not self._connection or self._connection.closed != 0:
+                self._connection = psycopg2.connect(**self._conn_kwargs)
+                self._connection.autocommit = True
+                self.server_version = self._connection.server_version
+        return self._connection
+
+    def cursor(self):
+        if not self._cursor_holder or self._cursor_holder.closed or self._cursor_holder.connection.closed != 0:
+            logger.info("establishing a new patroni connection to the postgres cluster")
+            self._cursor_holder = self.get().cursor()
+        return self._cursor_holder
+
+    def close(self):
+        if self._connection and self._connection.closed == 0:
+            self._connection.close()
+            logger.info("closed patroni connection to the postgresql cluster")
+        self._cursor_holder = self._connection = None
+
+
+@contextmanager
+def get_connection_cursor(**kwargs):
+    with psycopg2.connect(**kwargs) as conn:
+        conn.autocommit = True
+        with conn.cursor() as cur:
+            yield cur

patroni/postgresql/misc.py

@@ -0,0 +1,70 @@
+import logging
+
+from patroni.exceptions import PostgresException
+
+logger = logging.getLogger(__name__)
+
+
+def postgres_version_to_int(pg_version):
+    """Convert the server_version to integer
+
+    >>> postgres_version_to_int('9.5.3')
+    90503
+    >>> postgres_version_to_int('9.3.13')
+    90313
+    >>> postgres_version_to_int('10.1')
+    100001
+    >>> postgres_version_to_int('10')  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    PostgresException: 'Invalid PostgreSQL version format: X.Y or X.Y.Z is accepted: 10'
+    >>> postgres_version_to_int('9.6')  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    PostgresException: 'Invalid PostgreSQL version format: X.Y or X.Y.Z is accepted: 9.6'
+    >>> postgres_version_to_int('a.b.c')  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    PostgresException: 'Invalid PostgreSQL version: a.b.c'
+    """
+
+    try:
+        components = list(map(int, pg_version.split('.')))
+    except ValueError:
+        raise PostgresException('Invalid PostgreSQL version: {0}'.format(pg_version))
+
+    if len(components) < 2 or len(components) == 2 and components[0] < 10 or len(components) > 3:
+        raise PostgresException('Invalid PostgreSQL version format: X.Y or X.Y.Z is accepted: {0}'.format(pg_version))
+
+    if len(components) == 2:
+        # new style verion numbers, i.e. 10.1 becomes 100001
+        components.insert(1, 0)
+
+    return int(''.join('{0:02d}'.format(c) for c in components))
+
+
+def postgres_major_version_to_int(pg_version):
+    """
+    >>> postgres_major_version_to_int('10')
+    100000
+    >>> postgres_major_version_to_int('9.6')
+    90600
+    """
+    return postgres_version_to_int(pg_version + '.0')
+
+
+def parse_lsn(lsn):
+    t = lsn.split('/')
+    return int(t[0], 16) * 0x100000000 + int(t[1], 16)
+
+
+def parse_history(data):
+    for line in data.split('\n'):
+        values = line.strip().split('\t')
+        if len(values) == 3:
+            try:
+                values[0] = int(values[0])
+                values[1] = parse_lsn(values[1])
+                yield values
+            except (IndexError, ValueError):
+                logger.exception('Exception when parsing timeline history line "%s"', values)

patroni/postgresql/postmaster.py

@@ -5,6 +5,17 @@ import psutil
 import re
 import signal
 import subprocess
+import sys
+
+from patroni import PATRONI_ENV_PREFIX
+
+# avoid spawning the resource tracker process
+if sys.version_info >= (3, 8):  # pragma: no cover
+    import multiprocessing.resource_tracker
+    multiprocessing.resource_tracker.getfd = lambda: 0
+elif sys.version_info >= (3, 4):  # pragma: no cover
+    import multiprocessing.semaphore_tracker
+    multiprocessing.semaphore_tracker.getfd = lambda: 0
 
 logger = logging.getLogger(__name__)
 
@@ -153,7 +164,7 @@ class PostmasterProcess(psutil.Process):
         # In order to make everything portable we can't use fork&exec approach here, so  we will call
         # ourselves and pass list of arguments which must be used to start postgres.
         # On Windows, in order to run a side-by-side assembly the specified env must include a valid SYSTEMROOT.
-        env = {p: os.environ[p] for p in ('PATH', 'LD_LIBRARY_PATH', 'LC_ALL', 'LANG', 'SYSTEMROOT') if p in os.environ}
+        env = {p: os.environ[p] for p in os.environ if not p.startswith(PATRONI_ENV_PREFIX)}
         try:
             proc = PostmasterProcess._from_pidfile(data_dir)
             if proc and not proc._is_postmaster_process():
@@ -170,8 +181,9 @@ class PostmasterProcess(psutil.Process):
             pass
         cmdline = [pgcommand, '-D', data_dir, '--config-file={}'.format(conf)] + options
         logger.debug("Starting postgres: %s", " ".join(cmdline))
-        parent_conn, child_conn = multiprocessing.Pipe(False)
-        proc = multiprocessing.Process(target=pg_ctl_start, args=(child_conn, cmdline, env))
+        ctx = multiprocessing.get_context('spawn') if sys.version_info >= (3, 4) else multiprocessing
+        parent_conn, child_conn = ctx.Pipe(False)
+        proc = ctx.Process(target=pg_ctl_start, args=(child_conn, cmdline, env))
         proc.start()
         pid = parent_conn.recv()
         proc.join()

patroni/postgresql/rewind.py

@@ -0,0 +1,209 @@
+import logging
+import os
+import subprocess
+
+from patroni.dcs import Leader
+from patroni.postgresql.connection import get_connection_cursor
+from patroni.postgresql.misc import parse_history, parse_lsn
+
+logger = logging.getLogger(__name__)
+
+REWIND_STATUS = type('Enum', (), {'INITIAL': 0, 'CHECKPOINT': 1, 'CHECK': 2, 'NEED': 3,
+                                  'NOT_NEED': 4, 'SUCCESS': 5, 'FAILED': 6})
+
+
+class Rewind(object):
+
+    def __init__(self, postgresql):
+        self._postgresql = postgresql
+        self.reset_state()
+
+    @staticmethod
+    def configuration_allows_rewind(data):
+        return data.get('wal_log_hints setting', 'off') == 'on' or data.get('Data page checksum version', '0') != '0'
+
+    @property
+    def can_rewind(self):
+        """ check if pg_rewind executable is there and that pg_controldata indicates
+            we have either wal_log_hints or checksums turned on
+        """
+        # low-hanging fruit: check if pg_rewind configuration is there
+        if not self._postgresql.config.get('use_pg_rewind'):
+            return False
+
+        cmd = [self._postgresql.pgcommand('pg_rewind'), '--help']
+        try:
+            ret = subprocess.call(cmd, stdout=open(os.devnull, 'w'), stderr=subprocess.STDOUT)
+            if ret != 0:  # pg_rewind is not there, close up the shop and go home
+                return False
+        except OSError:
+            return False
+        return self.configuration_allows_rewind(self._postgresql.controldata())
+
+    @property
+    def can_rewind_or_reinitialize_allowed(self):
+        return self._postgresql.config.get('remove_data_directory_on_diverged_timelines') or self.can_rewind
+
+    def trigger_check_diverged_lsn(self):
+        if self.can_rewind_or_reinitialize_allowed and self._state != REWIND_STATUS.NEED:
+            self._state = REWIND_STATUS.CHECK
+
+    def check_leader_is_not_in_recovery(self, **kwargs):
+        if not kwargs.get('database'):
+            kwargs['database'] = self._postgresql.database
+        try:
+            with get_connection_cursor(connect_timeout=3, options='-c statement_timeout=2000', **kwargs) as cur:
+                cur.execute('SELECT pg_catalog.pg_is_in_recovery()')
+                if not cur.fetchone()[0]:
+                    return True
+                logger.info('Leader is still in_recovery and therefore can\'t be used for rewind')
+        except Exception:
+            return logger.exception('Exception when working with leader')
+
+    def _get_local_timeline_lsn_from_controldata(self):
+        timeline = lsn = None
+        data = self._postgresql.controldata()
+        try:
+            if data.get('Database cluster state') == 'shut down in recovery':
+                lsn = data.get('Minimum recovery ending location')
+                timeline = int(data.get("Min recovery ending loc's timeline"))
+                if lsn == '0/0' or timeline == 0:  # it was a master when it crashed
+                    data['Database cluster state'] = 'shut down'
+            if data.get('Database cluster state') == 'shut down':
+                lsn = data.get('Latest checkpoint location')
+                timeline = int(data.get("Latest checkpoint's TimeLineID"))
+        except (TypeError, ValueError):
+            logger.exception('Failed to get local timeline and lsn from pg_controldata output')
+        return timeline, lsn
+
+    def _get_local_timeline_lsn(self):
+        if self._postgresql.is_running():  # if postgres is running - get timeline and lsn from replication connection
+            timeline, lsn = self._postgresql.get_local_timeline_lsn_from_replication_connection()
+        else:  # otherwise analyze pg_controldata output
+            timeline, lsn = self._get_local_timeline_lsn_from_controldata()
+        logger.info('Local timeline=%s lsn=%s', timeline, lsn)
+        return timeline, lsn
+
+    def _check_timeline_and_lsn(self, leader):
+        local_timeline, local_lsn = self._get_local_timeline_lsn()
+        if local_timeline is None or local_lsn is None:
+            return
+
+        if isinstance(leader, Leader):
+            if leader.member.data.get('role') != 'master':
+                return
+        # standby cluster
+        elif not self.check_leader_is_not_in_recovery(**leader.conn_kwargs(self._postgresql.config.replication)):
+            return
+
+        history = need_rewind = None
+        try:
+            with self._postgresql.get_replication_connection_cursor(**leader.conn_kwargs()) as cur:
+                cur.execute('IDENTIFY_SYSTEM')
+                master_timeline = cur.fetchone()[1]
+                logger.info('master_timeline=%s', master_timeline)
+                if local_timeline > master_timeline:  # Not always supported by pg_rewind
+                    need_rewind = True
+                elif master_timeline > 1:
+                    cur.execute('TIMELINE_HISTORY %s', (master_timeline,))
+                    history = bytes(cur.fetchone()[1]).decode('utf-8')
+                    logger.info('master: history=%s', history)
+                else:  # local_timeline == master_timeline == 1
+                    need_rewind = False
+        except Exception:
+            return logger.exception('Exception when working with master via replication connection')
+
+        if history is not None:
+            for parent_timeline, switchpoint, _ in parse_history(history):
+                if parent_timeline == local_timeline:
+                    try:
+                        need_rewind = parse_lsn(local_lsn) >= switchpoint
+                    except (IndexError, ValueError):
+                        logger.exception('Exception when parsing lsn')
+                    break
+                elif parent_timeline > local_timeline:
+                    break
+
+        self._state = need_rewind and REWIND_STATUS.NEED or REWIND_STATUS.NOT_NEED
+
+    def rewind_or_reinitialize_needed_and_possible(self, leader):
+        if leader and leader.name != self._postgresql.name and leader.conn_url and self._state == REWIND_STATUS.CHECK:
+            self._check_timeline_and_lsn(leader)
+        return leader and leader.conn_url and self._state == REWIND_STATUS.NEED
+
+    def check_for_checkpoint_after_promote(self):
+        if self._state == REWIND_STATUS.INITIAL and self._postgresql.is_leader():
+            try:
+                timeline = int(self._postgresql.controldata().get("Latest checkpoint's TimeLineID"))
+                if self._postgresql.get_master_timeline() == timeline:
+                    self._state = REWIND_STATUS.CHECKPOINT
+            except (TypeError, ValueError):
+                logger.exception('Failed to parse timeline from pg_controldata output')
+
+    def checkpoint_after_promote(self):
+        return self._state == REWIND_STATUS.CHECKPOINT
+
+    def pg_rewind(self, r):
+        # prepare pg_rewind connection
+        env = self._postgresql.config.write_pgpass(r)
+        env['PGOPTIONS'] = '-c statement_timeout=0'
+        dsn = self._postgresql.config.format_dsn(r, True)
+        logger.info('running pg_rewind from %s', dsn)
+        try:
+            return self._postgresql.cancellable.call([self._postgresql.pgcommand('pg_rewind'), '-D',
+                                                      self._postgresql.data_dir, '--source-server', dsn], env=env) == 0
+        except OSError:
+            return False
+
+    def execute(self, leader):
+        if self._postgresql.is_running() and not self._postgresql.stop(checkpoint=False):
+            return logger.warning('Can not run pg_rewind because postgres is still running')
+
+        # prepare pg_rewind connection
+        r = leader.conn_kwargs(self._postgresql.config.rewind_credentials)
+
+        # 1. make sure that we are really trying to rewind from the master
+        # 2. make sure that pg_control contains the new timeline by:
+        #   running a checkpoint or
+        #   waiting until Patroni on the master will expose checkpoint_after_promote=True
+        checkpoint_status = leader.checkpoint_after_promote if isinstance(leader, Leader) else None
+        if checkpoint_status is None:  # master still runs the old Patroni
+            leader_status = self._postgresql.checkpoint(leader.conn_kwargs(self._postgresql.config.superuser))
+            if leader_status:
+                return logger.warning('Can not use %s for rewind: %s', leader.name, leader_status)
+        elif not checkpoint_status:
+            return logger.info('Waiting for checkpoint on %s before rewind', leader.name)
+        elif not self.check_leader_is_not_in_recovery(**r):
+            return
+
+        if self.pg_rewind(r):
+            self._state = REWIND_STATUS.SUCCESS
+        elif not self.check_leader_is_not_in_recovery(**r):
+            logger.warning('Failed to rewind because master %s become unreachable', leader.name)
+        else:
+            logger.error('Failed to rewind from healty master: %s', leader.name)
+
+            for name in ('remove_data_directory_on_rewind_failure', 'remove_data_directory_on_diverged_timelines'):
+                if self._postgresql.config.get(name):
+                    logger.warning('%s is set. removing...', name)
+                    self._postgresql.remove_data_directory()
+                    self._state = REWIND_STATUS.INITIAL
+                    break
+            else:
+                self._state = REWIND_STATUS.FAILED
+        return False
+
+    def reset_state(self):
+        self._state = REWIND_STATUS.INITIAL
+
+    @property
+    def is_needed(self):
+        return self._state in (REWIND_STATUS.CHECK, REWIND_STATUS.NEED)
+
+    @property
+    def executed(self):
+        return self._state > REWIND_STATUS.NOT_NEED
+
+    @property
+    def failed(self):
+        return self._state == REWIND_STATUS.FAILED

patroni/postgresql/slots.py

@@ -0,0 +1,103 @@
+import logging
+
+from patroni.postgresql.connection import get_connection_cursor
+from collections import defaultdict
+
+logger = logging.getLogger(__name__)
+
+
+def compare_slots(s1, s2):
+    return s1['type'] == s2['type'] and (s1['type'] == 'physical' or
+                                         s1['database'] == s2['database'] and s1['plugin'] == s2['plugin'])
+
+
+class SlotsHandler(object):
+
+    def __init__(self, postgresql):
+        self._postgresql = postgresql
+        self._replication_slots = {}  # already existing replication slots
+        self.schedule()
+
+    def _query(self, sql, *params):
+        return self._postgresql.query(sql, *params, retry=False)
+
+    def load_replication_slots(self):
+        if self._postgresql.major_version >= 90400 and self._schedule_load_slots:
+            replication_slots = {}
+            cursor = self._query('SELECT slot_name, slot_type, plugin, database FROM pg_catalog.pg_replication_slots')
+            for r in cursor:
+                value = {'type': r[1]}
+                if r[1] == 'logical':
+                    value.update({'plugin': r[2], 'database': r[3]})
+                replication_slots[r[0]] = value
+            self._replication_slots = replication_slots
+            self._schedule_load_slots = False
+
+    def drop_replication_slot(self, name):
+        cursor = self._query(('SELECT pg_catalog.pg_drop_replication_slot(%s) WHERE EXISTS (SELECT 1 ' +
+                              'FROM pg_catalog.pg_replication_slots WHERE slot_name = %s AND NOT active)'), name, name)
+        # In normal situation rowcount should be 1, otherwise either slot doesn't exists or it is still active
+        return cursor.rowcount == 1
+
+    def sync_replication_slots(self, cluster):
+        if self._postgresql.major_version >= 90400:
+            try:
+                self.load_replication_slots()
+
+                slots = cluster.get_replication_slots(self._postgresql.name, self._postgresql.role)
+
+                # drop old replication slots which are not presented in desired slots
+                for name in set(self._replication_slots) - set(slots):
+                    if not self.drop_replication_slot(name):
+                        logger.error("Failed to drop replication slot '%s'", name)
+                        self._schedule_load_slots = True
+
+                immediately_reserve = ', true' if self._postgresql.major_version >= 90600 else ''
+
+                logical_slots = defaultdict(dict)
+                for name, value in slots.items():
+                    if name in self._replication_slots and not compare_slots(value, self._replication_slots[name]):
+                        logger.info("Trying to drop replication slot '%s' because value is changing from %s to %s",
+                                    name, self._replication_slots[name], value)
+                        if not self.drop_replication_slot(name):
+                            logger.error("Failed to drop replication slot '%s'", name)
+                            self._schedule_load_slots = True
+                            continue
+                        self._replication_slots.pop(name)
+                    if name not in self._replication_slots:
+                        if value['type'] == 'physical':
+                            try:
+                                self._query(("SELECT pg_catalog.pg_create_physical_replication_slot(%s{0})" +
+                                             " WHERE NOT EXISTS (SELECT 1 FROM pg_catalog.pg_replication_slots" +
+                                             " WHERE slot_type = 'physical' AND slot_name = %s)").format(
+                                                 immediately_reserve), name, name)
+                            except Exception:
+                                logger.exception("Failed to create physical replication slot '%s'", name)
+                                self._schedule_load_slots = True
+                        elif value['type'] == 'logical' and name not in self._replication_slots:
+                            logical_slots[value['database']][name] = value
+
+                # create new logical slots
+                for database, values in logical_slots.items():
+                    conn_kwargs = self._postgresql.config.local_connect_kwargs
+                    conn_kwargs['database'] = database
+                    with get_connection_cursor(**conn_kwargs) as cur:
+                        for name, value in values.items():
+                            try:
+                                cur.execute("SELECT pg_catalog.pg_create_logical_replication_slot(%s, %s)" +
+                                            " WHERE NOT EXISTS (SELECT 1 FROM pg_catalog.pg_replication_slots" +
+                                            " WHERE slot_type = 'logical' AND slot_name = %s)",
+                                            (name, value['plugin'], name))
+                            except Exception:
+                                logger.exception("Failed to create logical replication slot '%s' plugin='%s'",
+                                                 name, value['plugin'])
+                                self._schedule_load_slots = True
+                self._replication_slots = slots
+            except Exception:
+                logger.exception('Exception when changing replication slots')
+                self._schedule_load_slots = True
+
+    def schedule(self, value=None):
+        if value is None:
+            value = self._postgresql.major_version >= 90400
+        self._schedule_load_slots = value

patroni/request.py

@@ -0,0 +1,58 @@
+import json
+import urllib3
+import six
+
+from six.moves.urllib_parse import urlparse, urlunparse
+
+from .utils import USER_AGENT
+
+
+class PatroniRequest(object):
+
+    def __init__(self, config, insecure=False):
+        cert_reqs = 'CERT_NONE' if insecure or config.get('ctl', {}).get('insecure', False) else 'CERT_REQUIRED'
+        self._pool = urllib3.PoolManager(num_pools=10, maxsize=10, cert_reqs=cert_reqs)
+        self.reload_config(config)
+
+    @staticmethod
+    def _get_cfg_value(config, name):
+        return config.get('ctl', {}).get(name) or config.get('restapi', {}).get(name)
+
+    def _apply_pool_param(self, param, value):
+        if value:
+            self._pool.connection_pool_kw[param] = value
+        else:
+            self._pool.connection_pool_kw.pop(param, None)
+
+    def _apply_ssl_file_param(self, config, name):
+        value = self._get_cfg_value(config, name + 'file')
+        self._apply_pool_param(name + '_file', value)
+        return value
+
+    def reload_config(self, config):
+        self._pool.headers = urllib3.make_headers(basic_auth=self._get_cfg_value(config, 'auth'), user_agent=USER_AGENT)
+
+        if self._apply_ssl_file_param(config, 'cert'):
+            self._apply_ssl_file_param(config, 'key')
+        else:
+            self._pool.connection_pool_kw.pop('key_file', None)
+
+        cacert = config.get('ctl', {}).get('cacert') or config.get('restapi', {}).get('cafile')
+        self._apply_pool_param('ca_certs', cacert)
+
+    def request(self, method, url, body=None, **kwargs):
+        if body is not None and not isinstance(body, six.string_types):
+            body = json.dumps(body)
+        return self._pool.request(method.upper(), url, body=body, **kwargs)
+
+    def __call__(self, member, method='GET', endpoint=None, data=None, **kwargs):
+        url = member.api_url
+        if endpoint:
+            scheme, netloc, _, _, _, _ = urlparse(url)
+            url = urlunparse((scheme, netloc, endpoint, '', '', ''))
+        return self.request(method, url, data, **kwargs)
+
+
+def get(url, verify=True, **kwargs):
+    http = PatroniRequest({}, not verify)
+    return http.request('GET', url, **kwargs)

patroni/scripts/aws.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python
 
+import json
 import logging
-import requests
-from requests.exceptions import RequestException
 import sys
 import boto.ec2
 
 from patroni.utils import Retry, RetryFailedError
+from patroni.request import get as requests_get
 
 logger = logging.getLogger(__name__)
 
@@ -19,14 +19,14 @@ class AWSConnection(object):
         self._retry = Retry(deadline=300, max_delay=30, max_tries=-1, retry_exceptions=(boto.exception.StandardError,))
         try:
             # get the instance id
-            r = requests.get('http://169.254.169.254/latest/dynamic/instance-identity/document', timeout=2.1)
-        except RequestException:
+            r = requests_get('http://169.254.169.254/latest/dynamic/instance-identity/document', timeout=2.1)
+        except Exception:
             logger.error('cannot query AWS meta-data')
             return
 
-        if r.ok:
+        if r.status < 400:
             try:
-                content = r.json()
+                content = json.loads(r.data.decode('utf-8'))
                 self.instance_id = content['instanceId']
                 self.region = content['region']
             except Exception:

patroni/utils.py

@@ -1,11 +1,24 @@
+import logging
+import platform
 import random
+import re
 import time
 
 from dateutil import tz
-from patroni.exceptions import PatroniException
+
+from .exceptions import PatroniException
+from .version import __version__
 
 tzutc = tz.tzutc()
 
+logger = logging.getLogger(__name__)
+
+USER_AGENT = 'Patroni/{0} Python/{1} {2}'.format(__version__, platform.python_version(), platform.system())
+OCT_RE = re.compile(r'^[-+]?0[0-7]*')
+DEC_RE = re.compile(r'^[-+]?(0|[1-9][0-9]*)')
+HEX_RE = re.compile(r'^[-+]?0x[0-9a-fA-F]+')
+DBL_RE = re.compile(r'^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?')
+
 
 def deep_compare(obj1, obj2):
     """
@@ -88,42 +101,77 @@ def strtol(value, strict=True):
     True
     >>> strtol(' d ') == (None, 'd')
     True
+    >>> strtol(' 1 d ') == (1, ' d')
+    True
     >>> strtol('9s', False) == (9, 's')
     True
     >>> strtol(' s ', False) == (1, 's')
     True
     """
     value = str(value).strip()
-    ln = len(value)
-    i = 0
-    # skip sign:
-    if i < ln and value[i] in ('-', '+'):
-        i += 1
+    for regex, base in ((HEX_RE, 16), (OCT_RE, 8), (DEC_RE, 10)):
+        match = regex.match(value)
+        if match:
+            end = match.end()
+            return int(value[:end], base), value[end:]
+    return (None if strict else 1), value
 
-    # we always expect to get digit in the beginning
-    if i < ln and value[i].isdigit():
-        if value[i] == '0':
-            i += 1
-            if i < ln and value[i] in ('x', 'X'):  # '0' followed by 'x': HEX
-                base = 16
-                i += 1
-            else:  # just starts with '0': OCT
-                base = 8
-        else:  # any other digit: DEC
-            base = 10
 
-        ret = None
-        while i <= ln:
-            try:  # try to find maximally long number
-                i += 1  # by giving to `int` longer and longer strings
-                ret = int(value[:i], base)
-            except ValueError:  # until we will not get an exception or end of the string
-                i -= 1
-                break
-        if ret is not None:  # yay! there is a number in the beginning of the string
-            return ret, value[i:].strip()  # return the number and the "rest"
+def strtod(value):
+    """As most as possible close equivalent of strtod(3) function used by postgres to parse parameter values.
+    >>> strtod(' A ') == (None, 'A')
+    True
+    """
+    value = str(value).strip()
+    match = DBL_RE.match(value)
+    if match:
+        end = match.end()
+        return float(value[:end]), value[end:]
+    return None, value
 
-    return (None if strict else 1), value.strip()
+
+def rint(value):
+    """
+    >>> rint(0.5) == 0
+    True
+    >>> rint(0.501) == 1
+    True
+    >>> rint(1.5) == 2
+    True
+    """
+
+    ret = round(value)
+    return 2.0 * round(value / 2.0) if abs(ret - value) == 0.5 else ret
+
+
+def convert_to_base_unit(value, unit, base_unit):
+    convert = {
+        'B': {'B': 1, 'kB': 1024, 'MB': 1024 * 1024, 'GB': 1024 * 1024 * 1024, 'TB': 1024 * 1024 * 1024 * 1024},
+        'kB': {'B': 1.0 / 1024, 'kB': 1, 'MB': 1024, 'GB': 1024 * 1024, 'TB': 1024 * 1024 * 1024},
+        'MB': {'B': 1.0 / (1024 * 1024), 'kB': 1.0 / 1024, 'MB': 1, 'GB': 1024, 'TB': 1024 * 1024},
+        'ms': {'us': 1.0 / 1000, 'ms': 1, 's': 1000, 'min': 1000 * 60, 'h': 1000 * 60 * 60, 'd': 1000 * 60 * 60 * 24},
+        's': {'us': 1.0 / (1000 * 1000), 'ms': 1.0 / 1000, 's': 1, 'min': 60, 'h': 60 * 60, 'd': 60 * 60 * 24},
+        'min': {'us': 1.0 / (1000 * 1000 * 60), 'ms': 1.0 / (1000 * 60), 's': 1.0 / 60, 'min': 1, 'h': 60, 'd': 60 * 24}
+    }
+
+    round_order = {
+        'TB': 'GB', 'GB': 'MB', 'MB': 'kB', 'kB': 'B',
+        'd': 'h', 'h': 'min', 'min': 's', 's': 'ms', 'ms': 'us'
+    }
+
+    if base_unit and base_unit not in convert:
+        base_value, base_unit = strtol(base_unit, False)
+    else:
+        base_value = 1
+
+    if base_unit in convert and unit in convert[base_unit]:
+        value *= convert[base_unit][unit] / float(base_value)
+
+        if unit in round_order:
+            multiplier = convert[base_unit][round_order[unit]]
+            value = rint(value / float(multiplier)) * multiplier
+
+        return value
 
 
 def parse_int(value, base_unit=None):
@@ -136,56 +184,71 @@ def parse_int(value, base_unit=None):
     True
     >>> parse_int('1000 ms', 's') == 1
     True
-    >>> parse_int('1GB', 'MB') is None
+    >>> parse_int('1TB', 'GB') is None
     True
     >>> parse_int(0) == 0
     True
+    >>> parse_int('6GB', '16MB') == 384
+    True
+    >>> parse_int('4097.4kB', 'kB') == 4097
+    True
+    >>> parse_int('4097.5kB', 'kB') == 4098
+    True
     """
 
-    convert = {
-        'kB': {'kB': 1, 'MB': 1024, 'GB': 1024 * 1024, 'TB': 1024 * 1024 * 1024},
-        'ms': {'ms': 1, 's': 1000, 'min': 1000 * 60, 'h': 1000 * 60 * 60, 'd': 1000 * 60 * 60 * 24},
-        's': {'ms': -1000, 's': 1, 'min': 60, 'h': 60 * 60, 'd': 60 * 60 * 24},
-        'min': {'ms': -1000 * 60, 's': -60, 'min': 1, 'h': 60, 'd': 60 * 24}
-    }
+    val, unit = strtol(value)
+    if val is None and unit.startswith('.') or unit and unit[0] in ('.', 'e', 'E'):
+        val, unit = strtod(value)
 
-    value, unit = strtol(value)
-    if value is not None:
+    if val is not None:
+        unit = unit.strip()
         if not unit:
-            return value
+            return int(rint(val))
 
-        if base_unit and base_unit not in convert:
-            base_value, base_unit = strtol(base_unit, False)
-        else:
-            base_value = 1
-        if base_unit in convert and unit in convert[base_unit]:
-            multiplier = convert[base_unit][unit]
-            if multiplier < 0:
-                value /= -multiplier
-            else:
-                value *= multiplier
-            return int(value/base_value)
+        val = convert_to_base_unit(val, unit, base_unit)
+        if val is not None:
+            return int(rint(val))
+
+
+def parse_real(value, base_unit=None):
+    """
+    >>> parse_real(' +0.0005 ') == 0.0005
+    True
+    >>> parse_real('0.0005ms', 'ms') == 0.0
+    True
+    >>> parse_real('0.00051ms', 'ms') == 0.001
+    True
+    """
+    val, unit = strtod(value)
+
+    if val is not None:
+        unit = unit.strip()
+        if not unit:
+            return val
+
+        return convert_to_base_unit(val, unit, base_unit)
 
 
 def compare_values(vartype, unit, old_value, new_value):
     """
     >>> compare_values('enum', None, 'remote_write', 'REMOTE_WRITE')
     True
-    >>> compare_values('real', None, '1.23', 1.23)
+    >>> compare_values('real', None, '1e-06', 0.000001)
     True
     """
 
-    # if the integer or bool new_value is not correct this function will return False
-    if vartype == 'bool':
-        old_value = parse_bool(old_value)
-        new_value = parse_bool(new_value)
-    elif vartype == 'integer':
-        old_value = parse_int(old_value)
-        new_value = parse_int(new_value, unit)
-    elif vartype == 'enum':
-        return str(old_value).lower() == str(new_value).lower()
-    else:  # ('string', 'real')
-        return str(old_value) == str(new_value)
+    converters = {
+        'bool': lambda v1, v2: parse_bool(v1),
+        'integer': parse_int,
+        'real': parse_real,
+        'enum': lambda v1, v2: str(v1).lower(),
+        'string': lambda v1, v2: str(v1)
+    }
+
+    convert = converters.get(vartype) or converters['string']
+    old_value = convert(old_value, None)
+    new_value = convert(new_value, unit)
+
     return old_value is not None and new_value is not None and old_value == new_value
 
 
@@ -237,6 +300,17 @@ class Retry(object):
                      max_jitter=self.max_jitter / 100.0, max_delay=self.max_delay, sleep_func=self.sleep_func,
                      deadline=self.deadline, retry_exceptions=self.retry_exceptions)
 
+    @property
+    def sleeptime(self):
+        return self._cur_delay + (random.randint(0, self.max_jitter) / 100.0)
+
+    def update_delay(self):
+        self._cur_delay = min(self._cur_delay * self.backoff, self.max_delay)
+
+    @property
+    def stoptime(self):
+        return self._cur_stoptime
+
     def __call__(self, func, *args, **kwargs):
         """Call a function with arguments until it completes without throwing a `retry_exceptions`
 
@@ -252,18 +326,20 @@ class Retry(object):
                 if self.deadline is not None and self._cur_stoptime is None:
                     self._cur_stoptime = time.time() + self.deadline
                 return func(*args, **kwargs)
-            except self.retry_exceptions:
+            except self.retry_exceptions as e:
                 # Note: max_tries == -1 means infinite tries.
                 if self._attempts == self.max_tries:
+                    logger.warning('Retry got exception: %s', e)
                     raise RetryFailedError("Too many retry attempts")
                 self._attempts += 1
-                sleeptime = self._cur_delay + (random.randint(0, self.max_jitter) / 100.0)
+                sleeptime = self.sleeptime
 
                 if self._cur_stoptime is not None and time.time() + sleeptime >= self._cur_stoptime:
+                    logger.warning('Retry got exception: %s', e)
                     raise RetryFailedError("Exceeded retry deadline")
-                else:
-                    self.sleep_func(sleeptime)
-                self._cur_delay = min(self._cur_delay * self.backoff, self.max_delay)
+                logger.debug('Retry got exception: %s', e)
+                self.sleep_func(sleeptime)
+                self.update_delay()
 
 
 def polling_loop(timeout, interval=1):
@@ -281,3 +357,62 @@ def split_host_port(value, default_port):
     t = value.rsplit(':', 1)
     t.append(default_port)
     return t[0], int(t[1])
+
+
+def uri(proto, netloc, path='', user=None):
+    host, port = netloc if isinstance(netloc, (list, tuple)) else split_host_port(netloc, 0)
+    if host and ':' in host and host[0] != '[' and host[-1] != ']':
+        host = '[{0}]'.format(host)
+    port = ':{0}'.format(port) if port else ''
+    path = '/{0}'.format(path) if path and not path.startswith('/') else path
+    user = '{0}@'.format(user) if user else ''
+    return '{0}://{1}{2}{3}{4}'.format(proto, user, host, port, path)
+
+
+def is_standby_cluster(config):
+    # Check whether or not provided configuration describes a standby cluster
+    return isinstance(config, dict) and (config.get('host') or config.get('port') or config.get('restore_command'))
+
+
+def cluster_as_json(cluster):
+    leader_name = cluster.leader.name if cluster.leader else None
+    xlog_location_cluster = cluster.last_leader_operation or 0
+
+    ret = {'members': []}
+    for m in cluster.members:
+        if m.name == leader_name:
+            config = cluster.config.data if cluster.config and cluster.config.modify_index else {}
+            role = 'standby_leader' if is_standby_cluster(config.get('standby_cluster')) else 'leader'
+        elif m.name == cluster.sync.sync_standby:
+            role = 'sync_standby'
+        else:
+            role = 'replica'
+
+        conn_kwargs = m.conn_kwargs()
+        member = {'name': m.name, 'host': conn_kwargs['host'], 'port': int(conn_kwargs['port']),
+                  'role': role, 'state': m.data.get('state', ''), 'api_url': m.api_url}
+        optional_attributes = ('timeline', 'pending_restart', 'scheduled_restart', 'tags')
+        member.update({n: m.data[n] for n in optional_attributes if n in m.data})
+
+        if m.name != leader_name:
+            xlog_location = m.data.get('xlog_location')
+            if xlog_location is None:
+                member['lag'] = 'unknown'
+            elif xlog_location_cluster >= xlog_location:
+                member['lag'] = xlog_location_cluster - xlog_location
+            else:
+                member['lag'] = 0
+
+        ret['members'].append(member)
+
+    # sort members by name for consistency
+    ret['members'].sort(key=lambda m: m['name'])
+    if cluster.is_paused():
+        ret['pause'] = True
+    if cluster.failover and cluster.failover.scheduled_at:
+        ret['scheduled_switchover'] = {'at': cluster.failover.scheduled_at.isoformat()}
+        if cluster.failover.leader:
+            ret['scheduled_switchover']['from'] = cluster.failover.leader
+        if cluster.failover.candidate:
+            ret['scheduled_switchover']['to'] = cluster.failover.candidate
+    return ret

patroni/version.py

@@ -1 +1 @@
-__version__ = '1.5.6'
+__version__ = '1.6.3'

patroni/watchdog/base.py

@@ -57,7 +57,7 @@ class WatchdogConfig(object):
         return not self == other
 
     def get_impl(self):
-        if self.driver == 'testing':
+        if self.driver == 'testing':  # pragma: no cover
             from patroni.watchdog.linux import TestingWatchdogDevice
             return TestingWatchdogDevice.from_config(self.driver_config)
         elif platform.system() == 'Linux' and self.driver == 'default':

patroni/watchdog/linux.py

@@ -16,11 +16,11 @@ IOC_DIRBITS = 2
 
 # Non-generic platform special cases
 machine = platform.machine()
-if machine in ['mips', 'sparc', 'powerpc', 'ppc64']:
+if machine in ['mips', 'sparc', 'powerpc', 'ppc64']:  # pragma: no cover
     IOC_SIZEBITS = 13
     IOC_DIRBITS = 3
     IOC_NONE, IOC_WRITE, IOC_READ = 1, 2, 4
-elif machine == 'parisc':
+elif machine == 'parisc':  # pragma: no cover
     IOC_WRITE, IOC_READ = 2, 1
 
 IOC_NRSHIFT = 0
@@ -218,7 +218,7 @@ class LinuxWatchdogDevice(WatchdogBase):
         return timeout.value
 
 
-class TestingWatchdogDevice(LinuxWatchdogDevice):
+class TestingWatchdogDevice(LinuxWatchdogDevice):  # pragma: no cover
     """Converts timeout ioctls to regular writes that can be intercepted from a named pipe."""
     timeout = 60
 

postgres0.yml

@@ -17,7 +17,18 @@ restapi:
 #   cacert: /etc/ssl/certs/ssl-cacert-snakeoil.pem
 
 etcd:
+  #Provide host to do the initial discovery of the cluster topology:
   host: 127.0.0.1:2379
+  #Or use "hosts" to provide multiple endpoints
+  #Could be a comma separated string:
+  #hosts: host1:port1,host2:port2
+  #or an actual yaml list:
+  #hosts:
+  #- host1:port1
+  #- host2:port2
+  #Once discovery is complete Patroni will use the list of advertised clientURLs
+  #It is possible to change this behavior through by setting:
+  #use_proxies: true
 
 bootstrap:
   # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
@@ -55,6 +66,9 @@ bootstrap:
   - data-checksums
 
   pg_hba:  # Add following lines to pg_hba.conf after running 'initdb'
+  # For kerberos gss based connectivity (discard @.*$)
+  #- host replication replicator 127.0.0.1/32 gss include_realm=0
+  #- host all all 0.0.0.0/0 gss include_realm=0
   - host replication replicator 127.0.0.1/32 md5
   - host all all 0.0.0.0/0 md5
 #  - hostssl all all 0.0.0.0/0 md5
@@ -84,7 +98,15 @@ postgresql:
     superuser:
       username: postgres
       password: zalando
+    rewind:  # Has no effect on postgres 10 and lower
+      username: rewind_user
+      password: rewind_password
+  # Server side kerberos spn
+#  krbsrvname: postgres
   parameters:
+    # Fully qualified kerberos ticket file for the running user
+    # same as KRB5CCNAME used by the GSS
+#   krb_server_keyfile: /var/spool/keytabs/postgres
     unix_socket_directories: '.'
 
 #watchdog:

postgres1.yml

@@ -17,7 +17,18 @@ restapi:
 #   cacert: /etc/ssl/certs/ssl-cacert-snakeoil.pem
 
 etcd:
+  #Provide host to do the initial discovery of the cluster topology:
   host: 127.0.0.1:2379
+  #Or use "hosts" to provide multiple endpoints
+  #Could be a comma separated string:
+  #hosts: host1:port1,host2:port2
+  #or an actual yaml list:
+  #hosts:
+  #- host1:port1
+  #- host2:port2
+  #Once discovery is complete Patroni will use the list of advertised clientURLs
+  #It is possible to change this behavior through by setting:
+  #use_proxies: true
 
 bootstrap:
   # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
@@ -49,6 +60,9 @@ bootstrap:
   - data-checksums
 
   pg_hba:  # Add following lines to pg_hba.conf after running 'initdb'
+  # For kerberos gss based connectivity (discard @.*$)
+  #- host replication replicator 127.0.0.1/32 gss include_realm=0
+  #- host all all 0.0.0.0/0 gss include_realm=0
   - host replication replicator 127.0.0.1/32 md5
   - host all all 0.0.0.0/0 md5
 #  - hostssl all all 0.0.0.0/0 md5
@@ -78,7 +92,15 @@ postgresql:
     superuser:
       username: postgres
       password: zalando
+    rewind:  # Has no effect on postgres 10 and lower
+      username: rewind_user
+      password: rewind_password
+  # Server side kerberos spn
+#  krbsrvname: postgres
   parameters:
+    # Fully qualified kerberos ticket file for the running user
+    # same as KRB5CCNAME used by the GSS
+#    krb_server_keyfile: /var/spool/keytabs/postgres
     unix_socket_directories: '.'
   basebackup:
       - verbose

postgres2.yml

@@ -17,7 +17,18 @@ restapi:
 #   cacert: /etc/ssl/certs/ssl-cacert-snakeoil.pem
 
 etcd:
+  #Provide host to do the initial discovery of the cluster topology:
   host: 127.0.0.1:2379
+  #Or use "hosts" to provide multiple endpoints
+  #Could be a comma separated string:
+  #hosts: host1:port1,host2:port2
+  #or an actual yaml list:
+  #hosts:
+  #- host1:port1
+  #- host2:port2
+  #Once discovery is complete Patroni will use the list of advertised clientURLs
+  #It is possible to change this behavior through by setting:
+  #use_proxies: true
 
 bootstrap:
   # this section will be written into Etcd:/<namespace>/<scope>/config after initializing new cluster
@@ -49,6 +60,9 @@ bootstrap:
   - data-checksums
 
   pg_hba:  # Add following lines to pg_hba.conf after running 'initdb'
+  # For kerberos gss based connectivity (discard @.*$)
+  #- host replication replicator 127.0.0.1/32 gss include_realm=0
+  #- host all all 0.0.0.0/0 gss include_realm=0
   - host replication replicator 127.0.0.1/32 md5
   - host all all 0.0.0.0/0 md5
 #  - hostssl all all 0.0.0.0/0 md5
@@ -75,7 +89,15 @@ postgresql:
     superuser:
       username: postgres
       password: zalando
+    rewind:  # Has no effect on postgres 10 and lower
+      username: rewind_user
+      password: rewind_password
+  # Server side kerberos spn
+#  krbsrvname: postgres
   parameters:
+    # Fully qualified kerberos ticket file for the running user
+    # same as KRB5CCNAME used by the GSS
+#    krb_server_keyfile: /var/spool/keytabs/postgres
     unix_socket_directories: '.'
 tags:
     nofailover: false

requirements.txt

@@ -1,16 +1,14 @@
 urllib3>=1.19.1,!=1.21
 boto
-psycopg2>=2.5.4
 PyYAML
-requests
 six >= 1.7
 kazoo>=1.3.1
 python-etcd>=0.4.3,<0.5
-python-consul>=0.7.0
+python-consul>=0.7.1
 click>=4.1
 prettytable>=0.7
 tzlocal
 python-dateutil
 psutil>=2.0.0
 cdiff
-kubernetes>=2.0.0,<=7.0.0,!=4.0.*,!=5.0.*
+kubernetes>=2.0.0,<=10.0.1,!=4.0.*,!=5.0.*

setup.py

@@ -8,27 +8,12 @@ import inspect
 import os
 import sys
 
-from setuptools.command.test import test as TestCommand
-from setuptools import find_packages, setup
-
-if sys.version_info < (2, 7, 0):
-    sys.stderr.write('FATAL: patroni needs to be run with Python 2.7+\n')
-    sys.exit(1)
+from setuptools import Command, find_packages, setup
 
 __location__ = os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe())))
 
-
-def read_version(package):
-    data = {}
-    with open(os.path.join(package, 'version.py'), 'r') as fd:
-        exec(fd.read(), data)
-    return data['__version__']
-
-
 NAME = 'patroni'
 MAIN_PACKAGE = NAME
-SCRIPTS = 'scripts'
-VERSION = read_version(MAIN_PACKAGE)
 DESCRIPTION = 'PostgreSQL High-Available orchestrator and CLI'
 LICENSE = 'The MIT License'
 URL = 'https://github.com/zalando/patroni'
@@ -37,9 +22,10 @@ AUTHOR_EMAIL = '[email protected], [email protected], alexk
 KEYWORDS = 'etcd governor patroni postgresql postgres ha haproxy confd' +\
     ' zookeeper exhibitor consul streaming replication kubernetes k8s'
 
+EXTRAS_REQUIRE = {'aws': ['boto'], 'etcd': ['python-etcd'], 'consul': ['python-consul'],
+                  'exhibitor': ['kazoo'], 'zookeeper': ['kazoo'], 'kubernetes': ['kubernetes']}
 COVERAGE_XML = True
 COVERAGE_HTML = False
-JUNIT_XML = True
 
 # Add here all kinds of additional classifiers as defined under
 # https://pypi.python.org/pypi?%3Aaction=list_classifiers
@@ -69,82 +55,74 @@ CONSOLE_SCRIPTS = ['patroni = patroni:main',
                    "patroni_aws = patroni.scripts.aws:main"]
 
 
-class PyTest(TestCommand):
+class PyTest(Command):
 
-    user_options = [('cov=', None, 'Run coverage'), ('cov-xml=', None, 'Generate junit xml report'), ('cov-html=',
-                    None, 'Generate junit html report'), ('junitxml=', None, 'Generate xml of test results')]
+    user_options = [('cov=', None, 'Run coverage'), ('cov-xml=', None, 'Generate junit xml report'),
+                    ('cov-html=', None, 'Generate junit html report')]
 
     def initialize_options(self):
-        TestCommand.initialize_options(self)
+        self.cov = []
         self.cov_xml = False
         self.cov_html = False
-        self.junitxml = None
 
     def finalize_options(self):
-        TestCommand.finalize_options(self)
         if self.cov_xml or self.cov_html:
             self.cov = ['--cov', MAIN_PACKAGE, '--cov-report', 'term-missing']
             if self.cov_xml:
                 self.cov.extend(['--cov-report', 'xml'])
             if self.cov_html:
                 self.cov.extend(['--cov-report', 'html'])
-        if self.junitxml is not None:
-            self.junitxml = ['--junitxml', self.junitxml]
 
     def run_tests(self):
         try:
             import pytest
         except Exception:
             raise RuntimeError('py.test is not installed, run: pip install pytest')
-        params = {'args': self.test_args}
-        if self.cov:
-            params['args'] += self.cov
-        if self.junitxml:
-            params['args'] += self.junitxml
-        params['args'] += ['--doctest-modules', MAIN_PACKAGE, '-vv']
 
         import logging
         silence = logging.WARNING
         logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=os.getenv('LOGLEVEL', silence))
-        params['args'] += ['-s' if logging.getLogger().getEffectiveLevel() < silence else '--capture=fd']
-        if not os.getenv('SYSTEMROOT'):
-            os.environ['SYSTEMROOT'] = '/'
-        errno = pytest.main(**params)
+
+        args = ['--verbose', 'tests', '--doctest-modules', MAIN_PACKAGE] +\
+            ['-s' if logging.getLogger().getEffectiveLevel() < silence else '--capture=fd']
+        if self.cov:
+            args += self.cov
+
+        errno = pytest.main(args=args)
         sys.exit(errno)
 
-
-def get_install_requirements(path):
-    content = open(os.path.join(__location__, path)).read()
-    return [req for req in content.split('\n') if req != '']
+    def run(self):
+        from pkg_resources import evaluate_marker
+        requirements = self.distribution.install_requires + ['mock>=2.0.0', 'pytest-cov', 'pytest'] +\
+            [v for k, v in self.distribution.extras_require.items() if not k.startswith(':') or evaluate_marker(k[1:])]
+        self.distribution.fetch_build_eggs(requirements)
+        self.run_tests()
 
 
 def read(fname):
-    return open(os.path.join(__location__, fname)).read()
+    with open(os.path.join(__location__, fname)) as fd:
+        return fd.read()
 
 
-def setup_package():
+def setup_package(version):
     # Assemble additional setup commands
     cmdclass = {'test': PyTest}
 
-    # Some helper variables
-    version = os.getenv('GO_PIPELINE_LABEL', VERSION)
-
     install_requires = []
-    extras_require = {'aws': ['boto'], 'etcd': ['python-etcd'], 'consul': ['python-consul'],
-                      'exhibitor': ['kazoo'], 'zookeeper': ['kazoo'], 'kubernetes': ['kubernetes']}
 
-    for r in get_install_requirements('requirements.txt'):
+    for r in read('requirements.txt').split('\n'):
+        r = r.strip()
+        if r == '':
+            continue
         extra = False
-        for e, v in extras_require.items():
+        for e, v in EXTRAS_REQUIRE.items():
             if r.startswith(v[0]):
-                extras_require[e] = [r]
+                EXTRAS_REQUIRE[e] = [r]
                 extra = True
         if not extra:
             install_requires.append(r)
 
-    command_options = {'test': {'test_suite': ('setup.py', 'tests')}}
-    if JUNIT_XML:
-        command_options['test']['junitxml'] = 'setup.py', 'junit.xml'
+    command_options = {'test': {}}
     if COVERAGE_XML:
         command_options['test']['cov_xml'] = 'setup.py', True
     if COVERAGE_HTML:
@@ -161,17 +139,28 @@ def setup_package():
         keywords=KEYWORDS,
         long_description=read('README.rst'),
         classifiers=CLASSIFIERS,
-        test_suite='tests',
         packages=find_packages(exclude=['tests', 'tests.*']),
         package_data={MAIN_PACKAGE: ["*.json"]},
+        python_requires='>=2.7',
         install_requires=install_requires,
-        extras_require=extras_require,
+        extras_require=EXTRAS_REQUIRE,
+        setup_requires='flake8',
         cmdclass=cmdclass,
-        tests_require=['flake8', 'mock>=2.0.0', 'pytest-cov', 'pytest'],
         command_options=command_options,
         entry_points={'console_scripts': CONSOLE_SCRIPTS},
     )
 
 
 if __name__ == '__main__':
-    setup_package()
+    old_modules = sys.modules.copy()
+    try:
+        from patroni import check_psycopg2, fatal, __version__
+    finally:
+        sys.modules.clear()
+        sys.modules.update(old_modules)
+
+    if sys.version_info < (2, 7, 0):
+        fatal('Patroni needs to be run with Python 2.7+')
+    check_psycopg2()
+
+    setup_package(__version__)

tests/__init__.py

@@ -0,0 +1,208 @@
+import datetime
+import os
+import shutil
+import unittest
+
+from mock import Mock, patch
+
+import psycopg2
+import urllib3
+
+from patroni.dcs import Leader, Member
+from patroni.postgresql import Postgresql
+from patroni.postgresql.config import ConfigHandler
+from patroni.utils import RetryFailedError
+
+
+class SleepException(Exception):
+    pass
+
+
+class MockResponse(object):
+
+    def __init__(self, status_code=200):
+        self.status_code = status_code
+        self.content = '{}'
+
+    @property
+    def data(self):
+        return self.content.encode('utf-8')
+
+    @property
+    def status(self):
+        return self.status_code
+
+    @staticmethod
+    def getheader(*args):
+        return ''
+
+
+def requests_get(url, **kwargs):
+    members = '[{"id":14855829450254237642,"peerURLs":["http://localhost:2380","http://localhost:7001"],' +\
+              '"name":"default","clientURLs":["http://localhost:2379","http://localhost:4001"]}]'
+    response = MockResponse()
+    if url.startswith('http://local'):
+        raise urllib3.exceptions.HTTPError()
+    elif ':8011/patroni' in url:
+        response.content = '{"role": "replica", "xlog": {"received_location": 0}, "tags": {}}'
+    elif url.endswith('/members'):
+        response.content = '[{}]' if url.startswith('http://error') else members
+    elif url.startswith('http://exhibitor'):
+        response.content = '{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}'
+    elif url.endswith(':8011/reinitialize'):
+        data = kwargs.get('data', '')
+        if ' false}' in data:
+            response.status_code = 503
+            response.content = 'restarting after failure already in progress'
+    else:
+        response.status_code = 404
+    return response
+
+
+class MockPostmaster(object):
+    def __init__(self, is_running=True, is_single_master=False):
+        self.is_running = Mock(return_value=is_running)
+        self.is_single_master = Mock(return_value=is_single_master)
+        self.wait_for_user_backends_to_close = Mock()
+        self.signal_stop = Mock(return_value=None)
+        self.wait = Mock()
+
+
+class MockCursor(object):
+
+    def __init__(self, connection):
+        self.connection = connection
+        self.closed = False
+        self.rowcount = 0
+        self.results = []
+
+    def execute(self, sql, *params):
+        if sql.startswith('blabla'):
+            raise psycopg2.ProgrammingError()
+        elif sql == 'CHECKPOINT' or sql.startswith('SELECT pg_catalog.pg_create_'):
+            raise psycopg2.OperationalError()
+        elif sql.startswith('RetryFailedError'):
+            raise RetryFailedError('retry')
+        elif sql.startswith('SELECT slot_name'):
+            self.results = [('blabla', 'physical'), ('foobar', 'physical'), ('ls', 'logical', 'a', 'b')]
+        elif sql.startswith('SELECT CASE WHEN pg_catalog.pg_is_in_recovery()'):
+            self.results = [(1, 2)]
+        elif sql.startswith('SELECT pg_catalog.pg_is_in_recovery()'):
+            self.results = [(False, 2)]
+        elif sql.startswith('SELECT pg_catalog.to_char'):
+            replication_info = '[{"application_name":"walreceiver","client_addr":"1.2.3.4",' +\
+                               '"state":"streaming","sync_state":"async","sync_priority":0}]'
+            self.results = [('', 0, '', '', '', '', False, replication_info)]
+        elif sql.startswith('SELECT name, setting'):
+            self.results = [('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
+                            ('wal_block_size', '8192', None, 'integer', 'internal'),
+                            ('shared_buffers', '16384', '8kB', 'integer', 'postmaster'),
+                            ('wal_buffers', '-1', '8kB', 'integer', 'postmaster'),
+                            ('search_path', 'public', None, 'string', 'user'),
+                            ('port', '5433', None, 'integer', 'postmaster'),
+                            ('listen_addresses', '*', None, 'string', 'postmaster'),
+                            ('autovacuum', 'on', None, 'bool', 'sighup'),
+                            ('unix_socket_directories', '/tmp', None, 'string', 'postmaster')]
+        elif sql.startswith('IDENTIFY_SYSTEM'):
+            self.results = [('1', 2, '0/402EEC0', '')]
+        elif sql.startswith('SELECT isdir, modification'):
+            self.results = [(False, datetime.datetime.now())]
+        elif sql.startswith('SELECT pg_catalog.pg_read_file'):
+            self.results = [('1\t0/40159C0\tno recovery target specified\n\n'
+                             '2\t1/40159C0\tno recovery target specified\n',)]
+        elif sql.startswith('TIMELINE_HISTORY '):
+            self.results = [('', b'x\t0/40159C0\tno recovery target specified\n\n'
+                                 b'1\t0/40159C0\tno recovery target specified\n\n'
+                                 b'2\t0/402DD98\tno recovery target specified\n\n'
+                                 b'3\t0/403DD98\tno recovery target specified\n')]
+        else:
+            self.results = [(None, None, None, None, None, None, None, None, None, None)]
+
+    def fetchone(self):
+        return self.results[0]
+
+    def fetchall(self):
+        return self.results
+
+    def __iter__(self):
+        for i in self.results:
+            yield i
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        pass
+
+
+class MockConnect(object):
+
+    server_version = 99999
+    autocommit = False
+    closed = 0
+
+    def cursor(self):
+        return MockCursor(self)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        pass
+
+    @staticmethod
+    def close():
+        pass
+
+
+def psycopg2_connect(*args, **kwargs):
+    return MockConnect()
+
+
+class PostgresInit(unittest.TestCase):
+    _PARAMETERS = {'wal_level': 'hot_standby', 'max_replication_slots': 5, 'f.oo': 'bar',
+                   'search_path': 'public', 'hot_standby': 'on', 'max_wal_senders': 5,
+                   'wal_keep_segments': 8, 'wal_log_hints': 'on', 'max_locks_per_transaction': 64,
+                   'max_worker_processes': 8, 'max_connections': 100, 'max_prepared_transactions': 0,
+                   'track_commit_timestamp': 'off', 'unix_socket_directories': '/tmp', 'trigger_file': 'bla'}
+
+    @patch('psycopg2.connect', psycopg2_connect)
+    @patch.object(ConfigHandler, 'write_postgresql_conf', Mock())
+    @patch.object(ConfigHandler, 'replace_pg_hba', Mock())
+    @patch.object(ConfigHandler, 'replace_pg_ident', Mock())
+    @patch.object(Postgresql, 'get_postgres_role_from_data_directory', Mock(return_value='master'))
+    def setUp(self):
+        data_dir = os.path.join('data', 'test0')
+        self.p = Postgresql({'name': 'postgresql0', 'scope': 'batman', 'data_dir': data_dir,
+                             'config_dir': data_dir, 'retry_timeout': 10,
+                             'krbsrvname': 'postgres', 'pgpass': os.path.join(data_dir, 'pgpass0'),
+                             'listen': '127.0.0.2, 127.0.0.3:5432', 'connect_address': '127.0.0.2:5432',
+                             'authentication': {'superuser': {'username': 'foo', 'password': 'test'},
+                                                'replication': {'username': '', 'password': 'rep-pass'}},
+                             'remove_data_directory_on_rewind_failure': True,
+                             'use_pg_rewind': True, 'pg_ctl_timeout': 'bla',
+                             'parameters': self._PARAMETERS,
+                             'recovery_conf': {'foo': 'bar'},
+                             'pg_hba': ['host all all 0.0.0.0/0 md5'],
+                             'pg_ident': ['krb realm postgres'],
+                             'callbacks': {'on_start': 'true', 'on_stop': 'true', 'on_reload': 'true',
+                                           'on_restart': 'true', 'on_role_change': 'true'}})
+
+
+class BaseTestPostgresql(PostgresInit):
+
+    def setUp(self):
+        super(BaseTestPostgresql, self).setUp()
+
+        if not os.path.exists(self.p.data_dir):
+            os.makedirs(self.p.data_dir)
+
+        self.leadermem = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres'})
+        self.leader = Leader(-1, 28, self.leadermem)
+        self.other = Member(0, 'test-1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres',
+                                              'tags': {'replicatefrom': 'leader'}})
+        self.me = Member(0, 'test0', 28, {'conn_url': 'postgres://replicator:[email protected]:5434/postgres'})
+
+    def tearDown(self):
+        if os.path.exists(self.p.data_dir):
+            shutil.rmtree(self.p.data_dir)

tests/test_api.py

@@ -2,6 +2,7 @@ import datetime
 import json
 import psycopg2
 import unittest
+import socket
 
 from mock import Mock, PropertyMock, patch
 from patroni.api import RestApiHandler, RestApiServer
@@ -10,7 +11,8 @@ from patroni.ha import _MemberStatus
 from patroni.utils import tzutc
 from six import BytesIO as IO
 from six.moves import BaseHTTPServer
-from test_postgresql import psycopg2_connect, MockCursor
+from . import psycopg2_connect, MockCursor
+from .test_ha import get_cluster_initialized_without_leader
 
 
 future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5)
@@ -60,7 +62,7 @@ class MockHa(object):
         return 'reinitialize'
 
     @staticmethod
-    def restart():
+    def restart(*args, **kwargs):
         return (True, '')
 
     @staticmethod
@@ -100,12 +102,20 @@ class MockHa(object):
         return False
 
 
+class MockLogger(object):
+
+    NORMAL_LOG_QUEUE_SIZE = 2
+    queue_size = 3
+    records_lost = 1
+
+
 class MockPatroni(object):
 
     ha = MockHa()
     config = Mock()
     postgresql = ha.state_handler
     dcs = Mock()
+    logger = MockLogger()
     tags = {}
     version = '0.00'
     noloadbalance = PropertyMock(return_value=False)
@@ -134,25 +144,29 @@ class MockRestApiServer(RestApiServer):
     def __init__(self, Handler, request, config=None):
         self.socket = 0
         self.serve_forever = Mock()
-        BaseHTTPServer.HTTPServer.__init__ = Mock()
         MockRestApiServer._BaseServer__is_shut_down = Mock()
         MockRestApiServer._BaseServer__shutdown_request = True
-        config = config or {'listen': '127.0.0.1:8008', 'auth': 'test:test', 'certfile': 'dumb'}
+        config = config or {'listen': '127.0.0.1:8008', 'auth': 'test:test', 'certfile': 'dumb', 'verify_client': 'a'}
         super(MockRestApiServer, self).__init__(MockPatroni(), config)
         Handler(MockRequest(request), ('0.0.0.0', 8080), self)
 
 
-@patch('ssl.wrap_socket', Mock(return_value=0))
+@patch('ssl.SSLContext.load_cert_chain', Mock())
+@patch('ssl.SSLContext.wrap_socket', Mock(return_value=0))
+@patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
 class TestRestApiHandler(unittest.TestCase):
 
     _authorization = '\nAuthorization: Basic dGVzdDp0ZXN0'
 
     def test_do_GET(self):
         MockRestApiServer(RestApiHandler, 'GET /replica')
+        MockRestApiServer(RestApiHandler, 'GET /read-only')
         with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={})):
             MockRestApiServer(RestApiHandler, 'GET /replica')
         with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'master'})):
             MockRestApiServer(RestApiHandler, 'GET /replica')
+        with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'state': 'running'})):
+            MockRestApiServer(RestApiHandler, 'GET /health')
         MockRestApiServer(RestApiHandler, 'GET /master')
         MockPatroni.dcs.cluster.sync.sync_standby = MockPostgresql.name
         MockPatroni.dcs.cluster.is_synchronous_mode = Mock(return_value=True)
@@ -185,6 +199,17 @@ class TestRestApiHandler(unittest.TestCase):
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0'))
         MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0\nAuthorization:')
 
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_GET_cluster(self, mock_dcs):
+        mock_dcs.cluster = get_cluster_initialized_without_leader()
+        mock_dcs.cluster.members[1].data['xlog_location'] = 11
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /cluster'))
+
+    @patch.object(MockPatroni, 'dcs')
+    def test_do_GET_history(self, mock_dcs):
+        mock_dcs.cluster = get_cluster_initialized_without_leader()
+        self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /history'))
+
     @patch.object(MockPatroni, 'dcs')
     def test_do_GET_config(self, mock_dcs):
         mock_dcs.cluster.config.data = {}
@@ -221,11 +246,8 @@ class TestRestApiHandler(unittest.TestCase):
         mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, config)
         MockRestApiServer(RestApiHandler, request)
 
-    @patch.object(MockPatroni, 'sighup_handler', Mock(side_effect=Exception))
+    @patch.object(MockPatroni, 'sighup_handler', Mock())
     def test_do_POST_reload(self):
-        with patch.object(MockPatroni, 'config') as mock_config:
-            mock_config.reload_local_configuration.return_value = False
-            MockRestApiServer(RestApiHandler, 'POST /reload HTTP/1.0' + self._authorization)
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /reload HTTP/1.0' + self._authorization))
 
     @patch.object(MockPatroni, 'dcs')
@@ -391,19 +413,32 @@ class TestRestApiHandler(unittest.TestCase):
         MockRestApiServer(RestApiHandler, post + '37\n\n{"candidate":"2","scheduled_at": "1"}')
 
 
-@patch('ssl.wrap_socket', Mock(return_value=0))
+@patch('ssl.SSLContext.load_cert_chain', Mock())
+@patch('ssl.SSLContext.wrap_socket', Mock(return_value=0))
+@patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
 class TestRestApiServer(unittest.TestCase):
 
     def test_reload_config(self):
         bad_config = {'listen': 'foo'}
         self.assertRaises(ValueError, MockRestApiServer, None, '', bad_config)
-        srv = MockRestApiServer(lambda a1, a2, a3: None, '')
+        srv = MockRestApiServer(Mock(), '', {'listen': '*:8008', 'certfile': 'a', 'verify_client': 'required'})
         self.assertRaises(ValueError, srv.reload_config, bad_config)
         self.assertRaises(ValueError, srv.reload_config, {})
-        srv.reload_config({'listen': '127.0.0.2:8008'})
+        with patch.object(socket.socket, 'setsockopt', Mock(side_effect=socket.error)):
+            srv.reload_config({'listen': ':8008'})
+
+    def test_check_auth(self):
+        srv = MockRestApiServer(Mock(), '', {'listen': '*:8008', 'certfile': 'a', 'verify_client': 'required'})
+        mock_rh = Mock()
+        mock_rh.request.getpeercert.return_value = None
+        self.assertIsNot(srv.check_auth(mock_rh), True)
 
     def test_handle_error(self):
         try:
             raise Exception()
         except Exception:
             self.assertIsNone(MockRestApiServer.handle_error(None, ('127.0.0.1', 55555)))
+
+    def test_socket_error(self):
+        with patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock(side_effect=socket.error)):
+            self.assertRaises(socket.error, MockRestApiServer, Mock(), '', {'listen': '*:8008'})

tests/test_aws.py

@@ -1,11 +1,11 @@
 import boto.ec2
 import sys
 import unittest
+import urllib3
 
 from mock import Mock, patch
 from collections import namedtuple
 from patroni.scripts.aws import AWSConnection, main as _main
-from requests.exceptions import RequestException
 
 
 class MockEc2Connection(object):
@@ -22,28 +22,11 @@ class MockEc2Connection(object):
         return True
 
 
-class MockResponse(object):
-    ok = True
-
-    def __init__(self, content):
-        self.content = content
-
-    def json(self):
-        return self.content
-
-
-def requests_get(url, **kwargs):
-    if url.split('/')[-1] == 'document':
-        result = {"instanceId": "012345", "region": "eu-west-1"}
-    else:
-        result = 'foo'
-    return MockResponse(result)
-
-
 @patch('boto.ec2.connect_to_region', Mock(return_value=MockEc2Connection()))
 class TestAWSConnection(unittest.TestCase):
 
-    @patch('requests.get', requests_get)
+    @patch('patroni.scripts.aws.requests_get', Mock(return_value=urllib3.HTTPResponse(
+        status=200, body=b'{"instanceId": "012345", "region": "eu-west-1"}')))
     def setUp(self):
         self.conn = AWSConnection('test')
 
@@ -53,17 +36,18 @@ class TestAWSConnection(unittest.TestCase):
             self.conn._retry.max_tries = 1
             self.assertFalse(self.conn.on_role_change('master'))
 
-    @patch('requests.get', Mock(side_effect=RequestException('foo')))
+    @patch('patroni.scripts.aws.requests_get', Mock(side_effect=Exception('foo')))
     def test_non_aws(self):
         conn = AWSConnection('test')
         self.assertFalse(conn.on_role_change("master"))
 
-    @patch('requests.get', Mock(return_value=MockResponse('foo')))
+    @patch('patroni.scripts.aws.requests_get', Mock(return_value=urllib3.HTTPResponse(status=200, body=b'foo')))
     def test_aws_bizare_response(self):
         conn = AWSConnection('test')
         self.assertFalse(conn.aws_available())
 
-    @patch('requests.get', requests_get)
+    @patch('patroni.scripts.aws.requests_get', Mock(return_value=urllib3.HTTPResponse(
+        status=200, body=b'{"instanceId": "012345", "region": "eu-west-1"}')))
     @patch('sys.exit', Mock())
     def test_main(self):
         self.assertIsNone(_main())

tests/test_bootstrap.py

@@ -0,0 +1,227 @@
+import os
+
+from mock import Mock, PropertyMock, patch
+
+from patroni.async_executor import CriticalTask
+from patroni.postgresql import Postgresql
+from patroni.postgresql.bootstrap import Bootstrap
+from patroni.postgresql.cancellable import CancellableSubprocess
+from patroni.postgresql.config import ConfigHandler
+
+from . import psycopg2_connect, BaseTestPostgresql
+
+
+@patch('subprocess.call', Mock(return_value=0))
+@patch('psycopg2.connect', psycopg2_connect)
+@patch('os.rename', Mock())
+class TestBootstrap(BaseTestPostgresql):
+
+    @patch('patroni.postgresql.CallbackExecutor', Mock())
+    def setUp(self):
+        super(TestBootstrap, self).setUp()
+        self.b = self.p.bootstrap
+
+    @patch('time.sleep', Mock())
+    @patch.object(CancellableSubprocess, 'call')
+    @patch.object(Postgresql, 'remove_data_directory', Mock(return_value=True))
+    @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
+    @patch.object(Bootstrap, '_post_restore', Mock(side_effect=OSError))
+    def test_create_replica(self, mock_cancellable_subprocess_call):
+        self.p.config._config['create_replica_methods'] = ['pgBackRest']
+        self.p.config._config['pgBackRest'] = {'command': 'pgBackRest', 'keep_data': True, 'no_params': True}
+        mock_cancellable_subprocess_call.return_value = 0
+        self.assertEqual(self.b.create_replica(self.leader), 0)
+
+        self.p.config._config['create_replica_methods'] = ['basebackup']
+        self.p.config._config['basebackup'] = [{'max_rate': '100M'}, 'no-sync']
+        self.assertEqual(self.b.create_replica(self.leader), 0)
+
+        self.p.config._config['basebackup'] = [{'max_rate': '100M', 'compress': '9'}]
+        with patch('patroni.postgresql.bootstrap.logger.error', new_callable=Mock()) as mock_logger:
+            self.b.create_replica(self.leader)
+            mock_logger.assert_called_once()
+            self.assertTrue("only one key-value is allowed and value should be a string" in mock_logger.call_args[0][0],
+                            "not matching {0}".format(mock_logger.call_args[0][0]))
+
+        self.p.config._config['basebackup'] = [42]
+        with patch('patroni.postgresql.bootstrap.logger.error', new_callable=Mock()) as mock_logger:
+            self.b.create_replica(self.leader)
+            mock_logger.assert_called_once()
+            self.assertTrue("value should be string value or a single key-value pair" in mock_logger.call_args[0][0],
+                            "not matching {0}".format(mock_logger.call_args[0][0]))
+
+        self.p.config._config['basebackup'] = {"foo": "bar"}
+        self.assertEqual(self.b.create_replica(self.leader), 0)
+
+        self.p.config._config['create_replica_methods'] = ['wale', 'basebackup']
+        del self.p.config._config['basebackup']
+        mock_cancellable_subprocess_call.return_value = 1
+        self.assertEqual(self.b.create_replica(self.leader), 1)
+
+        mock_cancellable_subprocess_call.side_effect = Exception('foo')
+        self.assertEqual(self.b.create_replica(self.leader), 1)
+
+        mock_cancellable_subprocess_call.side_effect = [1, 0]
+        self.assertEqual(self.b.create_replica(self.leader), 0)
+
+        mock_cancellable_subprocess_call.side_effect = [Exception(), 0]
+        self.assertEqual(self.b.create_replica(self.leader), 0)
+
+        self.p.cancellable.cancel()
+        self.assertEqual(self.b.create_replica(self.leader), 1)
+
+    @patch('time.sleep', Mock())
+    @patch.object(CancellableSubprocess, 'call')
+    @patch.object(Postgresql, 'remove_data_directory', Mock(return_value=True))
+    @patch.object(Bootstrap, '_post_restore', Mock(side_effect=OSError))
+    def test_create_replica_old_format(self, mock_cancellable_subprocess_call):
+        """ The same test as before but with old 'create_replica_method'
+            to test backward compatibility
+        """
+        self.p.config._config['create_replica_method'] = ['wale', 'basebackup']
+        self.p.config._config['wale'] = {'command': 'foo'}
+        mock_cancellable_subprocess_call.return_value = 0
+        self.assertEqual(self.b.create_replica(self.leader), 0)
+        del self.p.config._config['wale']
+        self.assertEqual(self.b.create_replica(self.leader), 0)
+
+        self.p.config._config['create_replica_method'] = ['wale']
+        mock_cancellable_subprocess_call.return_value = 1
+        self.assertEqual(self.b.create_replica(self.leader), 1)
+
+    def test_basebackup(self):
+        self.p.cancellable.cancel()
+        self.b.basebackup(None, None, {'foo': 'bar'})
+
+    def test__initdb(self):
+        self.assertRaises(Exception, self.b.bootstrap, {'initdb': [{'pgdata': 'bar'}]})
+        self.assertRaises(Exception, self.b.bootstrap, {'initdb': [{'foo': 'bar', 1: 2}]})
+        self.assertRaises(Exception, self.b.bootstrap, {'initdb': [1]})
+        self.assertRaises(Exception, self.b.bootstrap, {'initdb': 1})
+
+    @patch.object(CancellableSubprocess, 'call', Mock())
+    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
+    @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
+    @patch.object(Postgresql, 'controldata', Mock(return_value={'max_connections setting': 100,
+                                                                'max_prepared_xacts setting': 0,
+                                                                'max_locks_per_xact setting': 64}))
+    def test_bootstrap(self):
+        with patch('subprocess.call', Mock(return_value=1)):
+            self.assertFalse(self.b.bootstrap({}))
+
+        config = {'users': {'replicator': {'password': 'rep-pass', 'options': ['replication']}}}
+
+        with patch.object(Postgresql, 'is_running', Mock(return_value=False)),\
+                patch('multiprocessing.Process', Mock(side_effect=Exception)),\
+                patch('multiprocessing.get_context', Mock(side_effect=Exception), create=True):
+            self.assertRaises(Exception, self.b.bootstrap, config)
+        with open(os.path.join(self.p.data_dir, 'pg_hba.conf')) as f:
+            lines = f.readlines()
+            self.assertTrue('host all all 0.0.0.0/0 md5\n' in lines)
+
+        self.p.config._config.pop('pg_hba')
+        config.update({'post_init': '/bin/false',
+                       'pg_hba': ['host replication replicator 127.0.0.1/32 md5',
+                                  'hostssl all all 0.0.0.0/0 md5',
+                                  'host all all 0.0.0.0/0 md5']})
+        self.b.bootstrap(config)
+        with open(os.path.join(self.p.data_dir, 'pg_hba.conf')) as f:
+            lines = f.readlines()
+            self.assertTrue('host replication replicator 127.0.0.1/32 md5\n' in lines)
+
+    @patch.object(CancellableSubprocess, 'call')
+    @patch.object(Postgresql, 'get_major_version', Mock(return_value=90600))
+    @patch.object(Postgresql, 'controldata',  Mock(return_value={'Database cluster state': 'in production'}))
+    def test_custom_bootstrap(self, mock_cancellable_subprocess_call):
+        self.p.config._config.pop('pg_hba')
+        config = {'method': 'foo', 'foo': {'command': 'bar'}}
+
+        mock_cancellable_subprocess_call.return_value = 1
+        self.assertFalse(self.b.bootstrap(config))
+
+        mock_cancellable_subprocess_call.return_value = 0
+        with patch('multiprocessing.Process', Mock(side_effect=Exception("42"))),\
+                patch('multiprocessing.get_context', Mock(side_effect=Exception("42")), create=True),\
+                patch('os.path.isfile', Mock(return_value=True)),\
+                patch('os.unlink', Mock()),\
+                patch.object(ConfigHandler, 'save_configuration_files', Mock()),\
+                patch.object(ConfigHandler, 'restore_configuration_files', Mock()),\
+                patch.object(ConfigHandler, 'write_recovery_conf', Mock()):
+            with self.assertRaises(Exception) as e:
+                self.b.bootstrap(config)
+            self.assertEqual(str(e.exception), '42')
+
+            config['foo']['recovery_conf'] = {'foo': 'bar'}
+
+            with self.assertRaises(Exception) as e:
+                self.b.bootstrap(config)
+            self.assertEqual(str(e.exception), '42')
+
+        mock_cancellable_subprocess_call.side_effect = Exception
+        self.assertFalse(self.b.bootstrap(config))
+
+    @patch('time.sleep', Mock())
+    @patch('os.unlink', Mock())
+    @patch('shutil.copy', Mock())
+    @patch('os.path.isfile', Mock(return_value=True))
+    @patch.object(Bootstrap, 'call_post_bootstrap', Mock(return_value=True))
+    @patch.object(Bootstrap, '_custom_bootstrap', Mock(return_value=True))
+    @patch.object(Postgresql, 'start', Mock(return_value=True))
+    @patch.object(Postgresql, 'get_major_version', Mock(return_value=110000))
+    def test_post_bootstrap(self):
+        config = {'method': 'foo', 'foo': {'command': 'bar'}}
+        self.b.bootstrap(config)
+
+        task = CriticalTask()
+        with patch.object(Bootstrap, 'create_or_update_role', Mock(side_effect=Exception)):
+            self.b.post_bootstrap({}, task)
+            self.assertFalse(task.result)
+
+        self.p.config._config.pop('pg_hba')
+        self.b.post_bootstrap({}, task)
+        self.assertTrue(task.result)
+
+        self.b.bootstrap(config)
+        with patch.object(Postgresql, 'pending_restart', PropertyMock(return_value=True)), \
+                patch.object(Postgresql, 'restart', Mock()) as mock_restart:
+            self.b.post_bootstrap({}, task)
+            mock_restart.assert_called_once()
+
+        self.b.bootstrap(config)
+        self.p.set_state('stopped')
+        self.p.reload_config({'authentication': {'superuser': {'username': 'p', 'password': 'p'},
+                                                 'replication': {'username': 'r', 'password': 'r'},
+                                                 'rewind': {'username': 'rw', 'password': 'rw'}},
+                              'listen': '*', 'retry_timeout': 10, 'parameters': {'wal_level': '', 'hba_file': 'foo'}})
+        with patch.object(Postgresql, 'major_version', PropertyMock(return_value=110000)), \
+                patch.object(Postgresql, 'restart', Mock()) as mock_restart:
+            self.b.post_bootstrap({}, task)
+            mock_restart.assert_called_once()
+
+    @patch.object(CancellableSubprocess, 'call')
+    def test_call_post_bootstrap(self, mock_cancellable_subprocess_call):
+        mock_cancellable_subprocess_call.return_value = 1
+        self.assertFalse(self.b.call_post_bootstrap({'post_init': '/bin/false'}))
+
+        mock_cancellable_subprocess_call.return_value = 0
+        self.p.config.superuser.pop('username')
+        self.assertTrue(self.b.call_post_bootstrap({'post_init': '/bin/false'}))
+        mock_cancellable_subprocess_call.assert_called()
+        args, kwargs = mock_cancellable_subprocess_call.call_args
+        self.assertTrue('PGPASSFILE' in kwargs['env'])
+        self.assertEqual(args[0], ['/bin/false', 'dbname=postgres host=127.0.0.2 port=5432'])
+
+        mock_cancellable_subprocess_call.reset_mock()
+        self.p.config._local_address.pop('host')
+        self.assertTrue(self.b.call_post_bootstrap({'post_init': '/bin/false'}))
+        mock_cancellable_subprocess_call.assert_called()
+        self.assertEqual(mock_cancellable_subprocess_call.call_args[0][0], ['/bin/false', 'dbname=postgres port=5432'])
+
+        mock_cancellable_subprocess_call.side_effect = OSError
+        self.assertFalse(self.b.call_post_bootstrap({'post_init': '/bin/false'}))
+
+    @patch('os.path.exists', Mock(return_value=True))
+    @patch('os.unlink', Mock())
+    @patch.object(Bootstrap, 'create_replica', Mock(return_value=0))
+    def test_clone(self):
+        self.b.clone(self.leader)

tests/test_callback_executor.py

@@ -1,27 +1,34 @@
+import psutil
 import unittest
 
 from mock import Mock, patch
-from patroni.callback_executor import CallbackExecutor
+from patroni.postgresql.callback_executor import CallbackExecutor
 
 
 class TestCallbackExecutor(unittest.TestCase):
 
-    @patch('subprocess.Popen')
+    @patch('psutil.Popen')
     def test_callback_executor(self, mock_popen):
-        mock_popen.return_value.wait.side_effect = Exception
-        mock_popen.return_value.poll.return_value = None
+        mock_popen.return_value.children.return_value = []
+        mock_popen.return_value.is_running.return_value = True
 
         ce = CallbackExecutor()
+        ce._kill_children = Mock(side_effect=Exception)
         self.assertIsNone(ce.call([]))
         ce.join()
 
         self.assertIsNone(ce.call([]))
 
-        mock_popen.return_value.kill.side_effect = OSError
-        self.assertRaises(Exception, ce.call, [])
+        mock_popen.return_value.kill.side_effect = psutil.AccessDenied()
+        self.assertIsNone(ce.call([]))
+
+        ce._process_children = []
+        mock_popen.return_value.children.side_effect = psutil.Error()
+        mock_popen.return_value.kill.side_effect = psutil.NoSuchProcess(123)
+        self.assertIsNone(ce.call([]))
 
         mock_popen.side_effect = Exception
         ce = CallbackExecutor()
-        ce._callback_event.wait = Mock(side_effect=[None, Exception])
+        ce._condition.wait = Mock(side_effect=[None, Exception])
         self.assertIsNone(ce.call([]))
         ce.join()

tests/test_cancellable.py

@@ -0,0 +1,34 @@
+import psutil
+import unittest
+
+from mock import Mock, patch
+from patroni.exceptions import PostgresException
+from patroni.postgresql.cancellable import CancellableSubprocess
+
+
+class TestCancellableSubprocess(unittest.TestCase):
+
+    def setUp(self):
+        self.c = CancellableSubprocess()
+
+    def test_call(self):
+        self.c.cancel()
+        self.assertRaises(PostgresException, self.c.call, communicate_input=None)
+
+    def test__kill_children(self):
+        self.c._process_children = [Mock()]
+        self.c._kill_children()
+        self.c._process_children[0].kill.side_effect = psutil.AccessDenied()
+        self.c._kill_children()
+        self.c._process_children[0].kill.side_effect = psutil.NoSuchProcess(123)
+        self.c._kill_children()
+
+    @patch('patroni.postgresql.cancellable.polling_loop', Mock(return_value=[0, 0]))
+    def test_cancel(self):
+        self.c._process = Mock()
+        self.c._process.is_running.return_value = True
+        self.c._process.children.side_effect = psutil.Error()
+        self.c._process.suspend.side_effect = psutil.Error()
+        self.c.cancel()
+        self.c._process.is_running.side_effect = [True, False]
+        self.c.cancel()

tests/test_config.py

@@ -15,10 +15,7 @@ class TestConfig(unittest.TestCase):
     def setUp(self):
         sys.argv = ['patroni.py']
         os.environ[Config.PATRONI_CONFIG_VARIABLE] = 'restapi: {}\npostgresql: {data_dir: foo}'
-        self.config = Config()
-
-    def test_no_config(self):
-        self.assertRaises(SystemExit, Config)
+        self.config = Config(None)
 
     def test_set_dynamic_configuration(self):
         with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
@@ -66,12 +63,10 @@ class TestConfig(unittest.TestCase):
             'PATRONI_admin_PASSWORD': 'admin',
             'PATRONI_admin_OPTIONS': 'createrole,createdb'
         })
-        sys.argv = ['patroni.py', 'postgres0.yml']
-        config = Config()
+        config = Config('postgres0.yml')
         with patch.object(Config, '_load_config_file', Mock(return_value={'restapi': {}})):
             with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
-                self.assertRaises(Exception, config.reload_local_configuration, True)
-            self.assertTrue(config.reload_local_configuration(True))
+                config.reload_local_configuration()
             self.assertTrue(config.reload_local_configuration())
             self.assertIsNone(config.reload_local_configuration())
 

tests/test_consul.py

@@ -5,7 +5,7 @@ from consul import ConsulException, NotFound
 from mock import Mock, patch
 from patroni.dcs.consul import AbstractDCS, Cluster, Consul, ConsulInternalError, \
                                 ConsulError, HTTPClient, InvalidSessionTTL, InvalidSession
-from test_etcd import SleepException
+from . import SleepException
 
 
 def kv_get(self, key, **kwargs):
@@ -197,3 +197,6 @@ class TestConsul(unittest.TestCase):
         d['state'] = 'running'
         d['role'] = 'bla'
         self.assertIsNone(self.c.update_service({}, d))
+
+    def test_reload_config(self):
+        self.c.reload_config({'consul': {'token': 'foo'}, 'loop_wait': 10, 'ttl': 30, 'retry_timeout': 10})

tests/test_ctl.py

@@ -1,22 +1,22 @@
 import etcd
 import os
-import requests
-import sys
 import unittest
 
 from click.testing import CliRunner
 from datetime import datetime, timedelta
 from mock import patch, Mock
-from patroni.ctl import ctl, store_config, load_config, output_members, request_patroni, get_dcs, parse_dcs, \
+from patroni.ctl import ctl, store_config, load_config, output_members, get_dcs, parse_dcs, \
     get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException, apply_config_changes, \
-    format_config_for_editing, show_diff, invoke_editor, format_pg_version
+    format_config_for_editing, show_diff, invoke_editor, format_pg_version, find_executable
 from patroni.dcs.etcd import Client, Failover
 from patroni.utils import tzutc
 from psycopg2 import OperationalError
-from test_etcd import etcd_read, requests_get, socket_getaddrinfo, MockResponse
-from test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \
+from urllib3 import PoolManager
+
+from . import MockConnect, MockCursor, MockResponse, psycopg2_connect
+from .test_etcd import etcd_read, socket_getaddrinfo
+from .test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \
     get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader, get_cluster, Member
-from test_postgresql import MockConnect, psycopg2_connect
 
 CONFIG_FILE_PATH = './test-ctl.yaml'
 
@@ -24,9 +24,8 @@ CONFIG_FILE_PATH = './test-ctl.yaml'
 def test_rw_config():
     runner = CliRunner()
     with runner.isolated_filesystem():
-        store_config({'etcd': {'host': 'localhost:2379'}}, CONFIG_FILE_PATH + '/dummy')
-        sys.argv = ['patronictl.py', '']
         load_config(CONFIG_FILE_PATH + '/dummy', None)
+        store_config({'etcd': {'host': 'localhost:2379'}}, CONFIG_FILE_PATH + '/dummy')
         load_config(CONFIG_FILE_PATH + '/dummy', '0.0.0.0')
         os.remove(CONFIG_FILE_PATH + '/dummy')
         os.rmdir(CONFIG_FILE_PATH)
@@ -34,7 +33,7 @@ def test_rw_config():
 
 @patch('patroni.ctl.load_config',
        Mock(return_value={'scope': 'alpha', 'postgresql': {'data_dir': '.', 'parameters': {}, 'retry_timeout': 5},
-                          'restapi': {'auth': 'u:p', 'listen': ''}, 'etcd': {'host': 'localhost:2379'}}))
+                          'restapi': {'listen': '::', 'certfile': 'a'}, 'etcd': {'host': 'localhost:2379'}}))
 class TestCtl(unittest.TestCase):
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
@@ -74,7 +73,7 @@ class TestCtl(unittest.TestCase):
         self.assertIsNone(output_members(cluster, name='abc', fmt='tsv'))
 
     @patch('patroni.ctl.get_dcs')
-    @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
+    @patch.object(PoolManager, 'request', Mock(return_value=MockResponse()))
     def test_switchover(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
@@ -89,10 +88,15 @@ class TestCtl(unittest.TestCase):
             result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force', '--scheduled', '2015-01-01T12:00:00'])
             assert result.exit_code == 1
 
-        # Aborting switchover, as we anser NO to the confirmation
+        # Aborting switchover, as we answer NO to the confirmation
         result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\nN')
         assert result.exit_code == 1
 
+        # Aborting scheduled switchover, as we answer NO to the confirmation
+        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--scheduled', '2015-01-01T12:00:00+01:00'],
+                                    input='leader\nother\n\nN')
+        assert result.exit_code == 1
+
         # Target and source are equal
         result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nleader\n\ny')
         assert result.exit_code == 1
@@ -119,18 +123,18 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='dummy')
         assert result.exit_code == 1
 
-        with patch('patroni.ctl.request_patroni', Mock(side_effect=Exception)):
+        with patch.object(PoolManager, 'request', Mock(side_effect=Exception)):
             # Non-responding patroni
             result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
             assert 'falling back to DCS' in result.output
 
-        with patch('patroni.ctl.request_patroni') as mocked:
-            mocked.return_value.status_code = 500
+        with patch.object(PoolManager, 'request') as mocked:
+            mocked.return_value.status = 500
             result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
             assert 'Switchover failed' in result.output
 
-            mocked.return_value.status_code = 501
-            mocked.return_value.text = 'Server does not support this operation'
+            mocked.return_value.status = 501
+            mocked.return_value.data = b'Server does not support this operation'
             result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
             assert 'Switchover failed' in result.output
 
@@ -145,7 +149,7 @@ class TestCtl(unittest.TestCase):
         assert result.exit_code == 1
 
     @patch('patroni.ctl.get_dcs')
-    @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
+    @patch.object(PoolManager, 'request', Mock(return_value=MockResponse()))
     def test_failover(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
@@ -153,6 +157,7 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['failover', 'dummy'], input='\n')
         assert 'Failover could be performed only to a specific candidate' in result.output
 
+    @patch('patroni.dcs.dcs_modules', Mock(return_value=['patroni.dcs.dummy', 'patroni.dcs.etcd']))
     def test_get_dcs(self):
         self.assertRaises(PatroniCtlException, get_dcs, {'dummy': {}}, 'dummy')
 
@@ -198,7 +203,7 @@ class TestCtl(unittest.TestCase):
             rows = query_member(None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
             self.assertEqual(rows, (None, None))
 
-            with patch('test_postgresql.MockCursor.execute', Mock(side_effect=OperationalError('bla'))):
+            with patch.object(MockCursor, 'execute', Mock(side_effect=OperationalError('bla'))):
                 rows = query_member(None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
 
         with patch('patroni.ctl.get_cursor', Mock(return_value=None)):
@@ -225,7 +230,7 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['dsn', 'alpha', '--member', 'dummy'])
         assert result.exit_code == 1
 
-    @patch('requests.post')
+    @patch.object(PoolManager, 'request')
     @patch('patroni.ctl.get_dcs')
     def test_reload(self, mock_get_dcs, mock_post):
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
@@ -233,19 +238,20 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['reload', 'alpha'], input='y')
         assert 'Failed: reload for member' in result.output
 
-        mock_post.return_value.status_code = 200
+        mock_post.return_value.status = 200
         result = self.runner.invoke(ctl, ['reload', 'alpha'], input='y')
         assert 'No changes to apply on member' in result.output
 
-        mock_post.return_value.status_code = 202
+        mock_post.return_value.status = 202
         result = self.runner.invoke(ctl, ['reload', 'alpha'], input='y')
         assert 'Reload request received for member' in result.output
 
-    @patch('requests.post', requests_get)
+    @patch.object(PoolManager, 'request')
     @patch('patroni.ctl.get_dcs')
-    def test_restart_reinit(self, mock_get_dcs):
+    def test_restart_reinit(self, mock_get_dcs, mock_post):
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
-        result = self.runner.invoke(ctl, ['restart', 'alpha'], input='y\n\nnow')
+        mock_post.return_value.status = 503
+        result = self.runner.invoke(ctl, ['restart', 'alpha'], input='now\ny\n')
         assert 'Failed: restart for' in result.output
         assert result.exit_code == 0
 
@@ -257,76 +263,71 @@ class TestCtl(unittest.TestCase):
         assert result.exit_code == 0
 
         # Aborted restart
-        result = self.runner.invoke(ctl, ['restart', 'alpha'], input='N')
+        result = self.runner.invoke(ctl, ['restart', 'alpha'], input='now\nN')
         assert result.exit_code == 1
 
         result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force'])
         assert result.exit_code == 0
 
+        # Aborted scheduled restart
+        result = self.runner.invoke(ctl, ['restart', 'alpha', '--scheduled', '2019-10-01T14:30'], input='N')
+        assert result.exit_code == 1
+
         # Not a member
-        result = self.runner.invoke(ctl, ['restart', 'alpha', 'dummy', '--any'], input='y')
+        result = self.runner.invoke(ctl, ['restart', 'alpha', 'dummy', '--any'], input='now\ny')
         assert result.exit_code == 1
 
         # Wrong pg version
-        result = self.runner.invoke(ctl, ['restart', 'alpha', '--any', '--pg-version', '9.1'], input='y')
+        result = self.runner.invoke(ctl, ['restart', 'alpha', '--any', '--pg-version', '9.1'], input='now\ny')
         assert 'Error: Invalid PostgreSQL version format' in result.output
         assert result.exit_code == 1
 
         result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force', '--timeout', '10min'])
         assert result.exit_code == 0
 
-        with patch('requests.delete', Mock(return_value=MockResponse(500))):
-            # normal restart, the schedule is actually parsed, but not validated in patronictl
-            result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force',
-                                              '--scheduled', '2300-10-01T14:30'])
-            assert 'Failed: flush scheduled restart' in result.output
+        # normal restart, the schedule is actually parsed, but not validated in patronictl
+        result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
+        assert 'Failed: flush scheduled restart' in result.output
 
         with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
             result = self.runner.invoke(ctl,
                                         ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
             assert result.exit_code == 1
 
-        with patch('requests.post', Mock(return_value=MockResponse())):
-            # normal restart, the schedule is actually parsed, but not validated in patronictl
-            result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0.0',
-                                        '--scheduled', '2300-10-01T14:30'], input='y')
-            assert result.exit_code == 0
-
-        with patch('requests.post', Mock(return_value=MockResponse(204))):
-            # get restart with the non-200 return code
-            # normal restart, the schedule is actually parsed, but not validated in patronictl
-            result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0',
-                                        '--scheduled', '2300-10-01T14:30'], input='y')
-            assert result.exit_code == 0
-
         # force restart with restart already present
-        with patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse(204))):
-            result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force',
-                                              '--scheduled', '2300-10-01T14:30'])
-            assert result.exit_code == 0
+        result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
+        assert result.exit_code == 0
 
-        with patch('requests.post', Mock(return_value=MockResponse(202))):
-            # get restart with the non-200 return code
-            # normal restart, the schedule is actually parsed, but not validated in patronictl
-            result = self.runner.invoke(
-                ctl, ['restart', 'alpha', '--pg-version', '99.0.0', '--scheduled', '2300-10-01T14:30'], input='y'
-            )
-            assert 'Success: restart scheduled' in result.output
-            assert result.exit_code == 0
+        ctl_args = ['restart', 'alpha', '--pg-version', '99.0', '--scheduled', '2300-10-01T14:30']
+        # normal restart, the schedule is actually parsed, but not validated in patronictl
+        mock_post.return_value.status = 200
+        result = self.runner.invoke(ctl, ctl_args, input='y')
+        assert result.exit_code == 0
 
-        with patch('requests.post', Mock(return_value=MockResponse(409))):
-            # get restart with the non-200 return code
-            # normal restart, the schedule is actually parsed, but not validated in patronictl
-            result = self.runner.invoke(
-                ctl, ['restart', 'alpha', '--pg-version', '99.0.0', '--scheduled', '2300-10-01T14:30'], input='y'
-            )
-            assert 'Failed: another restart is already' in result.output
-            assert result.exit_code == 0
+        # get restart with the non-200 return code
+        # normal restart, the schedule is actually parsed, but not validated in patronictl
+        mock_post.return_value.status = 204
+        result = self.runner.invoke(ctl, ctl_args, input='y')
+        assert result.exit_code == 0
+
+        # get restart with the non-200 return code
+        # normal restart, the schedule is actually parsed, but not validated in patronictl
+        mock_post.return_value.status = 202
+        result = self.runner.invoke(ctl, ctl_args, input='y')
+        assert 'Success: restart scheduled' in result.output
+        assert result.exit_code == 0
+
+        # get restart with the non-200 return code
+        # normal restart, the schedule is actually parsed, but not validated in patronictl
+        mock_post.return_value.status = 409
+        result = self.runner.invoke(ctl, ctl_args, input='y')
+        assert 'Failed: another restart is already' in result.output
+        assert result.exit_code == 0
 
     @patch('patroni.ctl.get_dcs')
     def test_remove(self, mock_get_dcs):
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
-        result = self.runner.invoke(ctl, ['remove', 'alpha'], input='alpha\nslave')
+        result = self.runner.invoke(ctl, ['-k', 'remove', 'alpha'], input='alpha\nslave')
         assert 'Please confirm' in result.output
         assert 'You are about to remove all' in result.output
         # Not typing an exact confirmation
@@ -343,17 +344,6 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['remove', 'alpha'], input='alpha\nYes I am aware\nleader')
         assert result.exit_code == 0
 
-    @patch('requests.post', Mock(side_effect=requests.exceptions.ConnectionError('foo')))
-    @patch('click.get_current_context')
-    def test_request_patroni(self, mock_context):
-        member = get_cluster_initialized_with_leader().leader.member
-
-        mock_context.return_value.obj = {'ctl': {'cacert': 'cert.pem'}}
-        self.assertRaises(requests.exceptions.ConnectionError, request_patroni, member, 'post', 'dummy', {})
-
-        mock_context.return_value.obj = {'ctl': {'insecure': True}}
-        self.assertRaises(requests.exceptions.ConnectionError, request_patroni, member, 'post', 'dummy', {})
-
     def test_ctl(self):
         self.runner.invoke(ctl, ['list'])
 
@@ -427,7 +417,7 @@ class TestCtl(unittest.TestCase):
         assert 'Scheduled restart' in result.output
 
     @patch('patroni.ctl.get_dcs')
-    @patch('requests.delete', Mock(return_value=MockResponse()))
+    @patch.object(PoolManager, 'request', Mock(return_value=MockResponse()))
     def test_flush(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
@@ -437,58 +427,58 @@ class TestCtl(unittest.TestCase):
 
         result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '--force'])
         assert 'Success: flush scheduled restart' in result.output
-        with patch.object(requests, 'delete', return_value=MockResponse(404)):
+        with patch.object(PoolManager, 'request', return_value=MockResponse(404)):
             result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '--force'])
             assert 'Failed: flush scheduled restart' in result.output
 
+    @patch.object(PoolManager, 'request')
     @patch('patroni.ctl.get_dcs')
     @patch('patroni.ctl.polling_loop', Mock(return_value=[1]))
-    def test_pause_cluster(self, mock_get_dcs):
+    def test_pause_cluster(self, mock_get_dcs, mock_post):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
 
-        with patch('requests.patch', Mock(return_value=MockResponse(500))):
-            result = self.runner.invoke(ctl, ['pause', 'dummy'])
-            assert 'Failed' in result.output
+        mock_post.return_value.status = 500
+        result = self.runner.invoke(ctl, ['pause', 'dummy'])
+        assert 'Failed' in result.output
 
-        with patch('requests.patch', Mock(return_value=MockResponse(200))),\
-                patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+        mock_post.return_value.status = 200
+        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
             result = self.runner.invoke(ctl, ['pause', 'dummy'])
             assert 'Cluster is already paused' in result.output
 
-        with patch('requests.patch', Mock(return_value=MockResponse(200))):
-            result = self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
-            assert "'pause' request sent" in result.output
-            mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
-                                                                      get_cluster(None, None, [], None, None)])
-            self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
-            member = Member(1, 'other', 28, {})
-            mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
-                                                                      get_cluster(None, None, [member], None, None)])
-            self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
+        result = self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
+        assert "'pause' request sent" in result.output
+        mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
+                                                                  get_cluster(None, None, [], None, None)])
+        self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
+        member = Member(1, 'other', 28, {})
+        mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
+                                                                  get_cluster(None, None, [member], None, None)])
+        self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
 
+    @patch.object(PoolManager, 'request')
     @patch('patroni.ctl.get_dcs')
-    def test_resume_cluster(self, mock_get_dcs):
+    def test_resume_cluster(self, mock_get_dcs, mock_post):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
 
+        mock_post.return_value.status = 200
+        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=False)):
+            result = self.runner.invoke(ctl, ['resume', 'dummy'])
+            assert 'Cluster is not paused' in result.output
+
         with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
-            with patch('requests.patch', Mock(return_value=MockResponse(200))):
-                result = self.runner.invoke(ctl, ['resume', 'dummy'])
-                assert 'Success' in result.output
+            result = self.runner.invoke(ctl, ['resume', 'dummy'])
+            assert 'Success' in result.output
 
-            with patch('requests.patch', Mock(return_value=MockResponse(500))):
-                result = self.runner.invoke(ctl, ['resume', 'dummy'])
-                assert 'Failed' in result.output
+            mock_post.return_value.status = 500
+            result = self.runner.invoke(ctl, ['resume', 'dummy'])
+            assert 'Failed' in result.output
 
-            with patch('requests.patch', Mock(return_value=MockResponse(200))),\
-                    patch('patroni.dcs.Cluster.is_paused', Mock(return_value=False)):
-                result = self.runner.invoke(ctl, ['resume', 'dummy'])
-                assert 'Cluster is not paused' in result.output
-
-            with patch('requests.patch', Mock(side_effect=Exception)):
-                result = self.runner.invoke(ctl, ['resume', 'dummy'])
-                assert 'Can not find accessible cluster member' in result.output
+            mock_post.side_effect = Exception
+            result = self.runner.invoke(ctl, ['resume', 'dummy'])
+            assert 'Can not find accessible cluster member' in result.output
 
     def test_apply_config_changes(self):
         config = {"postgresql": {"parameters": {"work_mem": "4MB"}, "use_pg_rewind": True}, "ttl": 30}
@@ -535,9 +525,10 @@ class TestCtl(unittest.TestCase):
 
     @patch('subprocess.call', return_value=1)
     def test_invoke_editor(self, mock_subprocess_call):
-        for e in ('', 'false'):
-            os.environ['EDITOR'] = e
-            self.assertRaises(PatroniCtlException, invoke_editor, 'foo: bar\n', 'test')
+        os.environ.pop('EDITOR', None)
+        for e in ('', '/bin/vi'):
+            with patch('patroni.ctl.find_executable', Mock(return_value=e)):
+                self.assertRaises(PatroniCtlException, invoke_editor, 'foo: bar\n', 'test')
 
     @patch('patroni.ctl.get_dcs')
     def test_show_config(self, mock_get_dcs):
@@ -563,16 +554,39 @@ class TestCtl(unittest.TestCase):
     def test_version(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
-        with patch('patroni.ctl.request_patroni') as mocked:
+        with patch.object(PoolManager, 'request') as mocked:
             result = self.runner.invoke(ctl, ['version'])
             assert 'patronictl version' in result.output
-            mocked.return_value.json = lambda: {'patroni': {'version': '1.2.3'}, 'server_version': 100001}
+            mocked.return_value.data = b'{"patroni":{"version":"1.2.3"},"server_version": 100001}'
             result = self.runner.invoke(ctl, ['version', 'dummy'])
             assert '1.2.3' in result.output
-        with patch('requests.get', Mock(side_effect=Exception)):
+        with patch.object(PoolManager, 'request', Mock(side_effect=Exception)):
             result = self.runner.invoke(ctl, ['version', 'dummy'])
             assert 'failed to get version' in result.output
 
+    @patch('patroni.ctl.get_dcs')
+    def test_history(self, mock_get_dcs):
+        mock_get_dcs.return_value.get_cluster = Mock()
+        mock_get_dcs.return_value.get_cluster.return_value.history.lines = [[1, 67176, 'no recovery target specified']]
+        result = self.runner.invoke(ctl, ['history'])
+        assert 'Reason' in result.output
+
     def test_format_pg_version(self):
         self.assertEqual(format_pg_version(100001), '10.1')
         self.assertEqual(format_pg_version(90605), '9.6.5')
+
+    @patch('sys.platform', 'win32')
+    def test_find_executable(self):
+        with patch('os.path.isfile', Mock(return_value=True)):
+            self.assertEqual(find_executable('vim'), 'vim.exe')
+        with patch('os.path.isfile', Mock(return_value=False)):
+            self.assertIsNone(find_executable('vim'))
+        with patch('os.path.isfile', Mock(side_effect=[False, True])):
+            self.assertEqual(find_executable('vim', '/'), '/vim.exe')
+
+    @patch('patroni.ctl.get_dcs')
+    def test_get_members(self, mock_get_dcs):
+        mock_get_dcs.return_value = self.e
+        mock_get_dcs.return_value.get_cluster = get_cluster_not_initialized_without_leader
+        result = self.runner.invoke(ctl, ['reinit', 'dummy'])
+        assert "cluster doesn\'t have any members" in result.output

tests/test_etcd.py

@@ -1,7 +1,5 @@
 import etcd
-import json
 import urllib3.util.connection
-import requests
 import socket
 import unittest
 
@@ -9,58 +7,10 @@ from dns.exception import DNSException
 from mock import Mock, patch
 from patroni.dcs.etcd import AbstractDCS, Client, Cluster, Etcd, EtcdError, DnsCachingResolver
 from patroni.exceptions import DCSError
+from patroni.utils import Retry
 from urllib3.exceptions import ReadTimeoutError
 
-
-class MockResponse(object):
-
-    def __init__(self, status_code=200):
-        self.status_code = status_code
-        self.content = '{}'
-        self.ok = True
-
-    def json(self):
-        return json.loads(self.content)
-
-    @property
-    def data(self):
-        return self.content.encode('utf-8')
-
-    @property
-    def text(self):
-        return self.content
-
-    @property
-    def status(self):
-        return self.status_code
-
-    @staticmethod
-    def getheader(*args):
-        return ''
-
-
-def requests_get(url, **kwargs):
-    members = '[{"id":14855829450254237642,"peerURLs":["http://localhost:2380","http://localhost:7001"],' +\
-              '"name":"default","clientURLs":["http://localhost:2379","http://localhost:4001"]}]'
-    response = MockResponse()
-    if url.startswith('http://local'):
-        raise requests.exceptions.RequestException()
-    elif ':8011/patroni' in url:
-        response.content = '{"role": "replica", "xlog": {"received_location": 0}, "tags": {}}'
-    elif url.endswith('/members'):
-        response.content = '[{}]' if url.startswith('http://error') else members
-    elif url.startswith('http://exhibitor'):
-        response.content = '{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}'
-    elif url.endswith(':8011/reinitialize'):
-        data = kwargs.get('data', '')
-        if ' false}' in data:
-            response.status_code = 503
-            response.ok = False
-            response.content = 'restarting after failure already in progress'
-    else:
-        response.status_code = 404
-        response.ok = False
-    return response
+from . import SleepException, MockResponse, requests_get
 
 
 def etcd_watch(self, key, index=None, timeout=None, recursive=None):
@@ -122,10 +72,6 @@ def etcd_read(self, key, **kwargs):
     return result
 
 
-class SleepException(Exception):
-    pass
-
-
 def dns_query(name, _):
     if '-server' not in name or '-ssl' in name:
         return []
@@ -170,12 +116,12 @@ class TestDnsCachingResolver(unittest.TestCase):
 
 @patch('dns.resolver.query', dns_query)
 @patch('socket.getaddrinfo', socket_getaddrinfo)
-@patch('requests.get', requests_get)
+@patch('patroni.dcs.etcd.requests_get', requests_get)
 class TestClient(unittest.TestCase):
 
     @patch('dns.resolver.query', dns_query)
     @patch('socket.getaddrinfo', socket_getaddrinfo)
-    @patch('requests.get', requests_get)
+    @patch('patroni.dcs.etcd.requests_get', requests_get)
     def setUp(self):
         with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://localhost:2379', 'http://localhost:4001'])
@@ -204,7 +150,11 @@ class TestClient(unittest.TestCase):
         self.assertRaises(ValueError, self.client.api_execute, '', '')
         self.client._base_uri = 'http://localhost:4001'
         self.client._machines_cache = ['http://localhost:2379']
-        self.client.api_execute('/', 'POST', timeout=0)
+        self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'POST', timeout=0)
+        self.client._base_uri = 'http://localhost:4001'
+        self.client._machines_cache = ['http://localhost:2379']
+        rtry = Retry(deadline=10, max_delay=1, max_tries=-1, retry_exceptions=(etcd.EtcdLeaderElectionInProgress,))
+        rtry(self.client.api_execute, '/', 'POST', timeout=0, params={'retry': rtry})
         mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
         self.client._machines_cache_updated = 0
         self.client.api_execute('/', 'POST', timeout=0)
@@ -212,7 +162,11 @@ class TestClient(unittest.TestCase):
         self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
         self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
         self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '')
-        with patch.object(Client, '_load_machines_cache', Mock(side_effect=etcd.EtcdException)):
+
+        with patch.object(Client, '_do_http_request', Mock(side_effect=etcd.EtcdConnectionFailed)):
+            with patch.object(Client, '_calculate_timeouts', Mock(side_effect=[(1, 1, 0), (1, 1, 0), (0, 1, 0)])):
+                self.assertRaises(etcd.EtcdException, rtry, self.client.api_execute, '/', 'GET', params={'retry': rtry})
+            self.client._read_timeout = 0
             self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'GET')
 
     def test_get_srv_record(self):
@@ -245,7 +199,7 @@ class TestClient(unittest.TestCase):
                           socket_options=[(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)])
 
 
-@patch('requests.get', requests_get)
+@patch('patroni.dcs.etcd.requests_get', requests_get)
 @patch('socket.getaddrinfo', socket_getaddrinfo)
 @patch.object(etcd.Client, 'write', etcd_write)
 @patch.object(etcd.Client, 'read', etcd_read)

tests/test_exhibitor.py

@@ -1,13 +1,15 @@
 import unittest
+import urllib3
 
 from mock import Mock, patch
 from patroni.dcs.exhibitor import ExhibitorEnsembleProvider, Exhibitor
 from patroni.dcs.zookeeper import ZooKeeperError
-from test_etcd import SleepException, requests_get
-from test_zookeeper import MockKazooClient
+
+from . import SleepException, requests_get
+from .test_zookeeper import MockKazooClient
 
 
-@patch('requests.get', requests_get)
+@patch('patroni.dcs.exhibitor.requests_get', requests_get)
 @patch('time.sleep', Mock(side_effect=SleepException))
 class TestExhibitorEnsembleProvider(unittest.TestCase):
 
@@ -20,7 +22,8 @@ class TestExhibitorEnsembleProvider(unittest.TestCase):
 
 class TestExhibitor(unittest.TestCase):
 
-    @patch('requests.get', requests_get)
+    @patch('urllib3.PoolManager.request', Mock(return_value=urllib3.HTTPResponse(
+        status=200, body=b'{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}')))
     @patch('patroni.dcs.zookeeper.KazooClient', MockKazooClient)
     def setUp(self):
         self.e = Exhibitor({'hosts': ['localhost', 'exhibitor'], 'port': 8181, 'scope': 'test',

tests/test_ha.py

@@ -1,7 +1,6 @@
 import datetime
 import etcd
 import os
-import unittest
 import sys
 
 from mock import Mock, MagicMock, PropertyMock, patch
@@ -11,10 +10,16 @@ from patroni.dcs.etcd import Client
 from patroni.exceptions import DCSError, PostgresConnectionException, PatroniException
 from patroni.ha import Ha, _MemberStatus
 from patroni.postgresql import Postgresql
-from patroni.watchdog import Watchdog
+from patroni.postgresql.bootstrap import Bootstrap
+from patroni.postgresql.cancellable import CancellableSubprocess
+from patroni.postgresql.config import ConfigHandler
+from patroni.postgresql.rewind import Rewind
+from patroni.postgresql.slots import SlotsHandler
 from patroni.utils import tzutc
-from test_etcd import socket_getaddrinfo, etcd_read, etcd_write, requests_get
-from test_postgresql import psycopg2_connect, MockPostmaster
+from patroni.watchdog import Watchdog
+
+from . import PostgresInit, MockPostmaster, psycopg2_connect, requests_get
+from .test_etcd import socket_getaddrinfo, etcd_read, etcd_write
 
 SYSID = '12345678901'
 
@@ -117,8 +122,9 @@ zookeeper:
         # all the extra values that are coming from py.test
         sys.argv = sys.argv[:1]
 
-        self.config = Config()
+        self.config = Config(None)
         self.config.set_dynamic_configuration({'maximum_lag_on_failover': 5})
+        self.version = '1.5.7'
         self.postgresql = p
         self.dcs = d
         self.api = Mock()
@@ -131,10 +137,15 @@ zookeeper:
         self.scheduled_restart = {'schedule': future_restart_time,
                                   'postmaster_start_time': str(postmaster_start_time)}
         self.watchdog = Watchdog(self.config)
+        self.request = lambda member, **kwargs: requests_get(member.api_url, **kwargs)
 
 
 def run_async(self, func, args=()):
-    return func(*args) if args else func()
+    self.reset_scheduled_action()
+    if args:
+        func(*args)
+    else:
+        func()
 
 
 @patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
@@ -144,16 +155,17 @@ def run_async(self, func, args=()):
 @patch.object(Postgresql, 'call_nowait', Mock(return_value=True))
 @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
 @patch.object(Postgresql, 'controldata', Mock(return_value={'Database system identifier': SYSID}))
-@patch.object(Postgresql, 'sync_replication_slots', Mock())
-@patch.object(Postgresql, 'write_pg_hba', Mock())
-@patch.object(Postgresql, 'write_pgpass', Mock(return_value={}))
-@patch.object(Postgresql, 'write_recovery_conf', Mock())
+@patch.object(SlotsHandler, 'sync_replication_slots', Mock())
+@patch.object(ConfigHandler, 'append_pg_hba', Mock())
+@patch.object(ConfigHandler, 'write_pgpass', Mock(return_value={}))
+@patch.object(ConfigHandler, 'write_recovery_conf', Mock())
+@patch.object(ConfigHandler, 'write_postgresql_conf', Mock())
 @patch.object(Postgresql, 'query', Mock())
 @patch.object(Postgresql, 'checkpoint', Mock())
-@patch.object(Postgresql, 'cancellable_subprocess_call', Mock(return_value=0))
-@patch.object(Postgresql, '_get_local_timeline_lsn_from_replication_connection', Mock(return_value=[2, 10]))
+@patch.object(CancellableSubprocess, 'call', Mock(return_value=0))
+@patch.object(Postgresql, 'get_local_timeline_lsn_from_replication_connection', Mock(return_value=[2, 10]))
 @patch.object(Postgresql, 'get_master_timeline', Mock(return_value=2))
-@patch.object(Postgresql, 'restore_configuration_files', Mock())
+@patch.object(ConfigHandler, 'restore_configuration_files', Mock())
 @patch.object(etcd.Client, 'write', etcd_write)
 @patch.object(etcd.Client, 'read', etcd_read)
 @patch.object(etcd.Client, 'delete', Mock(side_effect=etcd.EtcdException))
@@ -162,21 +174,15 @@ def run_async(self, func, args=()):
 @patch('patroni.async_executor.AsyncExecutor.run_async', run_async)
 @patch('subprocess.call', Mock(return_value=0))
 @patch('time.sleep', Mock())
-class TestHa(unittest.TestCase):
+class TestHa(PostgresInit):
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
-    @patch('psycopg2.connect', psycopg2_connect)
-    @patch('patroni.dcs.dcs_modules', Mock(return_value=['patroni.dcs.foo', 'patroni.dcs.etcd']))
+    @patch('patroni.dcs.dcs_modules', Mock(return_value=['patroni.dcs.etcd']))
     @patch.object(etcd.Client, 'read', etcd_read)
     def setUp(self):
+        super(TestHa, self).setUp()
         with patch.object(Client, 'machines') as mock_machines:
             mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
-            self.p = Postgresql({'name': 'postgresql0', 'scope': 'dummy', 'listen': '127.0.0.1:5432',
-                                 'data_dir': 'data/postgresql0', 'retry_timeout': 10,
-                                 'authentication': {'superuser': {'username': 'foo', 'password': 'bar'},
-                                                    'replication': {'username': '', 'password': ''}},
-                                 'parameters': {'wal_level': 'hot_standby', 'max_replication_slots': 5, 'foo': 'bar',
-                                                'hot_standby': 'on', 'max_wal_senders': 5, 'wal_keep_segments': 8}})
             self.p.set_state('running')
             self.p.set_role('replica')
             self.p.postmaster_start_time = MagicMock(return_value=str(postmaster_start_time))
@@ -205,17 +211,23 @@ class TestHa(unittest.TestCase):
         self.assertEqual(self.ha.run_cycle(), 'starting as a secondary')
 
     @patch('patroni.dcs.etcd.Etcd.initialize', return_value=True)
-    def test_start_as_standby_leader(self, initialize):
+    def test_bootstrap_as_standby_leader(self, initialize):
         self.p.data_directory_empty = true
         self.ha.cluster = get_cluster_not_initialized_without_leader(cluster_config=ClusterConfig(0, {}, 0))
         self.ha.cluster.is_unlocked = true
         self.ha.patroni.config._dynamic_configuration = {"standby_cluster": {"port": 5432}}
         self.assertEqual(self.ha.run_cycle(), 'trying to bootstrap a new standby leader')
 
+    def test_bootstrap_waiting_for_standby_leader(self):
+        self.p.data_directory_empty = true
+        self.ha.cluster = get_cluster_initialized_without_leader()
+        self.ha.cluster.config.data.update({'standby_cluster': {'port': 5432}})
+        self.assertEqual(self.ha.run_cycle(), 'waiting for standby_leader to bootstrap')
+
     @patch.object(Cluster, 'get_clone_member',
                   Mock(return_value=Member(0, 'test', 1, {'api_url': 'http://127.0.0.1:8011/patroni',
                                                           'conn_url': 'postgres://127.0.0.1:5432/postgres'})))
-    @patch.object(Postgresql, 'create_replica', Mock(return_value=0))
+    @patch.object(Bootstrap, 'create_replica', Mock(return_value=0))
     def test_start_as_cascade_replica_in_standby_cluster(self):
         self.p.data_directory_empty = true
         self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
@@ -244,15 +256,15 @@ class TestHa(unittest.TestCase):
         self.p.controldata = lambda: {'Database cluster state': 'in production', 'Database system identifier': SYSID}
         self.assertEqual(self.ha.run_cycle(), 'doing crash recovery in a single user mode')
 
-    @patch.object(Postgresql, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
-    @patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
+    @patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
+    @patch.object(Rewind, 'can_rewind', PropertyMock(return_value=True))
     def test_recover_with_rewind(self):
         self.p.is_running = false
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.assertEqual(self.ha.run_cycle(), 'running pg_rewind from leader')
 
-    @patch.object(Postgresql, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
-    @patch.object(Postgresql, 'create_replica', Mock(return_value=1))
+    @patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
+    @patch.object(Bootstrap, 'create_replica', Mock(return_value=1))
     def test_recover_with_reinitialize(self):
         self.p.is_running = false
         self.ha.cluster = get_cluster_initialized_with_leader()
@@ -347,6 +359,7 @@ class TestHa(unittest.TestCase):
         self.p.is_leader = false
         self.assertEqual(self.ha.run_cycle(), 'no action.  i am a secondary and i am following a leader')
         self.ha.patroni.replicatefrom = "foo"
+        self.p.config.check_recovery_conf = Mock(return_value=(True, False))
         self.assertEqual(self.ha.run_cycle(), 'no action.  i am a secondary and i am following a leader')
 
     def test_follow_in_pause(self):
@@ -356,11 +369,11 @@ class TestHa(unittest.TestCase):
         self.p.is_leader = false
         self.assertEqual(self.ha.run_cycle(), 'PAUSE: no action')
 
-    @patch.object(Postgresql, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
-    @patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
+    @patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
+    @patch.object(Rewind, 'can_rewind', PropertyMock(return_value=True))
     def test_follow_triggers_rewind(self):
         self.p.is_leader = false
-        self.p.trigger_check_diverged_lsn()
+        self.ha._rewind.trigger_check_diverged_lsn()
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.assertEqual(self.ha.run_cycle(), 'running pg_rewind from leader')
 
@@ -419,7 +432,7 @@ class TestHa(unittest.TestCase):
 
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.assertIsNone(self.ha.reinitialize(True))
-
+        self.ha._async_executor.schedule('reinitialize')
         self.assertIsNotNone(self.ha.reinitialize())
 
         self.ha.state_handler.name = self.ha.cluster.leader.name
@@ -433,7 +446,7 @@ class TestHa(unittest.TestCase):
         self.p.restart = false
         self.assertEqual(self.ha.restart({}), (False, 'restart failed'))
         self.ha.cluster = get_cluster_initialized_with_leader()
-        self.ha.reinitialize()
+        self.ha._async_executor.schedule('reinitialize')
         self.assertEqual(self.ha.restart({}), (False, 'reinitialize already in progress'))
         with patch.object(self.ha, "restart_matches", return_value=False):
             self.assertEqual(self.ha.restart({'foo': 'bar'}), (False, "restart conditions are not satisfied"))
@@ -441,7 +454,7 @@ class TestHa(unittest.TestCase):
     @patch('os.kill', Mock())
     def test_restart_in_progress(self):
         with patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=True)):
-            self.ha.restart({}, run_async=True)
+            self.ha._async_executor.schedule('restart')
             self.assertTrue(self.ha.restart_scheduled())
             self.assertEqual(self.ha.run_cycle(), 'restart in progress')
 
@@ -458,7 +471,6 @@ class TestHa(unittest.TestCase):
                     self.assertEqual(self.ha.run_cycle(), 'lost leader lock during restart')
                     mock_terminate.assert_called()
 
-    @patch('requests.get', requests_get)
     def test_manual_failover_from_leader(self):
         self.ha.fetch_node_status = get_node_status()
         self.ha.has_lock = true
@@ -471,7 +483,7 @@ class TestHa(unittest.TestCase):
         f = Failover(0, self.p.name, '', None)
         self.ha.cluster = get_cluster_initialized_with_leader(f)
         self.assertEqual(self.ha.run_cycle(), 'manual failover: demoting myself')
-        self.p.rewind_or_reinitialize_needed_and_possible = true
+        self.ha._rewind.rewind_or_reinitialize_needed_and_possible = true
         self.assertEqual(self.ha.run_cycle(), 'manual failover: demoting myself')
         self.ha.fetch_node_status = get_node_status(nofailover=True)
         self.assertEqual(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
@@ -506,7 +518,6 @@ class TestHa(unittest.TestCase):
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
         self.assertEqual('no action.  i am the leader with the lock', self.ha.run_cycle())
 
-    @patch('requests.get', requests_get)
     def test_manual_failover_from_leader_in_pause(self):
         self.ha.has_lock = true
         self.ha.is_paused = true
@@ -516,7 +527,6 @@ class TestHa(unittest.TestCase):
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, '', None))
         self.assertEqual('PAUSE: no action.  i am the leader with the lock', self.ha.run_cycle())
 
-    @patch('requests.get', requests_get)
     def test_manual_failover_from_leader_in_synchronous_mode(self):
         self.p.is_leader = true
         self.ha.has_lock = true
@@ -529,7 +539,6 @@ class TestHa(unittest.TestCase):
         self.ha.is_failover_possible = true
         self.assertEqual('manual failover: demoting myself', self.ha.run_cycle())
 
-    @patch('requests.get', requests_get)
     def test_manual_failover_process_no_leader(self):
         self.p.is_leader = false
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', self.p.name, None))
@@ -579,8 +588,8 @@ class TestHa(unittest.TestCase):
         self.ha.is_paused = true
         self.assertFalse(self.ha.is_healthiest_node())
 
-    @patch('requests.get', requests_get)
     def test__is_healthiest_node(self):
+        self.ha.cluster = get_cluster_initialized_without_leader(sync=('postgresql1', self.p.name))
         self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.p.is_leader = false
         self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
@@ -589,6 +598,9 @@ class TestHa(unittest.TestCase):
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.fetch_node_status = get_node_status(wal_position=11)  # accessible, in_recovery, wal position ahead
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
+        # in synchronous_mode consider itself healthy if the former leader is accessible in read-only and ahead of us
+        with patch.object(Ha, 'is_synchronous_mode', Mock(return_value=True)):
+            self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         with patch('patroni.postgresql.Postgresql.timeline_wal_position', return_value=(1, 1)):
             self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         with patch('patroni.postgresql.Postgresql.replica_cached_timeline', return_value=1):
@@ -597,7 +609,6 @@ class TestHa(unittest.TestCase):
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.patroni.nofailover = False
 
-    @patch('requests.get', requests_get)
     def test_fetch_node_status(self):
         member = Member(0, 'test', 1, {'api_url': 'http://127.0.0.1:8011/patroni'})
         self.ha.fetch_node_status(member)
@@ -653,7 +664,7 @@ class TestHa(unittest.TestCase):
 
     def test_restart_matches(self):
         self.p._role = 'replica'
-        self.p.server_version = 90500
+        self.p._connection.server_version = 90500
         self.p._pending_restart = True
         self.assertFalse(self.ha.restart_matches("master", "9.5.0", True))
         self.assertFalse(self.ha.restart_matches("replica", "9.4.3", True))
@@ -674,7 +685,7 @@ class TestHa(unittest.TestCase):
         self.p.is_leader = false
         self.p.name = 'leader'
         self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
-        self.p.check_recovery_conf = true
+        self.p.config.check_recovery_conf = Mock(return_value=(False, False))
         self.assertEqual(self.ha.run_cycle(), 'promoted self to a standby leader because i had the session lock')
         self.assertEqual(self.ha.run_cycle(), 'no action.  i am the standby leader with the lock')
 
@@ -686,7 +697,6 @@ class TestHa(unittest.TestCase):
         with patch.object(Leader, 'conn_url', PropertyMock(return_value='')):
             self.assertEqual(self.ha.run_cycle(), 'continue following the old known standby leader')
 
-    @patch('requests.get', requests_get)
     def test_process_unhealthy_standby_cluster_as_standby_leader(self):
         self.p.is_leader = false
         self.p.name = 'leader'
@@ -696,8 +706,8 @@ class TestHa(unittest.TestCase):
         self.p._sysid = True
         self.assertEqual(self.ha.run_cycle(), 'promoted self to a standby leader by acquiring session lock')
 
-    @patch.object(Postgresql, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
-    @patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
+    @patch.object(Rewind, 'rewind_or_reinitialize_needed_and_possible', Mock(return_value=True))
+    @patch.object(Rewind, 'can_rewind', PropertyMock(return_value=True))
     def test_process_unhealthy_standby_cluster_as_cascade_replica(self):
         self.p.is_leader = false
         self.p.name = 'replica'
@@ -808,7 +818,7 @@ class TestHa(unittest.TestCase):
 
     def test_process_sync_replication(self):
         self.ha.has_lock = true
-        mock_set_sync = self.p.set_synchronous_standby = Mock()
+        mock_set_sync = self.p.config.set_synchronous_standby = Mock()
         self.p.name = 'leader'
 
         # Test sync key removed when sync mode disabled
@@ -885,7 +895,7 @@ class TestHa(unittest.TestCase):
     def test_sync_replication_become_master(self):
         self.ha.is_synchronous_mode = true
 
-        mock_set_sync = self.p.set_synchronous_standby = Mock()
+        mock_set_sync = self.p.config.set_synchronous_standby = Mock()
         self.p.is_leader = false
         self.p.set_role('replica')
         self.ha.has_lock = true
@@ -1017,7 +1027,6 @@ class TestHa(unittest.TestCase):
             self.assertEqual(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
 
     @patch('sys.exit', return_value=1)
-    @patch('requests.get', requests_get)
     def test_abort_join(self, exit_mock):
         self.ha.cluster = get_cluster_not_initialized_without_leader()
         self.p.is_leader = false

tests/test_kubernetes.py

@@ -1,8 +1,11 @@
+import json
 import time
 import unittest
 
 from mock import Mock, patch
-from patroni.dcs.kubernetes import Kubernetes, KubernetesError, k8s_client, k8s_watch, RetryFailedError
+from patroni.dcs.kubernetes import Kubernetes, KubernetesError, k8s_client, RetryFailedError
+from threading import Thread
+from . import SleepException
 
 
 def mock_list_namespaced_config_map(self, *args, **kwargs):
@@ -16,33 +19,53 @@ def mock_list_namespaced_config_map(self, *args, **kwargs):
     metadata.update({'name': 'test-sync', 'annotations': {'leader': 'p-0'}})
     items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
     metadata = k8s_client.V1ObjectMeta(resource_version='1')
-    return k8s_client.V1ConfigMapList(metadata=metadata, items=items)
+    return k8s_client.V1ConfigMapList(metadata=metadata, items=items, kind='ConfigMapList')
 
 
 def mock_list_namespaced_pod(self, *args, **kwargs):
     metadata = k8s_client.V1ObjectMeta(resource_version='1', name='p-0', annotations={'status': '{}'})
     items = [k8s_client.V1Pod(metadata=metadata)]
-    return k8s_client.V1PodList(items=items)
+    return k8s_client.V1PodList(items=items, kind='PodList')
 
 
-@patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', Mock())
-@patch.object(k8s_client.CoreV1Api, 'create_namespaced_config_map', Mock())
+def mock_config_map(*args, **kwargs):
+    mock = Mock()
+    mock.metadata.resource_version = '2'
+    return mock
+
+
+@patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', mock_config_map)
+@patch.object(k8s_client.CoreV1Api, 'create_namespaced_config_map', mock_config_map)
+@patch('kubernetes.client.api_client.ThreadPool', Mock(), create=True)
+@patch.object(Thread, 'start', Mock())
 class TestKubernetes(unittest.TestCase):
 
     @patch('kubernetes.config.load_kube_config', Mock())
     @patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map)
     @patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', mock_list_namespaced_pod)
+    @patch('kubernetes.client.api_client.ThreadPool', Mock(), create=True)
+    @patch.object(Thread, 'start', Mock())
     def setUp(self):
         self.k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10, 'labels': {'f': 'b'}})
+        self.assertRaises(AttributeError, self.k._pods._build_cache)
+        self.k._pods._is_ready = True
+        self.assertRaises(AttributeError, self.k._kinds._build_cache)
+        self.k._kinds._is_ready = True
         self.k.get_cluster()
 
+    @patch('time.time', Mock(side_effect=[1, 10.9, 100]))
+    def test__wait_caches(self):
+        self.k._pods._is_ready = False
+        with self.k._condition:
+            self.assertRaises(RetryFailedError, self.k._wait_caches)
+
     def test_get_cluster(self):
         with patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map), \
                 patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', mock_list_namespaced_pod), \
                 patch('time.time', Mock(return_value=time.time() + 31)):
             self.k.get_cluster()
 
-        with patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', Mock(side_effect=Exception)):
+        with patch.object(Kubernetes, '_wait_caches', Mock(side_effect=Exception)):
             self.assertRaises(KubernetesError, self.k.get_cluster)
 
     @patch('kubernetes.config.load_kube_config', Mock())
@@ -105,17 +128,12 @@ class TestKubernetes(unittest.TestCase):
         self.k.set_ttl(10)
         self.k.watch(None, 0)
         self.k.watch(None, 0)
-        with patch.object(k8s_watch.Watch, 'stream',
-                          Mock(side_effect=[Exception, [], KeyboardInterrupt,
-                                            [{'raw_object': {'metadata': {'resourceVersion': '2'}}}]])):
-            self.assertFalse(self.k.watch('1', 2))
-            self.assertRaises(KeyboardInterrupt, self.k.watch, '1', 2)
-            self.assertTrue(self.k.watch('1', 2))
 
     def test_set_history_value(self):
         self.k.set_history_value('{}')
 
     @patch('kubernetes.config.load_kube_config', Mock())
+    @patch('patroni.dcs.kubernetes.ObjectCache', Mock())
     @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_pod', Mock(return_value=True))
     @patch.object(k8s_client.CoreV1Api, 'create_namespaced_endpoints', Mock())
     @patch.object(k8s_client.CoreV1Api, 'create_namespaced_service',
@@ -126,3 +144,31 @@ class TestKubernetes(unittest.TestCase):
         self.assertIsNotNone(k.patch_or_create_config({'foo': 'bar'}))
         self.assertIsNotNone(k.patch_or_create_config({'foo': 'bar'}))
         k.touch_member({'state': 'running', 'role': 'replica'})
+
+
+class TestCacheBuilder(unittest.TestCase):
+
+    @patch('kubernetes.config.load_kube_config', Mock())
+    @patch('kubernetes.client.api_client.ThreadPool', Mock(), create=True)
+    @patch.object(Thread, 'start', Mock())
+    def setUp(self):
+        self.k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10, 'labels': {'f': 'b'}})
+
+    @patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map)
+    @patch('patroni.dcs.kubernetes.ObjectCache._watch')
+    def test__build_cache(self, mock_response):
+        mock_response.return_value.read_chunked.return_value = [json.dumps(
+            {'type': 'MODIFIED', 'object': {'metadata': {
+                'name': self.k.config_path, 'resourceVersion': '2', 'annotations': {self.k._CONFIG: 'foo'}}}}
+        ).encode('utf-8'), ('\n' + json.dumps(
+            {'type': 'DELETED', 'object': {'metadata': {
+                'name': self.k.config_path, 'resourceVersion': '3'}}}
+        ) + '\n' + json.dumps(
+            {'type': 'MDIFIED', 'object': {'metadata': {'name': self.k.config_path}}}
+        ) + '\n' + json.dumps({'object': {'code': 410}}) + '\n').encode('utf-8')]
+        self.k._kinds._build_cache()
+
+    @patch('patroni.dcs.kubernetes.logger.error', Mock(side_effect=SleepException))
+    @patch('patroni.dcs.kubernetes.ObjectCache._build_cache', Mock(side_effect=Exception))
+    def test_run(self):
+        self.assertRaises(SleepException, self.k._pods.run)

tests/test_log.py

@@ -1,3 +1,4 @@
+import logging
 import os
 import sys
 import unittest
@@ -6,14 +7,25 @@ import yaml
 from mock import Mock, patch
 from patroni.config import Config
 from patroni.log import PatroniLogger
+from six.moves.queue import Queue, Full
+
+_LOG = logging.getLogger(__name__)
 
 
 class TestPatroniLogger(unittest.TestCase):
 
+    def setUp(self):
+        self._handlers = logging.getLogger().handlers[:]
+
+    def tearDown(self):
+        logging.getLogger().handlers[:] = self._handlers
+
     @patch('logging.FileHandler._open', Mock())
     def test_patroni_logger(self):
         config = {
             'log': {
+                'traceback_level': 'DEBUG',
+                'max_queue_size': 5,
                 'dir': 'foo',
                 'file_size': 4096,
                 'file_num': 5,
@@ -26,11 +38,27 @@ class TestPatroniLogger(unittest.TestCase):
         sys.argv = ['patroni.py']
         os.environ[Config.PATRONI_CONFIG_VARIABLE] = yaml.dump(config, default_flow_style=False)
         logger = PatroniLogger()
-        patroni_config = Config()
+        patroni_config = Config(None)
         logger.reload_config(patroni_config['log'])
+        _LOG.exception('test')
+        logger.start()
 
-        self.assertEqual(logger.handler.maxBytes, config['log']['file_size'])
-        self.assertEqual(logger.handler.backupCount, config['log']['file_num'])
+        with patch.object(logging.Handler, 'format', Mock(side_effect=Exception)):
+            logging.error('test')
 
+        self.assertEqual(logger.log_handler.maxBytes, config['log']['file_size'])
+        self.assertEqual(logger.log_handler.backupCount, config['log']['file_num'])
+
+        config['log']['level'] = 'DEBUG'
         config['log'].pop('dir')
-        logger.reload_config(config['log'])
+        with patch('logging.Handler.close', Mock(side_effect=Exception)):
+            logger.reload_config(config['log'])
+            with patch.object(logging.Logger, 'makeRecord',
+                              Mock(side_effect=[logging.LogRecord('', logging.INFO, '', 0, '', (), None), Exception])):
+                logging.exception('test')
+            logging.error('test')
+            with patch.object(Queue, 'put_nowait', Mock(side_effect=Full)):
+                self.assertRaises(SystemExit, logger.shutdown)
+            self.assertRaises(Exception, logger.shutdown)
+        self.assertLessEqual(logger.queue_size, 2)  # "Failed to close the old log handler" could be still in the queue
+        self.assertEqual(logger.records_lost, 0)

tests/test_patroni.py

@@ -1,18 +1,25 @@
 import etcd
+import logging
+import os
 import signal
-import sys
 import time
 import unittest
 
+import patroni.config as config
 from mock import Mock, PropertyMock, patch
 from patroni.api import RestApiServer
 from patroni.async_executor import AsyncExecutor
 from patroni.dcs.etcd import Client
 from patroni.exceptions import DCSError
-from patroni import Patroni, main as _main, patroni_main
-from six.moves import BaseHTTPServer
-from test_etcd import SleepException, etcd_read, etcd_write
-from test_postgresql import Postgresql, psycopg2_connect, MockPostmaster
+from patroni.postgresql import Postgresql
+from patroni.postgresql.config import ConfigHandler
+from patroni import Patroni, main as _main, patroni_main, check_psycopg2
+from six.moves import BaseHTTPServer, builtins
+from threading import Thread
+
+from . import psycopg2_connect, SleepException
+from .test_etcd import etcd_read, etcd_write
+from .test_postgresql import MockPostmaster
 
 
 class MockFrozenImporter(object):
@@ -23,9 +30,9 @@ class MockFrozenImporter(object):
 @patch('time.sleep', Mock())
 @patch('subprocess.call', Mock(return_value=0))
 @patch('psycopg2.connect', psycopg2_connect)
-@patch.object(Postgresql, 'write_pg_hba', Mock())
-@patch.object(Postgresql, '_write_postgresql_conf', Mock())
-@patch.object(Postgresql, 'write_recovery_conf', Mock())
+@patch.object(ConfigHandler, 'append_pg_hba', Mock())
+@patch.object(ConfigHandler, 'write_postgresql_conf', Mock())
+@patch.object(ConfigHandler, 'write_recovery_conf', Mock())
 @patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
 @patch.object(Postgresql, 'call_nowait', Mock())
 @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
@@ -34,17 +41,26 @@ class MockFrozenImporter(object):
 @patch.object(etcd.Client, 'read', etcd_read)
 class TestPatroni(unittest.TestCase):
 
+    def test_no_config(self):
+        self.assertRaises(SystemExit, patroni_main)
+
     @patch('pkgutil.get_importer', Mock(return_value=MockFrozenImporter()))
     @patch('sys.frozen', Mock(return_value=True), create=True)
+    @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
     @patch.object(etcd.Client, 'read', etcd_read)
+    @patch.object(Thread, 'start', Mock())
+    @patch.object(Client, 'machines', PropertyMock(return_value=['http://remotehost:2379']))
     def setUp(self):
+        self._handlers = logging.getLogger().handlers[:]
         RestApiServer._BaseServer__is_shut_down = Mock()
         RestApiServer._BaseServer__shutdown_request = True
         RestApiServer.socket = 0
-        with patch.object(Client, 'machines') as mock_machines:
-            mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
-            sys.argv = ['patroni.py', 'postgres0.yml']
-            self.p = Patroni()
+        os.environ['PATRONI_POSTGRESQL_DATA_DIR'] = 'data/test0'
+        conf = config.Config('postgres0.yml')
+        self.p = Patroni(conf)
+
+    def tearDown(self):
+        logging.getLogger().handlers[:] = self._handlers
 
     @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(side_effect=[None, DCSError('foo'), None]))
     def test_load_dynamic_configuration(self):
@@ -52,18 +68,19 @@ class TestPatroni(unittest.TestCase):
         self.p.load_dynamic_configuration()
         self.p.load_dynamic_configuration()
 
+    @patch('sys.argv', ['patroni.py', 'postgres0.yml'])
     @patch('time.sleep', Mock(side_effect=SleepException))
     @patch.object(etcd.Client, 'delete', Mock())
-    @patch.object(Client, 'machines')
-    def test_patroni_patroni_main(self, mock_machines):
+    @patch.object(Client, 'machines', PropertyMock(return_value=['http://remotehost:2379']))
+    @patch.object(Thread, 'join', Mock())
+    def test_patroni_patroni_main(self):
         with patch('subprocess.call', Mock(return_value=1)):
-            sys.argv = ['patroni.py', 'postgres0.yml']
-
-            mock_machines.__get__ = Mock(return_value=['http://remotehost:2379'])
             with patch.object(Patroni, 'run', Mock(side_effect=SleepException)):
+                os.environ['PATRONI_POSTGRESQL_DATA_DIR'] = 'data/test0'
                 self.assertRaises(SleepException, patroni_main)
             with patch.object(Patroni, 'run', Mock(side_effect=KeyboardInterrupt())):
                 with patch('patroni.ha.Ha.is_paused', Mock(return_value=True)):
+                    os.environ['PATRONI_POSTGRESQL_DATA_DIR'] = 'data/test0'
                     patroni_main()
 
     @patch('os.getpid')
@@ -107,8 +124,12 @@ class TestPatroni(unittest.TestCase):
         self.p.sighup_handler()
         self.p.ha.dcs.watch = Mock(side_effect=SleepException)
         self.p.api.start = Mock()
+        self.p.logger.start = Mock()
         self.p.config._dynamic_configuration = {}
         self.assertRaises(SleepException, self.p.run)
+        with patch('patroni.config.Config.reload_local_configuration', Mock(return_value=False)):
+            self.p.sighup_handler()
+            self.assertRaises(SleepException, self.p.run)
         with patch('patroni.config.Config.set_dynamic_configuration', Mock(return_value=True)):
             self.assertRaises(SleepException, self.p.run)
         with patch('patroni.postgresql.Postgresql.data_directory_empty', Mock(return_value=False)):
@@ -150,6 +171,13 @@ class TestPatroni(unittest.TestCase):
         self.p.tags['nosync'] = None
         self.assertFalse(self.p.nosync)
 
+    @patch.object(Thread, 'join', Mock())
     def test_shutdown(self):
         self.p.api.shutdown = Mock(side_effect=Exception)
         self.p.shutdown()
+
+    def test_check_psycopg2(self):
+        with patch.object(builtins, '__import__', Mock(side_effect=ImportError)):
+            self.assertRaises(SystemExit, check_psycopg2)
+        with patch('psycopg2.__version__', '2.5.3.dev1 a b c'):
+            self.assertRaises(SystemExit, check_psycopg2)

tests/test_postgresql.py

@@ -1,114 +1,33 @@
-import datetime
 import mock  # for the mock.call method, importing it without a namespace breaks python3
 import os
 import psycopg2
-import shutil
+import re
 import subprocess
-import unittest
+import time
 
 from mock import Mock, MagicMock, PropertyMock, patch, mock_open
 from patroni.async_executor import CriticalTask
-from patroni.dcs import Cluster, ClusterConfig, Leader, Member, RemoteMember, SyncState
-from patroni.exceptions import PostgresConnectionException, PostgresException
+from patroni.dcs import Cluster, ClusterConfig, Member, RemoteMember, SyncState
+from patroni.exceptions import PostgresConnectionException
 from patroni.postgresql import Postgresql, STATE_REJECT, STATE_NO_RESPONSE
-from patroni.postmaster import PostmasterProcess
+from patroni.postgresql.postmaster import PostmasterProcess
+from patroni.postgresql.slots import SlotsHandler
 from patroni.utils import RetryFailedError
 from six.moves import builtins
 from threading import Thread, current_thread
-from tempfile import gettempdir
+
+from . import BaseTestPostgresql, MockCursor, MockPostmaster, psycopg2_connect
 
 
-class MockCursor(object):
-
-    def __init__(self, connection):
-        self.connection = connection
-        self.closed = False
-        self.rowcount = 0
-        self.results = []
-
-    def execute(self, sql, *params):
-        if sql.startswith('blabla'):
-            raise psycopg2.ProgrammingError()
-        elif sql == 'CHECKPOINT' or sql.startswith('SELECT pg_catalog.pg_create_'):
-            raise psycopg2.OperationalError()
-        elif sql.startswith('RetryFailedError'):
-            raise RetryFailedError('retry')
-        elif sql.startswith('SELECT slot_name'):
-            self.results = [('blabla', 'physical'), ('foobar', 'physical'), ('ls', 'logical', 'a', 'b')]
-        elif sql.startswith('SELECT CASE WHEN pg_catalog.pg_is_in_recovery()'):
-            self.results = [(1, 2)]
-        elif sql.startswith('SELECT pg_catalog.pg_is_in_recovery()'):
-            self.results = [(False, 2)]
-        elif sql.startswith('WITH replication_info AS ('):
-            replication_info = '[{"application_name":"walreceiver","client_addr":"1.2.3.4",' +\
-                               '"state":"streaming","sync_state":"async","sync_priority":0}]'
-            self.results = [('', 0, '', '', '', '', False, replication_info)]
-        elif sql.startswith('SELECT name, setting'):
-            self.results = [('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
-                            ('search_path', 'public', None, 'string', 'user'),
-                            ('port', '5433', None, 'integer', 'postmaster'),
-                            ('listen_addresses', '*', None, 'string', 'postmaster'),
-                            ('autovacuum', 'on', None, 'bool', 'sighup'),
-                            ('unix_socket_directories', '/tmp', None, 'string', 'postmaster')]
-        elif sql.startswith('IDENTIFY_SYSTEM'):
-            self.results = [('1', 2, '0/402EEC0', '')]
-        elif sql.startswith('SELECT isdir, modification'):
-            self.results = [(False, datetime.datetime.now())]
-        elif sql.startswith('SELECT pg_catalog.pg_read_file'):
-            self.results = [('1\t0/40159C0\tno recovery target specified\n\n' +
-                             '2\t1/40159C0\tno recovery target specified\n',)]
-        elif sql.startswith('TIMELINE_HISTORY '):
-            self.results = [('', b'x\t0/40159C0\tno recovery target specified\n\n' +
-                                 b'1\t0/40159C0\tno recovery target specified\n\n' +
-                                 b'2\t0/402DD98\tno recovery target specified\n\n' +
-                                 b'3\t0/403DD98\tno recovery target specified\n')]
-        else:
-            self.results = [(None, None, None, None, None, None, None, None, None, None)]
-
-    def fetchone(self):
-        return self.results[0]
-
-    def fetchall(self):
-        return self.results
-
-    def __iter__(self):
-        for i in self.results:
-            yield i
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *args):
-        pass
+mtime_ret = {}
 
 
-class MockConnect(object):
-
-    server_version = 99999
-    autocommit = False
-    closed = 0
-
-    def cursor(self):
-        return MockCursor(self)
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *args):
-        pass
-
-    @staticmethod
-    def close():
-        pass
-
-
-class MockPostmaster(object):
-    def __init__(self, is_running=True, is_single_master=False):
-        self.is_running = Mock(return_value=is_running)
-        self.is_single_master = Mock(return_value=is_single_master)
-        self.wait_for_user_backends_to_close = Mock()
-        self.signal_stop = Mock(return_value=None)
-        self.wait = Mock()
+def mock_mtime(filename):
+    if filename not in mtime_ret:
+        mtime_ret[filename] = time.time()
+    else:
+        mtime_ret[filename] += 1
+    return mtime_ret[filename]
 
 
 def pg_controldata_string(*args, **kwargs):
@@ -166,67 +85,25 @@ Data page checksum version:           0
 """
 
 
-def psycopg2_connect(*args, **kwargs):
-    return MockConnect()
-
-
 @patch('subprocess.call', Mock(return_value=0))
 @patch('psycopg2.connect', psycopg2_connect)
-class TestPostgresql(unittest.TestCase):
-    _PARAMETERS = {'wal_level': 'hot_standby', 'max_replication_slots': 5, 'f.oo': 'bar',
-                   'search_path': 'public', 'hot_standby': 'on', 'max_wal_senders': 5,
-                   'wal_keep_segments': 8, 'wal_log_hints': 'on', 'max_locks_per_transaction': 64,
-                   'max_worker_processes': 8, 'max_connections': 100, 'max_prepared_transactions': 0,
-                   'track_commit_timestamp': 'off', 'unix_socket_directories': '/tmp'}
+class TestPostgresql(BaseTestPostgresql):
 
     @patch('subprocess.call', Mock(return_value=0))
-    @patch('psycopg2.connect', psycopg2_connect)
     @patch('os.rename', Mock())
-    @patch.object(Postgresql, 'get_major_version', Mock(return_value=90600))
+    @patch('patroni.postgresql.CallbackExecutor', Mock())
+    @patch.object(Postgresql, 'get_major_version', Mock(return_value=120000))
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def setUp(self):
-        self.data_dir = 'data/test0'
-        self.config_dir = self.data_dir
-        if not os.path.exists(self.data_dir):
-            os.makedirs(self.data_dir)
-        self.p = Postgresql({'name': 'test0', 'scope': 'batman', 'data_dir': self.data_dir,
-                             'config_dir': self.config_dir, 'retry_timeout': 10,
-                             'pgpass': os.path.join(gettempdir(), 'pgpass0'),
-                             'listen': '127.0.0.2, 127.0.0.3:5432', 'connect_address': '127.0.0.2:5432',
-                             'authentication': {'superuser': {'username': 'test', 'password': 'test'},
-                                                'replication': {'username': 'replicator', 'password': 'rep-pass'}},
-                             'remove_data_directory_on_rewind_failure': True,
-                             'use_pg_rewind': True, 'pg_ctl_timeout': 'bla',
-                             'parameters': self._PARAMETERS,
-                             'recovery_conf': {'foo': 'bar'},
-                             'pg_hba': ['host all all 0.0.0.0/0 md5'],
-                             'callbacks': {'on_start': 'true', 'on_stop': 'true', 'on_reload': 'true',
-                                           'on_restart': 'true', 'on_role_change': 'true'}})
+        super(TestPostgresql, self).setUp()
+        self.p.config.write_postgresql_conf()
         self.p._callback_executor = Mock()
-        self.leadermem = Member(0, 'leader', 28, {'conn_url': 'postgres://replicator:[email protected]:5435/postgres'})
-        self.leader = Leader(-1, 28, self.leadermem)
-        self.other = Member(0, 'test-1', 28, {'conn_url': 'postgres://replicator:[email protected]:5433/postgres',
-                            'tags': {'replicatefrom': 'leader'}})
-        self.me = Member(0, 'test0', 28, {'conn_url': 'postgres://replicator:[email protected]:5434/postgres'})
-
-    def tearDown(self):
-        shutil.rmtree('data')
-
-    def test__initdb(self):
-        self.assertRaises(Exception, self.p.bootstrap, {'initdb': [{'pgdata': 'bar'}]})
-        self.assertRaises(Exception, self.p.bootstrap, {'initdb': [{'foo': 'bar', 1: 2}]})
-        self.assertRaises(Exception, self.p.bootstrap, {'initdb': [1]})
-        self.assertRaises(Exception, self.p.bootstrap, {'initdb': 1})
-
-    @patch('os.path.exists', Mock(return_value=True))
-    @patch('os.unlink', Mock())
-    def test_delete_trigger_file(self):
-        self.p.delete_trigger_file()
 
     @patch('subprocess.Popen')
     @patch.object(Postgresql, 'wait_for_startup')
     @patch.object(Postgresql, 'wait_for_port_open')
     @patch.object(Postgresql, 'is_running')
+    @patch.object(Postgresql, 'controldata', Mock())
     def test_start(self, mock_is_running, mock_wait_for_port_open, mock_wait_for_startup, mock_popen):
         mock_is_running.return_value = MockPostmaster()
         mock_wait_for_port_open.return_value = True
@@ -237,7 +114,7 @@ class TestPostgresql(unittest.TestCase):
 
         mock_postmaster = MockPostmaster()
         with patch.object(PostmasterProcess, 'start', return_value=mock_postmaster):
-            pg_conf = os.path.join(self.data_dir, 'postgresql.conf')
+            pg_conf = os.path.join(self.p.data_dir, 'postgresql.conf')
             open(pg_conf, 'w').close()
             self.assertFalse(self.p.start(task=CriticalTask()))
 
@@ -255,7 +132,7 @@ class TestPostgresql(unittest.TestCase):
             task.cancel()
             self.assertFalse(self.p.start(task=task))
 
-        self.p.cancel()
+        self.p.cancellable.cancel()
         self.assertFalse(self.p.start())
 
     @patch.object(Postgresql, 'pg_isready')
@@ -277,7 +154,7 @@ class TestPostgresql(unittest.TestCase):
         self.assertTrue(self.p.wait_for_port_open(mock_postmaster, 1))
 
         # cancelled
-        self.p.cancel()
+        self.p.cancellable.cancel()
         self.assertFalse(self.p.wait_for_port_open(mock_postmaster, 1))
 
     @patch('time.sleep', Mock())
@@ -312,10 +189,11 @@ class TestPostgresql(unittest.TestCase):
         self.assertFalse(self.p.restart())
         self.assertEqual(self.p.state, 'restart failed (restarting)')
 
+    @patch('os.chmod', Mock())
     @patch.object(builtins, 'open', MagicMock())
     def test_write_pgpass(self):
-        self.p.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo'})
-        self.p.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo', 'password': 'bar'})
+        self.p.config.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo'})
+        self.p.config.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo', 'password': 'bar'})
 
     def test_checkpoint(self):
         with patch.object(MockCursor, 'fetchone', Mock(return_value=(True, ))):
@@ -324,186 +202,79 @@ class TestPostgresql(unittest.TestCase):
             self.assertIsNone(self.p.checkpoint())
         self.assertEqual(self.p.checkpoint(), 'not accessible or not healty')
 
-    @patch.object(Postgresql, 'cancellable_subprocess_call')
-    @patch('patroni.postgresql.Postgresql.write_pgpass', MagicMock(return_value=dict()))
-    def test_pg_rewind(self, mock_cancellable_subprocess_call):
-        r = {'user': '', 'host': '', 'port': '', 'database': '', 'password': ''}
-        mock_cancellable_subprocess_call.return_value = 0
-        self.assertTrue(self.p.pg_rewind(r))
-        mock_cancellable_subprocess_call.side_effect = OSError
-        self.assertFalse(self.p.pg_rewind(r))
+    @patch('patroni.postgresql.config.mtime', mock_mtime)
+    @patch('patroni.postgresql.config.ConfigHandler._get_pg_settings')
+    def test_check_recovery_conf(self, mock_get_pg_settings):
+        mock_get_pg_settings.return_value = {
+            'primary_conninfo': ['primary_conninfo', 'foo=', None, 'string', 'postmaster'],
+            'recovery_min_apply_delay': ['recovery_min_apply_delay', '0', 'ms', 'integer', 'sighup']
+        }
+        self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
+        self.p.config.write_recovery_conf({'standby_mode': 'on'})
+        self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
+        mock_get_pg_settings.return_value['primary_conninfo'][1] = ''
+        mock_get_pg_settings.return_value['recovery_min_apply_delay'][1] = '1'
+        self.assertEqual(self.p.config.check_recovery_conf(None), (True, False))
+        mock_get_pg_settings.return_value['recovery_min_apply_delay'][1] = '0'
+        self.assertEqual(self.p.config.check_recovery_conf(None), (False, False))
+        conninfo = {'host': '1', 'password': 'bar'}
+        with patch('patroni.postgresql.config.ConfigHandler.primary_conninfo_params', Mock(return_value=conninfo)):
+            mock_get_pg_settings.return_value['recovery_min_apply_delay'][1] = '1'
+            self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
+            mock_get_pg_settings.return_value['primary_conninfo'][1] = 'host=1 passfile='\
+                + re.sub(r'([\'\\ ])', r'\\\1', self.p.config._pgpass)
+            mock_get_pg_settings.return_value['recovery_min_apply_delay'][1] = '0'
+            self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
+            self.p.config.write_recovery_conf({'standby_mode': 'on', 'primary_conninfo': conninfo.copy()})
+            self.p.config.write_postgresql_conf()
+            self.assertEqual(self.p.config.check_recovery_conf(None), (False, False))
 
-    def test_check_recovery_conf(self):
-        self.p.write_recovery_conf({'primary_conninfo': 'foo'})
-        self.assertFalse(self.p.check_recovery_conf(None))
-        self.p.write_recovery_conf({})
-        self.assertTrue(self.p.check_recovery_conf(None))
+    @patch.object(Postgresql, 'major_version', PropertyMock(return_value=120000))
+    @patch.object(Postgresql, 'is_running', MockPostmaster)
+    @patch.object(MockPostmaster, 'create_time', Mock(return_value=1234567), create=True)
+    @patch('patroni.postgresql.config.ConfigHandler._get_pg_settings')
+    def test__read_recovery_params(self, mock_get_pg_settings):
+        mock_get_pg_settings.return_value = {'primary_conninfo': ['primary_conninfo', '', None, 'string', 'postmaster']}
+        self.p.config.write_recovery_conf({'standby_mode': 'on', 'primary_conninfo': {'password': 'foo'}})
+        self.p.config.write_postgresql_conf()
+        self.assertEqual(self.p.config.check_recovery_conf(None), (False, False))
+        self.assertEqual(self.p.config.check_recovery_conf(None), (False, False))
+        mock_get_pg_settings.side_effect = Exception
+        with patch('patroni.postgresql.config.mtime', mock_mtime):
+            self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
 
-    @patch.object(Postgresql, 'start', Mock())
-    @patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
-    def test__get_local_timeline_lsn(self):
-        self.p.trigger_check_diverged_lsn()
-        with patch.object(Postgresql, 'controldata',
-                          Mock(return_value={'Database cluster state': 'shut down in recovery',
-                                             'Minimum recovery ending location': '0/0',
-                                             "Min recovery ending loc's timeline": '0'})):
-            self.p.rewind_or_reinitialize_needed_and_possible(self.leader)
-        with patch.object(Postgresql, 'is_running', Mock(return_value=True)):
-            with patch.object(MockCursor, 'fetchone', Mock(side_effect=[(False, ), Exception])):
-                self.p.rewind_or_reinitialize_needed_and_possible(self.leader)
+    @patch.object(Postgresql, 'major_version', PropertyMock(return_value=100000))
+    def test__read_recovery_params_pre_v12(self):
+        self.p.config.write_recovery_conf({'standby_mode': 'on', 'primary_conninfo': {'password': 'foo'}})
+        self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
+        self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
+        self.p.config.write_recovery_conf({'standby_mode': '\n'})
+        with patch('patroni.postgresql.config.mtime', mock_mtime):
+            self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
 
-    @patch.object(Postgresql, 'start', Mock())
-    @patch.object(Postgresql, 'can_rewind', PropertyMock(return_value=True))
-    @patch.object(Postgresql, '_get_local_timeline_lsn', Mock(return_value=(2, '40159C1')))
-    @patch.object(Postgresql, 'check_leader_is_not_in_recovery')
-    def test__check_timeline_and_lsn(self, mock_check_leader_is_not_in_recovery):
-        mock_check_leader_is_not_in_recovery.return_value = False
-        self.p.trigger_check_diverged_lsn()
-        self.assertFalse(self.p.rewind_or_reinitialize_needed_and_possible(self.leader))
-        self.leader = self.leader.member
-        self.assertFalse(self.p.rewind_or_reinitialize_needed_and_possible(self.leader))
-        mock_check_leader_is_not_in_recovery.return_value = True
-        self.assertFalse(self.p.rewind_or_reinitialize_needed_and_possible(self.leader))
-        self.p.trigger_check_diverged_lsn()
-        with patch('psycopg2.connect', Mock(side_effect=Exception)):
-            self.assertFalse(self.p.rewind_or_reinitialize_needed_and_possible(self.leader))
-        self.p.trigger_check_diverged_lsn()
-        with patch.object(MockCursor, 'fetchone', Mock(side_effect=[('', 2, '0/0'), ('', b'3\t0/40159C0\tn\n')])):
-            self.assertFalse(self.p.rewind_or_reinitialize_needed_and_possible(self.leader))
-        self.p.trigger_check_diverged_lsn()
-        with patch.object(MockCursor, 'fetchone', Mock(return_value=('', 1, '0/0'))):
-            with patch.object(Postgresql, '_get_local_timeline_lsn', Mock(return_value=(1, '0/0'))):
-                self.assertFalse(self.p.rewind_or_reinitialize_needed_and_possible(self.leader))
-            self.p.trigger_check_diverged_lsn()
-            self.assertTrue(self.p.rewind_or_reinitialize_needed_and_possible(self.leader))
+    def test_write_postgresql_and_sanitize_auto_conf(self):
+        read_data = 'primary_conninfo = foo\nfoo = bar\n'
+        with open(os.path.join(self.p.data_dir, 'postgresql.auto.conf'), 'w') as f:
+            f.write(read_data)
 
-    @patch.object(MockCursor, 'fetchone', Mock(side_effect=[(True,), Exception]))
-    def test_check_leader_is_not_in_recovery(self):
-        self.p.check_leader_is_not_in_recovery()
-        self.p.check_leader_is_not_in_recovery()
+        mock_read_auto = mock_open(read_data=read_data)
+        mock_read_auto.return_value.__iter__ = lambda o: iter(o.readline, '')
+        with patch.object(builtins, 'open', Mock(side_effect=[mock_open()(), mock_read_auto(), IOError])),\
+                patch('os.chmod', Mock()):
+            self.p.config.write_postgresql_conf()
 
-    @patch.object(Postgresql, 'cancellable_subprocess_call', Mock(return_value=0))
-    @patch.object(Postgresql, 'checkpoint', side_effect=['', '1'])
-    @patch.object(Postgresql, 'stop', Mock(return_value=False))
-    @patch.object(Postgresql, 'start', Mock())
-    def test_rewind(self, mock_checkpoint):
-        self.p.rewind(self.leader)
-        with patch.object(Postgresql, 'pg_rewind', Mock(return_value=False)):
-            mock_checkpoint.side_effect = ['1', '', '', '']
-            self.p.rewind(self.leader)
-            self.p.rewind(self.leader)
-            with patch.object(Postgresql, 'check_leader_is_not_in_recovery', Mock(return_value=False)):
-                self.p.rewind(self.leader)
-            self.p.config['remove_data_directory_on_rewind_failure'] = False
-            self.p.trigger_check_diverged_lsn()
-            self.p.rewind(self.leader)
-        with patch.object(Postgresql, 'is_running', Mock(return_value=True)):
-            self.p.rewind(self.leader)
-            self.p.is_leader = Mock(return_value=False)
-            self.p.rewind(self.leader)
+        with patch.object(builtins, 'open', Mock(side_effect=[mock_open()(), IOError])), patch('os.chmod', Mock()):
+            self.p.config.write_postgresql_conf()
+        self.p.config.write_recovery_conf({'foo': 'bar'})
+        self.p.config.write_postgresql_conf()
 
     @patch.object(Postgresql, 'is_running', Mock(return_value=False))
     @patch.object(Postgresql, 'start', Mock())
     def test_follow(self):
         self.p.call_nowait('on_start')
-        m = RemoteMember('1', {'restore_command': '2', 'recovery_min_apply_delay': 3, 'archive_cleanup_command': '4'})
+        m = RemoteMember('1', {'restore_command': '2', 'primary_slot_name': 'foo', 'conn_kwargs': {'host': 'bar'}})
         self.p.follow(m)
 
-    @patch('subprocess.check_output', Mock(return_value=0, side_effect=pg_controldata_string))
-    def test_can_rewind(self):
-        with patch('subprocess.call', MagicMock(return_value=1)):
-            self.assertFalse(self.p.can_rewind)
-        with patch('subprocess.call', side_effect=OSError):
-            self.assertFalse(self.p.can_rewind)
-        with patch.object(Postgresql, 'controldata', Mock(return_value={'wal_log_hints setting': 'on'})):
-            self.assertTrue(self.p.can_rewind)
-        self.p.config['use_pg_rewind'] = False
-        self.assertFalse(self.p.can_rewind)
-
-    @patch('time.sleep', Mock())
-    @patch.object(Postgresql, 'cancellable_subprocess_call')
-    @patch.object(Postgresql, 'remove_data_directory', Mock(return_value=True))
-    def test_create_replica(self, mock_cancellable_subprocess_call):
-        self.p.delete_trigger_file = Mock(side_effect=OSError)
-
-        self.p.config['create_replica_methods'] = ['pgBackRest']
-        self.p.config['pgBackRest'] = {'command': 'pgBackRest', 'keep_data': True, 'no_params': True}
-        mock_cancellable_subprocess_call.return_value = 0
-        self.assertEqual(self.p.create_replica(self.leader), 0)
-
-        self.p.config['create_replica_methods'] = ['wale', 'basebackup']
-        self.p.config['wale'] = {'command': 'foo'}
-        self.assertEqual(self.p.create_replica(self.leader), 0)
-        del self.p.config['wale']
-        self.assertEqual(self.p.create_replica(self.leader), 0)
-
-        self.p.config['create_replica_methods'] = ['basebackup']
-        self.p.config['basebackup'] = [{'max_rate': '100M'}, 'no-sync']
-        self.assertEqual(self.p.create_replica(self.leader), 0)
-
-        self.p.config['basebackup'] = [{'max_rate': '100M', 'compress': '9'}]
-        with mock.patch('patroni.postgresql.logger.error', new_callable=Mock()) as mock_logger:
-            self.p.create_replica(self.leader)
-            mock_logger.assert_called_once()
-            self.assertTrue("only one key-value is allowed and value should be a string" in mock_logger.call_args[0][0],
-                            "not matching {0}".format(mock_logger.call_args[0][0]))
-
-        self.p.config['basebackup'] = [42]
-        with mock.patch('patroni.postgresql.logger.error', new_callable=Mock()) as mock_logger:
-            self.p.create_replica(self.leader)
-            mock_logger.assert_called_once()
-            self.assertTrue("value should be string value or a single key-value pair" in mock_logger.call_args[0][0],
-                            "not matching {0}".format(mock_logger.call_args[0][0]))
-
-        self.p.config['basebackup'] = {"foo": "bar"}
-        self.assertEqual(self.p.create_replica(self.leader), 0)
-
-        self.p.config['create_replica_methods'] = ['wale', 'basebackup']
-        del self.p.config['basebackup']
-        mock_cancellable_subprocess_call.return_value = 1
-        self.assertEqual(self.p.create_replica(self.leader), 1)
-
-        mock_cancellable_subprocess_call.side_effect = Exception('foo')
-        self.assertEqual(self.p.create_replica(self.leader), 1)
-
-        mock_cancellable_subprocess_call.side_effect = [1, 0]
-        self.assertEqual(self.p.create_replica(self.leader), 0)
-
-        mock_cancellable_subprocess_call.side_effect = [Exception(), 0]
-        self.assertEqual(self.p.create_replica(self.leader), 0)
-
-        self.p.cancel()
-        self.assertEqual(self.p.create_replica(self.leader), 1)
-
-    @patch('time.sleep', Mock())
-    @patch.object(Postgresql, 'cancellable_subprocess_call')
-    @patch.object(Postgresql, 'remove_data_directory', Mock(return_value=True))
-    def test_create_replica_old_format(self, mock_cancellable_subprocess_call):
-        """ The same test as before but with old 'create_replica_method'
-            to test backward compatibility
-        """
-        self.p.delete_trigger_file = Mock(side_effect=OSError)
-
-        self.p.config['create_replica_method'] = ['wale', 'basebackup']
-        self.p.config['wale'] = {'command': 'foo'}
-        mock_cancellable_subprocess_call.return_value = 0
-        self.assertEqual(self.p.create_replica(self.leader), 0)
-        del self.p.config['wale']
-        self.assertEqual(self.p.create_replica(self.leader), 0)
-
-        self.p.config['create_replica_method'] = ['basebackup']
-        self.p.config['basebackup'] = [{'max_rate': '100M'}, 'no-sync']
-        self.assertEqual(self.p.create_replica(self.leader), 0)
-
-        self.p.config['create_replica_method'] = ['wale', 'basebackup']
-        del self.p.config['basebackup']
-        mock_cancellable_subprocess_call.return_value = 1
-        self.assertEqual(self.p.create_replica(self.leader), 1)
-
-    def test_basebackup(self):
-        self.p.cancel()
-        self.p.basebackup(None, None, {'foo': 'bar'})
-
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def test_sync_replication_slots(self):
         self.p.start()
@@ -511,17 +282,16 @@ class TestPostgresql(unittest.TestCase):
                                              'A': 0, 'test_3': 0, 'b': {'type': 'logical', 'plugin': '1'}}}, 1)
         cluster = Cluster(True, config, self.leader, 0, [self.me, self.other, self.leadermem], None, None, None)
         with mock.patch('patroni.postgresql.Postgresql._query', Mock(side_effect=psycopg2.OperationalError)):
-            self.p.sync_replication_slots(cluster)
-        self.p.sync_replication_slots(cluster)
+            self.p.slots_handler.sync_replication_slots(cluster)
+        self.p.slots_handler.sync_replication_slots(cluster)
         with mock.patch('patroni.postgresql.Postgresql.role', new_callable=PropertyMock(return_value='replica')):
-            self.p.sync_replication_slots(cluster)
-        with patch.object(Postgresql, 'drop_replication_slot', Mock(return_value=True)),\
+            self.p.slots_handler.sync_replication_slots(cluster)
+        with patch.object(SlotsHandler, 'drop_replication_slot', Mock(return_value=True)),\
                 patch('patroni.dcs.logger.error', new_callable=Mock()) as errorlog_mock:
-            self.p.query = Mock()
             alias1 = Member(0, 'test-3', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres'})
             alias2 = Member(0, 'test.3', 28, {'conn_url': 'postgres://replicator:[email protected]:5436/postgres'})
             cluster.members.extend([alias1, alias2])
-            self.p.sync_replication_slots(cluster)
+            self.p.slots_handler.sync_replication_slots(cluster)
             self.assertEqual(errorlog_mock.call_count, 5)
             ca = errorlog_mock.call_args_list[0][0][1]
             self.assertTrue("test-3" in ca, "non matching {0}".format(ca))
@@ -604,121 +374,9 @@ class TestPostgresql(unittest.TestCase):
         with patch('os.rename', Mock(side_effect=OSError)):
             self.p.move_data_directory()
 
-    @patch.object(Postgresql, 'is_running', Mock(return_value=True))
-    def test_bootstrap(self):
-        with patch('subprocess.call', Mock(return_value=1)):
-            self.assertFalse(self.p.bootstrap({}))
-
-        config = {'users': {'replicator': {'password': 'rep-pass', 'options': ['replication']}}}
-
-        self.p.bootstrap(config)
-        with open(os.path.join(self.config_dir, 'pg_hba.conf')) as f:
-            lines = f.readlines()
-            self.assertTrue('host all all 0.0.0.0/0 md5\n' in lines)
-
-        self.p.config.pop('pg_hba')
-        config.update({'post_init': '/bin/false',
-                       'pg_hba': ['host replication replicator 127.0.0.1/32 md5',
-                                  'hostssl all all 0.0.0.0/0 md5',
-                                  'host all all 0.0.0.0/0 md5']})
-        self.p.bootstrap(config)
-        with open(os.path.join(self.data_dir, 'pg_hba.conf')) as f:
-            lines = f.readlines()
-            self.assertTrue('host replication replicator 127.0.0.1/32 md5\n' in lines)
-
-    @patch.object(Postgresql, 'cancellable_subprocess_call')
-    @patch.object(Postgresql, 'get_major_version', Mock(return_value=90600))
-    def test_custom_bootstrap(self, mock_cancellable_subprocess_call):
-        self.p.config.pop('pg_hba')
-        config = {'method': 'foo', 'foo': {'command': 'bar'}}
-
-        mock_cancellable_subprocess_call.return_value = 1
-        self.assertFalse(self.p.bootstrap(config))
-
-        mock_cancellable_subprocess_call.return_value = 0
-        with patch('multiprocessing.Process', Mock(side_effect=Exception("42"))),\
-                patch('os.path.isfile', Mock(return_value=True)),\
-                patch('os.unlink', Mock()),\
-                patch.object(Postgresql, 'save_configuration_files', Mock()),\
-                patch.object(Postgresql, 'restore_configuration_files', Mock()),\
-                patch.object(Postgresql, 'write_recovery_conf', Mock()):
-            with self.assertRaises(Exception) as e:
-                self.p.bootstrap(config)
-            self.assertEqual(str(e.exception), '42')
-
-            config['foo']['recovery_conf'] = {'foo': 'bar'}
-
-            with self.assertRaises(Exception) as e:
-                self.p.bootstrap(config)
-            self.assertEqual(str(e.exception), '42')
-
-        mock_cancellable_subprocess_call.side_effect = Exception
-        self.assertFalse(self.p.bootstrap(config))
-
-    @patch('time.sleep', Mock())
-    @patch('os.unlink', Mock())
-    @patch('shutil.copy', Mock())
-    @patch('os.path.isfile', Mock(return_value=True))
-    @patch.object(Postgresql, 'run_bootstrap_post_init', Mock(return_value=True))
-    @patch.object(Postgresql, '_custom_bootstrap', Mock(return_value=True))
-    @patch.object(Postgresql, 'start', Mock(return_value=True))
-    @patch.object(Postgresql, 'get_major_version', Mock(return_value=90600))
-    def test_post_bootstrap(self):
-        config = {'method': 'foo', 'foo': {'command': 'bar'}}
-        self.p.bootstrap(config)
-
-        task = CriticalTask()
-        with patch.object(Postgresql, 'create_or_update_role', Mock(side_effect=Exception)):
-            self.p.post_bootstrap({}, task)
-            self.assertFalse(task.result)
-
-        self.p.config.pop('pg_hba')
-        self.p.post_bootstrap({}, task)
-        self.assertTrue(task.result)
-
-        self.p.bootstrap(config)
-        with patch.object(Postgresql, 'pending_restart', PropertyMock(return_value=True)), \
-                patch.object(Postgresql, 'restart', Mock()) as mock_restart:
-            self.p.post_bootstrap({}, task)
-            mock_restart.assert_called_once()
-
-        self.p.bootstrap(config)
-        self.p.set_state('stopped')
-        self.p.reload_config({'authentication': {'superuser': {'username': 'p', 'password': 'p'},
-                                                 'replication': {'username': 'r', 'password': 'r'}},
-                              'listen': '*', 'retry_timeout': 10, 'parameters': {'wal_level': '', 'hba_file': 'foo'}})
-        with patch.object(Postgresql, 'restart', Mock()) as mock_restart:
-            self.p.post_bootstrap({}, task)
-            mock_restart.assert_called_once()
-
-    @patch.object(Postgresql, 'cancellable_subprocess_call')
-    def test_run_bootstrap_post_init(self, mock_cancellable_subprocess_call):
-        mock_cancellable_subprocess_call.return_value = 1
-        self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
-
-        mock_cancellable_subprocess_call.return_value = 0
-        self.p._superuser.pop('username')
-        self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
-        mock_cancellable_subprocess_call.assert_called()
-        args, kwargs = mock_cancellable_subprocess_call.call_args
-        self.assertTrue('PGPASSFILE' in kwargs['env'])
-        self.assertEqual(args[0], ['/bin/false', 'postgres://127.0.0.2:5432/postgres'])
-
-        mock_cancellable_subprocess_call.reset_mock()
-        self.p._local_address.pop('host')
-        self.assertTrue(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
-        mock_cancellable_subprocess_call.assert_called()
-        self.assertEqual(mock_cancellable_subprocess_call.call_args[0][0], ['/bin/false', 'postgres://:5432/postgres'])
-
-        mock_cancellable_subprocess_call.side_effect = OSError
-        self.assertFalse(self.p.run_bootstrap_post_init({'post_init': '/bin/false'}))
-
-    @patch('patroni.postgresql.Postgresql.create_replica', Mock(return_value=0))
-    def test_clone(self):
-        self.p.clone(self.leader)
-
     @patch('os.listdir', Mock(return_value=['recovery.conf']))
     @patch('os.path.exists', Mock(return_value=True))
+    @patch.object(Postgresql, 'controldata', Mock())
     def test_get_postgres_role_from_data_directory(self):
         self.assertEqual(self.p.get_postgres_role_from_data_directory(), 'replica')
 
@@ -729,12 +387,12 @@ class TestPostgresql(unittest.TestCase):
             except OSError:
                 if os.name == 'nt':  # os.symlink under Windows needs admin rights skip it
                     pass
-        os.makedirs(os.path.join(self.data_dir, 'foo'))
-        _symlink('foo', os.path.join(self.data_dir, 'pg_wal'))
+        os.makedirs(os.path.join(self.p.data_dir, 'foo'))
+        _symlink('foo', os.path.join(self.p.data_dir, 'pg_wal'))
         self.p.remove_data_directory()
-        open(self.data_dir, 'w').close()
+        open(self.p.data_dir, 'w').close()
         self.p.remove_data_directory()
-        _symlink('unexisting', self.data_dir)
+        _symlink('unexisting', self.p.data_dir)
         with patch('os.unlink', Mock(side_effect=OSError)):
             self.p.remove_data_directory()
         self.p.remove_data_directory()
@@ -759,35 +417,40 @@ class TestPostgresql(unittest.TestCase):
     @patch('os.path.isfile', Mock(return_value=True))
     @patch('shutil.copy', Mock(side_effect=IOError))
     def test_save_configuration_files(self):
-        self.p.save_configuration_files()
+        self.p.config.save_configuration_files()
 
     @patch('os.path.isfile', Mock(side_effect=[False, True]))
     @patch('shutil.copy', Mock(side_effect=IOError))
     def test_restore_configuration_files(self):
-        self.p.restore_configuration_files()
+        self.p.config.restore_configuration_files()
 
     def test_can_create_replica_without_replication_connection(self):
-        self.p.config['create_replica_method'] = []
+        self.p.config._config['create_replica_method'] = []
         self.assertFalse(self.p.can_create_replica_without_replication_connection())
-        self.p.config['create_replica_method'] = ['wale', 'basebackup']
-        self.p.config['wale'] = {'command': 'foo', 'no_master': 1}
+        self.p.config._config['create_replica_method'] = ['wale', 'basebackup']
+        self.p.config._config['wale'] = {'command': 'foo', 'no_master': 1}
         self.assertTrue(self.p.can_create_replica_without_replication_connection())
 
     def test_replica_method_can_work_without_replication_connection(self):
         self.assertFalse(self.p.replica_method_can_work_without_replication_connection('basebackup'))
         self.assertFalse(self.p.replica_method_can_work_without_replication_connection('foobar'))
-        self.p.config['foo'] = {'command': 'bar', 'no_master': 1}
+        self.p.config._config['foo'] = {'command': 'bar', 'no_master': 1}
         self.assertTrue(self.p.replica_method_can_work_without_replication_connection('foo'))
-        self.p.config['foo'] = {'command': 'bar'}
+        self.p.config._config['foo'] = {'command': 'bar'}
         self.assertFalse(self.p.replica_method_can_work_without_replication_connection('foo'))
 
+    @patch('time.sleep', Mock())
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
-    def test_reload_config(self):
+    @patch.object(MockCursor, 'fetchone')
+    def test_reload_config(self, mock_fetchone):
+        mock_fetchone.return_value = (1,)
         parameters = self._PARAMETERS.copy()
         parameters.pop('f.oo')
-        config = {'pg_hba': [''], 'use_unix_socket': True, 'authentication': {},
-                  'retry_timeout': 10, 'listen': '*', 'parameters': parameters}
+        parameters['wal_buffers'] = '512'
+        config = {'pg_hba': [''], 'pg_ident': [''], 'use_unix_socket': True, 'authentication': {},
+                  'retry_timeout': 10, 'listen': '*', 'krbsrvname': 'postgres', 'parameters': parameters}
         self.p.reload_config(config)
+        mock_fetchone.side_effect = Exception
         parameters['b.ar'] = 'bar'
         self.p.reload_config(config)
         parameters['autovacuum'] = 'on'
@@ -798,7 +461,7 @@ class TestPostgresql(unittest.TestCase):
         self.p.reload_config(config)
         parameters['unix_socket_directories'] = '.'
         self.p.reload_config(config)
-        self.p.resolve_connection_addresses()
+        self.p.config.resolve_connection_addresses()
 
     @patch.object(Postgresql, '_version_file_exists', Mock(return_value=True))
     def test_get_major_version(self):
@@ -810,6 +473,10 @@ class TestPostgresql(unittest.TestCase):
     def test_postmaster_start_time(self):
         with patch.object(MockCursor, "fetchone", Mock(return_value=('foo', True, '', '', '', '', False))):
             self.assertEqual(self.p.postmaster_start_time(), 'foo')
+            t = Thread(target=self.p.postmaster_start_time)
+            t.start()
+            t.join()
+
         with patch.object(MockCursor, "execute", side_effect=psycopg2.Error):
             self.assertIsNone(self.p.postmaster_start_time())
 
@@ -885,7 +552,7 @@ class TestPostgresql(unittest.TestCase):
                 self.assertEqual(state['sleeps'], 3)
 
         with patch.object(Postgresql, 'check_startup_state_changed', Mock(return_value=False)):
-            self.p.cancel()
+            self.p.cancellable.cancel()
             self.p._state = 'starting'
             self.assertIsNone(self.p.wait_for_startup())
 
@@ -925,37 +592,37 @@ class TestPostgresql(unittest.TestCase):
 
     def test_set_sync_standby(self):
         def value_in_conf():
-            with open(os.path.join(self.data_dir, 'postgresql.conf')) as f:
+            with open(os.path.join(self.p.data_dir, 'postgresql.conf')) as f:
                 for line in f:
                     if line.startswith('synchronous_standby_names'):
                         return line.strip()
 
         mock_reload = self.p.reload = Mock()
-        self.p.set_synchronous_standby('n1')
+        self.p.config.set_synchronous_standby('n1')
         self.assertEqual(value_in_conf(), "synchronous_standby_names = 'n1'")
         mock_reload.assert_called()
 
         mock_reload.reset_mock()
-        self.p.set_synchronous_standby('n1')
+        self.p.config.set_synchronous_standby('n1')
         mock_reload.assert_not_called()
         self.assertEqual(value_in_conf(), "synchronous_standby_names = 'n1'")
 
-        self.p.set_synchronous_standby('n2')
+        self.p.config.set_synchronous_standby('n2')
         mock_reload.assert_called()
         self.assertEqual(value_in_conf(), "synchronous_standby_names = 'n2'")
 
         mock_reload.reset_mock()
-        self.p.set_synchronous_standby(None)
+        self.p.config.set_synchronous_standby(None)
         mock_reload.assert_called()
         self.assertEqual(value_in_conf(), None)
 
     def test_get_server_parameters(self):
         config = {'synchronous_mode': True, 'parameters': {'wal_level': 'hot_standby'}, 'listen': '0'}
-        self.p.get_server_parameters(config)
+        self.p.config.get_server_parameters(config)
         config['synchronous_mode_strict'] = True
-        self.p.get_server_parameters(config)
-        self.p.set_synchronous_standby('foo')
-        self.p.get_server_parameters(config)
+        self.p.config.get_server_parameters(config)
+        self.p.config.set_synchronous_standby('foo')
+        self.assertTrue(str(self.p.config.get_server_parameters(config)).startswith('{'))
 
     @patch('time.sleep', Mock())
     def test__wait_for_connection_close(self):
@@ -989,7 +656,7 @@ class TestPostgresql(unittest.TestCase):
             data = self.p.read_postmaster_opts()
             self.assertEqual(data, dict())
 
-    @patch('subprocess.Popen')
+    @patch('psutil.Popen')
     def test_single_user_mode(self, subprocess_popen_mock):
         subprocess_popen_mock.return_value.wait.return_value = 0
         self.assertEqual(self.p.single_user_mode('CHECKPOINT', {'archive_mode': 'on'}), 0)
@@ -1017,25 +684,14 @@ class TestPostgresql(unittest.TestCase):
     def test_get_master_timeline(self):
         self.assertEqual(self.p.get_master_timeline(), 1)
 
-    def test_cancellable_subprocess_call(self):
-        self.p.cancel()
-        self.assertRaises(PostgresException, self.p.cancellable_subprocess_call, communicate_input=None)
-
-    @patch('patroni.postgresql.polling_loop', Mock(return_value=[0, 0]))
-    def test_cancel(self):
-        self.p._cancellable = Mock()
-        self.p._cancellable.returncode = None
-        self.p.cancel()
-        type(self.p._cancellable).returncode = PropertyMock(side_effect=[None, -15])
-        self.p.cancel()
-
     @patch.object(Postgresql, 'get_postgres_role_from_data_directory', Mock(return_value='replica'))
     def test__build_effective_configuration(self):
         with patch.object(Postgresql, 'controldata',
                           Mock(return_value={'max_connections setting': '200',
                                              'max_worker_processes setting': '20',
                                              'max_prepared_xacts setting': '100',
-                                             'max_locks_per_xact setting': '100'})):
-            self.p.cancel()
+                                             'max_locks_per_xact setting': '100',
+                                             'max_wal_senders setting': 10})):
+            self.p.cancellable.cancel()
             self.assertFalse(self.p.start())
             self.assertTrue(self.p.pending_restart)

tests/test_postmaster.py

@@ -1,8 +1,9 @@
+import multiprocessing
 import psutil
 import unittest
 
 from mock import Mock, patch, mock_open
-from patroni.postmaster import PostmasterProcess
+from patroni.postgresql.postmaster import PostmasterProcess
 from six.moves import builtins
 
 
@@ -26,7 +27,7 @@ class TestPostmasterProcess(unittest.TestCase):
 
     @patch('psutil.Process.create_time')
     @patch('psutil.Process.__init__')
-    @patch('patroni.postmaster.PostmasterProcess._read_postmaster_pidfile')
+    @patch.object(PostmasterProcess, '_read_postmaster_pidfile')
     def test_from_pidfile(self, mock_read, mock_init, mock_create_time):
         mock_init.side_effect = psutil.NoSuchProcess(123)
         mock_read.return_value = {}
@@ -96,6 +97,7 @@ class TestPostmasterProcess(unittest.TestCase):
     @patch('subprocess.Popen')
     @patch('os.setsid', Mock(), create=True)
     @patch('multiprocessing.Process', MockProcess)
+    @patch('multiprocessing.get_context', Mock(return_value=multiprocessing), create=True)
     @patch.object(PostmasterProcess, 'from_pid')
     @patch.object(PostmasterProcess, '_from_pidfile')
     def test_start(self, mock_frompidfile, mock_frompid, mock_popen):

tests/test_rewind.py

@@ -0,0 +1,107 @@
+from mock import Mock, PropertyMock, patch
+
+from patroni.postgresql import Postgresql
+from patroni.postgresql.cancellable import CancellableSubprocess
+from patroni.postgresql.rewind import Rewind
+
+from . import BaseTestPostgresql, MockCursor, psycopg2_connect
+
+
+@patch('subprocess.call', Mock(return_value=0))
+@patch('psycopg2.connect', psycopg2_connect)
+class TestRewind(BaseTestPostgresql):
+
+    def setUp(self):
+        super(TestRewind, self).setUp()
+        self.r = Rewind(self.p)
+
+    def test_can_rewind(self):
+        with patch.object(Postgresql, 'controldata', Mock(return_value={'wal_log_hints setting': 'on'})):
+            self.assertTrue(self.r.can_rewind)
+        with patch('subprocess.call', Mock(return_value=1)):
+            self.assertFalse(self.r.can_rewind)
+        with patch('subprocess.call', side_effect=OSError):
+            self.assertFalse(self.r.can_rewind)
+        self.p.config._config['use_pg_rewind'] = False
+        self.assertFalse(self.r.can_rewind)
+
+    @patch.object(CancellableSubprocess, 'call')
+    def test_pg_rewind(self, mock_cancellable_subprocess_call):
+        r = {'user': '', 'host': '', 'port': '', 'database': '', 'password': ''}
+        mock_cancellable_subprocess_call.return_value = 0
+        self.assertTrue(self.r.pg_rewind(r))
+        mock_cancellable_subprocess_call.side_effect = OSError
+        self.assertFalse(self.r.pg_rewind(r))
+
+    @patch.object(Rewind, 'can_rewind', PropertyMock(return_value=True))
+    def test__get_local_timeline_lsn(self):
+        self.r.trigger_check_diverged_lsn()
+        with patch.object(Postgresql, 'controldata',
+                          Mock(return_value={'Database cluster state': 'shut down in recovery',
+                                             'Minimum recovery ending location': '0/0',
+                                             "Min recovery ending loc's timeline": '0'})):
+            self.r.rewind_or_reinitialize_needed_and_possible(self.leader)
+
+        with patch.object(Postgresql, 'is_running', Mock(return_value=True)):
+            with patch.object(MockCursor, 'fetchone', Mock(side_effect=[(False, ), Exception])):
+                self.r.rewind_or_reinitialize_needed_and_possible(self.leader)
+
+    @patch.object(CancellableSubprocess, 'call', Mock(return_value=0))
+    @patch.object(Postgresql, 'checkpoint', side_effect=['', '1'],)
+    @patch.object(Postgresql, 'stop', Mock(return_value=False))
+    @patch.object(Postgresql, 'start', Mock())
+    def test_execute(self, mock_checkpoint):
+        self.r.execute(self.leader)
+        with patch.object(Rewind, 'pg_rewind', Mock(return_value=False)):
+            mock_checkpoint.side_effect = ['1', '', '', '']
+            self.r.execute(self.leader)
+            self.r.execute(self.leader)
+            with patch.object(Rewind, 'check_leader_is_not_in_recovery', Mock(return_value=False)):
+                self.r.execute(self.leader)
+            self.p.config._config['remove_data_directory_on_rewind_failure'] = False
+            self.r.trigger_check_diverged_lsn()
+            self.r.execute(self.leader)
+
+        self.leader.member.data.update(version='1.5.7', checkpoint_after_promote=False, role='master')
+        self.assertIsNone(self.r.execute(self.leader))
+
+        del self.leader.member.data['checkpoint_after_promote']
+        with patch.object(Rewind, 'check_leader_is_not_in_recovery', Mock(return_value=False)):
+            self.assertIsNone(self.r.execute(self.leader))
+
+        with patch.object(Postgresql, 'is_running', Mock(return_value=True)):
+            self.r.execute(self.leader)
+
+    @patch.object(Postgresql, 'start', Mock())
+    @patch.object(Rewind, 'can_rewind', PropertyMock(return_value=True))
+    @patch.object(Rewind, '_get_local_timeline_lsn', Mock(return_value=(2, '40159C1')))
+    @patch.object(Rewind, 'check_leader_is_not_in_recovery')
+    def test__check_timeline_and_lsn(self, mock_check_leader_is_not_in_recovery):
+        mock_check_leader_is_not_in_recovery.return_value = False
+        self.r.trigger_check_diverged_lsn()
+        self.assertFalse(self.r.rewind_or_reinitialize_needed_and_possible(self.leader))
+        self.leader = self.leader.member
+        self.assertFalse(self.r.rewind_or_reinitialize_needed_and_possible(self.leader))
+        mock_check_leader_is_not_in_recovery.return_value = True
+        self.assertFalse(self.r.rewind_or_reinitialize_needed_and_possible(self.leader))
+        self.r.trigger_check_diverged_lsn()
+        with patch('psycopg2.connect', Mock(side_effect=Exception)):
+            self.assertFalse(self.r.rewind_or_reinitialize_needed_and_possible(self.leader))
+        self.r.trigger_check_diverged_lsn()
+        with patch.object(MockCursor, 'fetchone', Mock(side_effect=[('', 2, '0/0'), ('', b'3\t0/40159C0\tn\n')])):
+            self.assertFalse(self.r.rewind_or_reinitialize_needed_and_possible(self.leader))
+        self.r.trigger_check_diverged_lsn()
+        with patch.object(MockCursor, 'fetchone', Mock(return_value=('', 1, '0/0'))):
+            with patch.object(Rewind, '_get_local_timeline_lsn', Mock(return_value=(1, '0/0'))):
+                self.assertFalse(self.r.rewind_or_reinitialize_needed_and_possible(self.leader))
+            self.r.trigger_check_diverged_lsn()
+            self.assertTrue(self.r.rewind_or_reinitialize_needed_and_possible(self.leader))
+
+    @patch.object(MockCursor, 'fetchone', Mock(side_effect=[(True,), Exception]))
+    def test_check_leader_is_not_in_recovery(self):
+        self.r.check_leader_is_not_in_recovery()
+        self.r.check_leader_is_not_in_recovery()
+
+    @patch.object(Postgresql, 'controldata', Mock(return_value={"Latest checkpoint's TimeLineID": 1}))
+    def test_check_for_checkpoint_after_promote(self):
+        self.r.check_for_checkpoint_after_promote()

tests/test_wale_restore.py

@@ -6,9 +6,9 @@ from mock import Mock, PropertyMock, patch, mock_open
 from patroni.scripts import wale_restore
 from patroni.scripts.wale_restore import WALERestore, main as _main, get_major_version
 from six.moves import builtins
-from test_postgresql import MockConnect, psycopg2_connect
 from threading import current_thread
 
+from . import MockConnect, psycopg2_connect
 
 wale_output_header = (
     b'name\tlast_modified\t'