first commit

Dockerfile

@@ -25,7 +25,6 @@ RUN set -ex \
             | grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \
             | xargs apt-get install -y vim curl less jq locales haproxy sudo \
                             python3-etcd python3-kazoo python3-pip busybox \
-                            net-tools iputils-ping --fix-missing \
     && pip3 install dumb-init \
 \
     # Cleanup all locales but en_US.UTF-8
@@ -150,7 +149,6 @@ RUN sed -i 's/env python/&3/' /patroni*.py \
     && sed -i 's/^    \(replication\|superuser\|rewind\|unix_socket_directories\|\(\(  \)\{0,1\}\(username\|password\)\)\):/#&/' postgres?.yml \
     && sed -i 's/^      parameters:/      pg_hba:\n      - local all all trust\n      - host replication all all md5\n      - host all all all md5\n&\n        max_connections: 100/'  postgres?.yml \
     && if [ "$COMPRESS" = "true" ]; then chmod u+s /usr/bin/sudo; fi \
-    && chmod +s /bin/ping \
     && chown -R postgres:postgres $PGHOME /run /etc/haproxy
 
 USER postgres

docs/ENVIRONMENT.rst

@@ -11,9 +11,6 @@ Global/Universal
 -  **PATRONI\_NAME**: name of the node where the current instance of Patroni is running. Must be unique for the cluster.
 -  **PATRONI\_NAMESPACE**: path within the configuration store where Patroni will keep information about the cluster. Default value: "/service"
 -  **PATRONI\_SCOPE**: cluster name
-
-Log
----
 -  **PATRONI\_LOG\_LEVEL**: sets the general logging level. Default value is **INFO** (see `the docs for Python logging <https://docs.python.org/3.6/library/logging.html#levels>`_)
 -  **PATRONI\_LOG\_FORMAT**: sets the log formatting string. Default value is **%(asctime)s %(levelname)s: %(message)s** (see `the LogRecord attributes <https://docs.python.org/3.6/library/logging.html#logrecord-attributes>`_)
 -  **PATRONI\_LOG\_DATEFORMAT**: sets the datetime formatting string. (see the `formatTime() documentation <https://docs.python.org/3.6/library/logging.html#logging.Formatter.formatTime>`_)
@@ -22,6 +19,8 @@ Log
 -  **PATRONI\_LOG\_FILE\_NUM**: The number of application logs to retain.
 -  **PATRONI\_LOG\_FILE\_SIZE**: Size of patroni.log file (in bytes) that triggers a log rolling.
 -  **PATRONI\_LOG\_LOGGERS**: Redefine logging level per python module. Example ``PATRONI_LOG_LOGGERS="{patroni.postmaster: WARNING, urllib3: DEBUG}"``
+-  **PATRONI\_DEBUG\_MODE**: When set to a non-empty value, makes Patroni run in the special debug mode that enables stepping with a debugger through some execution paths within Patroni `
+Example ``PATRONI_DEBUG_MODE="on"``
 
 Bootstrap configuration
 -----------------------
@@ -65,10 +64,6 @@ Etcd
 -  **PATRONI\_ETCD\_CERT**: File with the client certificate.
 -  **PATRONI\_ETCD\_KEY**: File with the client key. Can be empty if the key is part of certificate.
 
-ZooKeeper
----------
--  **PATRONI\_ZOOKEEPER\_HOSTS**: comma separated list of ZooKeeper cluster members: "'host1:port1','host2:port2','etc...'". It is important to quote every single entity!
-
 Exhibitor
 ---------
 -  **PATRONI\_EXHIBITOR\_HOSTS**: initial list of Exhibitor (ZooKeeper) nodes in format: 'host1,host2,etc...'. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes.
@@ -84,7 +79,7 @@ Kubernetes
 -  **PATRONI\_KUBERNETES\_ROLE\_LABEL**: (optional) name of the label containing Postgres role (`master` or `replica`). Patroni will set this label on the pod it is running in. Default value is `role`.
 -  **PATRONI\_KUBERNETES\_USE\_ENDPOINTS**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
 -  **PATRONI\_KUBERNETES\_POD\_IP**: (optional) IP address of the pod Patroni is running in. This value is required when `PATRONI_KUBERNETES_USE_ENDPOINTS` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
--  **PATRONI\_KUBERNETES\_PORTS**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``PATRONI_KUBERNETES_PORTS='[{"name": "postgresql", "port": 5432}]'`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `PATRONI_KUBERNETES_USE_ENDPOINTS` is set.
+-  **PATRONI\_KUBERNETES\_PORTS**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``PATRONI_KUBERNETES_PORTS='{[{"name": "postgresql", "port": 5432}]}'`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `PATRONI_KUBERNETES_USE_ENDPOINTS` is set.
 
 PostgreSQL
 ----------
@@ -96,25 +91,10 @@ PostgreSQL
 -  **PATRONI\_POSTGRESQL\_PGPASS**: path to the `.pgpass <https://www.postgresql.org/docs/current/static/libpq-pgpass.html>`__ password file. Patroni creates this file before executing pg\_basebackup and under some other circumstances. The location must be writable by Patroni.
 -  **PATRONI\_REPLICATION\_USERNAME**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication
 -  **PATRONI\_REPLICATION\_PASSWORD**: replication password; the user will be created during initialization.
--  **PATRONI\_REPLICATION\_SSLMODE**: (optional) maps to the `sslmode <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE>`__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation <https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS>`__. The default mode is ``prefer``.
--  **PATRONI\_REPLICATION\_SSLKEY**: (optional) maps to the `sslkey <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY>`__ connection parameter, which specifies the location of the secret key used with the client's certificate.
--  **PATRONI\_REPLICATION\_SSLCERT**: (optional) maps to the `sslcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCERT>`__ connection parameter, which specifies the location of the client certificate.
--  **PATRONI\_REPLICATION\_SSLROOTCERT**: (optional) maps to the `sslrootcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLROOTCERT>`__ connection parameter, which specifies the location of a file containing one ore more certificate authorities (CA) certificates that the client will use to verify a server's certificate.
--  **PATRONI\_REPLICATION\_SSLCRL**: (optional) maps to the `sslcrl <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCRL>`__ connection parameter, which specifies the location of a file containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list.
 -  **PATRONI\_SUPERUSER\_USERNAME**: name for the superuser, set during initialization (initdb) and later used by Patroni to connect to the postgres. Also this user is used by pg_rewind.
 -  **PATRONI\_SUPERUSER\_PASSWORD**: password for the superuser, set during initialization (initdb).
--  **PATRONI\_SUPERUSER\_SSLMODE**: (optional) maps to the `sslmode <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE>`__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation <https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS>`__. The default mode is ``prefer``.
--  **PATRONI\_SUPERUSER\_SSLKEY**: (optional) maps to the `sslkey <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY>`__ connection parameter, which specifies the location of the secret key used with the client's certificate.
--  **PATRONI\_SUPERUSER\_SSLCERT**: (optional) maps to the `sslcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCERT>`__ connection parameter, which specifies the location of the client certificate.
--  **PATRONI\_SUPERUSER\_SSLROOTCERT**: (optional) maps to the `sslrootcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLROOTCERT>`__ connection parameter, which specifies the location of a file containing one ore more certificate authorities (CA) certificates that the client will use to verify a server's certificate.
--  **PATRONI\_SUPERUSER\_SSLCRL**: (optional) maps to the `sslcrl <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCRL>`__ connection parameter, which specifies the location of a file containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list.
 -  **PATRONI\_REWIND\_USERNAME**: name for the user for ``pg_rewind``; the user will be created during initialization of postgres 11+ and all necessary `permissions <https://www.postgresql.org/docs/11/app-pgrewind.html#id-1.9.5.8.8>`__ will be granted.
 -  **PATRONI\_REWIND\_PASSWORD**: password for the user for ``pg_rewind``; the user will be created during initialization.
--  **PATRONI\_REWIND\_SSLMODE**: (optional) maps to the `sslmode <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE>`__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation <https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS>`__. The default mode is ``prefer``.
--  **PATRONI\_REWIND\_SSLKEY**: (optional) maps to the `sslkey <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY>`__ connection parameter, which specifies the location of the secret key used with the client's certificate.
--  **PATRONI\_REWIND\_SSLCERT**: (optional) maps to the `sslcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCERT>`__ connection parameter, which specifies the location of the client certificate.
--  **PATRONI\_REWIND\_SSLROOTCERT**: (optional) maps to the `sslrootcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLROOTCERT>`__ connection parameter, which specifies the location of a file containing one ore more certificate authorities (CA) certificates that the client will use to verify a server's certificate.
--  **PATRONI\_REWIND\_SSLCRL**: (optional) maps to the `sslcrl <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCRL>`__ connection parameter, which specifies the location of a file containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list.
 
 REST API
 --------
@@ -124,12 +104,7 @@ REST API
 -  **PATRONI\_RESTAPI\_PASSWORD**: Basic-auth password to protect unsafe REST API endpoints.
 -  **PATRONI\_RESTAPI\_CERTFILE**: Specifies the file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL.
 -  **PATRONI\_RESTAPI\_KEYFILE**: Specifies the file with the secret key in the PEM format.
--  **PATRONI\_RESTAPI\_CAFILE**: Specifies the file with the CA_BUNDLE with certificates of trusted CAs to use while verifying client certs.
--  **PATRONI\_RESTAPI\_VERIFY\_CLIENT**: ``none``, ``optional`` or ``required``. When ``none`` REST API will not check client certificates. When ``required`` client certificates are required for all REST API calls. When ``optional`` client certificates are required for all unsafe REST API endpoints. If ``verify_client`` is set to ``optional`` or ``required`` basic-auth is not checked.
 
-CTL
+ZooKeeper
----
+---------
--  **PATRONI\_CTL\_INSECURE**: Allow connections to REST API without verifying SSL certs.
+-  **PATRONI\_ZOOKEEPER\_HOSTS**: comma separated list of ZooKeeper cluster members: "'host1:port1','host2:port2','etc...'". It is important to quote every single entity!
--  **PATRONI\_CTL\_CACERT**: Specifies the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs. If not provided patronictl will use the value provided for REST API "cafile" parameter.
--  **PATRONI\_CTL\_CERTFILE**: Specifies the file with the client certificate in the PEM format. If not provided patronictl will use the value provided for REST API "certfile" parameter.
--  **PATRONI\_CTL\_KEYFILE**: Specifies the file with the client secret key in the PEM format. If not provided patronictl will use the value provided for REST API "keyfile" parameter.

docs/SETTINGS.rst

@@ -4,39 +4,6 @@
 YAML Configuration Settings
 ===========================
 
-.. _dynamic_configuration_settings:
-
-Dynamic configuration settings
-------------------------------
-
-Dynamic configuration is stored in the DCS (Distributed Configuration Store) and applied on all cluster nodes. Some parameters, like **loop_wait**, **ttl**, **postgresql.parameters.max_connections**, **postgresql.parameters.max_worker_processes** and so on could be set only in the dynamic configuration. Some other parameters like **postgresql.listen**, **postgresql.data_dir** could be set only locally, i.e. in the Patroni config file or via :ref:`configuration <environment>` variable. In most cases the local configuration will override the dynamic configuration. In order to change the dynamic configuration you can use either ``patronictl edit-config`` tool or Patroni :ref:`REST API <rest_api>`.
-
--  **loop\_wait**: the number of seconds the loop will sleep. Default value: 10
--  **ttl**: the TTL to acquire the leader lock (in seconds). Think of it as the length of time before initiation of the automatic failover process. Default value: 30
--  **retry\_timeout**: timeout for DCS and PostgreSQL operation retries (in seconds). DCS or network issues shorter than this will not cause Patroni to demote the leader. Default value: 10
--  **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election.
--  **master\_start\_timeout**: the amount of time a master is allowed to recover from failures before failover is triggered (in seconds). Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Worst case failover time for master failure is: loop\_wait + master\_start\_timeout + loop\_wait, unless master\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff.
--  **synchronous\_mode**: turns on synchronous replication mode. In this mode a replica will be chosen as synchronous and only the latest leader and synchronous replica are able to participate in leader election. Synchronous mode makes sure that successfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See :ref:`replication modes documentation <replication_modes>` for details.
--  **synchronous\_mode\_strict**: prevents disabling synchronous replication if no synchronous replicas are available, blocking all client writes to the master. See :ref:`replication modes documentation <replication_modes>` for details.
--  **postgresql**:
-    -  **use\_pg\_rewind**: whether or not to use pg_rewind. Defaults to `false`.
-    -  **use\_slots**: whether or not to use replication_slots. Defaults to `true` on PostgreSQL 9.4+.
-    -  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower. There is no recovery.conf anymore in PostgreSQL 12, but you may continue using this section, because Patroni handles it transparently.
-    -  **parameters**: list of configuration settings for Postgres.
--  **standby\_cluster**: if this section is defined, we want to bootstrap a standby cluster.
-    -  **host**: an address of remote master
-    -  **port**: a port of remote master
-    -  **primary\_slot\_name**: which slot on the remote master to use for replication. This parameter is optional, the default value is derived from the instance name (see function `slot_name_from_member_name`).
-    -  **create\_replica\_methods**: an ordered list of methods that can be used to bootstrap standby leader from the remote master, can be different from the list defined in :ref:`postgresql_settings`
-    -  **restore\_command**: command to restore WAL records from the remote master to standby leader, can be different from the list defined in :ref:`postgresql_settings`
-    -  **archive\_cleanup\_command**: cleanup command for standby leader
-    -  **recovery\_min\_apply\_delay**: how long to wait before actually apply WAL records on a standby leader
--  **slots**: define permanent replication slots. These slots will be preserved during switchover/failover. Patroni will try to create slots before opening connections to the cluster.
-    -  **my_slot_name**: the name of replication slot. It is the responsibility of the operator to make sure that there are no clashes in names between replication slots automatically created by Patroni for members and permanent replication slots.
-        -  **type**: slot type. Could be ``physical`` or ``logical``. If the slot is logical, you have to additionally define ``database`` and ``plugin``.
-        -  **database**: the database name where logical slots should be created.
-        -  **plugin**: the plugin name for the logical slot.
-
 Global/Universal
 ----------------
 -  **name**: the name of the host. Must be unique for the cluster.
@@ -60,7 +27,32 @@ Log
 
 Bootstrap configuration
 -----------------------
--  **dcs**: This section will be written into `/<namespace>/<scope>/config` of the given configuration store after initializing of new cluster. The global dynamic configuration for the cluster. Under the ``bootstrap.dcs`` you can put any of the parameters described in the :ref:`Dynamic Configuration settings <dynamic_configuration_settings>` and after Patroni initialized (bootstrapped) the new cluster, it will write this section into `/<namespace>/<scope>/config` of the configuration store. All later changes of ``bootstrap.dcs`` will not take any effect! If you want to change them please use either ``patronictl edit-config`` or Patroni :ref:`REST API <rest_api>`.
+-  **dcs**: This section will be written into `/<namespace>/<scope>/config` of a given configuration store after initializing of new cluster. This is the global configuration for the cluster. If you want to change some parameters for all cluster nodes - just do it in DCS (or via Patroni API) and all nodes will apply this configuration.
+    -  **loop\_wait**: the number of seconds the loop will sleep. Default value: 10
+    -  **ttl**: the TTL to acquire the leader lock. Think of it as the length of time before initiation of the automatic failover process. Default value: 30
+    -  **retry\_timeout**: timeout for DCS and PostgreSQL operation retries. DCS or network issues shorter than this will not cause Patroni to demote the leader. Default value: 10
+    -  **maximum\_lag\_on\_failover**: the maximum bytes a follower may lag to be able to participate in leader election.
+    -  **master\_start\_timeout**: the amount of time a master is allowed to recover from failures before failover is triggered. Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Best worst case failover time for master failure is: loop\_wait + master\_start\_timeout + loop\_wait, unless master\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff.
+    -  **synchronous\_mode**: turns on synchronous replication mode. In this mode a replica will be chosen as synchronous and only the latest leader and synchronous replica are able to participate in leader election. Synchronous mode makes sure that successfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See :ref:`replication modes documentation <replication_modes>` for details.
+    -  **synchronous\_mode\_strict**: prevents disabling synchronous replication if no synchronous replicas are available, blocking all client writes to the master. See :ref:`replication modes documentation <replication_modes>` for details.
+    -  **postgresql**:
+        -  **use\_pg\_rewind**: whether or not to use pg_rewind
+        -  **use\_slots**: whether or not to use replication_slots. Must be False for PostgreSQL 9.3. You should comment out max_replication_slots before it becomes ineligible for leader status.
+        -  **recovery\_conf**: additional configuration settings written to recovery.conf when configuring follower.
+        -  **parameters**: list of configuration settings for Postgres. Many of these are required for replication to work.
+    -  **standby\_cluster**: if this section is defined, we want to bootstrap a standby cluster.
+        -  **host**: an address of remote master
+        -  **port**: a port of remote master
+        -  **primary\_slot\_name**: which slot on the remote master to use for replication. This parameter is optional, the default value is derived from the instance name (see function `slot_name_from_member_name`).
+        -  **create\_replica\_methods**: an ordered list of methods that can be used to bootstrap standby leader from the remote master, can be different from the list defined in :ref:`postgresql_settings`
+        -  **restore\_command**: command to restore WAL records from the remote master to standby leader, can be different from the list defined in :ref:`postgresql_settings`
+        -  **archive\_cleanup\_command**: cleanup command for standby leader
+        -  **recovery\_min\_apply\_delay**: how long to wait before actually apply WAL records on a standby leader
+    -  **slots**: define permanent replication slots. These slots will be preserved during switchover/failover. Patroni will try to create slots before opening connections to the cluster.
+        -  **my_slot_name**: the name of replication slot. It is the responsibility of the operator to make sure that there are no clashes in names between replication slots automatically created by Patroni for members and permanent replication slots.
+            -  **type**: slot type. Could be ``physical`` or ``logical``. If the slot is logical, you have to additionally define ``database`` and ``plugin``.
+            -  **database**: the database name where logical slots should be created.
+            -  **plugin**: the plugin name for the logical slot.
 -  **method**: custom script to use for bootstrapping this cluster.
    See :ref:`custom bootstrap methods documentation <custom_bootstrap>` for details.
    When ``initdb`` is specified revert to the default ``initdb`` command. ``initdb`` is also triggered when no ``method``
@@ -118,10 +110,6 @@ Most of the parameters are optional, but you have to specify one of the **host**
 -  **cert**: (optional) file with the client certificate.
 -  **key**: (optional) file with the client key. Can be empty if the key is part of **cert**.
 
-ZooKeeper
-----------
--  **hosts**: list of ZooKeeper cluster members in format: ['host1:port1', 'host2:port2', 'etc...'].
-
 Exhibitor
 ---------
 -  **hosts**: initial list of Exhibitor (ZooKeeper) nodes in format: 'host1,host2,etc...'. This list updates automatically whenever the Exhibitor (ZooKeeper) cluster topology changes.
@@ -138,7 +126,7 @@ Kubernetes
 -  **role\_label**: (optional) name of the label containing role (master or replica). Patroni will set this label on the pod it runs in. Default value is ``role``.
 -  **use\_endpoints**: (optional) if set to true, Patroni will use Endpoints instead of ConfigMaps to run leader elections and keep cluster state.
 -  **pod\_ip**: (optional) IP address of the pod Patroni is running in. This value is required when `use_endpoints` is enabled and is used to populate the leader endpoint subsets when the pod's PostgreSQL is promoted.
--  **ports**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``kubernetes.ports: [{"name": "postgresql", "port": 5432}]`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `kubernetes.use_endpoints` is set.
+-  **ports**: (optional) if the Service object has the name for the port, the same name must appear in the Endpoint object, otherwise service won't work. For example, if your service is defined as ``{Kind: Service, spec: {ports: [{name: postgresql, port: 5432, targetPort: 5432}]}}``, then you have to set ``kubernetes.ports: {[{"name": "postgresql", "port": 5432}]}`` and Patroni will use it for updating subsets of the leader Endpoint. This parameter is used only if `kubernetes.use_endpoints` is set.
 
 .. _postgresql_settings:
 
@@ -148,27 +136,12 @@ PostgreSQL
     -  **superuser**:
         -  **username**: name for the superuser, set during initialization (initdb) and later used by Patroni to connect to the postgres.
         -  **password**: password for the superuser, set during initialization (initdb).
-        -  **sslmode**: (optional) maps to the `sslmode <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE>`__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation <https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS>`__. The default mode is ``prefer``.
-        -  **sslkey**: (optional) maps to the `sslkey <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY>`__ connection parameter, which specifies the location of the secret key used with the client's certificate.
-        -  **sslcert**: (optional) maps to the `sslcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCERT>`__ connection parameter, which specifies the location of the client certificate.
-        -  **sslrootcert**: (optional) maps to the `sslrootcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLROOTCERT>`__ connection parameter, which specifies the location of a file containing one ore more certificate authorities (CA) certificates that the client will use to verify a server's certificate.
-        -  **sslcrl**: (optional) maps to the `sslcrl <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCRL>`__ connection parameter, which specifies the location of a file containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list.
     -  **replication**:
         -  **username**: replication username; the user will be created during initialization. Replicas will use this user to access master via streaming replication
         -  **password**: replication password; the user will be created during initialization.
-        -  **sslmode**: (optional) maps to the `sslmode <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE>`__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation <https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS>`__. The default mode is ``prefer``.
-        -  **sslkey**: (optional) maps to the `sslkey <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY>`__ connection parameter, which specifies the location of the secret key used with the client's certificate.
-        -  **sslcert**: (optional) maps to the `sslcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCERT>`__ connection parameter, which specifies the location of the client certificate.
-        -  **sslrootcert**: (optional) maps to the `sslrootcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLROOTCERT>`__ connection parameter, which specifies the location of a file containing one ore more certificate authorities (CA) certificates that the client will use to verify a server's certificate.
-        -  **sslcrl**: (optional) maps to the `sslcrl <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCRL>`__ connection parameter, which specifies the location of a file containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list.
     -  **rewind**:
         -  **username**: name for the user for ``pg_rewind``; the user will be created during initialization of postgres 11+ and all necessary `permissions <https://www.postgresql.org/docs/11/app-pgrewind.html#id-1.9.5.8.8>`__ will be granted.
         -  **password**: password for the user for ``pg_rewind``; the user will be created during initialization.
-        -  **sslmode**: (optional) maps to the `sslmode <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLMODE>`__ connection parameter, which allows a client to specify the type of TLS negotiation mode with the server. For more information on how each mode works, please visit the `PostgreSQL documentation <https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS>`__. The default mode is ``prefer``.
-        -  **sslkey**: (optional) maps to the `sslkey <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY>`__ connection parameter, which specifies the location of the secret key used with the client's certificate.
-        -  **sslcert**: (optional) maps to the `sslcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCERT>`__ connection parameter, which specifies the location of the client certificate.
-        -  **sslrootcert**: (optional) maps to the `sslrootcert <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLROOTCERT>`__ connection parameter, which specifies the location of a file containing one ore more certificate authorities (CA) certificates that the client will use to verify a server's certificate.
-        -  **sslcrl**: (optional) maps to the `sslcrl <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCRL>`__ connection parameter, which specifies the location of a file containing a certificate revocation list. A client will reject connecting to any server that has a certificate present in this list.
 -  **callbacks**: callback scripts to run on certain actions. Patroni will pass the action, role and cluster name. (See scripts/aws.py as an example of how to write them.)
         -  **on\_reload**: run this script when configuration reload is triggered.
         -  **on\_restart**: run this script when the postgres restarts (without changing role).
@@ -202,7 +175,7 @@ PostgreSQL
 
 REST API
 --------
--  **connect\_address**: IP address (or hostname) and port, to access the Patroni's :ref:`REST API <rest_api>`. All the members of the cluster must be able to connect to this address, so unless the Patroni setup is intended for a demo inside the localhost, this address must be a non "localhost" or loopback address (ie: "localhost" or "127.0.0.1"). It can serve as an endpoint for HTTP health checks (read below about the "listen" REST API parameter), and also for user queries (either directly or via the REST API), as well as for the health checks done by the cluster members during leader elections (for example, to determine whether the master is still running, or if there is a node which has a WAL position that is ahead of the one doing the query; etc.) The connect_address is put in the member key in DCS, making it possible to translate the member name into the address to connect to its REST API.
+-  **connect\_address**: IP address (or hostname) and port, to access the Patroni's REST API. All the members of the cluster must be able to connect to this address, so unless the Patroni setup is intended for a demo inside the localhost, this address must be a non "localhost" or loopback addres (ie: "localhost" or "127.0.0.1"). It can serve as a endpoint for HTTP health checks (read below about the "listen" REST API parameter), and also for user queries (either directly or via the REST API), as well as for the health checks done by the cluster members during leader elections (for example, to determine whether the master is still running, or if there is a node which has a WAL position that is ahead of the one doing the query; etc.) The connect_address is put in the member key in DCS, making it possible to translate the member name into the address to connect to its REST API.
 
 -  **listen**: IP address (or hostname) and port that Patroni will listen to for the REST API - to provide also the same health checks and cluster messaging between the participating nodes, as described above. to provide health-check information for HAProxy (or any other load balancer capable of doing a HTTP "OPTION" or "GET" checks).
 
@@ -213,8 +186,6 @@ REST API
 
         -  **certfile**: Specifies the file with the certificate in the PEM format. If the certfile is not specified or is left empty, the API server will work without SSL.
         -  **keyfile**: Specifies the file with the secret key in the PEM format.
-        -  **cafile**: Specifies the file with the CA_BUNDLE with certificates of trusted CAs to use while verifying client certs.
-        -  **verify\_client**: ``none``, ``optional`` or ``required``. When ``none`` REST API will not check client certificates. When ``required`` client certificates are required for all REST API calls. When ``optional`` client certificates are required for all unsafe REST API endpoints. If ``verify_client`` is set to ``optional`` or ``required`` basic-auth is not checked.
 
 .. _patronictl_settings:
 
@@ -222,20 +193,15 @@ CTL
 ---
 -  **Optional**:
     -  **insecure**: Allow connections to REST API without verifying SSL certs.
-    -  **cacert**: Specifies the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs. If not provided patronictl will use the value provided for REST API "cafile" parameter.
+    -  **cacert**: Specifices the file with the CA_BUNDLE file or directory with certificates of trusted CAs to use while verifying REST API SSL certs.
-    -  **certfile**: Specifies the file with the client certificate in the PEM format. If not provided patronictl will use the value provided for REST API "certfile" parameter.
+    -  **certfile**: Specifies the file with the certificate in the PEM format to use while verifying REST API SSL certs. If not provided patronictl will use the value provided for REST API "certfile" parameter.
-    -  **keyfile**: Specifies the file with the client secret key in the PEM format. If not provided patronictl will use the value provided for REST API "keyfile" parameter.
+
+ZooKeeper
+----------
+-  **hosts**: list of ZooKeeper cluster members in format: ['host1:port1', 'host2:port2', 'etc...'].
 
 Watchdog
 --------
 - **mode**: ``off``, ``automatic`` or ``required``. When ``off`` watchdog is disabled. When ``automatic`` watchdog will be used if available, but ignored if it is not. When ``required`` the node will not become a leader unless watchdog can be successfully enabled.
 - **device**: Path to watchdog device. Defaults to ``/dev/watchdog``.
 - **safety_margin**: Number of seconds of safety margin between watchdog triggering and leader key expiration.
-
-Tags
-----
-- **nofailover**: ``true`` or ``false``, controls whether this node is allowed to participate in the leader race and become a leader. Defaults to ``false``
-- **clonefrom**: ``true`` or ``false``. If set to ``true`` other nodes might prefer to use this node for bootstrap (take ``pg_basebackup`` from). If there are several nodes with ``clonefrom`` tag set to ``true`` the node to bootstrap from will be chosen randomly. The default value is ``false``.
-- **noloadbalance**: ``true`` or ``false``. If set to ``true`` the node will return HTTP Status Code 503 for the ``GET /replica`` REST API health-check and therefore will be excluded from the load-balancing. Defaults to ``false``.
-- **replicatefrom**: The IP address/hostname of another replica. Used to support cascading replication.
-- **nosync**: ``true`` or ``false``. If set to ``true`` the node will never be selected as a synchronous replica.

docs/dynamic_configuration.rst

@@ -14,7 +14,7 @@ Patroni configuration is stored in the DCS (Distributed Configuration Store). Th
 
 - Local :ref:`configuration <settings>` (patroni.yml).
 	These options are defined in the configuration file and take precedence over dynamic configuration.
-	patroni.yml could be changed and reloaded in runtime (without restart of Patroni) by sending SIGHUP to the Patroni process, performing ``POST /reload`` REST-API request or executing ``patronictl reload``.
+	patroni.yml could be changed and reload in runtime (without restart of Patroni) by sending SIGHUP to the Patroni process or by performing ``POST /reload`` REST-API request.
 
 - Environment :ref:`configuration <environment>`.
 	It is possible to set/override some of the "Local" configuration parameters with environment variables.
@@ -83,3 +83,147 @@ Upon changing these options, Patroni will read the relevant section of the confi
 run-time values.
 
 Patroni nodes are dumping the state of the DCS options to disk upon for every change of the configuration into the file ``patroni.dynamic.json`` located in the Postgres data directory. Only the master is allowed to restore these options from the on-disk dump if these are completely absent from the DCS or if they are invalid.
+
+REST API
+========
+
+We provide a REST API endpoint for working with dynamic configuration.
+
+GET /config
+-----------
+Get current version of dynamic configuration.
+
+.. code-block:: bash
+
+	$ curl -s localhost:8008/config | jq .
+	{
+	  "ttl": 30,
+	  "loop_wait": 10,
+	  "retry_timeout": 10,
+	  "maximum_lag_on_failover": 1048576,
+	  "postgresql": {
+	    "use_slots": true,
+	    "use_pg_rewind": true,
+	    "parameters": {
+	      "hot_standby": "on",
+	      "wal_log_hints": "on",
+	      "wal_keep_segments": 8,
+	      "wal_level": "hot_standby",
+	      "max_wal_senders": 5,
+	      "max_replication_slots": 5,
+	      "max_connections": "100"
+	    }
+	  }
+	}
+
+PATCH /config
+-------------
+Change existing configuration.
+
+.. code-block:: bash
+
+	$ curl -s -XPATCH -d \
+		'{"loop_wait":5,"ttl":20,"postgresql":{"parameters":{"max_connections":"101"}}}' \
+		http://localhost:8008/config | jq .
+	{
+	  "ttl": 20,
+	  "loop_wait": 5,
+	  "maximum_lag_on_failover": 1048576,
+	  "retry_timeout": 10,
+	  "postgresql": {
+	    "use_slots": true,
+	    "use_pg_rewind": true,
+	    "parameters": {
+	      "hot_standby": "on",
+	      "wal_log_hints": "on",
+	      "wal_keep_segments": 8,
+	      "wal_level": "hot_standby",
+	      "max_wal_senders": 5,
+	      "max_replication_slots": 5,
+	      "max_connections": "101"
+	    }
+	  }
+	}
+
+The above REST API call patches the existing configuration and returns the new configuration.
+
+Let's check that the node processed this configuration. First of all it should start printing log lines every 5 seconds (loop_wait=5). The change of "max_connections" requires a restart, so the "restart_pending" flag should be exposed:
+
+.. code-block:: bash
+
+	$ curl -s http://localhost:8008/patroni | jq .
+	{
+	  "pending_restart": true,
+	  "database_system_identifier": "6287881213849985952",
+	  "postmaster_start_time": "2016-06-13 13:13:05.211 CEST",
+	  "xlog": {
+	    "location": 2197818976
+	  },
+	  "patroni": {
+	    "scope": "batman",
+	    "version": "1.0"
+	  },
+	  "state": "running",
+	  "role": "master",
+	  "server_version": 90503
+	}
+
+Removing parameters:
+
+If you want to remove (reset) some setting just patch it with ``null``:
+
+.. code-block:: bash
+
+	$ curl -s -XPATCH -d \
+		'{"postgresql":{"parameters":{"max_connections":null}}}' \
+		http://localhost:8008/config | jq .
+	{
+	  "ttl": 20,
+	  "loop_wait": 5,
+	  "retry_timeout": 10,
+	  "maximum_lag_on_failover": 1048576,
+	  "postgresql": {
+	    "use_slots": true,
+	    "use_pg_rewind": true,
+	    "parameters": {
+	      "hot_standby": "on",
+	      "unix_socket_directories": ".",
+	      "wal_keep_segments": 8,
+	      "wal_level": "hot_standby",
+	      "wal_log_hints": "on",
+	      "max_wal_senders": 5,
+	      "max_replication_slots": 5
+	    }
+	  }
+	}
+
+Above call removes ``postgresql.parameters.max_connections`` from the dynamic configuration.
+
+PUT /config
+-----------
+
+It's also possible to perform the full rewrite of an existing dynamic configuration unconditionally:
+
+.. code-block:: bash
+
+	$ curl -s -XPUT -d \
+		'{"maximum_lag_on_failover":1048576,"retry_timeout":10,"postgresql":{"use_slots":true,"use_pg_rewind":true,"parameters":{"hot_standby":"on","wal_log_hints":"on","wal_keep_segments":8,"wal_level":"hot_standby","unix_socket_directories":".","max_wal_senders":5}},"loop_wait":3,"ttl":20}' \
+		http://localhost:8008/config | jq .
+	{
+	  "ttl": 20,
+	  "maximum_lag_on_failover": 1048576,
+	  "retry_timeout": 10,
+	  "postgresql": {
+	    "use_slots": true,
+	    "parameters": {
+	      "hot_standby": "on",
+	      "unix_socket_directories": ".",
+	      "wal_keep_segments": 8,
+	      "wal_level": "hot_standby",
+	      "wal_log_hints": "on",
+	      "max_wal_senders": 5
+	    },
+	    "use_pg_rewind": true
+	  },
+	  "loop_wait": 3
+	}

docs/index.rst

@@ -19,7 +19,6 @@ We call Patroni a "template" because it is far from being a one-size-fits-all or
 
    README
    dynamic_configuration
-   rest_api
    ENVIRONMENT
    SETTINGS
    replica_bootstrap

docs/releases.rst

@@ -3,134 +3,6 @@
 Release notes
 =============
 
-Version 1.6.1
-
-**New features**
-
-- Added ``PATRONICTL_CONFIG_FILE`` environment variable (msvechla)
-
-  It allows configuring the ``--config-file`` argument for ``patronictl`` from the environment.
-
-- Implement ``patronictl history`` (Alexander Kukushkin)
-
-  It shows the history of failovers/switchovers.
-
-- Pass ``-c statement_timeout=0`` in ``PGOPTIONS`` when doing ``pg_rewind`` (Alexander Kukushkin)
-
-  It protects from the case when ``statement_timeout`` on the server is set to some small value and one of the statements executed by pg_rewind is canceled.
-
-- Allow lower values for PostgreSQL configuration (Soulou)
-
-  Patroni didn't allow some of the PostgreSQL configuration parameters be set smaller than some hardcoded values. Now the minimal allowed values are smaller, default values have not been changed.
-
-- Allow for certificate-based authentication (Jonathan S. Katz)
-
-  This feature enables certificate-based authentication for superuser, replication, rewind accounts and allows the user to specify the ``sslmode`` they wish to connect with.
-
-- Use the ``passfile`` in the ``primary_conninfo`` instead of password (Alexander Kukushkin)
-
-  It allows to avoid setting ``600`` permissions on postgresql.conf
-
-- Perform ``pg_ctl reload`` regardless of config changes (Alexander Kukushkin)
-
-  It is possible that some config files are not controlled by Patroni. When somebody is doing a reload via the REST API or by sending SIGHUP to the Patroni process, the usual expectation is that Postgres will also be reloaded. Previously it didn't happen when there were no changes in the ``postgresql`` section of Patroni config.
-
-- Compare all recovery parameters, not only ``primary_conninfo`` (Alexander Kukushkin)
-
-  Previously the ``check_recovery_conf()`` method was only checking whether ``primary_conninfo`` has changed, never taking into account all other recovery parameters.
-
-- Make it possible to apply some recovery parameters without restart (Alexander Kukushkin)
-
-  Starting from PostgreSQL 12 the following recovery parameters could be changed without restart: ``archive_cleanup_command``, ``promote_trigger_file``, ``recovery_end_command``, and ``recovery_min_apply_delay``. In future Postgres releases this list will be extended and Patroni will support it automatically.
-
-- Make it possible to change ``use_slots`` online (Alexander Kukushkin)
-
-  Previously it required restarting Patroni and removing slots manually.
-
-- Remove only ``PATRONI_`` prefixed environment variables when starting up Postgres (Cody Coons)
-
-  It will solve a lot of problems with running different Foreign Data Wrappers.
-
-
-**Stability improvements**
-
-- Use LIST + WATCH when working with K8s API (Alexander Kukushkin)
-
-  It allows to efficiently receive object changes (pods, endpoints/configmaps) and makes less stress on K8s master nodes.
-
-- Improve the workflow when PGDATA is not empty during bootstrap (Alexander Kukushkin)
-
-  According to the ``initdb`` source code it might consider a PGDATA empty when there are only ``lost+found`` and ``.dotfiles`` in it. Now Patroni does the same. If ``PGDATA`` happens to be non-empty, and at the same time not valid from the ``pg_controldata`` point of view, Patroni will complain and exit.
-
-- Avoid calling expensive ``os.listdir()`` on every HA loop (Alexander Kukushkin)
-
-  When the system is under IO stress, ``os.listdir()`` could take a few seconds (or even minutes) to execute, badly affecting the HA loop of Patroni. This could even cause the leader key to disappear from DCS due to the lack of updates. There is a better and less expensive way to check that the PGDATA is not empty. Now we check the presence of the ``global/pg_control`` file in the PGDATA.
-
-- Some improvements in logging infrastructure (Alexander Kukushkin)
-
-  Previously threre was a possibility to loose the last few log lines on shutdown because the logging thread was a ``daemon`` thread.
-
-- Use ``spawn`` multiprocessing start method on python 3.4+ (Maciej Kowalczyk)
-
-  It is a known `issue <https://bugs.python.org/issue6721>`__ in Python that threading and multiprocessing do not mix well. Switching from the default method ``fork`` to the ``spawn`` is a recommended workaround. Not doing so might result in the Postmaster starting process hanging and Patroni indefinitely reporting ``INFO: restarting after failure in progress``, while  Postgres is actually up and running.
-
-**Improvements in REST API**
-
-- Make it possible to check client certificates in the REST API (Alexander Kukushkin)
-
-  If the ``verify_client`` is set to ``required``, Patroni will check client certificates for all REST API calls. When it is set to ``optional``, client certificates are checked for all unsafe REST API endpoints.
-
-- Return the response code 503 for the ``GET /replica`` health check request if Postgres is not running (Alexander Anikin)
-
-  Postgres might spend significant time in recovery before it starts accepting client connections.
-
-- Implement ``/history`` and ``/cluster`` endpoints (Alexander Kukushkin)
-
-  The ``/history`` endpoint shows the content of the ``history`` key in DCS. The ``/cluster`` endpoint shows all cluster members and some service info like pending and scheduled restarts or switchovers.
-
-
-**Improvements in Etcd support**
-
-- Retry on Etcd RAFT internal error (Alexander Kukushkin)
-
-  When the Etcd node is being shut down, it sends ``response code=300, data='etcdserver: server stopped'``, which was causing Patroni to demote the primary.
-
-- Don't give up on Etcd request retry too early (Alexander Kukushkin)
-
-  When there were some network problems, Patroni was quickly exhausting the list of Etcd nodes and giving up without using the whole ``retry_timeout``, potentially resulting in demoting the primary.
-
-
-**Bugfixes**
-
-- Disable ``synchronous_commit`` when granting execute permissions to the ``pg_rewind`` user (kremius)
-
-  If the bootstrap is done with ``synchronous_mode_strict: true`` the `GRANT EXECUTE` statement was waiting indefinitely due to the non-synchronous nodes being available.
-
-- Fix memory leak on python 3.7 (Alexander Kukushkin)
-
-  Patroni is using ``ThreadingMixIn`` to process REST API requests and python 3.7 made threads spawn for every request non-daemon by default.
-
-- Fix race conditions in asynchronous actions (Alexander Kukushkin)
-
-  There was a chance that ``patronictl reinit --force`` could be overwritten by the attempt to recover stopped Postgres. This ended up in a situation when Patroni was trying to start Postgres while basebackup was running.
-
-- Fix race condition in ``postmaster_start_time()`` method (Alexander Kukushkin)
-
-  If the method is executed from the REST API thread, it requires a separate cursor object to be created.
-
-- Fix the problem of not promoting the sync standby that had a name contaning upper case letters (Alexander Kukushkin)
-
-  We converted the name to the lower case because Postgres was doing the same while comparing the ``application_name`` with the value in ``synchronous_standby_names``.
-
-- Kill all children along with the callback process before starting the new one (Alexander Kukushkin)
-
-  Not doing so makes it hard to implement callbacks in bash and eventually can lead to the situation when two callbacks are running at the same time. 
-
-- Fix 'start failed' issue (Alexander Kukushkin)
-
-  Under certain conditions the Postgres state might be set to 'start failed' despite Postgres being up and running.
-
-
 Version 1.6.0
 -------------
 
@@ -168,7 +40,7 @@ This version adds compatibility with PostgreSQL 12, makes is possible to run pg_
   This functionality works similarly to ``pg_hba.conf``: if the ``postgresql.pg_ident`` is defined in the config file or DCS, Patroni will write its value to ``pg_ident.conf``, however, if ``postgresql.parameters.ident_file`` is defined, Patroni will assume that ``pg_ident`` is managed from outside and not update the file.
 
 
-**Improvements in REST API**
+  **Improvements in REST API**
 
 - Added ``/health`` endpoint (Wilfried Roset)
 

docs/replica_bootstrap.rst

@@ -127,7 +127,7 @@ A special ``no_master`` parameter, if defined, allows Patroni to call the replic
 running master or replicas. In that case, an empty string will be passed in a connection string. This is useful for
 restoring the formerly running cluster from the binary backup.
 
-A special ``keep_data`` parameter, if defined, will instruct Patroni to  not clean PGDATA folder before calling restore.
+A special ``keep_data`` parameter, if defined, will instuct Patroni to  not clean PGDATA folder before calling restore.
 
 A special ``no_params`` parameter, if defined, restricts passing parameters to custom command.
 
@@ -204,4 +204,4 @@ in a patroni configuration:
 Note, that these options will be applied only once during cluster bootstrap,
 and the only way to change them afterwards is through DCS.
 
-If you use replication slots on the standby cluster, you must also create the corresponding replication slot on the primary cluster.  It will not be done automatically by the standby cluster implementation.  You can use Patroni's permanent replication slots feature on the primary cluster to maintain a replication slot with the same name as ``primary_slot_name``, or its default value if ``primary_slot_name`` is not provided.
+If you use replication slots on the standby cluster, you must also create the corresponding replication slot on the primary cluster.  It will not be done automatically by the standby cluster implementation.  You can use Patroni's permenant replication slots feature on the primary cluster to maintain a replication slot with the same name as ``primary_slot_name``, or its default value if ``primary_slot_name`` is not provided.

docs/replication_modes.rst

@@ -46,7 +46,7 @@ When ``synchronous_mode`` is on and a standby crashes, commits will block until
 
 When it is absolutely necessary to guarantee that each write is stored durably
 on at least two nodes, enable ``synchronous_mode_strict`` in addition to the
-``synchronous_mode``. This parameter prevents Patroni from switching off the
+``synchronous_node``. This parameter prevents Patroni from switching off the
 synchronous replication on the primary when no synchronous standby candidates
 are available. As a downside, the primary is not be available for writes
 (unless the Postgres transaction explicitly turns of ``synchronous_mode``),

docs/rest_api.rst

@@ -1,336 +0,0 @@
-.. _rest_api:
-
-Patroni REST API
-================
-
-Patroni has a rich REST API, which is used by Patroni itself during the leader race, by the ``patronictl`` tool in order to perform failovers/switchovers/reinitialize/restarts/reloads, by HAProxy or any other kind of load balancer to perform HTTP health checks, and of course could also be used for monitoring. Below you will find the list of Patroni REST API endpoints.
-
-Health check endpoints
-----------------------
-For all health check ``GET`` requests Patroni returns a JSON document with the status of the node, along with the HTTP status code. If you don't want or don't need the JSON document, you might consider using the ``OPTIONS`` method instead of ``GET``.
-
-- The following requests to Patroni REST API will return HTTP status code **200** only when the Patroni node is running as the leader:
-
-  - ``GET /``
-  - ``GET /master``
-  - ``GET /leader``
-  - ``GET /primary``
-  - ``GET /read-write``
-
-- ``GET /replica``: replica health check endpoint. It returns HTTP status code **200** only when the Patroni node is in the state ``running``, the role is ``replica`` and ``noloadbalance`` tag is not set.
-
-- ``GET /read-only``: like the above endpoint, but also includes the primary.
-
-- ``GET /standby-leader``: returns HTTP status code **200** only when the Patroni node is running as the leader in a :ref:`standby cluster <standby_cluster>`.
-
-- ``GET /synchronous`` or ``GET /sync``: returns HTTP status code **200** only when the Patroni node is running as a synchronous standby.
-
-- ``GET /asynchronous`` or ``GET /async``: returns HTTP status code **200** only when the Patroni node is running as an asynchronous standby.
-
-- ``GET /health``: returns HTTP status code **200** only when PostgreSQL is up and running.
-
-Monitoring endpoint
--------------------
-
-The ``GET /patroni`` is used by Patroni during the leader race. It also could be used by your monitoring system. The JSON document produced by this endpoint has the same structure as the JSON produced by the health check endpoints.
-
-.. code-block:: bash
-
-    $ curl -s http://localhost:8008/patroni | jq .
-    {
-      "state": "running",
-      "postmaster_start_time": "2019-09-24 09:22:32.555 CEST",
-      "role": "master",
-      "server_version": 110005,
-      "cluster_unlocked": false,
-      "xlog": {
-        "location": 25624640
-      },
-      "timeline": 3,
-      "database_system_identifier": "6739877027151648096",
-      "patroni": {
-        "version": "1.6.0",
-        "scope": "batman"
-      }
-    }
-
-Cluster status endpoints
-------------------------
-
-- The ``GET /cluster`` endpoint generates a JSON document describing the current cluster topology and state:
-
-.. code-block:: bash
-
-    $ curl -s http://localhost:8008/cluster | jq .
-    {
-      "members": [
-        {
-          "name": "postgresql0",
-          "host": "127.0.0.1",
-          "port": 5432,
-          "role": "leader",
-          "state": "running",
-          "api_url": "http://127.0.0.1:8008/patroni",
-          "timeline": 5,
-          "tags": {
-            "clonefrom": true
-          }
-        },
-        {
-          "name": "postgresql1",
-          "host": "127.0.0.1",
-          "port": 5433,
-          "role": "replica",
-          "state": "running",
-          "api_url": "http://127.0.0.1:8009/patroni",
-          "timeline": 5,
-          "tags": {
-            "clonefrom": true
-          },
-          "lag": 0
-        }
-      ],
-      "scheduled_switchover": {
-        "at": "2019-09-24T10:36:00+02:00",
-        "from": "postgresql0"
-      }
-    }
-
-
-- The ``GET /history`` endpoint provides a view on the history of cluster switchovers/failovers. The format is very similar to the content of history files in the ``pg_wal`` directory. The only difference is the timestamp field showing when the new timeline was created.
-
-.. code-block:: bash
-
-    $ curl -s http://localhost:8008/history | jq .
-    [
-      [
-        1,
-        25623960,
-        "no recovery target specified",
-        "2019-09-23T16:57:57+02:00"
-      ],
-      [
-        2,
-        25624344,
-        "no recovery target specified",
-        "2019-09-24T09:22:33+02:00"
-      ],
-      [
-        3,
-        25624752,
-        "no recovery target specified",
-        "2019-09-24T09:26:15+02:00"
-      ],
-      [
-        4,
-        50331856,
-        "no recovery target specified",
-        "2019-09-24T09:35:52+02:00"
-      ]
-    ]
-
-
-Config endpoint
----------------
-
-``GET /config``: Get the current version of the dynamic configuration:
-
-.. code-block:: bash
-
-	$ curl -s localhost:8008/config | jq .
-	{
-	  "ttl": 30,
-	  "loop_wait": 10,
-	  "retry_timeout": 10,
-	  "maximum_lag_on_failover": 1048576,
-	  "postgresql": {
-	    "use_slots": true,
-	    "use_pg_rewind": true,
-	    "parameters": {
-	      "hot_standby": "on",
-	      "wal_log_hints": "on",
-	      "wal_keep_segments": 8,
-	      "wal_level": "hot_standby",
-	      "max_wal_senders": 5,
-	      "max_replication_slots": 5,
-	      "max_connections": "100"
-	    }
-	  }
-	}
-
-
-``PATCH /config``: Change the existing configuration.
-
-.. code-block:: bash
-
-	$ curl -s -XPATCH -d \
-		'{"loop_wait":5,"ttl":20,"postgresql":{"parameters":{"max_connections":"101"}}}' \
-		http://localhost:8008/config | jq .
-	{
-	  "ttl": 20,
-	  "loop_wait": 5,
-	  "maximum_lag_on_failover": 1048576,
-	  "retry_timeout": 10,
-	  "postgresql": {
-	    "use_slots": true,
-	    "use_pg_rewind": true,
-	    "parameters": {
-	      "hot_standby": "on",
-	      "wal_log_hints": "on",
-	      "wal_keep_segments": 8,
-	      "wal_level": "hot_standby",
-	      "max_wal_senders": 5,
-	      "max_replication_slots": 5,
-	      "max_connections": "101"
-	    }
-	  }
-	}
-
-The above REST API call patches the existing configuration and returns the new configuration.
-
-Let's check that the node processed this configuration. First of all it should start printing log lines every 5 seconds (loop_wait=5). The change of "max_connections" requires a restart, so the "pending_restart" flag should be exposed:
-
-.. code-block:: bash
-
-	$ curl -s http://localhost:8008/patroni | jq .
-	{
-	  "pending_restart": true,
-	  "database_system_identifier": "6287881213849985952",
-	  "postmaster_start_time": "2016-06-13 13:13:05.211 CEST",
-	  "xlog": {
-	    "location": 2197818976
-	  },
-	  "patroni": {
-	    "scope": "batman",
-	    "version": "1.0"
-	  },
-	  "state": "running",
-	  "role": "master",
-	  "server_version": 90503
-	}
-
-Removing parameters:
-
-If you want to remove (reset) some setting just patch it with ``null``:
-
-.. code-block:: bash
-
-	$ curl -s -XPATCH -d \
-		'{"postgresql":{"parameters":{"max_connections":null}}}' \
-		http://localhost:8008/config | jq .
-	{
-	  "ttl": 20,
-	  "loop_wait": 5,
-	  "retry_timeout": 10,
-	  "maximum_lag_on_failover": 1048576,
-	  "postgresql": {
-	    "use_slots": true,
-	    "use_pg_rewind": true,
-	    "parameters": {
-	      "hot_standby": "on",
-	      "unix_socket_directories": ".",
-	      "wal_keep_segments": 8,
-	      "wal_level": "hot_standby",
-	      "wal_log_hints": "on",
-	      "max_wal_senders": 5,
-	      "max_replication_slots": 5
-	    }
-	  }
-	}
-
-The above call removes ``postgresql.parameters.max_connections`` from the dynamic configuration.
-
-``PUT /config``: It's also possible to perform the full rewrite of an existing dynamic configuration unconditionally:
-
-.. code-block:: bash
-
-	$ curl -s -XPUT -d \
-		'{"maximum_lag_on_failover":1048576,"retry_timeout":10,"postgresql":{"use_slots":true,"use_pg_rewind":true,"parameters":{"hot_standby":"on","wal_log_hints":"on","wal_keep_segments":8,"wal_level":"hot_standby","unix_socket_directories":".","max_wal_senders":5}},"loop_wait":3,"ttl":20}' \
-		http://localhost:8008/config | jq .
-	{
-	  "ttl": 20,
-	  "maximum_lag_on_failover": 1048576,
-	  "retry_timeout": 10,
-	  "postgresql": {
-	    "use_slots": true,
-	    "parameters": {
-	      "hot_standby": "on",
-	      "unix_socket_directories": ".",
-	      "wal_keep_segments": 8,
-	      "wal_level": "hot_standby",
-	      "wal_log_hints": "on",
-	      "max_wal_senders": 5
-	    },
-	    "use_pg_rewind": true
-	  },
-	  "loop_wait": 3
-	}
-
-
-Switchover and failover endpoints
----------------------------------
-
-``POST /switchover`` or ``POST /failover``. These endpoints are very similar to each other. There are a couple of minor differences though:
-
-1. The failover endpoint allows to perform a manual failover when there are no healthy nodes, but at the same time it will not allow you to schedule a switchover.
-
-2. The switchover endpoint is the opposite. It works only when the cluster is healthy (there is a leader) and allows to schedule a switchover at a given time.
-
-
-In the JSON body of the ``POST`` request you must specify at least the ``leader`` or ``candidate`` fields and optionally the ``scheduled_at`` field if you want to schedule a switchover at a specific time.
-
-
-Example: perform a failover to the specific node:
-
-.. code-block:: bash
-
-    $ curl -s http://localhost:8009/failover -XPOST -d '{"candidate":"postgresql1"}'
-    Successfully failed over to "postgresql1"
-
-
-Example: schedule a switchover from the leader to any other healthy replica in the cluster at a specific time:
-
-.. code-block:: bash
-
-    $ curl -s http://localhost:8008/switchover -XPOST -d \
-	    '{"leader":"postgresql0","scheduled_at":"2019-09-24T12:00+00"}'
-    Switchover scheduled
-
-
-Depending on the situation the request might finish with a different HTTP status code and body. The status code **200** is returned when the switchover or failover successfully completed. If the switchover was successfully scheduled, Patroni will return HTTP status code **202**. In case something went wrong, the error status code (one of **400**, **412** or **503**) will be returned with some details in the response body. For more information please check the source code of ``patroni/api.py:do_POST_failover()`` method.
-
-The switchover and failover endpoints are used by ``patronictl switchover`` and ``patronictl failover``, respectively.
-
-
-Restart endpoint
-----------------
-
-- ``POST /restart``: You can restart Postgres on the specific node by performing the ``POST /restart`` call. In the JSON body of ``POST`` request it is possible to optionally specify some restart conditions:
-
-  - **restart_pending**: boolean, if set to ``true`` Patroni will restart PostgreSQL only when restart is pending in order to apply some changes in the PostgreSQL config.
-  - **role**: perform restart only if the current role of the node matches with the role from the POST request.
-  - **postgres_version**: perform restart only if the current version of postgres is smaller than specified in the POST request.
-  - **timeout**: how long we should wait before PostgreSQL starts accepting connections. Overrides ``master_start_timeout``.
-  - **schedule**: timestamp with time zone, schedule the restart somewhere in the future.
-
-- ``DELETE /restart``: delete the scheduled restart
-
-``POST /restart`` and ``DELETE /restart`` endpoints are used by ``patronictl restart`` and ``patronictl flush`` respectively.
-
-
-Reload endpoint
----------------
-
-The ``POST /reload`` call will order Patroni to re-read and apply the configuration file. This is the equivalent of sending the ``SIGHUP`` signal to the Patroni process. In case you changed some of the Postgres parameters which require a restart (like **shared_buffers**), you still have to explicitly do the restart of Postgres by either calling the ``POST /restart`` endpoint or with the help of ``patronictl restart``.
-
-The reload endpoint is used by ``patronictl reload``.
-
-
-Reinitialize endpoint
----------------------
-
-``POST /reinitialize``: reinitialize the PostgreSQL data directory on the specified node. It is allowed to be executed only on replicas. Once called, it will remove the data directory and start ``pg_basebackup`` or some alternative :ref:`replica creation method <custom_replica_creation>`.
-
-The call might fail if Patroni is in a loop trying to recover (restart) a failed Postgres. In order to overcome this problem one can specify ``{"force":true}`` in the request body.
-
-The reinitialize endpoint is used by ``patronictl reinitialize``.

features/environment.py

@@ -473,7 +473,7 @@ class KubernetesController(AbstractDcsController):
 
     def delete_pod(self, name):
         try:
-            self._api.delete_namespaced_pod(name, self._namespace, body=self._client.V1DeleteOptions())
+            self._api.delete_namespaced_pod(name, self._namespace, self._client.V1DeleteOptions())
         except:
             pass
         while True:

features/patroni_api.feature

@@ -28,7 +28,10 @@ Scenario: check API requests on a stand-alone server
 	And I receive a response text "Failover could be performed only to a specific candidate"
 
 Scenario: check local configuration reload
-	Given I add tag new_tag new_value to postgres0 config
+	Given I issue an empty POST request to http://127.0.0.1:8008/reload
+	Then I receive a response code 200
+	And I receive a response text nothing changed
+	When I add tag new_tag new_value to postgres0 config
 	And I issue an empty POST request to http://127.0.0.1:8008/reload
 	Then I receive a response code 202
 
@@ -70,7 +73,6 @@ Scenario: check the switchover via the API in the pause mode
 	And postgres1 role is the primary after 10 seconds
 	And postgres0 role is the secondary after 10 seconds
 	And replication works from postgres1 to postgres0 after 20 seconds
-	And "members/postgres0" key in DCS has state=running after 10 seconds
 	When I issue a GET request to http://127.0.0.1:8008/master
 	Then I receive a response code 503
 	When I issue a GET request to http://127.0.0.1:8008/replica
@@ -92,7 +94,6 @@ Scenario: check the scheduled switchover
 	And postgres0 role is the primary after 10 seconds
 	And postgres1 role is the secondary after 10 seconds
 	And replication works from postgres0 to postgres1 after 25 seconds
-	And "members/postgres1" key in DCS has state=running after 10 seconds
 	When I issue a GET request to http://127.0.0.1:8008/master
 	Then I receive a response code 200
 	When I issue a GET request to http://127.0.0.1:8008/replica

patroni.py

@@ -1,6 +1,14 @@
 #!/usr/bin/env python
 from patroni import main
 
+import os
 
 if __name__ == '__main__':
+
+    if os.getenv("PATRONI_DEBUG_MODE"):
+        # XXX Visual Code specific https://github.com/microsoft/ptvsd/issues/1443
+        # create processes by spawning new Python interpreters instead of forking the current one 
+        import multiprocessing
+        multiprocessing.set_start_method('spawn', True)
+
     main()

patroni/__init__.py

@@ -16,7 +16,6 @@ class Patroni(object):
         from patroni.ha import Ha
         from patroni.log import PatroniLogger
         from patroni.postgresql import Postgresql
-        from patroni.request import PatroniRequest
         from patroni.version import __version__
         from patroni.watchdog import Watchdog
 
@@ -32,8 +31,8 @@ class Patroni(object):
 
         self.postgresql = Postgresql(self.config['postgresql'])
         self.api = RestApiServer(self, self.config['restapi'])
-        self.request = PatroniRequest(self.config, True)
         self.ha = Ha(self)
+        self.is_in_debug_mode = bool(os.getenv("PATRONI_DEBUG_MODE"))
 
         self.tags = self.get_tags()
         self.next_run = time.time()
@@ -68,16 +67,14 @@ class Patroni(object):
     def nosync(self):
         return bool(self.tags.get('nosync', False))
 
-    def reload_config(self, sighup=False):
+    def reload_config(self):
         try:
             self.tags = self.get_tags()
             self.logger.reload_config(self.config.get('log', {}))
-            self.watchdog.reload_config(self.config)
-            if sighup:
-                self.request.reload_config(self.config)
-            self.api.reload_config(self.config['restapi'])
-            self.postgresql.reload_config(self.config['postgresql'], sighup)
             self.dcs.reload_config(self.config)
+            self.watchdog.reload_config(self.config)
+            self.api.reload_config(self.config['restapi'])
+            self.postgresql.reload_config(self.config['postgresql'])
         except Exception:
             logger.exception('Failed to reload config_file=%s', self.config.config_file)
 
@@ -106,8 +103,9 @@ class Patroni(object):
             self.next_run = current_time
             # Release the GIL so we don't starve anyone waiting on async_executor lock
             time.sleep(0.001)
-            # Warn user that Patroni is not keeping up
+            # Warn user that Patroni is not keeping up or runs in debug
-            logger.warning("Loop time exceeded, rescheduling immediately.")
+            msg = "Patroni runs in the debug mode: keys' TTL is infinite, loop wait disabled" if self.is_in_debug_mode else "Loop time exceeded, rescheduling immediately."
+            logger.warning(msg)
         elif self.ha.watch(nap_time):
             self.next_run = time.time()
 
@@ -118,16 +116,13 @@ class Patroni(object):
 
     def run(self):
         self.api.start()
-        self.logger.start()
         self.next_run = time.time()
 
         while not self.received_sigterm:
             if self._received_sighup:
                 self._received_sighup = False
                 if self.config.reload_local_configuration():
-                    self.reload_config(True)
+                    self.reload_config()
-                else:
-                    self.postgresql.config.reload_config(self.config['postgresql'], True)
 
             logger.info(self.ha.run_cycle())
 
@@ -176,13 +171,6 @@ def fatal(string, *args):
     sys.exit(1)
 
 
-def use_spawn_start_method():
-    if sys.version_info >= (3, 4):
-        # The default, forking, method is not a good idea in a multithreaded process: https://bugs.python.org/issue6721
-        import multiprocessing
-        multiprocessing.set_start_method('spawn')
-
-
 def check_psycopg2():
     min_psycopg2 = (2, 5, 4)
     min_psycopg2_str = '.'.join(map(str, min_psycopg2))
@@ -205,7 +193,6 @@ def check_psycopg2():
 
 
 def main():
-    use_spawn_start_method()
     check_psycopg2()
     if os.getpid() != 1:
         return patroni_main()

patroni/api.py

@@ -7,13 +7,12 @@ import traceback
 import dateutil.parser
 import datetime
 import os
-import six
 import socket
 
 from patroni.postgresql import PostgresConnectionException
 from patroni.postgresql.misc import postgres_version_to_int, PostgresException
 from patroni.utils import deep_compare, parse_bool, patch_config, Retry, \
-    RetryFailedError, parse_int, split_host_port, tzutc, uri, cluster_as_json
+    RetryFailedError, parse_int, split_host_port, tzutc, uri
 from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
 from six.moves.socketserver import ThreadingMixIn
 from threading import Thread
@@ -21,6 +20,20 @@ from threading import Thread
 logger = logging.getLogger(__name__)
 
 
+def check_auth(func):
+    """Decorator function to check authorization header.
+
+    Usage example:
+    @check_auth
+    def do_PUT_foo():
+        pass
+    """
+    def wrapper(handler, *args, **kwargs):
+        if handler.check_auth_header():
+            return func(handler, *args, **kwargs)
+    return wrapper
+
+
 class RestApiHandler(BaseHTTPRequestHandler):
 
     def _write_response(self, status_code, body, content_type='text/html', headers=None):
@@ -36,20 +49,14 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def _write_json_response(self, status_code, response):
         self._write_response(status_code, json.dumps(response), content_type='application/json')
 
-    def check_auth(func):
+    def send_auth_request(self, body):
-        """Decorator function to check authorization header or client certificates
+        headers = {'WWW-Authenticate': 'Basic realm="' + self.server.patroni.__class__.__name__ + '"'}
+        self._write_response(401, body, headers=headers)
 
-        Usage example:
+    def check_auth_header(self):
-        @check_auth
+        auth_header = self.headers.get('Authorization')
-        def do_PUT_foo():
+        status = self.server.check_auth_header(auth_header)
-            pass
+        return not status or self.send_auth_request(status)
-        """
-
-        def wrapper(self, *args, **kwargs):
-            if self.server.check_auth(self):
-                return func(self, *args, **kwargs)
-
-        return wrapper
 
     def _write_status_response(self, status_code, response):
         patroni = self.server.patroni
@@ -133,14 +140,6 @@ class RestApiHandler(BaseHTTPRequestHandler):
         response = self.get_postgresql_status(True)
         self._write_status_response(200, response)
 
-    def do_GET_cluster(self):
-        cluster = self.server.patroni.dcs.cluster or self.server.patroni.dcs.get_cluster()
-        self._write_json_response(200, cluster_as_json(cluster))
-
-    def do_GET_history(self):
-        cluster = self.server.patroni.dcs.cluster or self.server.patroni.dcs.get_cluster()
-        self._write_json_response(200, cluster.history and cluster.history.lines or [])
-
     def do_GET_config(self):
         cluster = self.server.patroni.dcs.cluster or self.server.patroni.dcs.get_cluster()
         if cluster.config:
@@ -188,8 +187,18 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
     @check_auth
     def do_POST_reload(self):
-        self.server.patroni.sighup_handler()
+        try:
-        self._write_response(202, 'reload scheduled')
+            if self.server.patroni.config.reload_local_configuration(True):
+                status_code = 202
+                response = 'reload scheduled'
+                self.server.patroni.sighup_handler()
+            else:
+                status_code = 200
+                response = 'nothing changed'
+        except Exception as e:
+            status_code = 500
+            response = str(e)
+        self._write_response(status_code, response)
 
     @staticmethod
     def parse_schedule(schedule, action):
@@ -423,7 +432,10 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
             if self.server.patroni.postgresql.state not in ('running', 'restarting', 'starting'):
                 raise RetryFailedError('')
-            stmt = ("SELECT pg_catalog.to_char(pg_catalog.pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),"
+            stmt = ("WITH replication_info AS ("
+                    "SELECT usename, application_name, client_addr, state, sync_state, sync_priority"
+                    " FROM pg_catalog.pg_stat_replication) SELECT"
+                    " pg_catalog.to_char(pg_catalog.pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),"
                     " CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0"
                     " ELSE ('x' || pg_catalog.substr(pg_catalog.pg_{0}file_name("
                     "pg_catalog.pg_current_{0}_{1}()), 1, 8))::bit(32)::int END,"
@@ -434,10 +446,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     " pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), '0/0')::bigint,"
                     " pg_catalog.to_char(pg_catalog.pg_last_xact_replay_timestamp(), 'YYYY-MM-DD HH24:MI:SS.MS TZ'),"
                     " pg_catalog.pg_is_in_recovery() AND pg_catalog.pg_is_{0}_replay_paused(), "
-                    " pg_catalog.array_to_json(pg_catalog.array_agg(pg_catalog.row_to_json(ri))) "
+                    "(SELECT pg_catalog.array_to_json(pg_catalog.array_agg("
-                    "FROM (SELECT (SELECT rolname FROM pg_authid WHERE oid = usesysid) AS usename,"
+                    "pg_catalog.row_to_json(ri))) FROM replication_info ri)")
-                    " application_name, client_addr, w.state, sync_state, sync_priority"
-                    " FROM pg_catalog.pg_stat_get_wal_senders() w, pg_catalog.pg_stat_get_activity(pid)) AS ri")
 
             row = self.query(stmt.format(self.server.patroni.postgresql.wal_name,
                                          self.server.patroni.postgresql.lsn_name), retry=retry)[0]
@@ -482,14 +492,12 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
 
 class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
-    # On 3.7+ the `ThreadingMixIn` gathers all non-daemon worker threads in order to join on them at server close.
-    daemon_threads = True  # Make worker threads "fire and forget" to prevent a memory leak.
 
     def __init__(self, patroni, config):
         self.patroni = patroni
         self.__listen = None
-        self.__ssl_options = None
+        self.__initialize(config)
-        self.reload_config(config)
+        self.__set_config_parameters(config)
         self.daemon = True
 
     def query(self, sql, *params):
@@ -520,16 +528,13 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
             if not auth_header.startswith('Basic ') or not self.check_basic_auth_key(auth_header[6:]):
                 return 'not authenticated'
 
-    def check_auth(self, rh):
+    @staticmethod
-        if not hasattr(rh.request, 'getpeercert') or not rh.request.getpeercert():  # valid client cert isn't present
+    def __get_ssl_options(config):
-            if self.__protocol == 'https' and self.__ssl_options.get('verify_client') in ('required', 'optional'):
+        return {option: config[option] for option in ['certfile', 'keyfile'] if option in config}
-                return rh._write_response(403, 'client certificate required')
 
-            reason = self.check_auth_header(rh.headers.get('Authorization'))
+    def __set_config_parameters(self, config):
-            if reason:
+        self.__auth_key = base64.b64encode(config['auth'].encode('utf-8')).decode('utf-8') if 'auth' in config else None
-                headers = {'WWW-Authenticate': 'Basic realm="' + self.patroni.__class__.__name__ + '"'}
+        self.connection_string = uri(self.__protocol, config.get('connect_address') or self.__listen, 'patroni')
-                return rh._write_response(401, reason, headers=headers)
-        return True
 
     @staticmethod
     def __has_dual_stack():
@@ -548,7 +553,7 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
 
     def __httpserver_init(self, host, port):
         dual_stack = self.__has_dual_stack()
-        if host in ('', '*'):
+        if host == '':
             host = None
 
         info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE)
@@ -558,56 +563,43 @@ class RestApiServer(ThreadingMixIn, HTTPServer, Thread):
         self.address_family = info[0][0]
         HTTPServer.__init__(self, info[0][-1][:2], RestApiHandler)
 
-    def __initialize(self, listen, ssl_options):
+    def __initialize(self, config):
         try:
-            host, port = split_host_port(listen, None)
+            host, port = split_host_port(config['listen'], None)
         except Exception:
             raise ValueError('Invalid "restapi" config: expected <HOST>:<PORT> for "listen", but got "{0}"'
-                             .format(listen))
+                             .format(config['listen']))
 
-        reloading_config = self.__listen is not None  # changing config in runtime
+        if self.__listen is not None:  # changing config in runtime
-        if reloading_config:
             self.shutdown()
 
-        self.__listen = listen
+        self.__listen = config['listen']
-        self.__ssl_options = ssl_options
+        self.__ssl_options = self.__get_ssl_options(config)
 
         self.__httpserver_init(host, port)
         Thread.__init__(self, target=self.serve_forever)
         self._set_fd_cloexec(self.socket)
 
+        self.__protocol = 'http'
+
         # wrap socket with ssl if 'certfile' is defined in a config.yaml
         # Sometime it's also needed to pass reference to a 'keyfile'.
-        self.__protocol = 'https' if ssl_options.get('certfile') else 'http'
+        if self.__ssl_options.get('certfile'):
-        if self.__protocol == 'https':
             import ssl
-            ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH, cafile=ssl_options.get('cafile'))
+            ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
-            ctx.load_cert_chain(certfile=ssl_options['certfile'], keyfile=ssl_options.get('keyfile'))
+            ctx.load_cert_chain(**self.__ssl_options)
-            verify_client = ssl_options.get('verify_client')
-            if verify_client:
-                modes = {'none': ssl.CERT_NONE, 'optional': ssl.CERT_OPTIONAL, 'required': ssl.CERT_REQUIRED}
-                if verify_client in modes:
-                    ctx.verify_mode = modes[verify_client]
-                else:
-                    logger.error('Bad value in the "restapi.verify_client": %s', verify_client)
             self.socket = ctx.wrap_socket(self.socket, server_side=True)
-        if reloading_config:
+            self.__protocol = 'https'
-            self.start()
+        return True
 
     def reload_config(self, config):
         if 'listen' not in config:  # changing config in runtime
             raise ValueError('Can not find "restapi.listen" config')
 
-        ssl_options = {n: config[n] for n in ('certfile', 'keyfile', 'cafile') if n in config}
+        elif (self.__listen != config['listen'] or self.__ssl_options != self.__get_ssl_options(config)) \
-
+                and self.__initialize(config):
-        if isinstance(config.get('verify_client'), six.string_types):
+            self.start()
-            ssl_options['verify_client'] = config['verify_client'].lower()
+        self.__set_config_parameters(config)
-
-        if self.__listen != config['listen'] or self.__ssl_options != ssl_options:
-            self.__initialize(config['listen'], ssl_options)
-
-        self.__auth_key = base64.b64encode(config['auth'].encode('utf-8')).decode('utf-8') if 'auth' in config else None
-        self.connection_string = uri(self.__protocol, config.get('connect_address') or self.__listen, 'patroni')
 
     @staticmethod
     def handle_error(request, client_address):

patroni/async_executor.py

@@ -110,12 +110,6 @@ class AsyncExecutor(object):
     def run_async(self, func, args=()):
         Thread(target=self.run, args=(func, args)).start()
 
-    def try_run_async(self, action, func, args=()):
-        prev = self.schedule(action)
-        if prev is None:
-            return self.run_async(func, args)
-        return 'Failed to run {0}, {1} is already in progress'.format(action, prev)
-
     def cancel(self):
         with self:
             with self._scheduled_action_lock:

patroni/config.py

@@ -15,16 +15,6 @@ from requests.structures import CaseInsensitiveDict
 
 logger = logging.getLogger(__name__)
 
-_AUTH_ALLOWED_PARAMETERS = (
-    'username',
-    'password',
-    'sslmode',
-    'sslcert',
-    'sslkey',
-    'sslrootcert',
-    'sslcrl'
-)
-
 
 class Config(object):
     """
@@ -168,19 +158,23 @@ class Config(object):
             except Exception:
                 logger.exception('Exception when setting dynamic_configuration')
 
-    def reload_local_configuration(self):
+    def reload_local_configuration(self, dry_run=False):
         if self.config_file:
             try:
                 configuration = self._load_config_file()
                 if not deep_compare(self._local_configuration, configuration):
                     new_configuration = self._build_effective_configuration(self._dynamic_configuration, configuration)
+                    if dry_run:
+                        return not deep_compare(new_configuration, self.__effective_configuration)
                     self._local_configuration = configuration
                     self.__effective_configuration = new_configuration
                     return True
                 else:
-                    logger.info('No local configuration items changed.')
+                    logger.info('No configuration items changed, nothing to reload.')
             except Exception:
                 logger.exception('Exception when reloading local configuration from %s', self.config_file)
+                if dry_run:
+                    raise
 
     @staticmethod
     def _process_postgresql_parameters(parameters, is_local=False):
@@ -236,8 +230,7 @@ class Config(object):
                 if value:
                     ret[section][param] = value
 
-        _set_section_values('restapi', ['listen', 'connect_address', 'certfile', 'keyfile', 'cafile', 'verify_client'])
+        _set_section_values('restapi', ['listen', 'connect_address', 'certfile', 'keyfile'])
-        _set_section_values('ctl', ['insecure', 'cacert', 'certfile', 'keyfile'])
         _set_section_values('postgresql', ['listen', 'connect_address', 'config_dir', 'data_dir', 'pgpass', 'bin_dir'])
         _set_section_values('log', ['level', 'format', 'dateformat', 'max_queue_size',
                                     'dir', 'file_size', 'file_num', 'loggers'])
@@ -257,9 +250,9 @@ class Config(object):
             if value:
                 ret['log']['loggers'] = value
 
-        def _get_auth(name, params=None):
+        def _get_auth(name):
             ret = {}
-            for param in params or _AUTH_ALLOWED_PARAMETERS[:2]:
+            for param in ('username', 'password'):
                 value = _popenv(name + '_' + param)
                 if value:
                     ret[param] = value
@@ -271,7 +264,7 @@ class Config(object):
 
         authentication = {}
         for user_type in ('replication', 'superuser', 'rewind'):
-            entry = _get_auth(user_type, _AUTH_ALLOWED_PARAMETERS)
+            entry = _get_auth(user_type)
             if entry:
                 authentication[user_type] = entry
 
@@ -339,6 +332,11 @@ class Config(object):
                         config['postgresql'][name] = deepcopy(value)
             elif name not in config or name in ['watchdog']:
                 config[name] = deepcopy(value) if value else {}
+        
+        if os.getenv("PATRONI_DEBUG_MODE"):
+            config['ttl'] = 24 * 60 * 60 # practical infinity for a debugging session
+            config['loop_wait'] = -1
+
 
         # restapi server expects to get restapi.auth = 'username:password'
         if 'authentication' in config['restapi']:
@@ -360,11 +358,6 @@ class Config(object):
         if 'superuser' not in pg_config['authentication'] and 'pg_rewind' in pg_config:
             pg_config['authentication']['superuser'] = pg_config['pg_rewind']
 
-        # handle setting additional connection parameters that may be available
-        # in the configuration file, such as SSL connection parameters
-        for name, value in pg_config['authentication'].items():
-            pg_config['authentication'][name] = {n: v for n, v in value.items() if n in _AUTH_ALLOWED_PARAMETERS}
-
         # no 'name' in config
         if 'name' not in config and 'name' in pg_config:
             config['name'] = pg_config['name']

patroni/ctl.py

@@ -2,6 +2,7 @@
 Patroni Control
 '''
 
+import base64
 import click
 import codecs
 import datetime
@@ -14,7 +15,7 @@ import json
 import logging
 import os
 import random
-import six
+import requests
 import subprocess
 import sys
 import tempfile
@@ -24,15 +25,16 @@ import yaml
 
 from click import ClickException
 from contextlib import contextmanager
+from patroni.config import Config
 from patroni.dcs import get_dcs as _get_dcs
 from patroni.exceptions import PatroniException
 from patroni.postgresql import Postgresql
 from patroni.postgresql.misc import postgres_version_to_int
-from patroni.utils import cluster_as_json, patch_config, polling_loop
+from patroni.utils import patch_config, polling_loop
-from patroni.request import PatroniRequest
 from patroni.version import __version__
 from prettytable import PrettyTable
 from six.moves.urllib_parse import urlparse
+from six import text_type
 
 CONFIG_DIR_PATH = click.get_app_dir('patroni')
 CONFIG_FILE_PATH = os.path.join(CONFIG_DIR_PATH, 'patronictl.yaml')
@@ -66,8 +68,6 @@ def parse_dcs(dcs):
 
 
 def load_config(path, dcs):
-    from patroni.config import Config
-
     if not (os.path.exists(path) and os.access(path, os.R_OK)):
         logging.debug('Ignoring configuration file "%s". It does not exists or is not readable.', path)
     else:
@@ -135,12 +135,36 @@ def get_dcs(config, scope):
         raise PatroniCtlException(str(e))
 
 
-def request_patroni(member, method='GET', endpoint=None, data=None):
+def auth_header(config):
+    if config.get('restapi', {}).get('auth', ''):
+        return {'Authorization': 'Basic ' + base64.b64encode(config['restapi']['auth'].encode('utf-8')).decode('utf-8')}
+
+
+def request_patroni(member, request_type, endpoint, content=None, headers=None):
     ctx = click.get_current_context()  # the current click context
-    request_executor = ctx.obj.get('__request_patroni')
+    headers = headers or {}
-    if not request_executor:
+    url_parts = urlparse(member.api_url)
-        request_executor = ctx.obj['__request_patroni'] = PatroniRequest(ctx.obj)
+    logging.debug(url_parts)
-    return request_executor(member, method, endpoint, data)
+    if 'Content-Type' not in headers:
+        headers['Content-Type'] = 'application/json'
+
+    url = '{0}://{1}/{2}'.format(url_parts.scheme, url_parts.netloc, endpoint)
+
+    insecure = ctx.obj.get('ctl', {}).get('insecure', False)
+    # Get certfile if any from several configuration namespace
+    cert = ctx.obj.get('ctl', {}).get('cacert') or \
+        ctx.obj.get('restapi', {}).get('cacert') or \
+        ctx.obj.get('restapi', {}).get('certfile')
+    # In the case we specificaly disable SSL cert verification we don't want to have the warning
+    if insecure:
+        verify = False
+    elif cert:
+        verify = cert
+    else:
+        verify = True
+    return getattr(requests, request_type)(url, headers=headers,
+                                           data=json.dumps(content) if content else None, timeout=60,
+                                           verify=verify)
 
 
 def print_output(columns, rows=None, alignment=None, fmt='pretty', header=True, delimiter='\t'):
@@ -433,9 +457,9 @@ def remove(obj, cluster_name, fmt):
 
 
 def check_response(response, member_name, action_name, silent_success=False):
-    if response.status >= 400:
+    if response.status_code >= 400:
         click.echo('Failed: {0} for member {1}, status code={2}, ({3})'.format(
-            action_name, member_name, response.status, response.data.decode('utf-8')
+            action_name, member_name, response.status_code, response.text
         ))
         return False
     elif not silent_success:
@@ -469,17 +493,18 @@ def reload(obj, cluster_name, member_names, force, role):
 
     members = get_members(cluster, cluster_name, member_names, role, force, 'reload')
 
+    content = {}
     for member in members:
-        r = request_patroni(member, 'post', 'reload')
+        r = request_patroni(member, 'post', 'reload', content, auth_header(obj))
-        if r.status == 200:
+        if r.status_code == 200:
             click.echo('No changes to apply on member {0}'.format(member.name))
-        elif r.status == 202:
+        elif r.status_code == 202:
             click.echo('Reload request received for member {0} and will be processed within {1} seconds'.format(
                 member.name, cluster.config.data.get('loop_wait'))
             )
         else:
             click.echo('Failed: reload for member {0}, status code={1}, ({2})'.format(
-                member.name, r.status, r.data.decode('utf-8'))
+                member.name, r.status_code, r.text)
             )
 
 
@@ -540,19 +565,19 @@ def restart(obj, cluster_name, member_names, force, role, p_any, scheduled, vers
     for member in members:
         if 'schedule' in content:
             if force and member.data.get('scheduled_restart'):
-                r = request_patroni(member, 'delete', 'restart')
+                r = request_patroni(member, 'delete', 'restart', headers=auth_header(obj))
                 check_response(r, member.name, 'flush scheduled restart', True)
 
-        r = request_patroni(member, 'post', 'restart', content)
+        r = request_patroni(member, 'post', 'restart', content, auth_header(obj))
-        if r.status == 200:
+        if r.status_code == 200:
             click.echo('Success: restart on member {0}'.format(member.name))
-        elif r.status == 202:
+        elif r.status_code == 202:
             click.echo('Success: restart scheduled on member {0}'.format(member.name))
-        elif r.status == 409:
+        elif r.status_code == 409:
             click.echo('Failed: another restart is already scheduled on member {0}'.format(member.name))
         else:
             click.echo('Failed: restart for member {0}, status code={1}, ({2})'.format(
-                member.name, r.status, r.data.decode('utf-8'))
+                member.name, r.status_code, r.text)
             )
 
 
@@ -568,8 +593,8 @@ def reinit(obj, cluster_name, member_names, force):
     for member in members:
         body = {'force': force}
         while True:
-            r = request_patroni(member, 'post', 'reinitialize', body)
+            r = request_patroni(member, 'post', 'reinitialize', body, auth_header(obj))
-            if not check_response(r, member.name, 'reinitialize') and r.data.endswith(b' already in progress') \
+            if not check_response(r, member.name, 'reinitialize') and r.text.endswith(' already in progress') \
                     and not force and click.confirm('Do you want to cancel it and reinitialize anyway?'):
                 body['force'] = True
                 continue
@@ -657,19 +682,19 @@ def _do_failover_or_switchover(obj, action, cluster_name, master, candidate, for
     try:
         member = cluster.leader.member if cluster.leader else cluster.get_member(candidate, False)
 
-        r = request_patroni(member, 'post', action, failover_value)
+        r = request_patroni(member, 'post', action, failover_value, auth_header(obj))
 
         # probably old patroni, which doesn't support switchover yet
-        if r.status == 501 and action == 'switchover' and b'Server does not support this operation' in r.data:
+        if r.status_code == 501 and action == 'switchover' and 'Server does not support this operation' in r.text:
-            r = request_patroni(member, 'post', 'failover', failover_value)
+            r = request_patroni(member, 'post', 'failover', failover_value, auth_header(obj))
 
-        if r.status in (200, 202):
+        if r.status_code in (200, 202):
             logging.debug(r)
             cluster = dcs.get_cluster()
             logging.debug(cluster)
-            click.echo('{0} {1}'.format(timestamp(), r.data.decode('utf-8')))
+            click.echo('{0} {1}'.format(timestamp(), r.text))
         else:
-            click.echo('{0} failed, details: {1}, {2}'.format(action.title(), r.status, r.data.decode('utf-8')))
+            click.echo('{0} failed, details: {1}, {2}'.format(action.title(), r.status_code, r.text))
             return
     except Exception:
         logging.exception(r)
@@ -706,51 +731,80 @@ def switchover(obj, cluster_name, master, candidate, force, scheduled):
 def output_members(cluster, name, extended=False, fmt='pretty'):
     rows = []
     logging.debug(cluster)
-    cluster = cluster_as_json(cluster)
+    leader_name = None
+    if cluster.leader:
+        leader_name = cluster.leader.name
 
-    columns = ['Cluster', 'Member', 'Host', 'Role', 'State', 'TL', 'Lag in MB']
+    xlog_location_cluster = cluster.last_leader_operation or 0
-    for c in ('Pending restart', 'Scheduled restart'):
+
-        if extended or any(m.get(c.lower().replace(' ', '_')) for m in cluster['members']):
+    # Mainly for consistent pretty printing and watching we sort the output
-            columns.append(c)
+    cluster.members.sort(key=lambda x: x.name)
+
+    has_scheduled_restarts = any(m.data.get('scheduled_restart') for m in cluster.members)
+    has_pending_restarts = any(m.data.get('pending_restart') for m in cluster.members)
 
     # Show Host as 'host:port' if somebody is running on non-standard port or two nodes are running on the same host
-    append_port = any(m['port'] != 5432 for m in cluster['members']) or\
+    append_port = any(str(m.conn_kwargs()['port']) != '5432' for m in cluster.members) or\
-        len(set(m['host'] for m in cluster['members'])) < len(cluster['members'])
+        len(set(m.conn_kwargs()['host'] for m in cluster.members)) < len(cluster.members)
 
-    for m in cluster['members']:
+    for m in cluster.members:
         logging.debug(m)
 
-        lag = m.get('lag', '')
+        role = ''
-        m.update(cluster=name, member=m['name'], tl=m.get('timeline', ''),
+        if m.name == leader_name:
-                 role='' if m['role'] == 'replica' else m['role'].replace('_', ' ').title(),
+            role = 'Leader'
-                 lag_in_mb=round(lag/1024/1024) if isinstance(lag, six.integer_types) else lag,
+        elif m.name == cluster.sync.sync_standby:
-                 pending_restart='*' if m.get('pending_restart') else '')
+            role = 'Sync standby'
 
+        xlog_location = m.data.get('xlog_location')
+        lag = ''
+        if xlog_location is None:
+            lag = 'unknown'
+        elif xlog_location_cluster >= xlog_location:
+            lag = round((xlog_location_cluster - xlog_location)/1024/1024)
+
+        host = m.conn_kwargs()['host']
         if append_port:
-            m['host'] = ':'.join([m['host'], str(m['port'])])
+            host += ':{0}'.format(m.conn_kwargs()['port'])
 
-        if 'scheduled_restart' in m:
+        row = [name, m.name, host, role, m.data.get('state', ''), m.data.get('timeline', ''), lag]
-            value = m['scheduled_restart']['schedule']
-            if 'postgres_version' in m['scheduled_restart']:
-                value += ' if version < {0}'.format(m['scheduled_restart']['postgres_version'])
-            m['scheduled_restart'] = value
 
-        rows.append([m.get(n.lower().replace(' ', '_'), '') for n in columns])
+        if extended or has_pending_restarts:
+            row.append('*' if m.data.get('pending_restart') else '')
 
-    print_output(columns, rows, {'Lag in MB': 'r', 'TL': 'r'}, fmt)
+        if extended or has_scheduled_restarts:
+            value = ''
+            scheduled_restart = m.data.get('scheduled_restart')
+            if scheduled_restart:
+                value = scheduled_restart['schedule']
+                if 'postgres_version' in scheduled_restart:
+                    value += ' if version < {0}'.format(scheduled_restart['postgres_version'])
 
-    if fmt != 'pretty':  # Omit service info when using machine-readable formats
+            row.append(value)
-        return
+
+        rows.append(row)
+
+    columns = ['Cluster', 'Member', 'Host', 'Role', 'State', 'TL', 'Lag in MB']
+    alignment = {'Lag in MB': 'r', 'TL': 'r'}
+
+    if extended or has_pending_restarts:
+        columns.append('Pending restart')
+
+    if extended or has_scheduled_restarts:
+        columns.append('Scheduled restart')
+
+    print_output(columns, rows, alignment, fmt)
 
     service_info = []
-    if cluster.get('pause'):
+    if cluster.is_paused():
         service_info.append('Maintenance mode: on')
 
-    if 'scheduled_switchover' in cluster:
+    if cluster.failover and cluster.failover.scheduled_at:
-        info = 'Switchover scheduled at: ' + cluster['scheduled_switchover']['at']
+        info = 'Switchover scheduled at: ' + cluster.failover.scheduled_at.isoformat()
-        for name in ('from', 'to'):
+        if cluster.failover.leader:
-            if name in cluster['scheduled_switchover']:
+            info += '\n                    from: ' + cluster.failover.leader
-                info += '\n{0:>24}: {1}'.format(name, cluster['scheduled_switchover'][name])
+        if cluster.failover.candidate:
+            info += '\n                      to: ' + cluster.failover.candidate
         service_info.append(info)
 
     if service_info:
@@ -865,7 +919,7 @@ def flush(obj, cluster_name, member_names, force, role, target):
     for member in members:
         if target == 'restart':
             if member.data.get('scheduled_restart'):
-                r = request_patroni(member, 'delete', 'restart')
+                r = request_patroni(member, 'delete', 'restart', None, auth_header(obj))
                 check_response(r, member.name, 'flush scheduled restart')
             else:
                 click.echo('No scheduled restart for member {0}'.format(member.name))
@@ -902,20 +956,20 @@ def toggle_pause(config, cluster_name, paused, wait):
 
     for member in members:
         try:
-            r = request_patroni(member, 'patch', 'config', {'pause': paused or None})
+            r = request_patroni(member, 'patch', 'config', {'pause': paused or None}, auth_header(config))
         except Exception as err:
             logging.warning(str(err))
             logging.warning('Member %s is not accessible', member.name)
             continue
 
-        if r.status == 200:
+        if r.status_code == 200:
             if wait:
                 wait_until_pause_is_applied(dcs, paused, cluster)
             else:
                 click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
         else:
             click.echo('Failed: {0} cluster management status code={1}, ({2})'.format(
-                       paused and 'pause' or 'resume', r.status, r.data.decode('utf-8')))
+                       paused and 'pause' or 'resume', r.status_code, r.text))
         break
     else:
         raise PatroniCtlException('Can not find accessible cluster member')
@@ -970,7 +1024,7 @@ def show_diff(before_editing, after_editing):
         buf = io.StringIO()
         for line in unified_diff:
             # Force cast to unicode as difflib on Python 2.7 returns a mix of unicode and str.
-            buf.write(six.text_type(line))
+            buf.write(text_type(line))
         buf.seek(0)
 
         class opts:
@@ -1179,8 +1233,8 @@ def version(obj, cluster_name, member_names):
         if m.api_url:
             if not member_names or m.name in member_names:
                 try:
-                    response = request_patroni(m)
+                    response = request_patroni(m, 'get', 'patroni')
-                    data = json.loads(response.data.decode('utf-8'))
+                    data = response.json()
                     version = data.get('patroni', {}).get('version')
                     pg_version = data.get('server_version')
                     pg_version_str = " PostgreSQL {0}".format(format_pg_version(pg_version)) if pg_version else ""
@@ -1189,19 +1243,6 @@ def version(obj, cluster_name, member_names):
                     click.echo("{0}: failed to get version: {1}".format(m.name, e))
 
 
-@ctl.command('history', help="Show the history of failovers/switchovers")
-@arg_cluster_name
-@option_format
-@click.pass_obj
-def history(obj, cluster_name, fmt):
-    cluster = get_dcs(obj, cluster_name).get_cluster()
-    history = cluster.history and cluster.history.lines or []
-    for line in history:
-        if len(line) < 4:
-            line.append('')
-    print_output(['TL', 'LSN', 'Reason', 'Timestamp'], history, {'TL': 'r', 'LSN': 'r'}, fmt)
-
-
 def format_pg_version(version):
     if version < 100000:
         return "{0}.{1}.{2}".format(version // 10000, version // 100 % 100, version % 100)

patroni/dcs/__init__.py

@@ -157,11 +157,11 @@ class Member(namedtuple('Member', 'index,name,session,data')):
             }
             self.data['conn_kwargs'] = ret.copy()
 
-        # apply any remaining authentication parameters
         if auth and isinstance(auth, dict):
-            ret.update(auth)
             if 'username' in auth:
-                ret['user'] = ret.pop('username')
+                ret['user'] = auth['username']
+            if 'password' in auth:
+                ret['password'] = auth['password']
         return ret
 
     @property
@@ -442,16 +442,14 @@ class Cluster(namedtuple('Cluster', 'initialize,config,leader,last_leader_operat
         # the current master, because that member would replicate from elsewhere. We still create the slot if
         # the replicatefrom destination member is currently not a member of the cluster (fallback to the
         # master), or if replicatefrom destination member happens to be the current master
-        use_slots = self.config and self.config.data.get('postgresql', {}).get('use_slots', True)
         if role in ('master', 'standby_leader'):
-            slot_members = [m.name for m in self.members if use_slots and m.name != name and
+            slot_members = [m.name for m in self.members if m.name != name and
                             (m.replicatefrom is None or m.replicatefrom == name or
                              not self.has_member(m.replicatefrom))]
             permanent_slots = (self.config and self.config.permanent_slots or {}).copy()
         else:
             # only manage slots for replicas that replicate from this one, except for the leader among them
-            slot_members = [m.name for m in self.members if use_slots and
+            slot_members = [m.name for m in self.members if m.replicatefrom == name and m.name != self.leader.name]
-                            m.replicatefrom == name and m.name != self.leader.name]
             permanent_slots = {}
 
         slots = {slot_name_from_member_name(name): {'type': 'physical'} for name in slot_members}

patroni/dcs/etcd.py

@@ -15,7 +15,7 @@ from dns import resolver
 from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
 from patroni.exceptions import DCSError
 from patroni.utils import Retry, RetryFailedError, split_host_port, uri
-from urllib3.exceptions import HTTPError, ReadTimeoutError, ProtocolError
+from urllib3.exceptions import HTTPError, ReadTimeoutError
 from requests.exceptions import RequestException
 from six.moves.queue import Queue
 from six.moves.http_client import HTTPException
@@ -25,10 +25,6 @@ from threading import Thread
 logger = logging.getLogger(__name__)
 
 
-class EtcdRaftInternal(etcd.EtcdException):
-    """Raft Internal Error"""
-
-
 class EtcdError(DCSError):
     pass
 
@@ -93,47 +89,22 @@ class Client(etcd.Client):
         self._config = config
         self._load_machines_cache()
         self._allow_reconnect = True
-        # allow passing retry argument to api_execute in params
-        self._comparison_conditions.add('retry')
-        self._read_options.add('retry')
-        self._del_conditions.add('retry')
 
-    def _calculate_timeouts(self, etcd_nodes=None, timeout=None):
+    def _build_request_parameters(self):
-        """Calculate a request timeout and number of retries per single etcd node.
-        In case if the timeout per node is too small (less than one second) we will reduce the number of nodes.
-        For the cluster with only one node we will try to do 2 retries.
-        For clusters with 2 nodes we will try to do 1 retry for every node.
-        No retries for clusters with 3 or more nodes. We better rely on switching to a different node."""
-
-        etcd_nodes = etcd_nodes or len(self._machines_cache) + 1
-        per_node_timeout = timeout = float(timeout or self.read_timeout)
-
-        max_retries = 4 - min(etcd_nodes, 3)
-        per_node_retries = 1
-        min_timeout = 1.0
-
-        while etcd_nodes > 0:
-            per_node_timeout = float(timeout) / etcd_nodes
-            if per_node_timeout >= min_timeout:
-                # for small clusters we will try to do more than on try on every node
-                while per_node_retries < max_retries and per_node_timeout / (per_node_retries + 1) >= min_timeout:
-                    per_node_retries += 1
-                per_node_timeout /= per_node_retries
-                break
-            # if the timeout per one node is to small try to reduce number of nodes
-            etcd_nodes -= 1
-            max_retries = 1
-
-        return etcd_nodes, per_node_timeout, per_node_retries - 1
-
-    def _build_request_parameters(self, timeout=None):
         kwargs = {'headers': self._get_headers(), 'redirect': self.allow_redirect}
 
-        if timeout is not None:
+        # calculate the number of retries and timeout *per node*
-            kwargs.update(retries=0, timeout=timeout)
+        # actual number of retries depends on the number of nodes
-        else:
+        etcd_nodes = len(self._machines_cache) + 1
-            _, per_node_timeout, per_node_retries = self._calculate_timeouts()
+        kwargs['retries'] = 0 if etcd_nodes > 3 else (1 if etcd_nodes > 1 else 2)
-            kwargs.update(timeout=per_node_timeout, retries=per_node_retries)
+
+        # if etcd_nodes > 3:
+        #     kwargs.update({'retries': 0, 'timeout': float(self.read_timeout)/etcd_nodes})
+        # elif etcd_nodes > 1:
+        #     kwargs.update({'retries': 1, 'timeout': self.read_timeout/2.0/etcd_nodes})
+        # else:
+        #     kwargs.update({'retries': 2, 'timeout': self.read_timeout/3.0})
+        kwargs['timeout'] = self.read_timeout/float(kwargs['retries'] + 1)/etcd_nodes
         return kwargs
 
     def set_machines_cache_ttl(self, cache_ttl):
@@ -191,12 +162,8 @@ class Client(etcd.Client):
             self._check_cluster_id(response)
         except (HTTPError, HTTPException, socket.error, socket.timeout) as e:
             if (isinstance(fields, dict) and fields.get("wait") == "true" and
-                    isinstance(e, (ReadTimeoutError, ProtocolError))):
+                    isinstance(e, ReadTimeoutError)):
                 logger.debug("Watch timed out.")
-                # switch to the next etcd node because we don't know exactly what happened,
-                # whether the key didn't received an update or there is a network problem.
-                self._machines_cache.insert(0, self._base_uri)
-                self._base_uri = self._next_server()
                 raise etcd.EtcdWatchTimedOut("Watch timed out: {0}".format(e), cause=e)
             logger.error("Request to server %s failed: %r", self._base_uri, e)
             logger.info("Reconnection allowed, looking for another server.")
@@ -208,7 +175,6 @@ class Client(etcd.Client):
         if not path.startswith('/'):
             raise ValueError('Path does not start with /')
 
-        retry = params.pop('retry', None) if isinstance(params, dict) else None
         kwargs = {'fields': params, 'preload_content': False}
 
         if method in [self._MGET, self._MDELETE]:
@@ -224,44 +190,31 @@ class Client(etcd.Client):
             self._load_machines_cache()
         elif not self._use_proxies and time.time() - self._machines_cache_updated > self._machines_cache_ttl:
             self._refresh_machines_cache()
+            self._machines_cache_updated = time.time()
 
-        kwargs.update(self._build_request_parameters(timeout))
+        kwargs.update(self._build_request_parameters())
 
-        if retry:
+        if timeout is not None:
-            machines_cache = [self._base_uri] + self._machines_cache
+            kwargs.update({'retries': 0, 'timeout': timeout})
 
         response = False
 
-        while True:
+        try:
-            try:
+            some_request_failed = False
-                some_request_failed = False
+            while not response:
-                while not response:
+                response = self._do_http_request(request_executor, method, self._base_uri + path, **kwargs)
-                    response = self._do_http_request(request_executor, method, self._base_uri + path, **kwargs)
-
-                    if response is False:
-                        if not retry:
-                            raise etcd.EtcdException('{0} {1} request failed'.format(method, path))
-                        some_request_failed = True
-                if some_request_failed:
-                    self._refresh_machines_cache()
-                if response:
-                    break
-            except etcd.EtcdConnectionFailed:
-                if not retry:
-                    raise
-                sleeptime = retry.sleeptime
-                remaining_time = retry.stoptime - sleeptime - time.time()
-                nodes, timeout, retries = self._calculate_timeouts(len(machines_cache), remaining_time)
-                if nodes == 0:
-                    self._update_machines_cache = True
-                    raise
-                retry.sleep_func(sleeptime)
-                retry.update_delay()
-                # We still have some time left. Partially restore `_machines_cache` and retry request
-                kwargs.update(timeout=timeout, retries=retries)
-                self._base_uri = machines_cache[0]
-                self._machines_cache = machines_cache[1:nodes]
 
+                if response is False:
+                    some_request_failed = True
+            if some_request_failed:
+                self._refresh_machines_cache()
+        except etcd.EtcdConnectionFailed as e:
+            if isinstance(e, etcd.EtcdWatchTimedOut) and self._machines_cache:
+                self._base_uri = self._next_server()
+            else:
+                self._update_machines_cache = True
+            if not response:
+                raise
         return self._handle_server_response(response)
 
     @staticmethod
@@ -343,15 +296,14 @@ class Client(etcd.Client):
         # After filling up initial list of machines_cache we should ask etcd-cluster about actual list
         self._base_uri = self._next_server()
         self._refresh_machines_cache()
+
         self._update_machines_cache = False
+        self._machines_cache_updated = time.time()
 
     def _refresh_machines_cache(self):
         self._machines_cache = self._get_machines_cache_from_config() if self._use_proxies else self.machines
         if self._base_uri in self._machines_cache:
             self._machines_cache.remove(self._base_uri)
-        elif self._machines_cache:
-            self._base_uri = self._next_server()
-        self._machines_cache_updated = time.time()
 
 
 class Etcd(AbstractDCS):
@@ -360,15 +312,15 @@ class Etcd(AbstractDCS):
         super(Etcd, self).__init__(config)
         self._ttl = int(config.get('ttl') or 30)
         self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
-                            retry_exceptions=(etcd.EtcdLeaderElectionInProgress, EtcdRaftInternal))
+                            retry_exceptions=(etcd.EtcdLeaderElectionInProgress,
+                                              etcd.EtcdWatcherCleared,
+                                              etcd.EtcdEventIndexCleared))
         self._client = self.get_etcd_client(config)
         self.__do_not_watch = False
         self._has_failed = False
 
     def retry(self, *args, **kwargs):
-        retry = self._retry.copy()
+        return self._retry.copy()(*args, **kwargs)
-        kwargs['retry'] = retry
-        return retry(*args, **kwargs)
 
     def _handle_exception(self, e, name='', do_sleep=False, raise_ex=None):
         if not self._has_failed:
@@ -554,7 +506,7 @@ class Etcd(AbstractDCS):
 
     @catch_etcd_errors
     def take_leader(self):
-        return self.retry(self._client.write, self.leader_path, self._name, ttl=self._ttl)
+        return self.retry(self._client.set, self.leader_path, self._name, self._ttl)
 
     def attempt_to_acquire_leader(self, permanent=False):
         try:
@@ -583,7 +535,7 @@ class Etcd(AbstractDCS):
 
     @catch_etcd_errors
     def _update_leader(self):
-        return self.retry(self._client.write, self.leader_path, self._name, prevValue=self._name, ttl=self._ttl)
+        return self.retry(self._client.test_and_set, self.leader_path, self._name, self._name, self._ttl)
 
     @catch_etcd_errors
     def initialize(self, create_new=True, sysid=""):
@@ -644,6 +596,3 @@ class Etcd(AbstractDCS):
             return super(Etcd, self).watch(None, timeout)
         finally:
             self.event.clear()
-
-
-etcd.EtcdError.error_exceptions[300] = EtcdRaftInternal

patroni/dcs/kubernetes.py

@@ -11,10 +11,8 @@ from kubernetes import client as k8s_client, config as k8s_config, watch as k8s_
 from patroni.dcs import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, SyncState, TimelineHistory
 from patroni.exceptions import DCSError
 from patroni.utils import deep_compare, tzutc, Retry, RetryFailedError
-from urllib3 import Timeout
 from urllib3.exceptions import HTTPError
 from six.moves.http_client import HTTPException
-from threading import Condition, Lock, Thread
 
 logger = logging.getLogger(__name__)
 
@@ -35,7 +33,6 @@ class CoreV1ApiProxy(object):
 
     def __init__(self, use_endpoints=False):
         self._api = k8s_client.CoreV1Api()
-        self._api.api_client.rest_client.pool_manager.connection_pool_kw['maxsize'] = 10
         self._request_timeout = None
         self._use_endpoints = use_endpoints
 
@@ -73,110 +70,6 @@ def catch_kubernetes_errors(func):
     return wrapper
 
 
-class ObjectCache(Thread):
-
-    def __init__(self, dcs, func, retry, condition):
-        Thread.__init__(self)
-        self.daemon = True
-        self._api_client = k8s_client.ApiClient()
-        self._dcs = dcs
-        self._func = func
-        self._retry = retry
-        self._condition = condition
-        self._is_ready = False
-        self._object_cache = {}
-        self._object_cache_lock = Lock()
-        self._annotations_map = {self._dcs.leader_path: self._dcs._LEADER, self._dcs.config_path: self._dcs._CONFIG}
-        self.start()
-
-    def _list(self):
-        return self._func(_request_timeout=(self._retry.deadline, Timeout.DEFAULT_TIMEOUT))
-
-    def _watch(self, resource_version):
-        return self._func(_request_timeout=(self._retry.deadline, Timeout.DEFAULT_TIMEOUT),
-                          _preload_content=False, watch=True, resource_version=resource_version)
-
-    def set(self, name, value):
-        with self._object_cache_lock:
-            old_value = self._object_cache.get(name)
-            ret = not old_value or int(old_value.metadata.resource_version) < int(value.metadata.resource_version)
-            if ret:
-                self._object_cache[name] = value
-        return ret, old_value
-
-    def delete(self, name, resource_version):
-        with self._object_cache_lock:
-            old_value = self._object_cache.get(name)
-            ret = old_value and int(old_value.metadata.resource_version) < int(resource_version)
-            if ret:
-                del self._object_cache[name]
-        return not old_value or ret, old_value
-
-    def copy(self):
-        with self._object_cache_lock:
-            return self._object_cache.copy()
-
-    def _build_cache(self):
-        objects = self._list()
-        return_type = 'V1' + objects.kind[:-4]
-        with self._object_cache_lock:
-            self._object_cache = {item.metadata.name: item for item in objects.items}
-        with self._condition:
-            self._is_ready = True
-            self._condition.notify()
-
-        response = self._watch(objects.metadata.resource_version)
-        try:
-            for line in k8s_watch.watch.iter_resp_lines(response):
-                event = json.loads(line)
-                obj = event['object']
-                if obj.get('code') == 410:
-                    break
-
-                ev_type = event['type']
-                name = obj['metadata']['name']
-
-                if ev_type in ('ADDED', 'MODIFIED'):
-                    obj = k8s_watch.watch.SimpleNamespace(data=json.dumps(obj))
-                    obj = self._api_client.deserialize(obj, return_type)
-                    success, old_value = self.set(name, obj)
-                    if success:
-                        new_value = (obj.metadata.annotations or {}).get(self._annotations_map.get(name))
-                elif ev_type == 'DELETED':
-                    success, old_value = self.delete(name, obj['metadata']['resourceVersion'])
-                    new_value = None
-                else:
-                    logger.warning('Unexpected event type: %s', ev_type)
-                    continue
-
-                if success and return_type != 'V1Pod':
-                    if old_value:
-                        old_value = (old_value.metadata.annotations or {}).get(self._annotations_map.get(name))
-
-                    if old_value != new_value and \
-                            (name != self._dcs.config_path or old_value is not None and new_value is not None):
-                        logger.debug('%s changed from %s to %s', name, old_value, new_value)
-                        self._dcs.event.set()
-        finally:
-            with self._condition:
-                self._is_ready = False
-            response.close()
-            response.release_conn()
-
-    def run(self):
-        while True:
-            try:
-                self._build_cache()
-            except Exception as e:
-                with self._condition:
-                    self._is_ready = False
-                logger.error('ObjectCache.run %r', e)
-
-    def is_ready(self):
-        """Must be called only when holding the lock on `_condition`"""
-        return self._is_ready
-
-
 class Kubernetes(AbstractDCS):
 
     def __init__(self, config):
@@ -217,16 +110,6 @@ class Kubernetes(AbstractDCS):
         self._config_resource_version = None
         self.__do_not_watch = False
 
-        self._condition = Condition()
-
-        pods_func = functools.partial(self._api.list_namespaced_pod, self._namespace,
-                                      label_selector=self._label_selector)
-        self._pods = ObjectCache(self, pods_func, self._retry, self._condition)
-
-        kinds_func = functools.partial(self._api.list_namespaced_kind, self._namespace,
-                                       label_selector=self._label_selector)
-        self._kinds = ObjectCache(self, kinds_func, self._retry, self._condition)
-
     def retry(self, *args, **kwargs):
         return self._retry.copy()(*args, **kwargs)
 
@@ -257,21 +140,14 @@ class Kubernetes(AbstractDCS):
         member.data['pod_labels'] = pod.metadata.labels
         return member
 
-    def _wait_caches(self):
-        stop_time = time.time() + self._retry.deadline
-        while not (self._pods.is_ready() and self._kinds.is_ready()):
-            timeout = stop_time - time.time()
-            if timeout <= 0:
-                raise RetryFailedError('Exceeded retry deadline')
-            self._condition.wait(timeout)
-
     def _load_cluster(self):
         try:
-            with self._condition:
+            # get list of members
-                self._wait_caches()
+            response = self.retry(self._api.list_namespaced_pod, self._namespace, label_selector=self._label_selector)
+            members = [self.member(pod) for pod in response.items]
 
-                members = [self.member(pod) for pod in self._pods.copy().values()]
+            response = self.retry(self._api.list_namespaced_kind, self._namespace, label_selector=self._label_selector)
-                nodes = self._kinds.copy()
+            nodes = {item.metadata.name: item for item in response.items}
 
             config = nodes.get(self.config_path)
             metadata = config and config.metadata
@@ -319,13 +195,12 @@ class Kubernetes(AbstractDCS):
             if metadata:
                 member = Member(-1, leader, None, {})
                 member = ([m for m in members if m.name == leader] or [member])[0]
-                leader = Leader(metadata.resource_version, None, member)
+                leader = Leader(response.metadata.resource_version, None, member)
 
             # failover key
             failover = nodes.get(self.failover_path)
             metadata = failover and failover.metadata
-            failover = Failover.from_node(metadata and metadata.resource_version,
+            failover = Failover.from_node(metadata and metadata.resource_version, metadata and metadata.annotations)
-                                          metadata and (metadata.annotations or {}).copy())
 
             # get synchronization state
             sync = nodes.get(self.sync_path)
@@ -404,10 +279,7 @@ class Kubernetes(AbstractDCS):
             body = k8s_client.V1Endpoints(**endpoints)
         else:
             body = k8s_client.V1ConfigMap(metadata=metadata)
-        ret = self.retry(func, self._namespace, body) if retry else func(self._namespace, body)
+        return self.retry(func, self._namespace, body) if retry else func(self._namespace, body)
-        if ret:
-            self._kinds.set(name, ret)
-        return ret
 
     def patch_or_create_config(self, annotations, resource_version=None, patch=False, retry=True):
         # SCOPE-config endpoint requires corresponding service otherwise it might be "cleaned" by k8s master
@@ -545,6 +417,24 @@ class Kubernetes(AbstractDCS):
             self.__do_not_watch = False
             return True
 
+        if leader_index:
+            end_time = time.time() + timeout
+            w = k8s_watch.Watch()
+            while timeout >= 1:
+                try:
+                    for event in w.stream(self._api.list_namespaced_kind, self._namespace,
+                                          resource_version=leader_index, timeout_seconds=int(timeout + 0.5),
+                                          field_selector='metadata.name=' + self.leader_path,
+                                          _request_timeout=(1, timeout + 1)):
+                        return event['raw_object'].get('metadata', {}).get('resourceVersion') != leader_index
+                    return False
+                except KeyboardInterrupt:
+                    raise
+                except Exception:
+                    logger.exception('watch')
+
+                timeout = end_time - time.time()
+
         try:
             return super(Kubernetes, self).watch(None, timeout)
         finally:

patroni/ha.py

@@ -3,6 +3,7 @@ import functools
 import json
 import logging
 import psycopg2
+import requests
 import sys
 import time
 import uuid
@@ -14,7 +15,7 @@ from patroni.exceptions import DCSError, PostgresConnectionException, PatroniExc
 from patroni.postgresql import ACTION_ON_START, ACTION_ON_ROLE_CHANGE
 from patroni.postgresql.misc import postgres_version_to_int
 from patroni.postgresql.rewind import Rewind
-from patroni.utils import polling_loop, tzutc, is_standby_cluster as _is_standby_cluster
+from patroni.utils import polling_loop, tzutc
 from patroni.dcs import RemoteMember
 from threading import RLock
 
@@ -108,7 +109,9 @@ class Ha(object):
         return config.get('standby_cluster')
 
     def is_standby_cluster(self):
-        return _is_standby_cluster(self.get_standby_cluster_config())
+        config = self.get_standby_cluster_config()
+        # Check whether or not provided configuration describes a standby cluster
+        return isinstance(config, dict) and (config.get('host') or config.get('port') or config.get('restore_command'))
 
     def is_leader(self):
         with self._is_leader_lock:
@@ -228,8 +231,9 @@ class Ha(object):
             clone_member = self.cluster.get_clone_member(self.state_handler.name)
             member_role = 'leader' if clone_member == self.cluster.leader else 'replica'
             msg = "from {0} '{1}'".format(member_role, clone_member.name)
-            ret = self._async_executor.try_run_async('bootstrap {0}'.format(msg), self.clone, args=(clone_member, msg))
+            self._async_executor.schedule('bootstrap {0}'.format(msg))
-            return ret or 'trying to bootstrap {0}'.format(msg)
+            self._async_executor.run_async(self.clone, args=(clone_member, msg))
+            return 'trying to bootstrap {0}'.format(msg)
 
         # no initialize key and node is allowed to be master and has 'bootstrap' section in a configuration file
         elif self.cluster.initialize is None and not self.patroni.nofailover and 'bootstrap' in self.patroni.config:
@@ -238,12 +242,16 @@ class Ha(object):
                 self._post_bootstrap_task = CriticalTask()
 
                 if self.is_standby_cluster():
-                    ret = self._async_executor.try_run_async('bootstrap_standby_leader', self.bootstrap_standby_leader)
+                    self._async_executor.schedule('bootstrap_standby_leader')
-                    return ret or 'trying to bootstrap a new standby leader'
+                    self._async_executor.run_async(self.bootstrap_standby_leader)
+                    return 'trying to bootstrap a new standby leader'
                 else:
-                    ret = self._async_executor.try_run_async('bootstrap', self.state_handler.bootstrap.bootstrap,
+                    self._async_executor.schedule('bootstrap')
-                                                             args=(self.patroni.config['bootstrap'],))
+                    self._async_executor.run_async(
-                    return ret or 'trying to bootstrap a new cluster'
+                        self.state_handler.bootstrap.bootstrap,
+                        args=(self.patroni.config['bootstrap'],)
+                    )
+                    return 'trying to bootstrap a new cluster'
             else:
                 return 'failed to acquire initialize lock'
         else:
@@ -251,7 +259,9 @@ class Ha(object):
                                      if self.is_standby_cluster() else None
             if self.state_handler.can_create_replica_without_replication_connection(create_replica_methods):
                 msg = 'bootstrap (without leader)'
-                return self._async_executor.try_run_async(msg, self.clone) or 'trying to ' + msg
+                self._async_executor.schedule(msg)
+                self._async_executor.run_async(self.clone)
+                return 'trying to ' + msg
             return 'waiting for {0}leader to bootstrap'.format('standby_' if self.is_standby_cluster() else '')
 
     def bootstrap_standby_leader(self):
@@ -274,13 +284,15 @@ class Ha(object):
             return None
 
         if self._rewind.can_rewind:
-            msg = 'running pg_rewind from ' + leader.name
+            self._async_executor.schedule('running pg_rewind from ' + leader.name)
-            return self._async_executor.try_run_async(msg, self._rewind.execute, args=(leader,)) or msg
+            self._async_executor.run_async(self._rewind.execute, (leader,))
+            return True
 
         # remove_data_directory_on_diverged_timelines is set
         if not self.is_standby_cluster():
-            msg = 'reinitializing due to diverged timelines'
+            self._async_executor.schedule('reinitializing due to diverged timelines')
-            return self._async_executor.try_run_async(msg, self._do_reinitialize, args=(self.cluster,)) or msg
+            self._async_executor.run_async(self._do_reinitialize, args=(self.cluster, ))
+            return True
 
     def recover(self):
         # Postgres is not running and we will restart in standby mode. Watchdog is not needed until we promote.
@@ -307,8 +319,9 @@ class Ha(object):
                 and not self._crash_recovery_executed and \
                 (self.cluster.is_unlocked() or self._rewind.can_rewind):
             self._crash_recovery_executed = True
-            msg = 'doing crash recovery in a single user mode'
+            self._async_executor.schedule('doing crash recovery in a single user mode')
-            return self._async_executor.try_run_async(msg, self.state_handler.fix_cluster_state) or msg
+            self._async_executor.run_async(self.state_handler.fix_cluster_state)
+            return self._async_executor.scheduled_action
 
         self.load_cluster_from_dcs()
 
@@ -316,9 +329,8 @@ class Ha(object):
         if self.is_standby_cluster() or not self.has_lock():
             if not self._rewind.executed:
                 self._rewind.trigger_check_diverged_lsn()
-            msg = self._handle_rewind_or_reinitialize()
+            if self._handle_rewind_or_reinitialize():
-            if msg:
+                return self._async_executor.scheduled_action
-                return msg
 
             if self.has_lock():  # in standby cluster
                 msg = "starting as a standby leader because i had the session lock"
@@ -334,9 +346,10 @@ class Ha(object):
             msg = "starting as readonly because i had the session lock"
             node_to_follow = None
 
-        if self._async_executor.try_run_async('restarting after failure', self.state_handler.follow,
+        self.recovering = True
-                                              args=(node_to_follow, role, timeout)) is None:
+
-            self.recovering = True
+        self._async_executor.schedule('restarting after failure')
+        self._async_executor.run_async(self.state_handler.follow, (node_to_follow, role, timeout))
         return msg
 
     def _get_node_to_follow(self, cluster):
@@ -371,26 +384,20 @@ class Ha(object):
             self.demote('immediate-nolock')
             return demote_reason
 
-        msg = self._handle_rewind_or_reinitialize()
+        if self._handle_rewind_or_reinitialize():
-        if msg:
+            return self._async_executor.scheduled_action
-            return msg
 
         role = 'standby_leader' if isinstance(node_to_follow, RemoteMember) and self.has_lock(False) else 'replica'
         # It might happen that leader key in the standby cluster references non-exiting member.
         # In this case it is safe to continue running without changing recovery.conf
         if self.is_standby_cluster() and role == 'replica' and not (node_to_follow and node_to_follow.conn_url):
             return 'continue following the old known standby leader'
-        else:
+        elif not self.state_handler.config.check_recovery_conf(node_to_follow):
-            change_required, restart_required = self.state_handler.config.check_recovery_conf(node_to_follow)
+            self._async_executor.schedule('changing primary_conninfo and restarting')
-            if change_required:
+            self._async_executor.run_async(self.state_handler.follow, (node_to_follow, role))
-                if restart_required:
+        elif role == 'standby_leader' and self.state_handler.role != role:
-                    self._async_executor.try_run_async('changing primary_conninfo and restarting',
+            self.state_handler.set_role(role)
-                                                       self.state_handler.follow, args=(node_to_follow, role))
+            self.state_handler.call_nowait(ACTION_ON_ROLE_CHANGE)
-                else:
-                    self.state_handler.follow(node_to_follow, role, do_reload=True)
-            elif role == 'standby_leader' and self.state_handler.role != role:
-                self.state_handler.set_role(role)
-                self.state_handler.call_nowait(ACTION_ON_ROLE_CHANGE)
 
         return follow_reason
 
@@ -546,21 +553,21 @@ class Ha(object):
                     self._rewind.reset_state()
                     logger.info("cleared rewind state after becoming the leader")
 
-                self._async_executor.try_run_async('promote', self.state_handler.promote,
+                self._async_executor.schedule('promote')
-                                                   args=(self.dcs.loop_wait, on_success,
+                self._async_executor.run_async(self.state_handler.promote,
-                                                         self._leader_access_is_restricted))
+                                               args=(self.dcs.loop_wait, on_success, self._leader_access_is_restricted))
             return promote_message
 
-    def fetch_node_status(self, member):
+    @staticmethod
+    def fetch_node_status(member):
         """This function perform http get request on member.api_url and fetches its status
         :returns: `_MemberStatus` object
         """
 
         try:
-            response = self.patroni.request(member, timeout=2, retries=0)
+            response = requests.get(member.api_url, timeout=2, verify=False)
-            data = response.data.decode('utf-8')
+            logger.info('Got response from %s %s: %s', member.name, member.api_url, response.content)
-            logger.info('Got response from %s %s: %s', member.name, member.api_url, data)
+            return _MemberStatus.from_api_response(member, response.json())
-            return _MemberStatus.from_api_response(member, json.loads(data))
         except Exception as e:
             logger.warning("Request failed to %s: GET %s (%s)", member.name, member.api_url, e)
         return _MemberStatus.unknown(member)
@@ -584,7 +591,6 @@ class Ha(object):
     def _is_healthiest_node(self, members, check_replication_lag=True):
         """This method tries to determine whether I am healthy enough to became a new leader candidate or not."""
 
-        # We don't call `last_operation()` here because it returns a string
         _, my_wal_position = self.state_handler.timeline_wal_position()
         if check_replication_lag and self.is_lagging(my_wal_position):
             logger.info('My wal position exceeds maximum replication lag')
@@ -768,7 +774,8 @@ class Ha(object):
         # there could be an async action already running, calling follow from here will lead
         # to racy state handler state updates.
         if mode_control['async_req']:
-            self._async_executor.try_run_async('starting after demotion', self.state_handler.follow, (node_to_follow,))
+            self._async_executor.schedule('starting after demotion')
+            self._async_executor.run_async(self.state_handler.follow, (node_to_follow,))
         else:
             if self.is_synchronous_mode():
                 self.state_handler.config.set_synchronous_standby(None)
@@ -841,8 +848,9 @@ class Ha(object):
                         members = [m for m in self.cluster.members
                                    if not failover.candidate or m.name == failover.candidate]
                     if self.is_failover_possible(members):  # check that there are healthy members
-                        ret = self._async_executor.try_run_async('manual failover: demote', self.demote, ('graceful',))
+                        self._async_executor.schedule('manual failover: demote')
-                        return ret or 'manual failover: demoting myself'
+                        self._async_executor.run_async(self.demote, ('graceful',))
+                        return 'manual failover: demoting myself'
                     else:
                         logger.warning('manual failover: no healthy members found, failover is not possible')
             else:
@@ -1145,9 +1153,10 @@ class Ha(object):
                 return 'waiting for end of recovery after bootstrap'
 
             self.state_handler.set_role('master')
-            ret = self._async_executor.try_run_async('post_bootstrap', self.state_handler.bootstrap.post_bootstrap,
+            self._async_executor.schedule('post_bootstrap')
-                                                     args=(self.patroni.config['bootstrap'], self._post_bootstrap_task))
+            self._async_executor.run_async(self.state_handler.bootstrap.post_bootstrap,
-            return ret or 'running post_bootstrap'
+                                           args=(self.patroni.config['bootstrap'], self._post_bootstrap_task))
+            return 'running post_bootstrap'
 
         self.state_handler.bootstrapping = False
         self.dcs.set_config_value(json.dumps(self.patroni.config.dynamic_configuration, separators=(',', ':')))
@@ -1263,6 +1272,15 @@ class Ha(object):
                     return 'released leader key voluntarily as data dir empty and currently leader'
 
                 return self.bootstrap()  # new node
+            # "bootstrap", but data directory is not empty
+            elif not self.sysid_valid(self.cluster.initialize) and self.cluster.is_unlocked() and not self.is_paused():
+                if not self.state_handler.cb_called and self.state_handler.is_running() \
+                        and not self.state_handler.is_leader():
+                    self._join_aborted = True
+                    logger.error('No initialize key in DCS and PostgreSQL is running as replica, aborting start')
+                    logger.error('Please first start Patroni on the node running as master')
+                    sys.exit(1)
+                self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=self.state_handler.sysid)
             else:
                 # check if we are allowed to join
                 data_sysid = self.state_handler.sysid
@@ -1270,20 +1288,10 @@ class Ha(object):
                     # data directory is not empty, but no valid sysid, cluster must be broken, suggest reinit
                     return "data dir for the cluster is not empty, but system ID is invalid; consider doing reinitalize"
 
-                if self.sysid_valid(self.cluster.initialize):
+                if self.sysid_valid(self.cluster.initialize) and self.cluster.initialize != self.state_handler.sysid:
-                    if self.cluster.initialize != data_sysid:
+                    logger.fatal("system ID mismatch, node %s belongs to a different cluster: %s != %s",
-                        logger.fatal("system ID mismatch, node %s belongs to a different cluster: %s != %s",
+                                 self.state_handler.name, self.cluster.initialize, self.state_handler.sysid)
-                                     self.state_handler.name, self.cluster.initialize, data_sysid)
+                    sys.exit(1)
-                        sys.exit(1)
-                elif self.cluster.is_unlocked() and not self.is_paused():
-                    # "bootstrap", but data directory is not empty
-                    if not self.state_handler.cb_called and self.state_handler.is_running() \
-                            and not self.state_handler.is_leader():
-                        self._join_aborted = True
-                        logger.error('No initialize key in DCS and PostgreSQL is running as replica, aborting start')
-                        logger.error('Please first start Patroni on the node running as master')
-                        sys.exit(1)
-                    self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=data_sysid)
 
             if not self.state_handler.is_healthy():
                 if self.is_paused():

patroni/log.py

@@ -48,16 +48,6 @@ class QueueHandler(logging.Handler):
         return self._records_lost
 
 
-class ProxyHandler(logging.Handler):
-
-    def __init__(self, patroni_logger):
-        logging.Handler.__init__(self)
-        self.patroni_logger = patroni_logger
-
-    def emit(self, record):
-        self.patroni_logger.log_handler.handle(record)
-
-
 class PatroniLogger(Thread):
 
     DEFAULT_LEVEL = 'INFO'
@@ -69,18 +59,16 @@ class PatroniLogger(Thread):
 
     def __init__(self):
         super(PatroniLogger, self).__init__()
+        self.daemon = True
         self._queue_handler = QueueHandler()
         self._root_logger = logging.getLogger()
+        self._root_logger.addHandler(self._queue_handler)
         self._config = None
-        self.log_handler = None
+        self._log_handler = None
-        self.log_handler_lock = Lock()
         self._old_handlers = []
+        self._log_handler_lock = Lock()
         self.reload_config({'level': 'DEBUG'})
-        # We will switch to the QueueHandler only when thread was started.
+        self.start()
-        # This is necessary to protect from the cases when Patroni constructor
-        # failed and PatroniLogger thread remain running and prevent shutdown.
-        self._proxy_handler = ProxyHandler(self)
-        self._root_logger.addHandler(self._proxy_handler)
 
     def update_loggers(self):
         loggers = deepcopy(self._config.get('loggers') or {})
@@ -102,15 +90,15 @@ class PatroniLogger(Thread):
 
             new_handler = None
             if 'dir' in config:
-                if not isinstance(self.log_handler, RotatingFileHandler):
+                if not isinstance(self._log_handler, RotatingFileHandler):
                     new_handler = RotatingFileHandler(os.path.join(config['dir'], __name__))
-                handler = new_handler or self.log_handler
+                handler = new_handler or self._log_handler
                 handler.maxBytes = int(config.get('file_size', 25000000))
                 handler.backupCount = int(config.get('file_num', 4))
             else:
-                if self.log_handler is None or isinstance(self.log_handler, RotatingFileHandler):
+                if self._log_handler is None or isinstance(self._log_handler, RotatingFileHandler):
                     new_handler = logging.StreamHandler()
-                handler = new_handler or self.log_handler
+                handler = new_handler or self._log_handler
 
             oldlogformat = (self._config or {}).get('format', PatroniLogger.DEFAULT_FORMAT)
             logformat = config.get('format', PatroniLogger.DEFAULT_FORMAT)
@@ -122,17 +110,17 @@ class PatroniLogger(Thread):
                 handler.setFormatter(logging.Formatter(logformat, dateformat))
 
             if new_handler:
-                with self.log_handler_lock:
+                with self._log_handler_lock:
-                    if self.log_handler:
+                    if self._log_handler:
-                        self._old_handlers.append(self.log_handler)
+                        self._old_handlers.append(self._log_handler)
-                    self.log_handler = new_handler
+                    self._log_handler = new_handler
 
             self._config = config.copy()
             self.update_loggers()
 
     def _close_old_handlers(self):
         while True:
-            with self.log_handler_lock:
+            with self._log_handler_lock:
                 if not self._old_handlers:
                     break
                 handler = self._old_handlers.pop()
@@ -142,11 +130,6 @@ class PatroniLogger(Thread):
                 _LOGGER.exception('Failed to close the old log handler %s', handler)
 
     def run(self):
-        # switch to QueueHandler only when the thread was started
-        with self.log_handler_lock:
-            self._root_logger.addHandler(self._queue_handler)
-            self._root_logger.removeHandler(self._proxy_handler)
-
         while True:
             self._close_old_handlers()
 
@@ -154,7 +137,7 @@ class PatroniLogger(Thread):
             if record is None:
                 break
 
-            self.log_handler.handle(record)
+            self._log_handler.handle(record)
             self._queue_handler.queue.task_done()
 
     def shutdown(self):

patroni/postgresql/__init__.py

@@ -18,6 +18,7 @@ from patroni.postgresql.postmaster import PostmasterProcess
 from patroni.postgresql.slots import SlotsHandler
 from patroni.exceptions import PostgresConnectionException
 from patroni.utils import Retry, RetryFailedError, polling_loop
+from patroni.dcs import slot_name_from_member_name, RemoteMember
 from threading import current_thread, Lock
 
 
@@ -37,6 +38,15 @@ STATE_UNKNOWN = 'unknown'
 
 STOP_POLLING_INTERVAL = 1
 
+cluster_info_query = ("SELECT CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0 "
+                      "ELSE ('x' || pg_catalog.substr(pg_catalog.pg_{0}file_name("
+                      "pg_catalog.pg_current_{0}_{1}()), 1, 8))::bit(32)::int END, "
+                      "CASE WHEN pg_catalog.pg_is_in_recovery() THEN GREATEST("
+                      " pg_catalog.pg_{0}_{1}_diff(COALESCE("
+                      "pg_catalog.pg_last_{0}_receive_{1}(), '0/0'), '0/0')::bigint,"
+                      " pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), '0/0')::bigint)"
+                      "ELSE pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}_{1}(), '0/0')::bigint END")
+
 
 @contextmanager
 def null_context():
@@ -51,7 +61,6 @@ class Postgresql(object):
         self._data_dir = config['data_dir']
         self._database = config.get('database', 'postgres')
         self._version_file = os.path.join(self._data_dir, 'PG_VERSION')
-        self._pg_control = os.path.join(self._data_dir, 'global', 'pg_control')
         self._major_version = self.get_major_version()
 
         self._state_lock = Lock()
@@ -68,6 +77,7 @@ class Postgresql(object):
 
         self.slots_handler = SlotsHandler(self)
 
+        self._pgpass = config.get('pgpass') or os.path.join(os.path.expanduser('~'), 'pgpass')
         self._callback_executor = CallbackExecutor()
         self.__cb_called = False
         self.__cb_pending = None
@@ -131,18 +141,6 @@ class Postgresql(object):
     def lsn_name(self):
         return 'lsn' if self._major_version >= 100000 else 'location'
 
-    @property
-    def cluster_info_query(self):
-        return ("SELECT CASE WHEN pg_catalog.pg_is_in_recovery() THEN 0 "
-                "ELSE ('x' || pg_catalog.substr(pg_catalog.pg_{0}file_name("
-                "pg_catalog.pg_current_{0}_{1}()), 1, 8))::bit(32)::int END, "
-                "CASE WHEN pg_catalog.pg_is_in_recovery() THEN GREATEST("
-                " pg_catalog.pg_{0}_{1}_diff(COALESCE("
-                "pg_catalog.pg_last_{0}_receive_{1}(), '0/0'), '0/0')::bigint,"
-                " pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_last_{0}_replay_{1}(), '0/0')::bigint)"
-                "ELSE pg_catalog.pg_{0}_{1}_diff(pg_catalog.pg_current_{0}_{1}(), '0/0')::bigint "
-                "END").format(self.wal_name, self.lsn_name)
-
     def _version_file_exists(self):
         return not self.data_directory_empty() and os.path.isfile(self._version_file)
 
@@ -190,8 +188,8 @@ class Postgresql(object):
                         3: STATE_UNKNOWN}
         return return_codes.get(ret, STATE_UNKNOWN)
 
-    def reload_config(self, config, sighup=False):
+    def reload_config(self, config):
-        self.config.reload_config(config, sighup)
+        self.config.reload_config(config)
         self._is_leader_retry.deadline = self.retry.deadline = config['retry_timeout']/2.0
 
     @property
@@ -209,7 +207,7 @@ class Postgresql(object):
         return self._sysid
 
     def get_postgres_role_from_data_directory(self):
-        if self.data_directory_empty() or not self.controldata():
+        if self.data_directory_empty():
             return 'uninitialized'
         elif self.config.recovery_conf_exists():
             return 'replica'
@@ -257,15 +255,21 @@ class Postgresql(object):
         except RetryFailedError as e:
             raise PostgresConnectionException(str(e))
 
-    def pg_control_exists(self):
-        return os.path.isfile(self._pg_control)
-
     def data_directory_empty(self):
-        if self.pg_control_exists():
+        return not os.path.exists(self._data_dir) or os.listdir(self._data_dir) == []
-            return False
+
-        if not os.path.exists(self._data_dir):
+    def write_pgpass(self, record):
-            return True
+        if 'user' not in record or 'password' not in record:
-        return all(os.name != 'nt' and (n.startswith('.') or n == 'lost+found') for n in os.listdir(self._data_dir))
+            return os.environ.copy()
+
+        with open(self._pgpass, 'w') as f:
+            if os.name != 'nt':
+                os.fchmod(f.fileno(), 0o600)
+            f.write('{host}:{port}:*:{user}:{password}\n'.format(**record))
+
+        env = os.environ.copy()
+        env['PGPASSFILE'] = self._pgpass
+        return env
 
     def replica_method_options(self, method):
         return deepcopy(self.config.get(method, {}))
@@ -286,8 +290,9 @@ class Postgresql(object):
 
     def _cluster_info_state_get(self, name):
         if not self._cluster_info_state:
+            stmt = cluster_info_query.format(self.wal_name, self.lsn_name)
             try:
-                result = self._is_leader_retry(self._query, self.cluster_info_query).fetchone()
+                result = self._is_leader_retry(self._query, stmt).fetchone()
                 self._cluster_info_state = dict(zip(['timeline', 'wal_position'], result))
             except RetryFailedError as e:  # SELECT failed two times
                 self._cluster_info_state = {'error': str(e)}
@@ -561,12 +566,10 @@ class Postgresql(object):
         if ready == STATE_REJECT:
             return False
         elif ready == STATE_NO_RESPONSE:
-            ret = not self.is_running()
+            self.set_state('start failed')
-            if ret:
+            self.slots_handler.schedule(False)  # TODO: can remove this?
-                self.set_state('start failed')
+            self.config.save_configuration_files(True)  # TODO: maybe remove this?
-                self.slots_handler.schedule(False)  # TODO: can remove this?
+            return True
-                self.config.save_configuration_files(True)  # TODO: maybe remove this?
-            return ret
         else:
             if ready != STATE_RUNNING:
                 # Bad configuration or unexpected OS error. No idea of PostgreSQL status.
@@ -642,10 +645,10 @@ class Postgresql(object):
 
     @contextmanager
     def get_replication_connection_cursor(self, host='localhost', port=5432, database=None, **kwargs):
-        conn_kwargs = self.config.replication.copy()
+        replication = self.config.replication
-        conn_kwargs.update(host=host, port=int(port), database=database or self._database, connect_timeout=3,
+        with get_connection_cursor(host=host, port=int(port), database=database or self._database, replication=1,
-                           user=conn_kwargs.pop('username'), replication=1, options='-c statement_timeout=2000')
+                                   user=replication['username'], password=replication.get('password'),
-        with get_connection_cursor(**conn_kwargs) as cur:
+                                   connect_timeout=3, options='-c statement_timeout=2000') as cur:
             yield cur
 
     def get_local_timeline_lsn_from_replication_connection(self):
@@ -684,25 +687,43 @@ class Postgresql(object):
         except Exception:
             logger.exception('Failed to read and parse %s', (history_path,))
 
-    def follow(self, member, role='replica', timeout=None, do_reload=False):
+    def follow(self, member, role='replica', timeout=None):
-        recovery_params = self.config.build_recovery_params(member)
+        is_remote_master = isinstance(member, RemoteMember)
+        no_replication_slot = is_remote_master and member.no_replication_slot
+        restore_command = is_remote_master and member.restore_command
+        min_apply_delay = is_remote_master and member.recovery_min_apply_delay
+        archive_cleanup = is_remote_master and member.archive_cleanup_command
+
+        primary_conninfo = self.config.primary_conninfo(member)
+        change_role = self.cb_called and (self.role in ('master', 'demoted') or
+                                          not {'standby_leader', 'replica'} - {self.role, role})
+
+        recovery_params = self.config.get('recovery_conf', {}).copy()
+        recovery_params.update({'standby_mode': 'on', 'recovery_target_timeline': 'latest'})
+        if primary_conninfo:
+            recovery_params['primary_conninfo'] = primary_conninfo
+        if self.slots_handler.use_slots and not no_replication_slot:
+            required_name = is_remote_master and member.data.get('primary_slot_name')
+            name = required_name or slot_name_from_member_name(self.name)
+            recovery_params['primary_slot_name'] = name
+        if restore_command:
+            recovery_params['restore_command'] = restore_command
+        if min_apply_delay:
+            recovery_params['recovery_min_apply_delay'] = min_apply_delay
+        if archive_cleanup:
+            recovery_params['archive_cleanup_command'] = archive_cleanup
+
         self.config.write_recovery_conf(recovery_params)
 
         # When we demoting the master or standby_leader to replica or promoting replica to a standby_leader
         # and we know for sure that postgres was already running before, we will only execute on_role_change
         # callback and prevent execution of on_restart/on_start callback.
         # If the role remains the same (replica or standby_leader), we will execute on_start or on_restart
-        change_role = self.cb_called and (self.role in ('master', 'demoted') or
-                                          not {'standby_leader', 'replica'} - {self.role, role})
         if change_role:
             self.__cb_pending = ACTION_NOOP
 
         if self.is_running():
-            if do_reload:
+            self.restart(block_callbacks=change_role, role=role)
-                self.config.write_postgresql_conf()
-                self.reload()
-            else:
-                self.restart(block_callbacks=change_role, role=role)
         else:
             self.start(timeout=timeout, block_callbacks=change_role, role=role)
 
@@ -737,17 +758,14 @@ class Postgresql(object):
             return self._cluster_info_state_get('timeline'), self._cluster_info_state_get('wal_position')
 
         with self.connection().cursor() as cursor:
-            cursor.execute(self.cluster_info_query)
+            cursor.execute(cluster_info_query.format(self.wal_name, self.lsn_name))
             return cursor.fetchone()[:2]
 
     def postmaster_start_time(self):
         try:
-            query = "SELECT pg_catalog.to_char(pg_catalog.pg_postmaster_start_time(), 'YYYY-MM-DD HH24:MI:SS.MS TZ')"
+            cursor = self.query("SELECT pg_catalog.to_char(pg_catalog.pg_postmaster_start_time(),"
-            if current_thread().ident == self.__thread_ident:
+                                " 'YYYY-MM-DD HH24:MI:SS.MS TZ')")
-                return self.query(query).fetchone()[0]
+            return cursor.fetchone()[0]
-            with self.connection().cursor() as cursor:
-                cursor.execute(query)
-                return cursor.fetchone()[0]
         except psycopg2.Error:
             return None
 
@@ -819,9 +837,9 @@ class Postgresql(object):
                 return app_name, True
             if sync_state == 'potential' and app_name == current:
                 # Prefer current even if not the best one any more to avoid indecisivness and spurious swaps.
-                return cluster.sync.sync_standby, False
+                return current, False
             if sync_state in ('async', 'potential'):
-                candidates.append(member.name)
+                candidates.append(app_name)
 
         if candidates:
             return candidates[0], False

patroni/postgresql/bootstrap.py

@@ -135,8 +135,16 @@ class Bootstrap(object):
                 # (pghost empty or the default socket directory) connections coming from the local machine.
                 r['host'] = 'localhost'  # set it to localhost to write into pgpass
 
-            connstring = uri('postgres', (host, r['port']), r['database'], r.get('user'))
+            if 'user' in r:
-            env = self._postgresql.config.write_pgpass(r) if 'password' in r else None
+                user = r['user']
+            else:
+                user = ''
+                if 'password' in r:
+                    import getpass
+                    r.setdefault('user', os.environ.get('PGUSER', getpass.getuser()))
+
+            connstring = uri('postgres', (host, r['port']), r['database'], user)
+            env = self._postgresql.write_pgpass(r) if 'password' in r else None
 
             try:
                 ret = self._postgresql.cancellable.call(shlex.split(cmd) + [connstring], env=env)
@@ -170,7 +178,7 @@ class Bootstrap(object):
             r = clone_member.conn_kwargs(self._postgresql.config.replication)
             connstring = uri('postgres', (r['host'], r['port']), r['database'], r['user'])
             # add the credentials to connect to the replica origin to pgpass.
-            env = self._postgresql.config.write_pgpass(r)
+            env = self._postgresql.write_pgpass(r)
         else:
             connstring = ''
             env = os.environ.copy()

patroni/postgresql/callback_executor.py

@@ -1,36 +1,40 @@
 import logging
-
+import subprocess
-from patroni.postgresql.cancellable import CancellableExecutor
+from threading import Event, Lock, Thread
-from threading import Condition, Thread
 
 logger = logging.getLogger(__name__)
 
 
-class CallbackExecutor(CancellableExecutor, Thread):
+class CallbackExecutor(Thread):
 
     def __init__(self):
-        CancellableExecutor.__init__(self)
+        super(CallbackExecutor, self).__init__()
-        Thread.__init__(self)
         self.daemon = True
+        self._lock = Lock()
         self._cmd = None
-        self._condition = Condition()
+        self._process = None
+        self._callback_event = Event()
         self.start()
 
     def call(self, cmd):
-        self._kill_process()
+        with self._lock:
-        with self._condition:
+            if self._process and self._process.poll() is None:
-            self._cmd = cmd
+                try:
-            self._condition.notify()
+                    self._process.kill()
+                    logger.warning('Killed the old callback process because it was still running: %s', self._cmd)
+                except OSError:
+                    logger.exception('Failed to kill the old callback')
+        self._cmd = cmd
+        self._callback_event.set()
 
     def run(self):
         while True:
-            with self._condition:
+            self._callback_event.wait()
-                if self._cmd is None:
+            self._callback_event.clear()
-                    self._condition.wait()
-                cmd, self._cmd = self._cmd, None
-
             with self._lock:
-                if not self._start_process(cmd, close_fds=True):
+                try:
+                    self._process = subprocess.Popen(self._cmd, close_fds=True)
+                except Exception:
+                    logger.exception('Failed to execute %s',  self._cmd)
                     continue
             self._process.wait()
-            self._kill_children()

patroni/postgresql/cancellable.py

@@ -1,6 +1,5 @@
 import logging
 import os
-import psutil
 import subprocess
 
 from patroni.exceptions import PostgresException
@@ -11,65 +10,12 @@ from threading import Lock
 logger = logging.getLogger(__name__)
 
 
-class CancellableExecutor(object):
+class CancellableSubprocess(object):
 
     def __init__(self):
-        self._process = None
-        self._process_cmd = None
-        self._process_children = []
-        self._lock = Lock()
-
-    def _start_process(self, cmd, *args, **kwargs):
-        """This method must be executed only when the `_lock` is acquired"""
-
-        try:
-            self._process_children = []
-            self._process_cmd = cmd
-            self._process = psutil.Popen(cmd, *args, **kwargs)
-        except Exception:
-            return logger.exception('Failed to execute %s',  cmd)
-        return True
-
-    def _kill_process(self):
-        with self._lock:
-            if self._process is not None and self._process.is_running() and not self._process_children:
-                try:
-                    self._process.suspend()  # Suspend the process before getting list of childrens
-                except psutil.Error as e:
-                    logger.info('Failed to suspend the process: %s', e.msg)
-
-                try:
-                    self._process_children = self._process.children(recursive=True)
-                except psutil.Error:
-                    pass
-
-                try:
-                    self._process.kill()
-                    logger.warning('Killed %s because it was still running', self._process_cmd)
-                except psutil.NoSuchProcess:
-                    pass
-                except psutil.AccessDenied as e:
-                    logger.warning('Failed to kill the process: %s', e.msg)
-
-    def _kill_children(self):
-        waitlist = []
-        with self._lock:
-            for child in self._process_children:
-                try:
-                    child.kill()
-                except psutil.NoSuchProcess:
-                    continue
-                except psutil.AccessDenied as e:
-                    logger.info('Failed to kill child process: %s', e.msg)
-                waitlist.append(child)
-        psutil.wait_procs(waitlist)
-
-
-class CancellableSubprocess(CancellableExecutor):
-
-    def __init__(self):
-        super(CancellableSubprocess, self).__init__()
         self._is_cancelled = False
+        self._process = None
+        self._lock = Lock()
 
     def call(self, *args, **kwargs):
         for s in ('stdin', 'stdout', 'stderr'):
@@ -92,18 +38,17 @@ class CancellableSubprocess(CancellableExecutor):
                     raise PostgresException('cancelled')
 
                 self._is_cancelled = False
-                started = self._start_process(*args, **kwargs)
+                self._process = subprocess.Popen(*args, **kwargs)
 
-            if started:
+            if communicate_input:
-                if communicate_input:
+                if input_data:
-                    if input_data:
+                    self._process.communicate(input_data)
-                        self._process.communicate(input_data)
+                self._process.stdin.close()
-                    self._process.stdin.close()
+
-                return self._process.wait()
+            return self._process.wait()
         finally:
             with self._lock:
                 self._process = None
-            self._kill_children()
 
     def reset_is_cancelled(self):
         with self._lock:
@@ -117,13 +62,15 @@ class CancellableSubprocess(CancellableExecutor):
     def cancel(self):
         with self._lock:
             self._is_cancelled = True
-            if self._process is None or not self._process.is_running():
+            if self._process is None or self._process.returncode is not None:
                 return
             self._process.terminate()
 
         for _ in polling_loop(10):
             with self._lock:
-                if self._process is None or not self._process.is_running():
+                if self._process is None or self._process.returncode is not None:
                     return
 
-        self._kill_process()
+        with self._lock:
+            if self._process is not None and self._process.returncode is None:
+                self._process.kill()

patroni/postgresql/config.py

@@ -4,12 +4,10 @@ import re
 import shutil
 import socket
 import stat
-import time
 
 from requests.structures import CaseInsensitiveDict
 from six.moves.urllib_parse import urlparse, parse_qsl, unquote
 
-from ..dcs import slot_name_from_member_name, RemoteMember
 from ..utils import compare_values, parse_bool, parse_int, split_host_port, uri
 
 logger = logging.getLogger(__name__)
@@ -59,15 +57,14 @@ def conninfo_uri_parse(dsn):
     return ret
 
 
-def read_param_value(value):
+def read_param_value(value, is_quoted=False):
     length = len(value)
     ret = ''
-    is_quoted = value[0] == "'"
+    i = 0
-    i = int(is_quoted)
     while i < length:
         if is_quoted:
             if value[i] == "'":
-                return ret, i + 1
+                return ret, i
         elif value[i].isspace():
             break
         if value[i] == '\\':
@@ -98,10 +95,12 @@ def conninfo_parse(dsn):
         if i >= length:
             return
 
-        value, end = read_param_value(dsn[i:])
+        is_quoted = dsn[i] == "'"
+        i += int(is_quoted)
+        value, end = read_param_value(dsn[i:], is_quoted)
         if value is None:
             return
-        i += end
+        i += end + int(is_quoted)
         ret[param] = value
     return ret
 
@@ -118,7 +117,7 @@ def parse_dsn(value):
     >>> r == {'application_name': 'mya/pp', 'host': ',/host2', 'sslmode': 'require',\
               'password': 'pass', 'port': '/123', 'user': 'u/se'}
     True
-    >>> r = parse_dsn(" host = 'host' dbname = db\\\\ name requiressl=1 ")
+    >>> r = parse_dsn(" host = 'host' dbname = db\\ name requiressl=1 ")
     >>> r == {'host': 'host', 'sslmode': 'require'}
     True
     >>> parse_dsn('requiressl = 0\\\\') == {'sslmode': 'prefer'}
@@ -148,70 +147,6 @@ def parse_dsn(value):
     return ret
 
 
-def strip_comment(value):
-    i = value.find('#')
-    if i > -1:
-        value = value[:i].strip()
-    return value
-
-
-def read_recovery_param_value(value):
-    """
-    >>> read_recovery_param_value('') is None
-    True
-    >>> read_recovery_param_value("'") is None
-    True
-    >>> read_recovery_param_value("''a") is None
-    True
-    >>> read_recovery_param_value('a b') is None
-    True
-    >>> read_recovery_param_value("'''") is None
-    True
-    >>> read_recovery_param_value("'\\\\") is None
-    True
-    >>> read_recovery_param_value("'a' s#") is None
-    True
-    >>> read_recovery_param_value("'\\\\'''' #a")
-    "''"
-    >>> read_recovery_param_value('asd')
-    'asd'
-    """
-    value = value.strip()
-    length = len(value)
-    if length == 0:
-        return None
-    elif value[0] == "'":
-        if length == 1:
-            return None
-        ret = ''
-        i = 1
-        while i < length:
-            if value[i] == '\\':
-                i += 1
-                if i >= length:
-                    return None
-            elif value[i] == "'":
-                i += 1
-                if i >= length:
-                    break
-                if value[i] in ('#', ' '):
-                    if strip_comment(value[i:]):
-                        return None
-                    break
-                if value[i] != "'":
-                    return None
-            ret += value[i]
-            i += 1
-        else:
-            return None
-        return ret
-    else:
-        value = strip_comment(value)
-        if not value or ' ' in value or '\\' in value:
-            return None
-    return value
-
-
 def mtime(filename):
     try:
         return os.stat(filename).st_mtime
@@ -219,37 +154,6 @@ def mtime(filename):
         return None
 
 
-class ConfigWriter(object):
-
-    def __init__(self, filename):
-        self._filename = filename
-        self._fd = None
-
-    def __enter__(self):
-        self._fd = open(self._filename, 'w')
-        self.writeline('# Do not edit this file manually!\n# It will be overwritten by Patroni!')
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if self._fd:
-            self._fd.close()
-
-    def writeline(self, line):
-        self._fd.write(line)
-        self._fd.write('\n')
-
-    def writelines(self, lines):
-        for line in lines:
-            self.writeline(line)
-
-    @staticmethod
-    def escape(value):  # Escape (by doubling) any single quotes or backslashes in given string
-        return re.sub(r'([\'\\])', r'\1\1', str(value))
-
-    def write_param(self, param, value):
-        self.writeline("{0} = '{1}'".format(param, self.escape(value)))
-
-
 class ConfigHandler(object):
 
     # List of parameters which must be always passed to postmaster as command line options
@@ -302,6 +206,8 @@ class ConfigHandler(object):
         'trigger_file'
     }
 
+    _CONFIG_WARNING_HEADER = '# Do not edit this file manually!\n# It will be overwritten by Patroni!\n'
+
     def __init__(self, postgresql, config):
         self._postgresql = postgresql
         self._config_dir = os.path.abspath(config.get('config_dir') or postgresql.data_dir)
@@ -318,12 +224,9 @@ class ConfigHandler(object):
         self._standby_signal = os.path.join(postgresql.data_dir, 'standby.signal')
         self._auto_conf = os.path.join(postgresql.data_dir, 'postgresql.auto.conf')
         self._auto_conf_mtime = None
-        self._pgpass = config.get('pgpass') or os.path.join(os.path.expanduser('~'), 'pgpass')
-        self._passfile = None
-        self._passfile_mtime = None
         self._synchronous_standby_names = None
         self._postmaster_ctime = None
-        self._current_recovery_params = None
+        self._primary_conninfo = None
         self._config = {}
         self._recovery_params = {}
         self.reload_config(config)
@@ -380,26 +283,28 @@ class ConfigHandler(object):
         if 'custom_conf' not in self._config and not os.path.exists(self._postgresql_base_conf):
             os.rename(self._postgresql_conf, self._postgresql_base_conf)
 
-        with ConfigWriter(self._postgresql_conf) as f:
+        with open(self._postgresql_conf, 'w') as f:
-            include = self._config.get('custom_conf') or self._postgresql_base_conf_name
+            os.chmod(self._postgresql_conf, stat.S_IWRITE | stat.S_IREAD)
-            f.writeline("include '{0}'\n".format(ConfigWriter.escape(include)))
+            f.write(self._CONFIG_WARNING_HEADER)
+            f.write("include '{0}'\n\n".format(self._config.get('custom_conf') or self._postgresql_base_conf_name))
             for name, value in sorted((configuration or self._server_parameters).items()):
                 if (not self._postgresql.bootstrap.running_custom_bootstrap or name != 'hba_file') \
                         and name not in self._RECOVERY_PARAMETERS:
-                    f.write_param(name, value)
+                    f.write("{0} = '{1}'\n".format(name, value))
             # when we are doing custom bootstrap we assume that we don't know superuser password
             # and in order to be able to change it, we are opening trust access from a certain address
             # therefore we need to make sure that hba_file is not overriden
             # after changing superuser password we will "revert" all these "changes"
             if self._postgresql.bootstrap.running_custom_bootstrap or 'hba_file' not in self._server_parameters:
-                f.write_param('hba_file', self._pg_hba_conf)
+                f.write("hba_file = '{0}'\n".format(self._pg_hba_conf.replace('\\', '\\\\')))
             if 'ident_file' not in self._server_parameters:
-                f.write_param('ident_file', self._pg_ident_conf)
+                f.write("ident_file = '{0}'\n".format(self._pg_ident_conf.replace('\\', '\\\\')))
 
             if self._postgresql.major_version >= 120000:
                 if self._recovery_params:
-                    f.writeline('\n# recovery.conf')
+                    f.write('\n# recovery.conf\n')
-                    self._write_recovery_params(f, self._recovery_params)
+                    for name, value in sorted(self._recovery_params.items()):
+                        f.write("{0} = '{1}'\n".format(name, value))
 
                 if not self._postgresql.bootstrap.keep_existing_recovery_conf:
                     self._sanitize_auto_conf()
@@ -427,15 +332,18 @@ class ConfigHandler(object):
                                   self.local_replication_address['host'], self.local_replication_address['port'],
                                   0, socket.SOCK_STREAM, socket.IPPROTO_TCP)})
 
-            with ConfigWriter(self._pg_hba_conf) as f:
+            with open(self._pg_hba_conf, 'w') as f:
+                f.write(self._CONFIG_WARNING_HEADER)
                 for address, t in addresses.items():
-                    f.writeline((
+                    f.write((
                         '{0}\treplication\t{1}\t{3}\ttrust\n'
-                        '{0}\tall\t{2}\t{3}\ttrust'
+                        '{0}\tall\t{2}\t{3}\ttrust\n'
                     ).format(t, self.replication['username'], self._superuser.get('username') or 'all', address))
         elif not self.hba_file and self._config.get('pg_hba'):
-            with ConfigWriter(self._pg_hba_conf) as f:
+            with open(self._pg_hba_conf, 'w') as f:
-                f.writelines(self._config['pg_hba'])
+                f.write(self._CONFIG_WARNING_HEADER)
+                for line in self._config['pg_hba']:
+                    f.write('{0}\n'.format(line))
             return True
 
     def replace_pg_ident(self):
@@ -447,250 +355,122 @@ class ConfigHandler(object):
         """
 
         if not self._server_parameters.get('ident_file') and self._config.get('pg_ident'):
-            with ConfigWriter(self._pg_ident_conf) as f:
+            with open(self._pg_ident_conf, 'w') as f:
-                f.writelines(self._config['pg_ident'])
+                f.write(self._CONFIG_WARNING_HEADER)
+                for line in self._config['pg_ident']:
+                    f.write('{0}\n'.format(line))
             return True
 
     def primary_conninfo_params(self, member):
-        if not (member and member.conn_url) or member.name == self._postgresql.name:
+        name = self._postgresql.name
+        if not (member and member.conn_url) or member.name == name:
             return None
         ret = member.conn_kwargs(self.replication)
-        ret['application_name'] = self._postgresql.name
+        ret.update(application_name=name, sslmode='prefer')
-        ret.setdefault('sslmode', 'prefer')
         if self._krbsrvname:
             ret['krbsrvname'] = self._krbsrvname
         if 'database' in ret:
             del ret['database']
         return ret
 
-    def format_dsn(self, params, include_dbname=False):
+    def primary_conninfo(self, member):
-        # A list of keywords that can be found in a conninfo string. Follows what is acceptable by libpq
+        r = self.primary_conninfo_params(member)
-        keywords = ('user', 'passfile', 'host', 'port', 'sslmode', 'sslcompression', 'sslcert',
+        if not r:
-                    'sslkey', 'sslrootcert', 'sslcrl', 'application_name', 'krbsrvname')
+            return None
-        if include_dbname:
+        keywords = 'user password host port sslmode application_name krbsrvname'.split()
-            params = params.copy()
+        return ' '.join('{0}={{{0}}}'.format(kw) for kw in keywords if r.get(kw)).format(**r)
-            params['dbname'] = params.get('database') or self._postgresql.database
-            keywords = ('dbname',) + keywords
-
-        def escape(value):
-            return re.sub(r'([\'\\ ])', r'\\\1', str(value))
-
-        return ' '.join('{0}={1}'.format(kw, escape(params[kw])) for kw in keywords if params.get(kw) is not None)
-
-    def _write_recovery_params(self, fd, recovery_params):
-        for name, value in sorted(recovery_params.items()):
-            if name == 'primary_conninfo':
-                if 'password' in value:
-                    self.write_pgpass(value)
-                    value['passfile'] = self._passfile = self._pgpass
-                    self._passfile_mtime = mtime(self._pgpass)
-                value = self.format_dsn(value)
-            fd.write_param(name, value)
-
-    def build_recovery_params(self, member):
-        recovery_params = CaseInsensitiveDict({p: v for p, v in self.get('recovery_conf', {}).items()
-                                               if not p.lower().startswith('recovery_target') and
-                                               p.lower() not in ('primary_conninfo', 'primary_slot_name')})
-        recovery_params.update({'standby_mode': 'on', 'recovery_target_timeline': 'latest'})
-        if self._postgresql.major_version >= 120000:
-            # on pg12 we want to protect from following params being set in one of included files
-            # not doing so might result in a standby being paused, promoted or shutted down.
-            recovery_params.update({'recovery_target': '', 'recovery_target_name': '', 'recovery_target_time': '',
-                                    'recovery_target_xid': '', 'recovery_target_lsn': ''})
-
-        is_remote_master = isinstance(member, RemoteMember)
-        primary_conninfo = self.primary_conninfo_params(member)
-        if primary_conninfo:
-            recovery_params['primary_conninfo'] = primary_conninfo
-            if self.get('use_slots', True) and self._postgresql.major_version >= 90400 \
-                    and not (is_remote_master and member.no_replication_slot):
-                recovery_params['primary_slot_name'] = member.primary_slot_name if is_remote_master \
-                        else slot_name_from_member_name(self._postgresql.name)
-
-        if is_remote_master:  # standby_cluster config might have different parameters, we want to override them
-            recovery_params.update({p: member.data.get(p) for p in ('restore_command', 'recovery_min_apply_delay',
-                                                                    'archive_cleanup_command') if member.data.get(p)})
-        return recovery_params
 
     def recovery_conf_exists(self):
         if self._postgresql.major_version >= 120000:
             return os.path.exists(self._standby_signal) or os.path.exists(self._recovery_signal)
         return os.path.exists(self._recovery_conf)
 
-    @property
+    def _read_primary_conninfo(self):
-    def _triggerfile_good_name(self):
-        return 'trigger_file' if self._postgresql.major_version < 120000 else 'promote_trigger_file'
-
-    @property
-    def _triggerfile_wrong_name(self):
-        return 'trigger_file' if self._postgresql.major_version >= 120000 else 'promote_trigger_file'
-
-    @property
-    def _recovery_parameters_to_compare(self):
-        skip_params = {'recovery_target_inclusive', 'recovery_target_action', self._triggerfile_wrong_name}
-        return self._RECOVERY_PARAMETERS - skip_params
-
-    def _read_recovery_params(self):
         pg_conf_mtime = mtime(self._postgresql_conf)
         auto_conf_mtime = mtime(self._auto_conf)
-        passfile_mtime = mtime(self._passfile) if self._passfile else False
         postmaster_ctime = self._postgresql.is_running()
         if postmaster_ctime:
             postmaster_ctime = postmaster_ctime.create_time()
 
         if self._postgresql_conf_mtime == pg_conf_mtime and self._auto_conf_mtime == auto_conf_mtime \
-                and self._passfile_mtime == passfile_mtime and self._postmaster_ctime == postmaster_ctime:
+                and self._postmaster_ctime == postmaster_ctime:
             return None, False
 
         try:
-            values = self._get_pg_settings(self._recovery_parameters_to_compare).values()
+            primary_conninfo = self._postgresql.query('SHOW primary_conninfo').fetchone()[0]
-            values = {p[0]: [p[1], p[4] == 'postmaster'] for p in values}
             self._postgresql_conf_mtime = pg_conf_mtime
             self._auto_conf_mtime = auto_conf_mtime
             self._postmaster_ctime = postmaster_ctime
         except Exception:
-            values = None
+            primary_conninfo = None
-        return values, True
+        return primary_conninfo, True
 
-    def _read_recovery_params_pre_v12(self):
+    def _read_primary_conninfo_pre_v12(self):
         recovery_conf_mtime = mtime(self._recovery_conf)
-        passfile_mtime = mtime(self._passfile) if self._passfile else False
+        if recovery_conf_mtime == self._recovery_conf_mtime:
-        if recovery_conf_mtime == self._recovery_conf_mtime and passfile_mtime == self._passfile_mtime:
             return None, False
 
-        values = {}
+        primary_conninfo = ''
         with open(self._recovery_conf, 'r') as f:
             for line in f:
-                line = line.strip()
-                if not line or line.startswith('#'):
-                    continue
-                value = None
                 match = PARAMETER_RE.match(line)
-                if match:
+                if match and match.group(1) == 'primary_conninfo':
-                    value = read_recovery_param_value(line[match.end():])
+                    i = match.end()
-                if value is None:
+                    if i < len(line):
-                    return None, True
+                        is_quoted = line[i] == "'"
-                values[match.group(1)] = [value, True]
+                        i += int(is_quoted)
+                        primary_conninfo, _ = read_param_value(line[i:], is_quoted)
             self._recovery_conf_mtime = recovery_conf_mtime
-        values.setdefault('recovery_min_apply_delay', ['0', True])
+        return primary_conninfo, True
-        values.update({param: ['', True] for param in self._recovery_parameters_to_compare if param not in values})
-        return values, True
-
-    def _check_passfile(self, passfile, wanted_primary_conninfo):
-        # If there is a passfile in the primary_conninfo try to figure out that
-        # the passfile contains the line allowing connection to the given node.
-        # We assume that the passfile was created by Patroni and therefore doing
-        # the full match and not covering cases when host, port or user are set to '*'
-        passfile_mtime = mtime(passfile)
-        if passfile_mtime:
-            try:
-                with open(passfile) as f:
-                    wanted_line = self._pgpass_line(wanted_primary_conninfo).strip()
-                    for raw_line in f:
-                        if raw_line.strip() == wanted_line:
-                            self._passfile = passfile
-                            self._passfile_mtime = passfile_mtime
-                            return True
-            except Exception:
-                logger.info('Failed to read %s', passfile)
-        return False
-
-    def _check_primary_conninfo(self, primary_conninfo, wanted_primary_conninfo):
-        # first we will cover corner cases, when we are replicating from somewhere while shouldn't
-        # or there is no primary_conninfo but we should replicate from some specific node.
-        if not wanted_primary_conninfo:
-            return not primary_conninfo
-        elif not primary_conninfo:
-            return False
-
-        if 'passfile' in primary_conninfo and 'password' not in primary_conninfo \
-                and 'password' in wanted_primary_conninfo:
-            if self._check_passfile(primary_conninfo['passfile'], wanted_primary_conninfo):
-                primary_conninfo['password'] = wanted_primary_conninfo['password']
-            else:
-                return False
-
-        return all(primary_conninfo.get(p) == str(v) for p, v in wanted_primary_conninfo.items())
-
-    def check_recovery_conf(self, member):
-        """Returns a tuple. The first boolean element indicates that recovery params don't match
-           and the second is set to `True` if the restart is required in order to apply new values"""
 
+    def check_recovery_conf(self, member):  # Name is confusing. In fact it checks the value of primary_conninfo
         # TODO: recovery.conf could be stale, would be nice to detect that.
         if self._postgresql.major_version >= 120000:
             if not os.path.exists(self._standby_signal):
-                return True, True
+                return False
 
-            _read_recovery_params = self._read_recovery_params
+            _read_primary_conninfo = self._read_primary_conninfo
         else:
             if not self.recovery_conf_exists():
-                return True, True
+                return False
 
-            _read_recovery_params = self._read_recovery_params_pre_v12
+            _read_primary_conninfo = self._read_primary_conninfo_pre_v12
 
-        params, updated = _read_recovery_params()
+        primary_conninfo, updated = _read_primary_conninfo()
-        # updated indicates that mtime of postgresql.conf, postgresql.auto.conf, or recovery.conf
+        # updated indicates that mtime of postgresql.conf, postgresql.auto.conf, or recovery.conf was changed
-        # was changed and params were read either from the config or from the database connection.
+        # and the primary_conninfo value was read either from config or from the database connection.
         if updated:
-            if params is None:  # exception or unparsable config
+            # primary_conninfo is one of:
-                return True, True
+            # - None (exception or unparsable config)
-
+            # - '' (not in config)
-            # We will cache parsed value until the next config change.
+            # - or the actual dsn value
-            self._current_recovery_params = params
+            self._primary_conninfo = primary_conninfo
-            primary_conninfo = params['primary_conninfo']
+            if primary_conninfo:
-            if primary_conninfo[0]:
+                # We will cache parsed value until the next config change.
-                primary_conninfo[0] = parse_dsn(params['primary_conninfo'][0])
+                self._primary_conninfo = parse_dsn(primary_conninfo)
                 # If we failed to parse non-empty connection string this indicates that config if broken.
-                if not primary_conninfo[0]:
+                if not self._primary_conninfo:
-                    return True, True
+                    return False
-            else:  # empty string, primary_conninfo is not in the config
+            elif primary_conninfo is not None:
-                primary_conninfo[0] = {}
+                self._primary_conninfo = {}
+            else:  # primary_conninfo is None, config is probably broken
+                return False
 
-        required = {'restart': 0, 'reload': 0}
+        wanted_primary_conninfo = self.primary_conninfo_params(member)
+        # first we will cover corner cases, when we are replicating from somewhere while shouldn't
+        # or there is no primary_conninfo but we should replicate from some specific node.
+        if not wanted_primary_conninfo:
+            return not self._primary_conninfo
+        elif not self._primary_conninfo:
+            return False
 
-        def record_missmatch(mtype):
+        return all(self._primary_conninfo.get(p) == str(v) for p, v in wanted_primary_conninfo.items())
-            required['restart' if mtype else 'reload'] += 1
-
-        wanted_recovery_params = self.build_recovery_params(member)
-        for param, value in self._current_recovery_params.items():
-            if param == 'recovery_min_apply_delay':
-                if not compare_values('integer', 'ms', value[0], wanted_recovery_params.get(param, 0)):
-                    record_missmatch(value[1])
-            elif param == 'primary_conninfo':
-                if not self._check_primary_conninfo(value[0], wanted_recovery_params.get('primary_conninfo', {})):
-                    record_missmatch(value[1])
-            elif (param != 'primary_slot_name' or wanted_recovery_params.get('primary_conninfo')) \
-                    and str(value[0]) != str(wanted_recovery_params.get(param, '')):
-                record_missmatch(value[1])
-        return required['restart'] + required['reload'] > 0, required['restart'] > 0
 
     @staticmethod
     def _remove_file_if_exists(name):
         if os.path.isfile(name) or os.path.islink(name):
             os.unlink(name)
 
-    @staticmethod
-    def _pgpass_line(record):
-        if 'password' in record:
-            def escape(value):
-                return re.sub(r'([:\\])', r'\\\1', str(value))
-
-            record = {n: escape(record.get(n, '*')) for n in ('host', 'port', 'user', 'password')}
-            return '{host}:{port}:*:{user}:{password}'.format(**record)
-
-    def write_pgpass(self, record):
-        line = self._pgpass_line(record)
-        if not line:
-            return os.environ.copy()
-
-        with open(self._pgpass, 'w') as f:
-            os.chmod(self._pgpass, stat.S_IWRITE | stat.S_IREAD)
-            f.write(line)
-
-        env = os.environ.copy()
-        env['PGPASSFILE'] = self._pgpass
-        return env
-
     def write_recovery_conf(self, recovery_params):
         if self._postgresql.major_version >= 120000:
             if parse_bool(recovery_params.pop('standby_mode', None)):
@@ -700,9 +480,10 @@ class ConfigHandler(object):
                 open(self._recovery_signal, 'w').close()
             self._recovery_params = recovery_params
         else:
-            with ConfigWriter(self._recovery_conf) as f:
+            with open(self._recovery_conf, 'w') as f:
                 os.chmod(self._recovery_conf, stat.S_IWRITE | stat.S_IREAD)
-                self._write_recovery_params(f, recovery_params)
+                for name, value in recovery_params.items():
+                    f.write("{0} = '{1}'\n".format(name, value))
 
     def remove_recovery_conf(self):
         for name in (self._recovery_conf, self._standby_signal, self._recovery_signal):
@@ -741,9 +522,13 @@ class ConfigHandler(object):
             self._config['recovery_conf'] = recovery_conf
 
         if self.get('recovery_conf'):
-            value = self._config['recovery_conf'].pop(self._triggerfile_wrong_name, None)
+            good_name, bad_name = 'trigger_file', 'promote_trigger_file'
-            if self._triggerfile_good_name not in self._config['recovery_conf'] and value:
+            if self._postgresql.major_version >= 120000:
-                self._config['recovery_conf'][self._triggerfile_good_name] = value
+                good_name, bad_name = bad_name, good_name
+
+            value = self._config['recovery_conf'].pop(bad_name, None)
+            if good_name not in self._config['recovery_conf'] and value:
+                self._config['recovery_conf'][good_name] = value
 
     def get_server_parameters(self, config):
         parameters = config['parameters'].copy()
@@ -783,18 +568,14 @@ class ConfigHandler(object):
     @property
     def local_connect_kwargs(self):
         ret = self._local_address.copy()
-        # add all of the other connection settings that are available
-        ret.update(self._superuser)
-        # if the "username" parameter is present, it actually needs to be "user"
-        # for connecting to PostgreSQL
-        if 'username' in self._superuser:
-            ret['user'] = self._superuser['username']
-            del ret['username']
-        # ensure certain Patroni configurations are available
         ret.update({'database': self._postgresql.database,
                     'fallback_application_name': 'Patroni',
                     'connect_timeout': 3,
                     'options': '-c statement_timeout=2000'})
+        if 'username' in self._superuser:
+            ret['user'] = self._superuser['username']
+        if 'password' in self._superuser:
+            ret['password'] = self._superuser['password']
         return ret
 
     def resolve_connection_addresses(self):
@@ -821,77 +602,51 @@ class ConfigHandler(object):
 
         self._postgresql.set_connection_kwargs(self.local_connect_kwargs)
 
-    def _get_pg_settings(self, names):
+    def reload_config(self, config):
-        return {r[0]: r for r in self._postgresql.query(('SELECT name, setting, unit, vartype, context '
-                                                         + ' FROM pg_catalog.pg_settings ' +
-                                                         ' WHERE pg_catalog.lower(name) = ANY(%s)'),
-                                                        [n.lower() for n in names])}
-
-    @staticmethod
-    def _handle_wal_buffers(old_values, changes):
-        wal_block_size = parse_int(old_values['wal_block_size'][1])
-        wal_segment_size = old_values['wal_segment_size']
-        wal_segment_size = parse_int(wal_segment_size[1]) * parse_int(wal_segment_size[2], 'B') / wal_block_size
-        default_wal_buffers = min(max(parse_int(old_values['shared_buffers'][1]) / 32, 8), wal_segment_size)
-
-        wal_buffers = old_values['wal_buffers']
-        new_value = str(changes['wal_buffers'] or -1)
-
-        new_value = default_wal_buffers if new_value == '-1' else parse_int(new_value, wal_buffers[2])
-        old_value = default_wal_buffers if wal_buffers[1] == '-1' else parse_int(*wal_buffers[1:3])
-
-        if new_value == old_value:
-            del changes['wal_buffers']
-
-    def reload_config(self, config, sighup=False):
         self._superuser = config['authentication'].get('superuser', {})
         server_parameters = self.get_server_parameters(config)
 
         conf_changed = hba_changed = ident_changed = local_connection_address_changed = pending_restart = False
         if self._postgresql.state == 'running':
-            changes = CaseInsensitiveDict({p: v for p, v in server_parameters.items()
+            changes = CaseInsensitiveDict({p: v for p, v in server_parameters.items() if '.' not in p})
-                                           if '.' not in p and p.lower() not in self._RECOVERY_PARAMETERS})
+            changes.update({p: None for p in self._server_parameters.keys() if not ('.' in p or p in changes)})
-            changes.update({p: None for p in self._server_parameters.keys()
-                            if not ('.' in p or p in changes or p.lower() in self._RECOVERY_PARAMETERS)})
             if changes:
-                if 'wal_buffers' in changes:  # we need to calculate the default value of wal_buffers
-                    undef = [p for p in ('shared_buffers', 'wal_segment_size', 'wal_block_size') if p not in changes]
-                    changes.update({p: None for p in undef})
                 # XXX: query can raise an exception
-                old_values = self._get_pg_settings(changes.keys())
+                for r in self._postgresql.query(('SELECT name, setting, unit, vartype, context '
-                if 'wal_buffers' in changes:
+                                                 + 'FROM pg_catalog.pg_settings ' +
-                    self._handle_wal_buffers(old_values, changes)
+                                                 ' WHERE pg_catalog.lower(name) IN ('
-                    for p in undef:
+                                                 + ', '.join(['%s'] * len(changes)) +
-                        del changes[p]
+                                                 ')'), *(k.lower() for k in changes.keys())):
-
-                for r in old_values.values():
                     if r[4] != 'internal' and r[0] in changes:
                         new_value = changes.pop(r[0])
                         if new_value is None or not compare_values(r[3], r[2], r[1], new_value):
-                            conf_changed = True
                             if r[4] == 'postmaster':
                                 pending_restart = True
-                                logger.info('Changed %s from %s to %s (restart might be required)',
+                                logger.info('Changed %s from %s to %s (restart required)', r[0], r[1], new_value)
-                                            r[0], r[1], new_value)
                                 if config.get('use_unix_socket') and r[0] == 'unix_socket_directories'\
                                         or r[0] in ('listen_addresses', 'port'):
                                     local_connection_address_changed = True
                             else:
                                 logger.info('Changed %s from %s to %s', r[0], r[1], new_value)
+                                conf_changed = True
                 for param in changes:
                     if param in server_parameters:
                         logger.warning('Removing invalid parameter `%s` from postgresql.parameters', param)
                         server_parameters.pop(param)
 
             # Check that user-defined-paramters have changed (parameters with period in name)
-            for p, v in server_parameters.items():
+            if not conf_changed:
-                if '.' in p and (p not in self._server_parameters or str(v) != str(self._server_parameters[p])):
+                for p, v in server_parameters.items():
-                    logger.info('Changed %s from %s to %s', p, self._server_parameters.get(p), v)
+                    if '.' in p and (p not in self._server_parameters or str(v) != str(self._server_parameters[p])):
-                    conf_changed = True
+                        logger.info('Changed %s from %s to %s', p, self._server_parameters.get(p), v)
-            for p, v in self._server_parameters.items():
+                        conf_changed = True
-                if '.' in p and (p not in server_parameters or str(v) != str(server_parameters[p])):
+                        break
-                    logger.info('Changed %s from %s to %s', p, v, server_parameters.get(p))
+                if not conf_changed:
-                    conf_changed = True
+                    for p, v in self._server_parameters.items():
+                        if '.' in p and (p not in server_parameters or str(v) != str(server_parameters[p])):
+                            logger.info('Changed %s from %s to %s', p, v, server_parameters.get(p))
+                            conf_changed = True
+                            break
 
             if not server_parameters.get('hba_file') and config.get('pg_hba'):
                 hba_changed = self._config.get('pg_hba', []) != config['pg_hba']
@@ -903,6 +658,7 @@ class ConfigHandler(object):
         self._postgresql.set_pending_restart(pending_restart)
         self._server_parameters = server_parameters
         self._adjust_recovery_parameters()
+        self._connect_address = config.get('connect_address')
         self._krbsrvname = config.get('krbsrvname')
 
         # for not so obvious connection attempts that may happen outside of pyscopg2
@@ -921,18 +677,10 @@ class ConfigHandler(object):
         if ident_changed:
             self.replace_pg_ident()
 
-        if sighup or conf_changed or hba_changed or ident_changed:
+        if conf_changed or hba_changed or ident_changed:
-            logger.info('Reloading PostgreSQL configuration.')
+            logger.info('PostgreSQL configuration items changed, reloading configuration.')
             self._postgresql.reload()
-            if self._postgresql.major_version >= 90500:
+        elif not pending_restart:
-                time.sleep(1)
-                try:
-                    pending_restart = self._postgresql.query('SELECT COUNT(*) FROM pg_catalog.pg_settings'
-                                                             ' WHERE pending_restart').fetchone()[0] > 0
-                    self._postgresql.set_pending_restart(pending_restart)
-                except Exception as e:
-                    logger.warning('Exception %r when running query', e)
-        else:
             logger.info('No PostgreSQL configuration items changed, nothing to reload.')
 
     def set_synchronous_standby(self, name):

patroni/postgresql/postmaster.py

@@ -6,8 +6,6 @@ import re
 import signal
 import subprocess
 
-from patroni.config import Config
-
 logger = logging.getLogger(__name__)
 
 STOP_SIGNALS = {
@@ -155,7 +153,7 @@ class PostmasterProcess(psutil.Process):
         # In order to make everything portable we can't use fork&exec approach here, so  we will call
         # ourselves and pass list of arguments which must be used to start postgres.
         # On Windows, in order to run a side-by-side assembly the specified env must include a valid SYSTEMROOT.
-        env = {p: os.environ[p] for p in os.environ if not p.startswith(Config.PATRONI_ENV_PREFIX)}
+        env = {p: os.environ[p] for p in ('PATH', 'LD_LIBRARY_PATH', 'LC_ALL', 'LANG', 'SYSTEMROOT') if p in os.environ}
         try:
             proc = PostmasterProcess._from_pidfile(data_dir)
             if proc and not proc._is_postmaster_process():

patroni/postgresql/rewind.py

@@ -145,9 +145,17 @@ class Rewind(object):
 
     def pg_rewind(self, r):
         # prepare pg_rewind connection
-        env = self._postgresql.config.write_pgpass(r)
+        env = self._postgresql.write_pgpass(r)
         env['PGOPTIONS'] = '-c statement_timeout=0'
-        dsn = self._postgresql.config.format_dsn(r, True)
+        dsn_attrs = [
+            ('user', r.get('user')),
+            ('host', r.get('host')),
+            ('port', r.get('port')),
+            ('dbname', r.get('database') or self._postgresql.database),
+            ('sslmode', 'prefer'),
+            ('sslcompression', '1'),
+        ]
+        dsn = " ".join("{0}={1}".format(k, v) for k, v in dsn_attrs if v is not None)
         logger.info('running pg_rewind from %s', dsn)
         try:
             return self._postgresql.cancellable.call([self._postgresql.pgcommand('pg_rewind'), '-D',

patroni/postgresql/slots.py

@@ -15,14 +15,19 @@ class SlotsHandler(object):
 
     def __init__(self, postgresql):
         self._postgresql = postgresql
+        self._use_slots = postgresql.config.get('use_slots', True)
         self._replication_slots = {}  # already existing replication slots
         self.schedule()
 
+    @property
+    def use_slots(self):
+        return self._use_slots and self._postgresql.major_version >= 90400
+
     def _query(self, sql, *params):
         return self._postgresql.query(sql, *params, retry=False)
 
     def load_replication_slots(self):
-        if self._postgresql.major_version >= 90400 and self._schedule_load_slots:
+        if self.use_slots and self._schedule_load_slots:
             replication_slots = {}
             cursor = self._query('SELECT slot_name, slot_type, plugin, database FROM pg_catalog.pg_replication_slots')
             for r in cursor:
@@ -40,7 +45,7 @@ class SlotsHandler(object):
         return cursor.rowcount == 1
 
     def sync_replication_slots(self, cluster):
-        if self._postgresql.major_version >= 90400:
+        if self.use_slots:
             try:
                 self.load_replication_slots()
 
@@ -99,5 +104,5 @@ class SlotsHandler(object):
 
     def schedule(self, value=None):
         if value is None:
-            value = self._postgresql.major_version >= 90400
+            value = self.use_slots
         self._schedule_load_slots = value

patroni/request.py

@@ -1,47 +0,0 @@
-import json
-import urllib3
-
-from six.moves.urllib_parse import urlparse, urlunparse
-
-
-class PatroniRequest(object):
-
-    def __init__(self, config, insecure=False):
-        cert_reqs = 'CERT_NONE' if insecure or config.get('ctl', {}).get('insecure', False) else 'CERT_REQUIRED'
-        self._pool = urllib3.PoolManager(cert_reqs=cert_reqs)
-        self.reload_config(config)
-
-    @staticmethod
-    def _get_cfg_value(config, name):
-        return config.get('ctl', {}).get(name) or config.get('restapi', {}).get(name)
-
-    def _apply_pool_param(self, param, value):
-        if value:
-            self._pool.connection_pool_kw[param] = value
-        else:
-            self._pool.connection_pool_kw.pop(param, None)
-
-    def _apply_ssl_file_param(self, config, name):
-        value = self._get_cfg_value(config, name + 'file')
-        self._apply_pool_param(name + '_file', value)
-        return value
-
-    def reload_config(self, config):
-        self._pool.headers = urllib3.make_headers(basic_auth=self._get_cfg_value(config, 'auth'))
-
-        if self._apply_ssl_file_param(config, 'cert'):
-            self._apply_ssl_file_param(config, 'key')
-        else:
-            self._pool.connection_pool_kw.pop('key_file', None)
-
-        cacert = config.get('ctl', {}).get('cacert') or config.get('restapi', {}).get('cafile')
-        self._apply_pool_param('ca_certs', cacert)
-
-    def __call__(self, member, method='GET', endpoint=None, data=None, **kwargs):
-        url = member.api_url
-        if endpoint:
-            scheme, netloc, _, _, _, _ = urlparse(url)
-            url = urlunparse((scheme, netloc, endpoint, '', '', ''))
-        if data is not None:
-            kwargs['body'] = json.dumps(data)
-        return self._pool.request(method.upper(), url, **kwargs)

patroni/utils.py

@@ -296,17 +296,6 @@ class Retry(object):
                      max_jitter=self.max_jitter / 100.0, max_delay=self.max_delay, sleep_func=self.sleep_func,
                      deadline=self.deadline, retry_exceptions=self.retry_exceptions)
 
-    @property
-    def sleeptime(self):
-        return self._cur_delay + (random.randint(0, self.max_jitter) / 100.0)
-
-    def update_delay(self):
-        self._cur_delay = min(self._cur_delay * self.backoff, self.max_delay)
-
-    @property
-    def stoptime(self):
-        return self._cur_stoptime
-
     def __call__(self, func, *args, **kwargs):
         """Call a function with arguments until it completes without throwing a `retry_exceptions`
 
@@ -328,14 +317,14 @@ class Retry(object):
                     logger.warning('Retry got exception: %s', e)
                     raise RetryFailedError("Too many retry attempts")
                 self._attempts += 1
-                sleeptime = self.sleeptime
+                sleeptime = self._cur_delay + (random.randint(0, self.max_jitter) / 100.0)
 
                 if self._cur_stoptime is not None and time.time() + sleeptime >= self._cur_stoptime:
                     logger.warning('Retry got exception: %s', e)
                     raise RetryFailedError("Exceeded retry deadline")
                 logger.debug('Retry got exception: %s', e)
                 self.sleep_func(sleeptime)
-                self.update_delay()
+                self._cur_delay = min(self._cur_delay * self.backoff, self.max_delay)
 
 
 def polling_loop(timeout, interval=1):
@@ -363,52 +352,3 @@ def uri(proto, netloc, path='', user=None):
     path = '/{0}'.format(path) if path and not path.startswith('/') else path
     user = '{0}@'.format(user) if user else ''
     return '{0}://{1}{2}{3}{4}'.format(proto, user, host, port, path)
-
-
-def is_standby_cluster(config):
-    # Check whether or not provided configuration describes a standby cluster
-    return isinstance(config, dict) and (config.get('host') or config.get('port') or config.get('restore_command'))
-
-
-def cluster_as_json(cluster):
-    leader_name = cluster.leader.name if cluster.leader else None
-    xlog_location_cluster = cluster.last_leader_operation or 0
-
-    ret = {'members': []}
-    for m in cluster.members:
-        if m.name == leader_name:
-            config = cluster.config.data if cluster.config and cluster.config.modify_index else {}
-            role = 'standby_leader' if is_standby_cluster(config.get('standby_cluster')) else 'leader'
-        elif m.name == cluster.sync.sync_standby:
-            role = 'sync_standby'
-        else:
-            role = 'replica'
-
-        conn_kwargs = m.conn_kwargs()
-        member = {'name': m.name, 'host': conn_kwargs['host'], 'port': int(conn_kwargs['port']),
-                  'role': role, 'state': m.data.get('state', ''), 'api_url': m.api_url}
-        optional_attributes = ('timeline', 'pending_restart', 'scheduled_restart', 'tags')
-        member.update({n: m.data[n] for n in optional_attributes if n in m.data})
-
-        if m.name != leader_name:
-            xlog_location = m.data.get('xlog_location')
-            if xlog_location is None:
-                member['lag'] = 'unknown'
-            elif xlog_location_cluster >= xlog_location:
-                member['lag'] = xlog_location_cluster - xlog_location
-            else:
-                member['lag'] = 0
-
-        ret['members'].append(member)
-
-    # sort members by name for consistency
-    ret['members'].sort(key=lambda m: m['name'])
-    if cluster.is_paused():
-        ret['pause'] = True
-    if cluster.failover and cluster.failover.scheduled_at:
-        ret['scheduled_switchover'] = {'at': cluster.failover.scheduled_at.isoformat()}
-        if cluster.failover.leader:
-            ret['scheduled_switchover']['from'] = cluster.failover.leader
-        if cluster.failover.candidate:
-            ret['scheduled_switchover']['to'] = cluster.failover.candidate
-    return ret

patroni/version.py

@@ -1 +1 @@
-__version__ = '1.6.1'
+__version__ = '1.6.0'

requirements.txt

@@ -5,11 +5,11 @@ requests
 six >= 1.7
 kazoo>=1.3.1
 python-etcd>=0.4.3,<0.5
-python-consul>=0.7.1
+python-consul>=0.7.0
 click>=4.1
 prettytable>=0.7
 tzlocal
 python-dateutil
 psutil>=2.0.0
 cdiff
-kubernetes>=2.0.0,<=10.0.1,!=4.0.*,!=5.0.*
+kubernetes>=2.0.0,<=7.0.0,!=4.0.*,!=5.0.*

tests/__init__.py

@@ -5,6 +5,7 @@ import shutil
 import unittest
 
 from mock import Mock, patch
+from tempfile import gettempdir
 
 import psycopg2
 import requests
@@ -100,15 +101,12 @@ class MockCursor(object):
             self.results = [(1, 2)]
         elif sql.startswith('SELECT pg_catalog.pg_is_in_recovery()'):
             self.results = [(False, 2)]
-        elif sql.startswith('SELECT pg_catalog.to_char'):
+        elif sql.startswith('WITH replication_info AS ('):
             replication_info = '[{"application_name":"walreceiver","client_addr":"1.2.3.4",' +\
                                '"state":"streaming","sync_state":"async","sync_priority":0}]'
             self.results = [('', 0, '', '', '', '', False, replication_info)]
         elif sql.startswith('SELECT name, setting'):
             self.results = [('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
-                            ('wal_block_size', '8192', None, 'integer', 'internal'),
-                            ('shared_buffers', '16384', '8kB', 'integer', 'postmaster'),
-                            ('wal_buffers', '-1', '8kB', 'integer', 'postmaster'),
                             ('search_path', 'public', None, 'string', 'user'),
                             ('port', '5433', None, 'integer', 'postmaster'),
                             ('listen_addresses', '*', None, 'string', 'postmaster'),
@@ -181,12 +179,11 @@ class PostgresInit(unittest.TestCase):
     @patch.object(ConfigHandler, 'write_postgresql_conf', Mock())
     @patch.object(ConfigHandler, 'replace_pg_hba', Mock())
     @patch.object(ConfigHandler, 'replace_pg_ident', Mock())
-    @patch.object(Postgresql, 'get_postgres_role_from_data_directory', Mock(return_value='master'))
     def setUp(self):
         data_dir = 'data/test0'
         self.p = Postgresql({'name': 'postgresql0', 'scope': 'batman', 'data_dir': data_dir,
                              'config_dir': data_dir, 'retry_timeout': 10,
-                             'krbsrvname': 'postgres', 'pgpass': os.path.join(data_dir, 'pgpass0'),
+                             'krbsrvname': 'postgres', 'pgpass': os.path.join(gettempdir(), 'pgpass0'),
                              'listen': '127.0.0.2, 127.0.0.3:5432', 'connect_address': '127.0.0.2:5432',
                              'authentication': {'superuser': {'username': 'foo', 'password': 'test'},
                                                 'replication': {'username': '', 'password': 'rep-pass'}},

tests/test_api.py

@@ -12,7 +12,6 @@ from patroni.utils import tzutc
 from six import BytesIO as IO
 from six.moves import BaseHTTPServer
 from . import psycopg2_connect, MockCursor
-from .test_ha import get_cluster_initialized_without_leader
 
 
 future_restart_time = datetime.datetime.now(tzutc) + datetime.timedelta(days=5)
@@ -146,7 +145,7 @@ class MockRestApiServer(RestApiServer):
         self.serve_forever = Mock()
         MockRestApiServer._BaseServer__is_shut_down = Mock()
         MockRestApiServer._BaseServer__shutdown_request = True
-        config = config or {'listen': '127.0.0.1:8008', 'auth': 'test:test', 'certfile': 'dumb', 'verify_client': 'a'}
+        config = config or {'listen': '127.0.0.1:8008', 'auth': 'test:test', 'certfile': 'dumb'}
         super(MockRestApiServer, self).__init__(MockPatroni(), config)
         Handler(MockRequest(request), ('0.0.0.0', 8080), self)
 
@@ -160,7 +159,6 @@ class TestRestApiHandler(unittest.TestCase):
 
     def test_do_GET(self):
         MockRestApiServer(RestApiHandler, 'GET /replica')
-        MockRestApiServer(RestApiHandler, 'GET /read-only')
         with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={})):
             MockRestApiServer(RestApiHandler, 'GET /replica')
         with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'master'})):
@@ -199,17 +197,6 @@ class TestRestApiHandler(unittest.TestCase):
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0'))
         MockRestApiServer(RestApiHandler, 'POST /restart HTTP/1.0\nAuthorization:')
 
-    @patch.object(MockPatroni, 'dcs')
-    def test_do_GET_cluster(self, mock_dcs):
-        mock_dcs.cluster = get_cluster_initialized_without_leader()
-        mock_dcs.cluster.members[1].data['xlog_location'] = 11
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /cluster'))
-
-    @patch.object(MockPatroni, 'dcs')
-    def test_do_GET_history(self, mock_dcs):
-        mock_dcs.cluster = get_cluster_initialized_without_leader()
-        self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /history'))
-
     @patch.object(MockPatroni, 'dcs')
     def test_do_GET_config(self, mock_dcs):
         mock_dcs.cluster.config.data = {}
@@ -246,8 +233,11 @@ class TestRestApiHandler(unittest.TestCase):
         mock_dcs.get_cluster.return_value.config = ClusterConfig.from_node(1, config)
         MockRestApiServer(RestApiHandler, request)
 
-    @patch.object(MockPatroni, 'sighup_handler', Mock())
+    @patch.object(MockPatroni, 'sighup_handler', Mock(side_effect=Exception))
     def test_do_POST_reload(self):
+        with patch.object(MockPatroni, 'config') as mock_config:
+            mock_config.reload_local_configuration.return_value = False
+            MockRestApiServer(RestApiHandler, 'POST /reload HTTP/1.0' + self._authorization)
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'POST /reload HTTP/1.0' + self._authorization))
 
     @patch.object(MockPatroni, 'dcs')
@@ -421,18 +411,12 @@ class TestRestApiServer(unittest.TestCase):
     def test_reload_config(self):
         bad_config = {'listen': 'foo'}
         self.assertRaises(ValueError, MockRestApiServer, None, '', bad_config)
-        srv = MockRestApiServer(Mock(), '', {'listen': '*:8008', 'certfile': 'a', 'verify_client': 'required'})
+        srv = MockRestApiServer(lambda a1, a2, a3: None, '')
         self.assertRaises(ValueError, srv.reload_config, bad_config)
         self.assertRaises(ValueError, srv.reload_config, {})
         with patch.object(socket.socket, 'setsockopt', Mock(side_effect=socket.error)):
             srv.reload_config({'listen': ':8008'})
 
-    def test_check_auth(self):
-        srv = MockRestApiServer(Mock(), '', {'listen': '*:8008', 'certfile': 'a', 'verify_client': 'required'})
-        mock_rh = Mock()
-        mock_rh.request.getpeercert.return_value = None
-        self.assertIsNot(srv.check_auth(mock_rh), True)
-
     def test_handle_error(self):
         try:
             raise Exception()

tests/test_bootstrap.py

@@ -16,7 +16,6 @@ from . import psycopg2_connect, BaseTestPostgresql
 @patch('os.rename', Mock())
 class TestBootstrap(BaseTestPostgresql):
 
-    @patch('patroni.postgresql.CallbackExecutor', Mock())
     def setUp(self):
         super(TestBootstrap, self).setUp()
         self.b = self.p.bootstrap
@@ -102,9 +101,6 @@ class TestBootstrap(BaseTestPostgresql):
     @patch.object(CancellableSubprocess, 'call', Mock())
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     @patch.object(Postgresql, 'data_directory_empty', Mock(return_value=False))
-    @patch.object(Postgresql, 'controldata', Mock(return_value={'max_connections setting': 100,
-                                                                'max_prepared_xacts setting': 0,
-                                                                'max_locks_per_xact setting': 64}))
     def test_bootstrap(self):
         with patch('subprocess.call', Mock(return_value=1)):
             self.assertFalse(self.b.bootstrap({}))
@@ -130,7 +126,6 @@ class TestBootstrap(BaseTestPostgresql):
 
     @patch.object(CancellableSubprocess, 'call')
     @patch.object(Postgresql, 'get_major_version', Mock(return_value=90600))
-    @patch.object(Postgresql, 'controldata',  Mock(return_value={'Database cluster state': 'in production'}))
     def test_custom_bootstrap(self, mock_cancellable_subprocess_call):
         self.p.config._config.pop('pg_hba')
         config = {'method': 'foo', 'foo': {'command': 'bar'}}

tests/test_callback_executor.py

@@ -1,4 +1,3 @@
-import psutil
 import unittest
 
 from mock import Mock, patch
@@ -7,28 +6,22 @@ from patroni.postgresql.callback_executor import CallbackExecutor
 
 class TestCallbackExecutor(unittest.TestCase):
 
-    @patch('psutil.Popen')
+    @patch('subprocess.Popen')
     def test_callback_executor(self, mock_popen):
-        mock_popen.return_value.children.return_value = []
+        mock_popen.return_value.wait.side_effect = Exception
-        mock_popen.return_value.is_running.return_value = True
+        mock_popen.return_value.poll.return_value = None
 
         ce = CallbackExecutor()
-        ce._kill_children = Mock(side_effect=Exception)
         self.assertIsNone(ce.call([]))
         ce.join()
 
         self.assertIsNone(ce.call([]))
 
-        mock_popen.return_value.kill.side_effect = psutil.AccessDenied()
+        mock_popen.return_value.kill.side_effect = OSError
-        self.assertIsNone(ce.call([]))
-
-        ce._process_children = []
-        mock_popen.return_value.children.side_effect = psutil.Error()
-        mock_popen.return_value.kill.side_effect = psutil.NoSuchProcess(123)
         self.assertIsNone(ce.call([]))
 
         mock_popen.side_effect = Exception
         ce = CallbackExecutor()
-        ce._condition.wait = Mock(side_effect=[None, Exception])
+        ce._callback_event.wait = Mock(side_effect=[None, Exception])
         self.assertIsNone(ce.call([]))
         ce.join()

tests/test_cancellable.py

@@ -1,7 +1,6 @@
-import psutil
 import unittest
 
-from mock import Mock, patch
+from mock import Mock, PropertyMock, patch
 from patroni.exceptions import PostgresException
 from patroni.postgresql.cancellable import CancellableSubprocess
 
@@ -15,20 +14,10 @@ class TestCancellableSubprocess(unittest.TestCase):
         self.c.cancel()
         self.assertRaises(PostgresException, self.c.call, communicate_input=None)
 
-    def test__kill_children(self):
-        self.c._process_children = [Mock()]
-        self.c._kill_children()
-        self.c._process_children[0].kill.side_effect = psutil.AccessDenied()
-        self.c._kill_children()
-        self.c._process_children[0].kill.side_effect = psutil.NoSuchProcess(123)
-        self.c._kill_children()
-
     @patch('patroni.postgresql.cancellable.polling_loop', Mock(return_value=[0, 0]))
     def test_cancel(self):
         self.c._process = Mock()
-        self.c._process.is_running.return_value = True
+        self.c._process.returncode = None
-        self.c._process.children.side_effect = psutil.Error()
-        self.c._process.suspend.side_effect = psutil.Error()
         self.c.cancel()
-        self.c._process.is_running.side_effect = [True, False]
+        type(self.c._process).returncode = PropertyMock(side_effect=[None, -15])
         self.c.cancel()

tests/test_config.py

@@ -70,7 +70,8 @@ class TestConfig(unittest.TestCase):
         config = Config()
         with patch.object(Config, '_load_config_file', Mock(return_value={'restapi': {}})):
             with patch.object(Config, '_build_effective_configuration', Mock(side_effect=Exception)):
-                config.reload_local_configuration()
+                self.assertRaises(Exception, config.reload_local_configuration, True)
+            self.assertTrue(config.reload_local_configuration(True))
             self.assertTrue(config.reload_local_configuration())
             self.assertIsNone(config.reload_local_configuration())
 

tests/test_ctl.py

@@ -1,20 +1,20 @@
 import etcd
 import os
+import requests
 import sys
 import unittest
 
 from click.testing import CliRunner
 from datetime import datetime, timedelta
 from mock import patch, Mock
-from patroni.ctl import ctl, store_config, load_config, output_members, get_dcs, parse_dcs, \
+from patroni.ctl import ctl, store_config, load_config, output_members, request_patroni, get_dcs, parse_dcs, \
     get_all_members, get_any_member, get_cursor, query_member, configure, PatroniCtlException, apply_config_changes, \
     format_config_for_editing, show_diff, invoke_editor, format_pg_version, find_executable
 from patroni.dcs.etcd import Client, Failover
 from patroni.utils import tzutc
 from psycopg2 import OperationalError
-from urllib3 import PoolManager
 
-from . import MockConnect, MockCursor, MockResponse, psycopg2_connect
+from . import MockConnect, MockCursor, MockResponse, psycopg2_connect, requests_get
 from .test_etcd import etcd_read, socket_getaddrinfo
 from .test_ha import get_cluster_initialized_without_leader, get_cluster_initialized_with_leader, \
     get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader, get_cluster, Member
@@ -35,7 +35,7 @@ def test_rw_config():
 
 @patch('patroni.ctl.load_config',
        Mock(return_value={'scope': 'alpha', 'postgresql': {'data_dir': '.', 'parameters': {}, 'retry_timeout': 5},
-                          'restapi': {'listen': '::', 'certfile': 'a'}, 'etcd': {'host': 'localhost:2379'}}))
+                          'restapi': {'auth': 'u:p', 'listen': ''}, 'etcd': {'host': 'localhost:2379'}}))
 class TestCtl(unittest.TestCase):
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
@@ -75,7 +75,7 @@ class TestCtl(unittest.TestCase):
         self.assertIsNone(output_members(cluster, name='abc', fmt='tsv'))
 
     @patch('patroni.ctl.get_dcs')
-    @patch.object(PoolManager, 'request', Mock(return_value=MockResponse()))
+    @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
     def test_switchover(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
@@ -125,18 +125,18 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='dummy')
         assert result.exit_code == 1
 
-        with patch.object(PoolManager, 'request', Mock(side_effect=Exception)):
+        with patch('patroni.ctl.request_patroni', Mock(side_effect=Exception)):
             # Non-responding patroni
             result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n2300-01-01T12:23:00\ny')
             assert 'falling back to DCS' in result.output
 
-        with patch.object(PoolManager, 'request') as mocked:
+        with patch('patroni.ctl.request_patroni') as mocked:
-            mocked.return_value.status = 500
+            mocked.return_value.status_code = 500
             result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
             assert 'Switchover failed' in result.output
 
-            mocked.return_value.status = 501
+            mocked.return_value.status_code = 501
-            mocked.return_value.data = b'Server does not support this operation'
+            mocked.return_value.text = 'Server does not support this operation'
             result = self.runner.invoke(ctl, ['switchover', 'dummy'], input='leader\nother\n\ny')
             assert 'Switchover failed' in result.output
 
@@ -151,7 +151,7 @@ class TestCtl(unittest.TestCase):
         assert result.exit_code == 1
 
     @patch('patroni.ctl.get_dcs')
-    @patch.object(PoolManager, 'request', Mock(return_value=MockResponse()))
+    @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
     def test_failover(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
@@ -231,7 +231,7 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['dsn', 'alpha', '--member', 'dummy'])
         assert result.exit_code == 1
 
-    @patch.object(PoolManager, 'request')
+    @patch('requests.post')
     @patch('patroni.ctl.get_dcs')
     def test_reload(self, mock_get_dcs, mock_post):
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
@@ -239,19 +239,18 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['reload', 'alpha'], input='y')
         assert 'Failed: reload for member' in result.output
 
-        mock_post.return_value.status = 200
+        mock_post.return_value.status_code = 200
         result = self.runner.invoke(ctl, ['reload', 'alpha'], input='y')
         assert 'No changes to apply on member' in result.output
 
-        mock_post.return_value.status = 202
+        mock_post.return_value.status_code = 202
         result = self.runner.invoke(ctl, ['reload', 'alpha'], input='y')
         assert 'Reload request received for member' in result.output
 
-    @patch.object(PoolManager, 'request')
+    @patch('requests.post', requests_get)
     @patch('patroni.ctl.get_dcs')
-    def test_restart_reinit(self, mock_get_dcs, mock_post):
+    def test_restart_reinit(self, mock_get_dcs):
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
-        mock_post.return_value.status = 503
         result = self.runner.invoke(ctl, ['restart', 'alpha'], input='now\ny\n')
         assert 'Failed: restart for' in result.output
         assert result.exit_code == 0
@@ -286,49 +285,58 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['restart', 'alpha', '--pending', '--force', '--timeout', '10min'])
         assert result.exit_code == 0
 
-        # normal restart, the schedule is actually parsed, but not validated in patronictl
+        with patch('requests.delete', Mock(return_value=MockResponse(500))):
-        result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
-        assert 'Failed: flush scheduled restart' in result.output
+            result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force',
+                                              '--scheduled', '2300-10-01T14:30'])
+            assert 'Failed: flush scheduled restart' in result.output
 
         with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
             result = self.runner.invoke(ctl,
                                         ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
             assert result.exit_code == 1
 
+        with patch('requests.post', Mock(return_value=MockResponse())):
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0.0',
+                                              '--scheduled', '2300-10-01T14:30'], input='y')
+            assert result.exit_code == 0
+
+        with patch('requests.post', Mock(return_value=MockResponse(204))):
+            # get restart with the non-200 return code
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
+            result = self.runner.invoke(ctl, ['restart', 'alpha', '--pg-version', '42.0',
+                                              '--scheduled', '2300-10-01T14:30'], input='y')
+            assert result.exit_code == 0
+
         # force restart with restart already present
-        result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
+        with patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse(204))):
-        assert result.exit_code == 0
+            result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force',
+                                              '--scheduled', '2300-10-01T14:30'])
+            assert result.exit_code == 0
 
-        ctl_args = ['restart', 'alpha', '--pg-version', '99.0', '--scheduled', '2300-10-01T14:30']
+        with patch('requests.post', Mock(return_value=MockResponse(202))):
-        # normal restart, the schedule is actually parsed, but not validated in patronictl
+            # get restart with the non-200 return code
-        mock_post.return_value.status = 200
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
-        result = self.runner.invoke(ctl, ctl_args, input='y')
+            result = self.runner.invoke(
-        assert result.exit_code == 0
+                ctl, ['restart', 'alpha', '--pg-version', '99.0.0', '--scheduled', '2300-10-01T14:30'], input='y'
+            )
+            assert 'Success: restart scheduled' in result.output
+            assert result.exit_code == 0
 
-        # get restart with the non-200 return code
+        with patch('requests.post', Mock(return_value=MockResponse(409))):
-        # normal restart, the schedule is actually parsed, but not validated in patronictl
+            # get restart with the non-200 return code
-        mock_post.return_value.status = 204
+            # normal restart, the schedule is actually parsed, but not validated in patronictl
-        result = self.runner.invoke(ctl, ctl_args, input='y')
+            result = self.runner.invoke(
-        assert result.exit_code == 0
+                ctl, ['restart', 'alpha', '--pg-version', '99.0.0', '--scheduled', '2300-10-01T14:30'], input='y'
-
+            )
-        # get restart with the non-200 return code
+            assert 'Failed: another restart is already' in result.output
-        # normal restart, the schedule is actually parsed, but not validated in patronictl
+            assert result.exit_code == 0
-        mock_post.return_value.status = 202
-        result = self.runner.invoke(ctl, ctl_args, input='y')
-        assert 'Success: restart scheduled' in result.output
-        assert result.exit_code == 0
-
-        # get restart with the non-200 return code
-        # normal restart, the schedule is actually parsed, but not validated in patronictl
-        mock_post.return_value.status = 409
-        result = self.runner.invoke(ctl, ctl_args, input='y')
-        assert 'Failed: another restart is already' in result.output
-        assert result.exit_code == 0
 
     @patch('patroni.ctl.get_dcs')
     def test_remove(self, mock_get_dcs):
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
-        result = self.runner.invoke(ctl, ['-k', 'remove', 'alpha'], input='alpha\nslave')
+        result = self.runner.invoke(ctl, ['remove', 'alpha'], input='alpha\nslave')
         assert 'Please confirm' in result.output
         assert 'You are about to remove all' in result.output
         # Not typing an exact confirmation
@@ -345,6 +353,17 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['remove', 'alpha'], input='alpha\nYes I am aware\nleader')
         assert result.exit_code == 0
 
+    @patch('requests.post', Mock(side_effect=requests.exceptions.ConnectionError('foo')))
+    @patch('click.get_current_context')
+    def test_request_patroni(self, mock_context):
+        member = get_cluster_initialized_with_leader().leader.member
+
+        mock_context.return_value.obj = {'ctl': {'cacert': 'cert.pem'}}
+        self.assertRaises(requests.exceptions.ConnectionError, request_patroni, member, 'post', 'dummy', {})
+
+        mock_context.return_value.obj = {'ctl': {'insecure': True}}
+        self.assertRaises(requests.exceptions.ConnectionError, request_patroni, member, 'post', 'dummy', {})
+
     def test_ctl(self):
         self.runner.invoke(ctl, ['list'])
 
@@ -418,7 +437,7 @@ class TestCtl(unittest.TestCase):
         assert 'Scheduled restart' in result.output
 
     @patch('patroni.ctl.get_dcs')
-    @patch.object(PoolManager, 'request', Mock(return_value=MockResponse()))
+    @patch('requests.delete', Mock(return_value=MockResponse()))
     def test_flush(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
@@ -428,58 +447,58 @@ class TestCtl(unittest.TestCase):
 
         result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '--force'])
         assert 'Success: flush scheduled restart' in result.output
-        with patch.object(PoolManager, 'request', return_value=MockResponse(404)):
+        with patch.object(requests, 'delete', return_value=MockResponse(404)):
             result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '--force'])
             assert 'Failed: flush scheduled restart' in result.output
 
-    @patch.object(PoolManager, 'request')
     @patch('patroni.ctl.get_dcs')
     @patch('patroni.ctl.polling_loop', Mock(return_value=[1]))
-    def test_pause_cluster(self, mock_get_dcs, mock_post):
+    def test_pause_cluster(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
 
-        mock_post.return_value.status = 500
+        with patch('requests.patch', Mock(return_value=MockResponse(500))):
-        result = self.runner.invoke(ctl, ['pause', 'dummy'])
+            result = self.runner.invoke(ctl, ['pause', 'dummy'])
-        assert 'Failed' in result.output
+            assert 'Failed' in result.output
 
-        mock_post.return_value.status = 200
+        with patch('requests.patch', Mock(return_value=MockResponse(200))),\
-        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
+                patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
             result = self.runner.invoke(ctl, ['pause', 'dummy'])
             assert 'Cluster is already paused' in result.output
 
-        result = self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
+        with patch('requests.patch', Mock(return_value=MockResponse(200))):
-        assert "'pause' request sent" in result.output
+            result = self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
-        mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
+            assert "'pause' request sent" in result.output
-                                                                  get_cluster(None, None, [], None, None)])
+            mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
-        self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
+                                                                      get_cluster(None, None, [], None, None)])
-        member = Member(1, 'other', 28, {})
+            self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
-        mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
+            member = Member(1, 'other', 28, {})
-                                                                  get_cluster(None, None, [member], None, None)])
+            mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
-        self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
+                                                                      get_cluster(None, None, [member], None, None)])
+            self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
 
-    @patch.object(PoolManager, 'request')
     @patch('patroni.ctl.get_dcs')
-    def test_resume_cluster(self, mock_get_dcs, mock_post):
+    def test_resume_cluster(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
 
-        mock_post.return_value.status = 200
-        with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=False)):
-            result = self.runner.invoke(ctl, ['resume', 'dummy'])
-            assert 'Cluster is not paused' in result.output
-
         with patch('patroni.dcs.Cluster.is_paused', Mock(return_value=True)):
-            result = self.runner.invoke(ctl, ['resume', 'dummy'])
+            with patch('requests.patch', Mock(return_value=MockResponse(200))):
-            assert 'Success' in result.output
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Success' in result.output
 
-            mock_post.return_value.status = 500
+            with patch('requests.patch', Mock(return_value=MockResponse(500))):
-            result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
-            assert 'Failed' in result.output
+                assert 'Failed' in result.output
 
-            mock_post.side_effect = Exception
+            with patch('requests.patch', Mock(return_value=MockResponse(200))),\
-            result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                    patch('patroni.dcs.Cluster.is_paused', Mock(return_value=False)):
-            assert 'Can not find accessible cluster member' in result.output
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Cluster is not paused' in result.output
+
+            with patch('requests.patch', Mock(side_effect=Exception)):
+                result = self.runner.invoke(ctl, ['resume', 'dummy'])
+                assert 'Can not find accessible cluster member' in result.output
 
     def test_apply_config_changes(self):
         config = {"postgresql": {"parameters": {"work_mem": "4MB"}, "use_pg_rewind": True}, "ttl": 30}
@@ -555,23 +574,16 @@ class TestCtl(unittest.TestCase):
     def test_version(self, mock_get_dcs):
         mock_get_dcs.return_value = self.e
         mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
-        with patch.object(PoolManager, 'request') as mocked:
+        with patch('patroni.ctl.request_patroni') as mocked:
             result = self.runner.invoke(ctl, ['version'])
             assert 'patronictl version' in result.output
-            mocked.return_value.data = b'{"patroni":{"version":"1.2.3"},"server_version": 100001}'
+            mocked.return_value.json = lambda: {'patroni': {'version': '1.2.3'}, 'server_version': 100001}
             result = self.runner.invoke(ctl, ['version', 'dummy'])
             assert '1.2.3' in result.output
-        with patch.object(PoolManager, 'request', Mock(side_effect=Exception)):
+        with patch('requests.get', Mock(side_effect=Exception)):
             result = self.runner.invoke(ctl, ['version', 'dummy'])
             assert 'failed to get version' in result.output
 
-    @patch('patroni.ctl.get_dcs')
-    def test_history(self, mock_get_dcs):
-        mock_get_dcs.return_value.get_cluster = Mock()
-        mock_get_dcs.return_value.get_cluster.return_value.history.lines = [[1, 67176, 'no recovery target specified']]
-        result = self.runner.invoke(ctl, ['history'])
-        assert 'Reason' in result.output
-
     def test_format_pg_version(self):
         self.assertEqual(format_pg_version(100001), '10.1')
         self.assertEqual(format_pg_version(90605), '9.6.5')

tests/test_etcd.py

@@ -7,7 +7,6 @@ from dns.exception import DNSException
 from mock import Mock, patch
 from patroni.dcs.etcd import AbstractDCS, Client, Cluster, Etcd, EtcdError, DnsCachingResolver
 from patroni.exceptions import DCSError
-from patroni.utils import Retry
 from urllib3.exceptions import ReadTimeoutError
 
 from . import SleepException, MockResponse, requests_get
@@ -150,11 +149,7 @@ class TestClient(unittest.TestCase):
         self.assertRaises(ValueError, self.client.api_execute, '', '')
         self.client._base_uri = 'http://localhost:4001'
         self.client._machines_cache = ['http://localhost:2379']
-        self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'POST', timeout=0)
+        self.client.api_execute('/', 'POST', timeout=0)
-        self.client._base_uri = 'http://localhost:4001'
-        self.client._machines_cache = ['http://localhost:2379']
-        rtry = Retry(deadline=10, max_delay=1, max_tries=-1, retry_exceptions=(etcd.EtcdLeaderElectionInProgress,))
-        rtry(self.client.api_execute, '/', 'POST', timeout=0, params={'retry': rtry})
         mock_machines.__get__ = Mock(return_value=['http://localhost:2379'])
         self.client._machines_cache_updated = 0
         self.client.api_execute('/', 'POST', timeout=0)
@@ -162,11 +157,7 @@ class TestClient(unittest.TestCase):
         self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
         self.assertRaises(etcd.EtcdWatchTimedOut, self.client.api_execute, '/timeout', 'POST', params={'wait': 'true'})
         self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', '')
-
+        with patch.object(Client, '_load_machines_cache', Mock(side_effect=etcd.EtcdException)):
-        with patch.object(Client, '_do_http_request', Mock(side_effect=etcd.EtcdConnectionFailed)):
-            with patch.object(Client, '_calculate_timeouts', Mock(side_effect=[(1, 1, 0), (1, 1, 0), (0, 1, 0)])):
-                self.assertRaises(etcd.EtcdException, rtry, self.client.api_execute, '/', 'GET', params={'retry': rtry})
-            self.client._read_timeout = 0
             self.assertRaises(etcd.EtcdException, self.client.api_execute, '/', 'GET')
 
     def test_get_srv_record(self):

tests/test_ha.py

@@ -137,15 +137,10 @@ zookeeper:
         self.scheduled_restart = {'schedule': future_restart_time,
                                   'postmaster_start_time': str(postmaster_start_time)}
         self.watchdog = Watchdog(self.config)
-        self.request = lambda member, **kwargs: requests_get(member.api_url, **kwargs)
 
 
 def run_async(self, func, args=()):
-    self.reset_scheduled_action()
+    return func(*args) if args else func()
-    if args:
-        func(*args)
-    else:
-        func()
 
 
 @patch.object(Postgresql, 'is_running', Mock(return_value=MockPostmaster()))
@@ -157,9 +152,8 @@ def run_async(self, func, args=()):
 @patch.object(Postgresql, 'controldata', Mock(return_value={'Database system identifier': SYSID}))
 @patch.object(SlotsHandler, 'sync_replication_slots', Mock())
 @patch.object(ConfigHandler, 'append_pg_hba', Mock())
-@patch.object(ConfigHandler, 'write_pgpass', Mock(return_value={}))
+@patch.object(Postgresql, 'write_pgpass', Mock(return_value={}))
 @patch.object(ConfigHandler, 'write_recovery_conf', Mock())
-@patch.object(ConfigHandler, 'write_postgresql_conf', Mock())
 @patch.object(Postgresql, 'query', Mock())
 @patch.object(Postgresql, 'checkpoint', Mock())
 @patch.object(CancellableSubprocess, 'call', Mock(return_value=0))
@@ -359,7 +353,6 @@ class TestHa(PostgresInit):
         self.p.is_leader = false
         self.assertEqual(self.ha.run_cycle(), 'no action.  i am a secondary and i am following a leader')
         self.ha.patroni.replicatefrom = "foo"
-        self.p.config.check_recovery_conf = Mock(return_value=(True, False))
         self.assertEqual(self.ha.run_cycle(), 'no action.  i am a secondary and i am following a leader')
 
     def test_follow_in_pause(self):
@@ -432,7 +425,7 @@ class TestHa(PostgresInit):
 
         self.ha.cluster = get_cluster_initialized_with_leader()
         self.assertIsNone(self.ha.reinitialize(True))
-        self.ha._async_executor.schedule('reinitialize')
+
         self.assertIsNotNone(self.ha.reinitialize())
 
         self.ha.state_handler.name = self.ha.cluster.leader.name
@@ -446,7 +439,7 @@ class TestHa(PostgresInit):
         self.p.restart = false
         self.assertEqual(self.ha.restart({}), (False, 'restart failed'))
         self.ha.cluster = get_cluster_initialized_with_leader()
-        self.ha._async_executor.schedule('reinitialize')
+        self.ha.reinitialize()
         self.assertEqual(self.ha.restart({}), (False, 'reinitialize already in progress'))
         with patch.object(self.ha, "restart_matches", return_value=False):
             self.assertEqual(self.ha.restart({'foo': 'bar'}), (False, "restart conditions are not satisfied"))
@@ -454,7 +447,7 @@ class TestHa(PostgresInit):
     @patch('os.kill', Mock())
     def test_restart_in_progress(self):
         with patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=True)):
-            self.ha._async_executor.schedule('restart')
+            self.ha.restart({}, run_async=True)
             self.assertTrue(self.ha.restart_scheduled())
             self.assertEqual(self.ha.run_cycle(), 'restart in progress')
 
@@ -471,6 +464,7 @@ class TestHa(PostgresInit):
                     self.assertEqual(self.ha.run_cycle(), 'lost leader lock during restart')
                     mock_terminate.assert_called()
 
+    @patch('requests.get', requests_get)
     def test_manual_failover_from_leader(self):
         self.ha.fetch_node_status = get_node_status()
         self.ha.has_lock = true
@@ -518,6 +512,7 @@ class TestHa(PostgresInit):
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, 'blabla', self.p.name, scheduled))
         self.assertEqual('no action.  i am the leader with the lock', self.ha.run_cycle())
 
+    @patch('requests.get', requests_get)
     def test_manual_failover_from_leader_in_pause(self):
         self.ha.has_lock = true
         self.ha.is_paused = true
@@ -527,6 +522,7 @@ class TestHa(PostgresInit):
         self.ha.cluster = get_cluster_initialized_with_leader(Failover(0, self.p.name, '', None))
         self.assertEqual('PAUSE: no action.  i am the leader with the lock', self.ha.run_cycle())
 
+    @patch('requests.get', requests_get)
     def test_manual_failover_from_leader_in_synchronous_mode(self):
         self.p.is_leader = true
         self.ha.has_lock = true
@@ -539,6 +535,7 @@ class TestHa(PostgresInit):
         self.ha.is_failover_possible = true
         self.assertEqual('manual failover: demoting myself', self.ha.run_cycle())
 
+    @patch('requests.get', requests_get)
     def test_manual_failover_process_no_leader(self):
         self.p.is_leader = false
         self.ha.cluster = get_cluster_initialized_without_leader(failover=Failover(0, '', self.p.name, None))
@@ -588,6 +585,7 @@ class TestHa(PostgresInit):
         self.ha.is_paused = true
         self.assertFalse(self.ha.is_healthiest_node())
 
+    @patch('requests.get', requests_get)
     def test__is_healthiest_node(self):
         self.ha.cluster = get_cluster_initialized_without_leader(sync=('postgresql1', self.p.name))
         self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
@@ -609,6 +607,7 @@ class TestHa(PostgresInit):
         self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.patroni.nofailover = False
 
+    @patch('requests.get', requests_get)
     def test_fetch_node_status(self):
         member = Member(0, 'test', 1, {'api_url': 'http://127.0.0.1:8011/patroni'})
         self.ha.fetch_node_status(member)
@@ -685,7 +684,7 @@ class TestHa(PostgresInit):
         self.p.is_leader = false
         self.p.name = 'leader'
         self.ha.cluster = get_standby_cluster_initialized_with_only_leader()
-        self.p.config.check_recovery_conf = Mock(return_value=(False, False))
+        self.p.config.check_recovery_conf = true
         self.assertEqual(self.ha.run_cycle(), 'promoted self to a standby leader because i had the session lock')
         self.assertEqual(self.ha.run_cycle(), 'no action.  i am the standby leader with the lock')
 
@@ -697,6 +696,7 @@ class TestHa(PostgresInit):
         with patch.object(Leader, 'conn_url', PropertyMock(return_value='')):
             self.assertEqual(self.ha.run_cycle(), 'continue following the old known standby leader')
 
+    @patch('requests.get', requests_get)
     def test_process_unhealthy_standby_cluster_as_standby_leader(self):
         self.p.is_leader = false
         self.p.name = 'leader'
@@ -1027,6 +1027,7 @@ class TestHa(PostgresInit):
             self.assertEqual(self.ha.run_cycle(), 'no action.  i am the leader with the lock')
 
     @patch('sys.exit', return_value=1)
+    @patch('requests.get', requests_get)
     def test_abort_join(self, exit_mock):
         self.ha.cluster = get_cluster_not_initialized_without_leader()
         self.p.is_leader = false

tests/test_kubernetes.py

@@ -1,11 +1,8 @@
-import json
 import time
 import unittest
 
 from mock import Mock, patch
-from patroni.dcs.kubernetes import Kubernetes, KubernetesError, k8s_client, RetryFailedError
+from patroni.dcs.kubernetes import Kubernetes, KubernetesError, k8s_client, k8s_watch, RetryFailedError
-from threading import Thread
-from . import SleepException
 
 
 def mock_list_namespaced_config_map(self, *args, **kwargs):
@@ -19,53 +16,33 @@ def mock_list_namespaced_config_map(self, *args, **kwargs):
     metadata.update({'name': 'test-sync', 'annotations': {'leader': 'p-0'}})
     items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
     metadata = k8s_client.V1ObjectMeta(resource_version='1')
-    return k8s_client.V1ConfigMapList(metadata=metadata, items=items, kind='ConfigMapList')
+    return k8s_client.V1ConfigMapList(metadata=metadata, items=items)
 
 
 def mock_list_namespaced_pod(self, *args, **kwargs):
     metadata = k8s_client.V1ObjectMeta(resource_version='1', name='p-0', annotations={'status': '{}'})
     items = [k8s_client.V1Pod(metadata=metadata)]
-    return k8s_client.V1PodList(items=items, kind='PodList')
+    return k8s_client.V1PodList(items=items)
 
 
-def mock_config_map(*args, **kwargs):
+@patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', Mock())
-    mock = Mock()
+@patch.object(k8s_client.CoreV1Api, 'create_namespaced_config_map', Mock())
-    mock.metadata.resource_version = '2'
-    return mock
-
-
-@patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', mock_config_map)
-@patch.object(k8s_client.CoreV1Api, 'create_namespaced_config_map', mock_config_map)
-@patch('kubernetes.client.api_client.ThreadPool', Mock(), create=True)
-@patch.object(Thread, 'start', Mock())
 class TestKubernetes(unittest.TestCase):
 
     @patch('kubernetes.config.load_kube_config', Mock())
     @patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map)
     @patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', mock_list_namespaced_pod)
-    @patch('kubernetes.client.api_client.ThreadPool', Mock(), create=True)
-    @patch.object(Thread, 'start', Mock())
     def setUp(self):
         self.k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10, 'labels': {'f': 'b'}})
-        self.assertRaises(AttributeError, self.k._pods._build_cache)
-        self.k._pods._is_ready = True
-        self.assertRaises(AttributeError, self.k._kinds._build_cache)
-        self.k._kinds._is_ready = True
         self.k.get_cluster()
 
-    @patch('time.time', Mock(side_effect=[1, 10.9, 100]))
-    def test__wait_caches(self):
-        self.k._pods._is_ready = False
-        with self.k._condition:
-            self.assertRaises(RetryFailedError, self.k._wait_caches)
-
     def test_get_cluster(self):
         with patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map), \
                 patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', mock_list_namespaced_pod), \
                 patch('time.time', Mock(return_value=time.time() + 31)):
             self.k.get_cluster()
 
-        with patch.object(Kubernetes, '_wait_caches', Mock(side_effect=Exception)):
+        with patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', Mock(side_effect=Exception)):
             self.assertRaises(KubernetesError, self.k.get_cluster)
 
     @patch('kubernetes.config.load_kube_config', Mock())
@@ -128,12 +105,17 @@ class TestKubernetes(unittest.TestCase):
         self.k.set_ttl(10)
         self.k.watch(None, 0)
         self.k.watch(None, 0)
+        with patch.object(k8s_watch.Watch, 'stream',
+                          Mock(side_effect=[Exception, [], KeyboardInterrupt,
+                                            [{'raw_object': {'metadata': {'resourceVersion': '2'}}}]])):
+            self.assertFalse(self.k.watch('1', 2))
+            self.assertRaises(KeyboardInterrupt, self.k.watch, '1', 2)
+            self.assertTrue(self.k.watch('1', 2))
 
     def test_set_history_value(self):
         self.k.set_history_value('{}')
 
     @patch('kubernetes.config.load_kube_config', Mock())
-    @patch('patroni.dcs.kubernetes.ObjectCache', Mock())
     @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_pod', Mock(return_value=True))
     @patch.object(k8s_client.CoreV1Api, 'create_namespaced_endpoints', Mock())
     @patch.object(k8s_client.CoreV1Api, 'create_namespaced_service',
@@ -144,30 +126,3 @@ class TestKubernetes(unittest.TestCase):
         self.assertIsNotNone(k.patch_or_create_config({'foo': 'bar'}))
         self.assertIsNotNone(k.patch_or_create_config({'foo': 'bar'}))
         k.touch_member({'state': 'running', 'role': 'replica'})
-
-
-class TestCacheBuilder(unittest.TestCase):
-
-    @patch('kubernetes.config.load_kube_config', Mock())
-    @patch('kubernetes.client.api_client.ThreadPool', Mock(), create=True)
-    @patch.object(Thread, 'start', Mock())
-    def setUp(self):
-        self.k = Kubernetes({'ttl': 30, 'scope': 'test', 'name': 'p-0', 'retry_timeout': 10, 'labels': {'f': 'b'}})
-
-    @patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map)
-    @patch('patroni.dcs.kubernetes.ObjectCache._watch')
-    def test__build_cache(self, mock_response):
-        mock_response.return_value.read_chunked.return_value = json.dumps(
-            {'type': 'MODIFIED', 'object': {'metadata': {
-                'name': self.k.config_path, 'resourceVersion': '2', 'annotations': {self.k._CONFIG: 'foo'}}}}
-        ) + '\n' + json.dumps(
-            {'type': 'DELETED', 'object': {'metadata': {
-                'name': self.k.config_path, 'resourceVersion': '3'}}}
-        ) + '\n' + json.dumps(
-            {'type': 'MDIFIED', 'object': {'metadata': {'name': self.k.config_path}}}
-        ) + '\n' + json.dumps({'object': {'code': 410}}) + '\n'
-        self.k._kinds._build_cache()
-
-    @patch('patroni.dcs.kubernetes.logger.error', Mock(side_effect=SleepException))
-    def test_run(self):
-        self.assertRaises(SleepException, self.k._pods.run)

tests/test_log.py

@@ -19,6 +19,7 @@ class TestPatroniLogger(unittest.TestCase):
         logging.getLogger().handlers[:] = self._handlers
 
     @patch('logging.FileHandler._open', Mock())
+    @patch('logging.Handler.close', Mock(side_effect=Exception))
     def test_patroni_logger(self):
         config = {
             'log': {
@@ -37,23 +38,21 @@ class TestPatroniLogger(unittest.TestCase):
         logger = PatroniLogger()
         patroni_config = Config()
         logger.reload_config(patroni_config['log'])
-        logger.start()
 
         with patch.object(logging.Handler, 'format', Mock(side_effect=Exception)):
             logging.error('test')
 
-        self.assertEqual(logger.log_handler.maxBytes, config['log']['file_size'])
+        self.assertEqual(logger._log_handler.maxBytes, config['log']['file_size'])
-        self.assertEqual(logger.log_handler.backupCount, config['log']['file_num'])
+        self.assertEqual(logger._log_handler.backupCount, config['log']['file_num'])
 
         config['log'].pop('dir')
-        with patch('logging.Handler.close', Mock(side_effect=Exception)):
+        logger.reload_config(config['log'])
-            logger.reload_config(config['log'])
+        with patch.object(logging.Logger, 'makeRecord',
-            with patch.object(logging.Logger, 'makeRecord',
+                          Mock(side_effect=[logging.LogRecord('', logging.INFO, '', 0, '', (), None), Exception])):
-                              Mock(side_effect=[logging.LogRecord('', logging.INFO, '', 0, '', (), None), Exception])):
-                logging.error('test')
             logging.error('test')
-            with patch.object(Queue, 'put_nowait', Mock(side_effect=Full)):
+        logging.error('test')
-                self.assertRaises(SystemExit, logger.shutdown)
+        with patch.object(Queue, 'put_nowait', Mock(side_effect=Full)):
-            self.assertRaises(Exception, logger.shutdown)
+            self.assertRaises(SystemExit, logger.shutdown)
+        self.assertRaises(Exception, logger.shutdown)
         self.assertLessEqual(logger.queue_size, 2)  # "Failed to close the old log handler" could be still in the queue
         self.assertEqual(logger.records_lost, 0)

tests/test_patroni.py

@@ -39,13 +39,13 @@ class MockFrozenImporter(object):
 @patch.object(AsyncExecutor, 'run', Mock())
 @patch.object(etcd.Client, 'write', etcd_write)
 @patch.object(etcd.Client, 'read', etcd_read)
+@patch.object(Thread, 'start', Mock())
 class TestPatroni(unittest.TestCase):
 
     @patch('pkgutil.get_importer', Mock(return_value=MockFrozenImporter()))
     @patch('sys.frozen', Mock(return_value=True), create=True)
     @patch.object(BaseHTTPServer.HTTPServer, '__init__', Mock())
     @patch.object(etcd.Client, 'read', etcd_read)
-    @patch.object(Thread, 'start', Mock())
     def setUp(self):
         self._handlers = logging.getLogger().handlers[:]
         RestApiServer._BaseServer__is_shut_down = Mock()
@@ -83,7 +83,6 @@ class TestPatroni(unittest.TestCase):
 
     @patch('os.getpid')
     @patch('multiprocessing.Process')
-    @patch('patroni.use_spawn_start_method', Mock())
     @patch('patroni.patroni_main', Mock())
     def test_patroni_main(self, mock_process, mock_getpid):
         mock_getpid.return_value = 2
@@ -123,12 +122,8 @@ class TestPatroni(unittest.TestCase):
         self.p.sighup_handler()
         self.p.ha.dcs.watch = Mock(side_effect=SleepException)
         self.p.api.start = Mock()
-        self.p.logger.start = Mock()
         self.p.config._dynamic_configuration = {}
         self.assertRaises(SleepException, self.p.run)
-        with patch('patroni.config.Config.reload_local_configuration', Mock(return_value=False)):
-            self.p.sighup_handler()
-            self.assertRaises(SleepException, self.p.run)
         with patch('patroni.config.Config.set_dynamic_configuration', Mock(return_value=True)):
             self.assertRaises(SleepException, self.p.run)
         with patch('patroni.postgresql.Postgresql.data_directory_empty', Mock(return_value=False)):
@@ -170,7 +165,6 @@ class TestPatroni(unittest.TestCase):
         self.p.tags['nosync'] = None
         self.assertFalse(self.p.nosync)
 
-    @patch.object(Thread, 'join', Mock())
     def test_shutdown(self):
         self.p.api.shutdown = Mock(side_effect=Exception)
         self.p.shutdown()

tests/test_postgresql.py

@@ -90,7 +90,6 @@ class TestPostgresql(BaseTestPostgresql):
 
     @patch('subprocess.call', Mock(return_value=0))
     @patch('os.rename', Mock())
-    @patch('patroni.postgresql.CallbackExecutor', Mock())
     @patch.object(Postgresql, 'get_major_version', Mock(return_value=120000))
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def setUp(self):
@@ -102,7 +101,6 @@ class TestPostgresql(BaseTestPostgresql):
     @patch.object(Postgresql, 'wait_for_startup')
     @patch.object(Postgresql, 'wait_for_port_open')
     @patch.object(Postgresql, 'is_running')
-    @patch.object(Postgresql, 'controldata', Mock())
     def test_start(self, mock_is_running, mock_wait_for_port_open, mock_wait_for_startup, mock_popen):
         mock_is_running.return_value = MockPostmaster()
         mock_wait_for_port_open.return_value = True
@@ -188,11 +186,10 @@ class TestPostgresql(BaseTestPostgresql):
         self.assertFalse(self.p.restart())
         self.assertEqual(self.p.state, 'restart failed (restarting)')
 
-    @patch('os.chmod', Mock())
     @patch.object(builtins, 'open', MagicMock())
     def test_write_pgpass(self):
-        self.p.config.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo'})
+        self.p.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo'})
-        self.p.config.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo', 'password': 'bar'})
+        self.p.write_pgpass({'host': 'localhost', 'port': '5432', 'user': 'foo', 'password': 'bar'})
 
     def test_checkpoint(self):
         with patch.object(MockCursor, 'fetchone', Mock(return_value=(True, ))):
@@ -202,53 +199,41 @@ class TestPostgresql(BaseTestPostgresql):
         self.assertEqual(self.p.checkpoint(), 'not accessible or not healty')
 
     @patch('patroni.postgresql.config.mtime', mock_mtime)
-    @patch('patroni.postgresql.config.ConfigHandler._get_pg_settings')
+    @patch.object(MockCursor, 'fetchone', Mock(side_effect=[('foo=bar',), ('',), ('',), ('a=b',)]))
-    def test_check_recovery_conf(self, mock_get_pg_settings):
+    def test_check_recovery_conf(self):
-        mock_get_pg_settings.return_value = {
+        for version in (120000, 100000):
-            'primary_conninfo': ['primary_conninfo', 'foo=', None, 'string', 'postmaster'],
+            with patch.object(Postgresql, 'major_version', PropertyMock(return_value=version)):
-            'recovery_min_apply_delay': ['recovery_min_apply_delay', '0', 'ms', 'integer', 'sighup']
+                self.p.config.write_recovery_conf({'standby_mode': 'on', 'primary_conninfo': 'foo'})
-        }
+                self.assertFalse(self.p.config.check_recovery_conf(None))
-        self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
+                self.p.config.write_recovery_conf({'primary_conninfo': 'foo'})
-        self.p.config.write_recovery_conf({'standby_mode': 'on'})
+                self.p.config.write_postgresql_conf()
-        self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
+                self.assertFalse(self.p.config.check_recovery_conf(None))
-        mock_get_pg_settings.return_value['primary_conninfo'][1] = ''
+                self.p.config.write_recovery_conf({'standby_mode': 'on'})
-        mock_get_pg_settings.return_value['recovery_min_apply_delay'][1] = '1'
+                self.assertTrue(self.p.config.check_recovery_conf(None))
-        self.assertEqual(self.p.config.check_recovery_conf(None), (True, False))
+                with patch('patroni.postgresql.config.ConfigHandler.primary_conninfo_params',
-        mock_get_pg_settings.return_value['recovery_min_apply_delay'][1] = '0'
+                           Mock(return_value={'a': 'b'})):
-        self.assertEqual(self.p.config.check_recovery_conf(None), (False, False))
+                    self.assertFalse(self.p.config.check_recovery_conf(None))
-        conninfo = {'host': '1', 'password': 'bar'}
+                    self.p.config.write_recovery_conf({'standby_mode': 'on', 'primary_conninfo': 'a=b'})
-        with patch('patroni.postgresql.config.ConfigHandler.primary_conninfo_params', Mock(return_value=conninfo)):
+                    self.assertTrue(self.p.config.check_recovery_conf(None))
-            mock_get_pg_settings.return_value['recovery_min_apply_delay'][1] = '1'
-            self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
-            mock_get_pg_settings.return_value['primary_conninfo'][1] = 'host=1 passfile=' + self.p.config._pgpass
-            mock_get_pg_settings.return_value['recovery_min_apply_delay'][1] = '0'
-            self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
-            self.p.config.write_recovery_conf({'standby_mode': 'on', 'primary_conninfo': conninfo.copy()})
-            self.p.config.write_postgresql_conf()
-            self.assertEqual(self.p.config.check_recovery_conf(None), (False, False))
 
     @patch.object(Postgresql, 'major_version', PropertyMock(return_value=120000))
     @patch.object(Postgresql, 'is_running', MockPostmaster)
     @patch.object(MockPostmaster, 'create_time', Mock(return_value=1234567), create=True)
-    @patch('patroni.postgresql.config.ConfigHandler._get_pg_settings')
+    @patch.object(MockCursor, 'fetchone', Mock(return_value=('',)))
-    def test__read_recovery_params(self, mock_get_pg_settings):
+    def test__read_primary_conninfo(self):
-        mock_get_pg_settings.return_value = {'primary_conninfo': ['primary_conninfo', '', None, 'string', 'postmaster']}
+        self.p.config.write_recovery_conf({'standby_mode': 'on'})
-        self.p.config.write_recovery_conf({'standby_mode': 'on', 'primary_conninfo': {'password': 'foo'}})
         self.p.config.write_postgresql_conf()
-        self.assertEqual(self.p.config.check_recovery_conf(None), (False, False))
+        self.assertTrue(self.p.config.check_recovery_conf(None))
-        self.assertEqual(self.p.config.check_recovery_conf(None), (False, False))
+        self.assertTrue(self.p.config.check_recovery_conf(None))
-        mock_get_pg_settings.side_effect = Exception
+        with patch.object(Postgresql, 'query', Mock(side_effect=Exception)),\
-        with patch('patroni.postgresql.config.mtime', mock_mtime):
+                patch('patroni.postgresql.config.mtime', mock_mtime):
-            self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
+            self.assertFalse(self.p.config.check_recovery_conf(None))
 
     @patch.object(Postgresql, 'major_version', PropertyMock(return_value=100000))
-    def test__read_recovery_params_pre_v12(self):
+    def test__read_primary_conninfo_pre_v12(self):
-        self.p.config.write_recovery_conf({'standby_mode': 'on', 'primary_conninfo': {'password': 'foo'}})
+        self.p.config.write_recovery_conf({'standby_mode': 'on'})
-        self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
+        self.assertTrue(self.p.config.check_recovery_conf(None))
-        self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
+        self.assertTrue(self.p.config.check_recovery_conf(None))
-        self.p.config.write_recovery_conf({'standby_mode': '\n'})
-        with patch('patroni.postgresql.config.mtime', mock_mtime):
-            self.assertEqual(self.p.config.check_recovery_conf(None), (True, True))
 
     def test_write_postgresql_and_sanitize_auto_conf(self):
         read_data = 'primary_conninfo = foo\nfoo = bar\n'
@@ -270,7 +255,7 @@ class TestPostgresql(BaseTestPostgresql):
     @patch.object(Postgresql, 'start', Mock())
     def test_follow(self):
         self.p.call_nowait('on_start')
-        m = RemoteMember('1', {'restore_command': '2', 'primary_slot_name': 'foo', 'conn_kwargs': {'host': 'bar'}})
+        m = RemoteMember('1', {'restore_command': '2', 'recovery_min_apply_delay': 3, 'archive_cleanup_command': '4'})
         self.p.follow(m)
 
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
@@ -374,7 +359,6 @@ class TestPostgresql(BaseTestPostgresql):
 
     @patch('os.listdir', Mock(return_value=['recovery.conf']))
     @patch('os.path.exists', Mock(return_value=True))
-    @patch.object(Postgresql, 'controldata', Mock())
     def test_get_postgres_role_from_data_directory(self):
         self.assertEqual(self.p.get_postgres_role_from_data_directory(), 'replica')
 
@@ -437,18 +421,13 @@ class TestPostgresql(BaseTestPostgresql):
         self.p.config._config['foo'] = {'command': 'bar'}
         self.assertFalse(self.p.replica_method_can_work_without_replication_connection('foo'))
 
-    @patch('time.sleep', Mock())
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
-    @patch.object(MockCursor, 'fetchone')
+    def test_reload_config(self):
-    def test_reload_config(self, mock_fetchone):
-        mock_fetchone.return_value = (1,)
         parameters = self._PARAMETERS.copy()
         parameters.pop('f.oo')
-        parameters['wal_buffers'] = '512'
         config = {'pg_hba': [''], 'pg_ident': [''], 'use_unix_socket': True, 'authentication': {},
                   'retry_timeout': 10, 'listen': '*', 'krbsrvname': 'postgres', 'parameters': parameters}
         self.p.reload_config(config)
-        mock_fetchone.side_effect = Exception
         parameters['b.ar'] = 'bar'
         self.p.reload_config(config)
         parameters['autovacuum'] = 'on'
@@ -471,10 +450,6 @@ class TestPostgresql(BaseTestPostgresql):
     def test_postmaster_start_time(self):
         with patch.object(MockCursor, "fetchone", Mock(return_value=('foo', True, '', '', '', '', False))):
             self.assertEqual(self.p.postmaster_start_time(), 'foo')
-            t = Thread(target=self.p.postmaster_start_time)
-            t.start()
-            t.join()
-
         with patch.object(MockCursor, "execute", side_effect=psycopg2.Error):
             self.assertIsNone(self.p.postmaster_start_time())
 
@@ -654,7 +629,7 @@ class TestPostgresql(BaseTestPostgresql):
             data = self.p.read_postmaster_opts()
             self.assertEqual(data, dict())
 
-    @patch('psutil.Popen')
+    @patch('subprocess.Popen')
     def test_single_user_mode(self, subprocess_popen_mock):
         subprocess_popen_mock.return_value.wait.return_value = 0
         self.assertEqual(self.p.single_user_mode('CHECKPOINT', {'archive_mode': 'on'}), 0)