Merge branch 'master' of github.com:zalando/patroni into feature/quorum-commit

PatchStream class was removed as it was effectively doing nothing useful.

Dockerfile

@@ -125,6 +125,8 @@ RUN if [ "$COMPRESS" = "true" ]; then \
         && /bin/busybox sh -c "(find $save_dirs -not -type d && cat /exclude /exclude && echo exclude) | sort | uniq -u | xargs /bin/busybox rm" \
         && /bin/busybox --install -s \
         && /bin/busybox sh -c "find $save_dirs -type d -depth -exec rmdir -p {} \; 2> /dev/null"; \
+    else \
+        /bin/busybox --install -s; \
     fi
 
 FROM scratch

docs/citus.rst

@@ -34,6 +34,8 @@ There are only a few simple rules you need to follow:
 
 After that you just need to start Patroni and it will handle the rest:
 
+0. Patroni will set ``bootstrap.dcs.synchronous_mode`` to :ref:`quorum <quorum_mode>`
+   if it is not explicitly set to any other value.
 1. ``citus`` extension will be automatically added to ``shared_preload_libraries``.
 2. If ``max_prepared_transactions`` isn't explicitly set in the global
    :ref:`dynamic configuration <dynamic_configuration>` Patroni will
@@ -77,36 +79,36 @@ It results in two major differences in :ref:`patronictl` behaviour when
 An example of :ref:`patronictl_list` output for the Citus cluster::
 
     postgres@coord1:~$ patronictl list demo
-    + Citus cluster: demo ----------+--------------+---------+----+-----------+
-    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
-    +-------+---------+-------------+--------------+---------+----+-----------+
-    |     0 | coord1  | 172.27.0.10 | Replica      | running |  1 |         0 |
-    |     0 | coord2  | 172.27.0.6  | Sync Standby | running |  1 |         0 |
-    |     0 | coord3  | 172.27.0.4  | Leader       | running |  1 |           |
-    |     1 | work1-1 | 172.27.0.8  | Sync Standby | running |  1 |         0 |
-    |     1 | work1-2 | 172.27.0.2  | Leader       | running |  1 |           |
-    |     2 | work2-1 | 172.27.0.5  | Sync Standby | running |  1 |         0 |
-    |     2 | work2-2 | 172.27.0.7  | Leader       | running |  1 |           |
-    +-------+---------+-------------+--------------+---------+----+-----------+
+    + Citus cluster: demo ----------+----------------+---------+----+-----------+
+    | Group | Member  | Host        | Role           | State   | TL | Lag in MB |
+    +-------+---------+-------------+----------------+---------+----+-----------+
+    |     0 | coord1  | 172.27.0.10 | Replica        | running |  1 |         0 |
+    |     0 | coord2  | 172.27.0.6  | Quorum Standby | running |  1 |         0 |
+    |     0 | coord3  | 172.27.0.4  | Leader         | running |  1 |           |
+    |     1 | work1-1 | 172.27.0.8  | Quorum Standby | running |  1 |         0 |
+    |     1 | work1-2 | 172.27.0.2  | Leader         | running |  1 |           |
+    |     2 | work2-1 | 172.27.0.5  | Quorum Standby | running |  1 |         0 |
+    |     2 | work2-2 | 172.27.0.7  | Leader         | running |  1 |           |
+    +-------+---------+-------------+----------------+---------+----+-----------+
 
 If we add the ``--group`` option, the output will change to::
 
     postgres@coord1:~$ patronictl list demo --group 0
-    + Citus cluster: demo (group: 0, 7179854923829112860) -----------+
-    | Member | Host        | Role         | State   | TL | Lag in MB |
-    +--------+-------------+--------------+---------+----+-----------+
-    | coord1 | 172.27.0.10 | Replica      | running |  1 |         0 |
-    | coord2 | 172.27.0.6  | Sync Standby | running |  1 |         0 |
-    | coord3 | 172.27.0.4  | Leader       | running |  1 |           |
-    +--------+-------------+--------------+---------+----+-----------+
+    + Citus cluster: demo (group: 0, 7179854923829112860) -+-----------+
+    | Member | Host        | Role           | State   | TL | Lag in MB |
+    +--------+-------------+----------------+---------+----+-----------+
+    | coord1 | 172.27.0.10 | Replica        | running |  1 |         0 |
+    | coord2 | 172.27.0.6  | Quorum Standby | running |  1 |         0 |
+    | coord3 | 172.27.0.4  | Leader         | running |  1 |           |
+    +--------+-------------+----------------+---------+----+-----------+
 
     postgres@coord1:~$ patronictl list demo --group 1
-    + Citus cluster: demo (group: 1, 7179854923881963547) -----------+
-    | Member  | Host       | Role         | State   | TL | Lag in MB |
-    +---------+------------+--------------+---------+----+-----------+
-    | work1-1 | 172.27.0.8 | Sync Standby | running |  1 |         0 |
-    | work1-2 | 172.27.0.2 | Leader       | running |  1 |           |
-    +---------+------------+--------------+---------+----+-----------+
+    + Citus cluster: demo (group: 1, 7179854923881963547) -+-----------+
+    | Member  | Host       | Role           | State   | TL | Lag in MB |
+    +---------+------------+----------------+---------+----+-----------+
+    | work1-1 | 172.27.0.8 | Quorum Standby | running |  1 |         0 |
+    | work1-2 | 172.27.0.2 | Leader         | running |  1 |           |
+    +---------+------------+----------------+---------+----+-----------+
 
 Citus worker switchover
 -----------------------
@@ -122,28 +124,28 @@ new primary worker node is ready to accept read-write queries.
 An example of :ref:`patronictl_switchover` on the worker cluster::
 
     postgres@coord1:~$ patronictl switchover demo
-    + Citus cluster: demo ----------+--------------+---------+----+-----------+
-    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
-    +-------+---------+-------------+--------------+---------+----+-----------+
-    |     0 | coord1  | 172.27.0.10 | Replica      | running |  1 |         0 |
-    |     0 | coord2  | 172.27.0.6  | Sync Standby | running |  1 |         0 |
-    |     0 | coord3  | 172.27.0.4  | Leader       | running |  1 |           |
-    |     1 | work1-1 | 172.27.0.8  | Leader       | running |  1 |           |
-    |     1 | work1-2 | 172.27.0.2  | Sync Standby | running |  1 |         0 |
-    |     2 | work2-1 | 172.27.0.5  | Sync Standby | running |  1 |         0 |
-    |     2 | work2-2 | 172.27.0.7  | Leader       | running |  1 |           |
-    +-------+---------+-------------+--------------+---------+----+-----------+
+    + Citus cluster: demo ----------+----------------+---------+----+-----------+
+    | Group | Member  | Host        | Role           | State   | TL | Lag in MB |
+    +-------+---------+-------------+----------------+---------+----+-----------+
+    |     0 | coord1  | 172.27.0.10 | Replica        | running |  1 |         0 |
+    |     0 | coord2  | 172.27.0.6  | Quorum Standby | running |  1 |         0 |
+    |     0 | coord3  | 172.27.0.4  | Leader         | running |  1 |           |
+    |     1 | work1-1 | 172.27.0.8  | Leader         | running |  1 |           |
+    |     1 | work1-2 | 172.27.0.2  | Quorum Standby | running |  1 |         0 |
+    |     2 | work2-1 | 172.27.0.5  | Quorum Standby | running |  1 |         0 |
+    |     2 | work2-2 | 172.27.0.7  | Leader         | running |  1 |           |
+    +-------+---------+-------------+----------------+---------+----+-----------+
     Citus group: 2
     Primary [work2-2]:
     Candidate ['work2-1'] []:
     When should the switchover take place (e.g. 2022-12-22T08:02 )  [now]:
     Current cluster topology
-    + Citus cluster: demo (group: 2, 7179854924063375386) -----------+
-    | Member  | Host       | Role         | State   | TL | Lag in MB |
-    +---------+------------+--------------+---------+----+-----------+
-    | work2-1 | 172.27.0.5 | Sync Standby | running |  1 |         0 |
-    | work2-2 | 172.27.0.7 | Leader       | running |  1 |           |
-    +---------+------------+--------------+---------+----+-----------+
+    + Citus cluster: demo (group: 2, 7179854924063375386) -+-----------+
+    | Member  | Host       | Role           | State   | TL | Lag in MB |
+    +---------+------------+----------------+---------+----+-----------+
+    | work2-1 | 172.27.0.5 | Quorum Standby | running |  1 |         0 |
+    | work2-2 | 172.27.0.7 | Leader         | running |  1 |           |
+    +---------+------------+----------------+---------+----+-----------+
     Are you sure you want to switchover cluster demo, demoting current primary work2-2? [y/N]: y
     2022-12-22 07:02:40.33003 Successfully switched over to "work2-1"
     + Citus cluster: demo (group: 2, 7179854924063375386) ------+
@@ -154,17 +156,17 @@ An example of :ref:`patronictl_switchover` on the worker cluster::
     +---------+------------+---------+---------+----+-----------+
 
     postgres@coord1:~$ patronictl list demo
-    + Citus cluster: demo ----------+--------------+---------+----+-----------+
-    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
-    +-------+---------+-------------+--------------+---------+----+-----------+
-    |     0 | coord1  | 172.27.0.10 | Replica      | running |  1 |         0 |
-    |     0 | coord2  | 172.27.0.6  | Sync Standby | running |  1 |         0 |
-    |     0 | coord3  | 172.27.0.4  | Leader       | running |  1 |           |
-    |     1 | work1-1 | 172.27.0.8  | Leader       | running |  1 |           |
-    |     1 | work1-2 | 172.27.0.2  | Sync Standby | running |  1 |         0 |
-    |     2 | work2-1 | 172.27.0.5  | Leader       | running |  2 |           |
-    |     2 | work2-2 | 172.27.0.7  | Sync Standby | running |  2 |         0 |
-    +-------+---------+-------------+--------------+---------+----+-----------+
+    + Citus cluster: demo ----------+----------------+---------+----+-----------+
+    | Group | Member  | Host        | Role           | State   | TL | Lag in MB |
+    +-------+---------+-------------+----------------+---------+----+-----------+
+    |     0 | coord1  | 172.27.0.10 | Replica        | running |  1 |         0 |
+    |     0 | coord2  | 172.27.0.6  | Quorum Standby | running |  1 |         0 |
+    |     0 | coord3  | 172.27.0.4  | Leader         | running |  1 |           |
+    |     1 | work1-1 | 172.27.0.8  | Leader         | running |  1 |           |
+    |     1 | work1-2 | 172.27.0.2  | Quorum Standby | running |  1 |         0 |
+    |     2 | work2-1 | 172.27.0.5  | Leader         | running |  2 |           |
+    |     2 | work2-2 | 172.27.0.7  | Quorum Standby | running |  2 |         0 |
+    +-------+---------+-------------+----------------+---------+----+-----------+
 
 And this is how it looks on the coordinator side::
 

docs/dynamic_configuration.rst

@@ -25,7 +25,7 @@ In order to change the dynamic configuration you can use either :ref:`patronictl
 -  **max\_timelines\_history**: maximum number of timeline history items kept in DCS.  Default value: 0. When set to 0, it keeps the full history in DCS.
 -  **primary\_start\_timeout**: the amount of time a primary is allowed to recover from failures before failover is triggered (in seconds). Default is 300 seconds. When set to 0 failover is done immediately after a crash is detected if possible. When using asynchronous replication a failover can cause lost transactions. Worst case failover time for primary failure is: loop\_wait + primary\_start\_timeout + loop\_wait, unless primary\_start\_timeout is zero, in which case it's just loop\_wait. Set the value according to your durability/availability tradeoff.
 -  **primary\_stop\_timeout**: The number of seconds Patroni is allowed to wait when stopping Postgres and effective only when synchronous_mode is enabled. When set to > 0 and the synchronous_mode is enabled, Patroni sends SIGKILL to the postmaster if the stop operation is running for more than the value set by primary\_stop\_timeout. Set the value according to your durability/availability tradeoff. If the parameter is not set or set <= 0, primary\_stop\_timeout does not apply.
--  **synchronous\_mode**: turns on synchronous replication mode. In this mode a replica will be chosen as synchronous and only the latest leader and synchronous replica are able to participate in leader election. Synchronous mode makes sure that successfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See :ref:`replication modes documentation <replication_modes>` for details.
+-  **synchronous\_mode**: turns on synchronous replication mode. Possible values: ``off``, ``on``, ``quorum``. In this mode the leader takes care of management of ``synchronous_standby_names``, and only the last known leader, or one of synchronous replicas, are allowed to participate in leader race. Synchronous mode makes sure that successfully committed transactions will not be lost at failover, at the cost of losing availability for writes when Patroni cannot ensure transaction durability. See :ref:`replication modes documentation <replication_modes>` for details.
 -  **synchronous\_mode\_strict**: prevents disabling synchronous replication if no synchronous replicas are available, blocking all client writes to the primary. See :ref:`replication modes documentation <replication_modes>` for details.
 -  **failsafe\_mode**: Enables :ref:`DCS Failsafe Mode <dcs_failsafe_mode>`. Defaults to `false`.
 -  **postgresql**:
@@ -108,3 +108,7 @@ Note: if cluster topology is static (fixed number of nodes that never change the
 .. warning::
    Permanent replication slots are synchronized only from the ``primary``/``standby_leader`` to replica nodes. That means, applications are supposed to be using them only from the leader node. Using them on replica nodes will cause indefinite growth of ``pg_wal`` on all other nodes in the cluster.
    An exception to that rule are permanent physical slots that match the Patroni member names, if you happen to configure any. Those will be synchronized among all nodes as they are used for replication among them.
+
+
+.. warning::
+   Setting ``nostream`` tag on standby disables copying and synchronization of permanent logical replication slots on the node itself and all its cascading replicas if any.

docs/index.rst

@@ -28,12 +28,14 @@ Currently supported PostgreSQL versions: 9.3 to 16.
    patronictl
    replica_bootstrap
    replication_modes
+   standby_cluster
    watchdog
    pause
    dcs_failsafe_mode
    kubernetes
    citus
    existing_data
+   tools_integration
    security
    ha_multi_dc
    faq

docs/replica_bootstrap.rst

@@ -1,3 +1,5 @@
+.. _replica_imaging_and_bootstrap:
+
 Replica imaging and bootstrap
 =============================
 
@@ -79,14 +81,13 @@ As an example, you are able to bootstrap a fresh Patroni cluster from a Barman b
         method: barman
         barman:
             keep_existing_recovery_conf: true
-            command: patroni_barman_recover
-            api-url: https://barman-host:7480
+            command: patroni_barman --api-url https://barman-host:7480 recover
             barman-server: my_server
             ssh-command: ssh postgres@patroni-host
 
 .. note::
-    ``patroni_barman_recover`` requires that you have both Barman and ``pg-backup-api`` configured in the Barman host, so it can execute a remote ``barman recover`` through the backup API.
-    The above example uses a subset of the available parameters. You can get more information running ``patroni_barman_recover --help``.
+    ``patroni_barman recover`` requires that you have both Barman and ``pg-backup-api`` configured in the Barman host, so it can execute a remote ``barman recover`` through the backup API.
+    The above example uses a subset of the available parameters. You can get more information running ``patroni_barman recover --help``.
 
 .. _custom_replica_creation:
 
@@ -150,16 +151,15 @@ example: Barman
             - barman
             - basebackup
         barman:
-            command: patroni_barman_recover
-            api-url: https://barman-host:7480
+            command: patroni_barman --api-url https://barman-host:7480 recover
             barman-server: my_server
             ssh-command: ssh postgres@patroni-host
         basebackup:
             max-rate: '100M'
 
 .. note::
-    ``patroni_barman_recover`` requires that you have both Barman and ``pg-backup-api`` configured in the Barman host, so it can execute a remote ``barman recover`` through the backup API.
-    The above example uses a subset of the available parameters. You can get more information running ``patroni_barman_recover --help``.
+    ``patroni_barman recover`` requires that you have both Barman and ``pg-backup-api`` configured in the Barman host, so it can execute a remote ``barman recover`` through the backup API.
+    The above example uses a subset of the available parameters. You can get more information running ``patroni_barman recover --help``.
 
 The ``create_replica_methods`` defines available replica creation methods and the order of executing them. Patroni will
 stop on the first one that returns 0. Each method should define a separate section in the configuration file, listing the command
@@ -219,59 +219,3 @@ and
             - waldir: /pg-wal-mount/external-waldir
 
 If all replica creation methods fail, Patroni will try again all methods in order during the next event loop cycle.
-
-.. _standby_cluster:
-
-Standby cluster
----------------
-
-Another available option is to run a "standby cluster", that contains only of
-standby nodes replicating from some remote node. This type of clusters has:
-
-* "standby leader", that behaves pretty much like a regular cluster leader,
-  except it replicates from a remote node.
-
-* cascade replicas, that are replicating from standby leader.
-
-Standby leader holds and updates a leader lock in DCS. If the leader lock
-expires, cascade replicas will perform an election to choose another leader
-from the standbys.
-
-There is no further relationship between the standby cluster and the primary
-cluster it replicates from, in particular, they must not share the same DCS
-scope if they use the same DCS. They do not know anything else from each other
-apart from replication information. Also, the standby cluster is not being
-displayed in :ref:`patronictl_list` or :ref:`patronictl_topology` output on the
-primary cluster.
-
-For the sake of flexibility, you can specify methods of creating a replica and
-recovery WAL records when a cluster is in the "standby mode" by providing
-`create_replica_methods` key in `standby_cluster` section. It is distinct from
-creating replicas, when cluster is detached and functions as a normal cluster,
-which is controlled by `create_replica_methods` in `postgresql` section. Both
-"standby" and "normal" `create_replica_methods` reference  keys in `postgresql`
-section.
-
-To configure such cluster you need to specify the section ``standby_cluster``
-in a patroni configuration:
-
-.. code:: YAML
-
-    bootstrap:
-        dcs:
-            standby_cluster:
-                host: 1.2.3.4
-                port: 5432
-                primary_slot_name: patroni
-                create_replica_methods:
-                - basebackup
-
-Note, that these options will be applied only once during cluster bootstrap,
-and the only way to change them afterwards is through DCS.
-
-Patroni expects to find `postgresql.conf` or `postgresql.conf.backup` in PGDATA
-of the remote primary and will not start if it does not find it after a
-basebackup. If the remote primary keeps its `postgresql.conf` elsewhere, it is
-your responsibility to copy it to PGDATA.
-
-If you use replication slots on the standby cluster, you must also create the corresponding replication slot on the primary cluster.  It will not be done automatically by the standby cluster implementation.  You can use Patroni's permanent replication slots feature on the primary cluster to maintain a replication slot with the same name as ``primary_slot_name``, or its default value if ``primary_slot_name`` is not provided.

docs/replication_modes.rst

@@ -6,8 +6,9 @@ Replication modes
 
 Patroni uses PostgreSQL streaming replication. For more information about streaming replication, see the `Postgres documentation <http://www.postgresql.org/docs/current/static/warm-standby.html#STREAMING-REPLICATION>`__. By default Patroni configures PostgreSQL for asynchronous replication. Choosing your replication schema is dependent on your business considerations. Investigate both async and sync replication, as well as other HA solutions, to determine which solution is best for you.
 
+
 Asynchronous mode durability
-----------------------------
+============================
 
 In asynchronous mode the cluster is allowed to lose some committed transactions to ensure availability. When the primary server fails or becomes unavailable for any other reason Patroni will automatically promote a sufficiently healthy standby to primary. Any transactions that have not been replicated to that standby remain in a "forked timeline" on the primary, and are effectively unrecoverable [1]_.
 
@@ -15,10 +16,11 @@ The amount of transactions that can be lost is controlled via ``maximum_lag_on_f
 
 By default, when running leader elections, Patroni does not take into account the current timeline of replicas, what in some cases could be undesirable behavior. You can prevent the node not having the same timeline as a former primary become the new leader by changing the value of ``check_timeline`` parameter to ``true``.
 
-PostgreSQL synchronous replication
-----------------------------------
 
-You can use Postgres's `synchronous replication <http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION>`__ with Patroni. Synchronous replication ensures consistency across a cluster by confirming that writes are written to a secondary before returning to the connecting client with a success. The cost of synchronous replication: reduced throughput on writes. This throughput will be entirely based on network performance.
+PostgreSQL synchronous replication
+==================================
+
+You can use Postgres's `synchronous replication <http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION>`__ with Patroni. Synchronous replication ensures consistency across a cluster by confirming that writes are written to a secondary before returning to the connecting client with a success. The cost of synchronous replication: increased latency and reduced throughput on writes. This throughput will be entirely based on network performance.
 
 In hosted datacenter environments (like AWS, Rackspace, or any network you do not control), synchronous replication significantly increases the variability of write performance. If followers become inaccessible from the leader, the leader effectively becomes read-only.
 
@@ -33,10 +35,11 @@ When using PostgreSQL synchronous replication, use at least three Postgres data
 
 Using PostgreSQL synchronous replication does not guarantee zero lost transactions under all circumstances. When the primary and the secondary that is currently acting as a synchronous replica fail simultaneously a third node that might not contain all transactions will be promoted.
 
+
 .. _synchronous_mode:
 
 Synchronous mode
-----------------
+================
 
 For use cases where losing committed transactions is not permissible you can turn on Patroni's ``synchronous_mode``. When ``synchronous_mode`` is turned on Patroni will not promote a standby unless it is certain that the standby contains all transactions that may have returned a successful commit status to client [2]_. This means that the system may be unavailable for writes even though some servers are available. System administrators can still use manual failover commands to promote a standby even if it results in transaction loss.
 
@@ -53,30 +56,124 @@ are available. As a downside, the primary is not be available for writes
 blocking all client write requests until at least one synchronous replica comes
 up.
 
-You can ensure that a standby never becomes the synchronous standby by setting ``nosync`` tag to true. This is recommended to set for standbys that are behind slow network connections and would cause performance degradation when becoming a synchronous standby.
+You can ensure that a standby never becomes the synchronous standby by setting ``nosync`` tag to true. This is recommended to set for standbys that are behind slow network connections and would cause performance degradation when becoming a synchronous standby. Setting tag ``nostream`` to true will also have the same effect.
 
-Synchronous mode can be switched on and off via Patroni REST interface. See :ref:`dynamic configuration <dynamic_configuration>` for instructions.
+Synchronous mode can be switched on and off using ``patronictl edit-config`` command or via Patroni REST interface. See :ref:`dynamic configuration <dynamic_configuration>` for instructions.
 
 Note: Because of the way synchronous replication is implemented in PostgreSQL it is still possible to lose transactions even when using ``synchronous_mode_strict``. If the PostgreSQL backend is cancelled while waiting to acknowledge replication (as a result of packet cancellation due to client timeout or backend failure) transaction changes become visible for other backends. Such changes are not yet replicated and may be lost in case of standby promotion.
 
+
 Synchronous Replication Factor
-------------------------------
-The parameter ``synchronous_node_count`` is used by Patroni to manage number of synchronous standby databases. It is set to 1 by default. It has no effect when ``synchronous_mode`` is set to off. When enabled, Patroni manages precise number of synchronous standby databases based on parameter ``synchronous_node_count`` and adjusts the state in DCS & synchronous_standby_names as members join and leave.
+==============================
+
+The parameter ``synchronous_node_count`` is used by Patroni to manage the number of synchronous standby databases. It is set to ``1`` by default. It has no effect when ``synchronous_mode`` is set to ``off``. When enabled, Patroni manages the precise number of synchronous standby databases based on parameter ``synchronous_node_count`` and adjusts the state in DCS & ``synchronous_standby_names`` in PostgreSQL as members join and leave. If the parameter is set to a value higher than the number of eligible nodes it will be automatically reduced by Patroni.
+
+
+Maximum lag on synchronous node
+===============================
+
+By default Patroni sticks to nodes that are declared as ``synchronous``, according to the ``pg_stat_replication`` view, even when there are other nodes ahead of it. This is done to minimize the number of changes of ``synchronous_standby_names``. To change this behavior one may use ``maximum_lag_on_syncnode`` parameter. It controls how much lag the replica can have to still be considered as "synchronous".
+
+Patroni utilizes the max replica LSN if there is more than one standby, otherwise it will use leader's current wal LSN. The default is ``-1``, and Patroni will not take action to swap a synchronous unhealthy standby when the value is set to ``0`` or less. Please set the value high enough so that Patroni won't swap synchronous standbys frequently during high transaction volume.
+
 
 Synchronous mode implementation
--------------------------------
+===============================
 
-When in synchronous mode Patroni maintains synchronization state in the DCS, containing the latest primary and current synchronous standby databases. This state is updated with strict ordering constraints to ensure the following invariants:
+When in synchronous mode Patroni maintains synchronization state in the DCS (``/sync`` key), containing the latest primary and current synchronous standby databases. This state is updated with strict ordering constraints to ensure the following invariants:
 
 - A node must be marked as the latest leader whenever it can accept write transactions. Patroni crashing or PostgreSQL not shutting down can cause violations of this invariant.
 
-- A node must be set as the synchronous standby in PostgreSQL as long as it is published as the synchronous standby.
+- A node must be set as the synchronous standby in PostgreSQL as long as it is published as the synchronous standby in the ``/sync`` key in DCS..
 
 - A node that is not the leader or current synchronous standby is not allowed to promote itself automatically.
 
 Patroni will only assign one or more synchronous standby nodes based on ``synchronous_node_count`` parameter to ``synchronous_standby_names``.
 
-On each HA loop iteration Patroni re-evaluates synchronous standby nodes choice. If the current list of synchronous standby nodes are connected and has not requested its synchronous status to be removed it remains picked. Otherwise the cluster member available for sync that is furthest ahead in replication is picked.
+On each HA loop iteration Patroni re-evaluates synchronous standby nodes choice. If the current list of synchronous standby nodes are connected and has not requested its synchronous status to be removed it remains picked. Otherwise the cluster members available for sync that are furthest ahead in replication are picked.
+
+Example:
+---------
+
+``/config`` key in DCS
+^^^^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: YAML
+
+    synchronous_mode: on
+    synchronous_node_count: 2
+    ...
+
+``/sync`` key in DCS
+^^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: JSON
+
+    {
+        "leader": "node0",
+        "sync_standby": "node1,node2"
+    }
+
+postgresql.conf
+^^^^^^^^^^^^^^^
+
+.. code-block:: INI
+
+    synchronous_standby_names = 'FIRST 2 (node1,node2)'
+
+
+In the above examples only nodes ``node1`` and ``node2`` are known to be synchronous and allowed to be automatically promoted if the primary (``node0``) fails.
+
+
+.. _quorum_mode:
+
+Quorum commit mode
+==================
+
+Starting from PostgreSQL v10 Patroni supports quorum-based synchronous replication.
+
+In this mode, Patroni maintains synchronization state in the DCS, containing the latest known primary, the number of nodes required for quorum, and the nodes currently eligible to vote on quorum. In steady state, the nodes voting on quorum are the leader and all synchronous standbys. This state is updated with strict ordering constraints, with regards to node promotion and ``synchronous_standby_names``, to ensure that at all times any subset of voters that can achieve quorum includes at least one node with the latest successful commit.
+
+On each iteration of HA loop, Patroni re-evaluates synchronous standby choices and quorum, based on node availability and requested cluster configuration. In PostgreSQL versions above 9.6 all eligible nodes are added as synchronous standbys as soon as their replication catches up to leader.
+
+Quorum commit helps to reduce worst case latencies, even during normal operation, as a higher latency of replicating to one standby can be compensated by other standbys.
+
+The quorum-based synchronous mode could be enabled by setting ``synchronous_mode`` to ``quorum`` using ``patronictl edit-config`` command or via Patroni REST interface. See :ref:`dynamic configuration <dynamic_configuration>` for instructions.
+
+Other parameters, like ``synchronous_node_count``, ``maximum_lag_on_syncnode``, and ``synchronous_mode_strict`` continue to work the same way as with ``synchronous_mode=on``.
+
+Example:
+---------
+
+``/config`` key in DCS
+^^^^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: YAML
+
+    synchronous_mode: quorum
+    synchronous_node_count: 2
+    ...
+
+``/sync`` key in DCS
+^^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: JSON
+
+    {
+        "leader": "node0",
+        "sync_standby": "node1,node2,node3",
+        "quorum": 1
+    }
+
+postgresql.conf
+^^^^^^^^^^^^^^^
+
+.. code-block:: INI
+
+    synchronous_standby_names = 'ANY 2 (node1,node2,node3)'
+
+
+If the primary (``node0``) failed, in the above example two of the ``node1``, ``node2``, ``node3`` will have the latest transaction received, but we don't know which ones. To figure out whether the node ``node1`` has received the latest transaction, we need to compare its LSN with the LSN on **at least** one node (``quorum=1`` in the ``/sync`` key) among ``node2`` and ``node3``. If ``node1`` isn't behind of at least one of them, we can guarantee that there will be no user visible data loss if ``node1`` is promoted.
 
 
 .. [1] The data is still there, but recovering it requires a manual recovery effort by data recovery specialists. When Patroni is allowed to rewind with ``use_pg_rewind`` the forked timeline will be automatically erased to rejoin the failed primary with the cluster.

docs/rest_api.rst

@@ -45,6 +45,10 @@ For all health check ``GET`` requests Patroni returns a JSON document with the s
 
 - ``GET /read-only-sync``: like the above endpoint, but also includes the primary.
 
+- ``GET /quorum``: returns HTTP status code **200** only when this Patroni node is listed as a quorum node in ``synchronous_standby_names`` on the primary.
+
+- ``GET /read-only-quorum``: like the above endpoint, but also includes the primary.
+
 - ``GET /asynchronous`` or ``GET /async``: returns HTTP status code **200** only when the Patroni node is running as an asynchronous standby.
 
 
@@ -308,6 +312,9 @@ Retrieve the Patroni metrics in Prometheus format through the ``GET /metrics`` e
 	# HELP patroni_sync_standby Value is 1 if this node is a sync standby replica, 0 otherwise.
 	# TYPE patroni_sync_standby gauge
 	patroni_sync_standby{scope="batman",name="patroni1"} 0
+	# HELP patroni_quorum_standby Value is 1 if this node is a quorum standby replica, 0 otherwise.
+	# TYPE patroni_quorum_standby gauge
+	patroni_quorum_standby{scope="batman",name="patroni1"} 0
 	# HELP patroni_xlog_received_location Current location of the received Postgres transaction log, 0 if this node is not a replica.
 	# TYPE patroni_xlog_received_location counter
 	patroni_xlog_received_location{scope="batman",name="patroni1"} 0

docs/standby_cluster.rst

@@ -0,0 +1,78 @@
+.. _standby_cluster:
+
+Standby cluster
+---------------
+
+Patroni also support running cascading replication to a remote datacenter
+(region) using a feature that is called "standby cluster". This type of
+clusters has:
+
+* "standby leader", that behaves pretty much like a regular cluster leader,
+  except it replicates from a remote node.
+
+* cascade replicas, that are replicating from standby leader.
+
+Standby leader holds and updates a leader lock in DCS. If the leader lock
+expires, cascade replicas will perform an election to choose another leader
+from the standbys.
+
+There is no further relationship between the standby cluster and the primary
+cluster it replicates from, in particular, they must not share the same DCS
+scope if they use the same DCS. They do not know anything else from each other
+apart from replication information. Also, the standby cluster is not being
+displayed in :ref:`patronictl_list` or :ref:`patronictl_topology` output on the
+primary cluster.
+
+For the sake of flexibility, you can specify methods of creating a replica and
+recovery WAL records when a cluster is in the "standby mode" by providing
+:ref:`create_replica_methods <custom_replica_creation>` key in
+`standby_cluster` section. It is distinct from creating replicas, when cluster
+is detached and functions as a normal cluster, which is controlled by
+`create_replica_methods` in `postgresql` section. Both "standby" and "normal"
+`create_replica_methods` reference  keys in `postgresql` section.
+
+To configure such cluster you need to specify the section ``standby_cluster``
+in a patroni configuration:
+
+.. code:: YAML
+
+    bootstrap:
+        dcs:
+            standby_cluster:
+                host: 1.2.3.4
+                port: 5432
+                primary_slot_name: patroni
+                create_replica_methods:
+                - basebackup
+
+Note, that these options will be applied only once during cluster bootstrap,
+and the only way to change them afterwards is through DCS.
+
+Patroni expects to find `postgresql.conf` or `postgresql.conf.backup` in PGDATA
+of the remote primary and will not start if it does not find it after a
+basebackup. If the remote primary keeps its `postgresql.conf` elsewhere, it is
+your responsibility to copy it to PGDATA.
+
+If you use replication slots on the standby cluster, you must also create the
+corresponding replication slot on the primary cluster.  It will not be done
+automatically by the standby cluster implementation.  You can use Patroni's
+permanent replication slots feature on the primary cluster to maintain a
+replication slot with the same name as ``primary_slot_name``, or its default
+value if ``primary_slot_name`` is not provided.
+
+In case the remote site doesn't provide a single endpoint that connects to a
+primary, one could list all hosts of the source cluster in the
+``standby_cluster.host`` section.  When ``standby_cluster.host`` contains
+multiple hosts separated by commas, Patroni will:
+
+* add ``target_session_attrs=read-write`` to the ``primary_conninfo`` on the
+  standby leader node.
+* use ``target_session_attrs=read-write`` when trying to determine whether we
+  need to run ``pg_rewind`` or when executing ``pg_rewind`` on all nodes of the
+  standby cluster.
+  
+There is also a possibility to replicate the standby cluster from another
+standby cluster or from a standby member of the primary cluster: for that, you
+need to define a single host in the ``standby_cluster.host`` section. However,
+you need to beware that in this case ``pg_rewind`` will fail to execute on the
+standby cluster.

docs/tools_integration.rst

@@ -0,0 +1,62 @@
+Integration with other tools
+============================
+
+Patroni is able to integrate with other tools in your stack. In this section you
+will find a list of examples, which although not an exhaustive list, might
+provide you with ideas on how Patroni can integrate with other tools.
+
+Barman
+------
+
+Patroni delivers an application named ``patroni_barman`` which has logic to
+communicate with ``pg-backup-api``, so you are able to perform Barman operations
+remotely.
+
+This application currently has a couple of sub-commands: ``recover`` and
+``config-switch``.
+
+patroni_barman recover
+^^^^^^^^^^^^^^^^^^^^^^
+
+The ``recover`` sub-command can be used as a custom bootstrap or custom replica
+creation method. You can find more information about that in
+:ref:`replica_imaging_and_bootstrap`.
+
+patroni_barman config-switch
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The ``config-switch`` sub-command is designed to be used as an ``on_role_change``
+callback in Patroni. As an example, assume you are streaming WALs from your
+current primary to your Barman host. In the event of a failover in the cluster
+you might want to start streaming WALs from the new primary. You can accomplish
+this by using ``patroni_barman config-switch`` as the ``on_role_change`` callback.
+
+.. note::
+    That sub-command relies on the ``barman config-switch`` command, which is in
+    charge of overriding the configuration of a Barman server by applying a
+    pre-defined model on top of it. This command is available since Barman 3.10.
+    Please consult the Barman documentation for more details.
+
+This is an example of how you can configure Patroni to apply a configuration
+model in case this Patroni node is promoted to primary:
+
+.. code:: YAML
+
+    postgresql:
+        callbacks:
+            on_role_change: >
+                patroni_barman
+                    --api-url YOUR_API_URL
+                    config-switch
+                    --barman-server YOUR_BARMAN_SERVER_NAME
+                    --barman-model YOUR_BARMAN_MODEL_NAME
+                    --switch-when promoted
+
+.. note::
+    ``patroni_barman config-switch`` requires that you have both Barman and
+    ``pg-backup-api`` configured in the Barman host, so it can execute a remote
+    ``barman config-switch`` through the backup API. Also, it requires that you
+    have pre-configured Barman models to be applied. The above example uses a
+    subset of the available parameters. You can get more information running
+    ``patroni_barman config-switch --help``, and by consulting the Barman
+    documentation.

docs/yaml_configuration.rst

@@ -399,6 +399,7 @@ Tags
 -  **nosync**: ``true`` or ``false``. If set to ``true`` the node will never be selected as a synchronous replica.
 -  **nofailover**: ``true`` or ``false``, controls whether this node is allowed to participate in the leader race and become a leader. Defaults to ``false``, meaning this node _can_ participate in leader races. 
 -  **failover_priority**: integer, controls the priority that this node should have during failover. Nodes with higher priority will be preferred over lower priority nodes if they received/replayed the same amount of WAL. However, nodes with higher values of receive/replay LSN are preferred regardless of their priority. If the ``failover_priority`` is 0 or negative - such node is not allowed to participate in the leader race and to become a leader (similar to ``nofailover: true``).
+-  **nostream**: ``true`` or ``false``. If set to ``true`` the node will not use replication protocol to stream WAL. It will rely instead on archive recovery (if ``restore_command`` is configured) and ``pg_wal``/``pg_xlog`` polling. It also disables copying and synchronization of permanent logical replication slots on the node itself and all its cascading replicas. Setting this tag on primary node has no effect.
 
 .. warning::
    Provide only one of ``nofailover`` or ``failover_priority``. Providing ``nofailover: true`` is the same as ``failover_priority: 0``, and providing ``nofailover: false`` will give the node priority 1. 

features/environment.py

@@ -256,10 +256,18 @@ class PatroniController(AbstractController):
                         'parameters': {
                             'wal_keep_segments': 100,
                             'archive_mode': 'on',
-                            'archive_command': (PatroniPoolController.ARCHIVE_RESTORE_SCRIPT
-                                                + ' --mode archive '
-                                                + '--dirname {} --filename %f --pathname %p').format(
-                                                    os.path.join(self._work_directory, 'data', 'wal_archive'))
+                            'archive_command':
+                                (PatroniPoolController.ARCHIVE_RESTORE_SCRIPT
+                                 + ' --mode archive '
+                                 + '--dirname {} --filename %f --pathname %p').format(
+                                     os.path.join(self._work_directory, 'data',
+                                                  f'wal_archive{str(self._citus_group or "")}')).replace('\\', '/'),
+                            'restore_command':
+                                (PatroniPoolController.ARCHIVE_RESTORE_SCRIPT
+                                 + ' --mode restore '
+                                 + '--dirname {} --filename %f --pathname %p').format(
+                                     os.path.join(self._work_directory, 'data',
+                                                  f'wal_archive{str(self._citus_group or "")}')).replace('\\', '/')
                         }
                     }
                 }
@@ -928,11 +936,6 @@ class PatroniPoolController(object):
         custom_config = {
             'scope': cluster_name,
             'postgresql': {
-                'recovery_conf': {
-                    'restore_command': (self.ARCHIVE_RESTORE_SCRIPT + ' --mode restore '
-                                        + '--dirname {} --filename %f --pathname %p')
-                    .format(os.path.join(self.patroni_path, 'data', 'wal_archive').replace('\\', '/'))
-                },
                 'create_replica_methods': ['no_leader_bootstrap'],
                 'no_leader_bootstrap': self.backup_restore_config({'no_leader': '1'})
             }

features/nostream_node.feature

@@ -0,0 +1,18 @@
+Feature: nostream node
+
+Scenario: check nostream node is recovering from archive
+    When I start postgres0
+    And I configure and start postgres1 with a tag nostream true
+    Then "members/postgres1" key in DCS has replication_state=in archive recovery after 10 seconds
+    And replication works from postgres0 to postgres1 after 30 seconds
+
+@slot-advance
+Scenario: check permanent logical replication slots are not copied
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"wal_level": "logical"}}, "slots":{"test_logical":{"type":"logical","database":"postgres","plugin":"test_decoding"}}}
+    Then I receive a response code 200
+    When I run patronictl.py restart batman postgres0 --force
+    Then postgres0 has a logical replication slot named test_logical with the test_decoding plugin after 10 seconds
+    When I configure and start postgres2 with a tag replicatefrom postgres1
+    Then "members/postgres2" key in DCS has replication_state=streaming after 10 seconds 
+    And postgres1 does not have a replication slot named test_logical
+    And postgres2 does not have a replication slot named test_logical

features/quorum_commit.feature

@@ -0,0 +1,68 @@
+Feature: quorum commit
+  Check basic workfrlows when quorum commit is enabled
+
+  Scenario: check enable quorum commit and that the only leader promotes after restart
+    Given I start postgres0
+    Then postgres0 is a leader after 10 seconds
+    And there is a non empty initialize key in DCS after 15 seconds
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"ttl": 20, "synchronous_mode": "quorum"}
+    Then I receive a response code 200
+    And sync key in DCS has leader=postgres0 after 20 seconds
+    And sync key in DCS has quorum=0 after 2 seconds
+    And synchronous_standby_names on postgres0 is set to "_empty_str_" after 2 seconds
+    When I shut down postgres0
+    And sync key in DCS has leader=postgres0 after 2 seconds
+    When I start postgres0
+    Then postgres0 role is the primary after 10 seconds
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"synchronous_mode_strict": true}
+    Then synchronous_standby_names on postgres0 is set to "ANY 1 (*)" after 10 seconds
+
+  Scenario: check failover with one quorum standby
+    Given I start postgres1
+    Then sync key in DCS has sync_standby=postgres1 after 10 seconds
+    And synchronous_standby_names on postgres0 is set to "ANY 1 (postgres1)" after 2 seconds
+    When I shut down postgres0
+    Then postgres1 role is the primary after 10 seconds
+    And sync key in DCS has quorum=0 after 10 seconds
+    Then synchronous_standby_names on postgres1 is set to "ANY 1 (*)" after 10 seconds
+    When I start postgres0
+    Then sync key in DCS has leader=postgres1 after 10 seconds
+    Then sync key in DCS has sync_standby=postgres0 after 10 seconds
+    And synchronous_standby_names on postgres1 is set to "ANY 1 (postgres0)" after 2 seconds
+
+  Scenario: check behavior with three nodes and different replication factor
+    Given I start postgres2
+    Then sync key in DCS has sync_standby=postgres0,postgres2 after 10 seconds
+    And sync key in DCS has quorum=1 after 2 seconds
+    And synchronous_standby_names on postgres1 is set to "ANY 1 (postgres0,postgres2)" after 2 seconds
+    When I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_node_count": 2}
+    Then sync key in DCS has quorum=0 after 10 seconds
+    And synchronous_standby_names on postgres1 is set to "ANY 2 (postgres0,postgres2)" after 2 seconds
+
+  Scenario: switch from quorum replication to good old multisync and back
+    Given I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_mode": true, "synchronous_node_count": 1}
+    And I shut down postgres0
+    Then synchronous_standby_names on postgres1 is set to "postgres2" after 10 seconds
+    And sync key in DCS has sync_standby=postgres2 after 10 seconds
+    Then sync key in DCS has quorum=0 after 2 seconds
+    When I issue a PATCH request to http://127.0.0.1:8009/config with {"synchronous_mode": "quorum"}
+    And I start postgres0
+    Then synchronous_standby_names on postgres1 is set to "ANY 1 (postgres0,postgres2)" after 10 seconds
+    And sync key in DCS has sync_standby=postgres0,postgres2 after 10 seconds
+    Then sync key in DCS has quorum=1 after 2 seconds
+
+  Scenario: REST API and patronictl
+    Given I run patronictl.py list batman
+    Then I receive a response returncode 0
+    And I receive a response output "Quorum Standby"
+    And Status code on GET http://127.0.0.1:8008/quorum is 200 after 3 seconds
+    And Status code on GET http://127.0.0.1:8010/quorum is 200 after 3 seconds
+
+  Scenario: nosync node is removed from voters and synchronous_standby_names
+    Given I add tag nosync true to postgres2 config
+    When I issue an empty POST request to http://127.0.0.1:8010/reload
+    Then I receive a response code 202
+    And sync key in DCS has quorum=0 after 10 seconds
+    And sync key in DCS has sync_standby=postgres0 after 10 seconds
+    And synchronous_standby_names on postgres1 is set to "ANY 1 (postgres0)" after 2 seconds
+    And Status code on GET http://127.0.0.1:8010/quorum is 503 after 10 seconds

features/standby_cluster.feature

@@ -26,7 +26,7 @@ Feature: standby cluster
   Scenario: Detach exiting node from the cluster
     When I shut down postgres1
     Then postgres0 is a leader after 10 seconds
-    And "members/postgres0" key in DCS has role=master after 3 seconds
+    And "members/postgres0" key in DCS has role=master after 5 seconds
     When I issue a GET request to http://127.0.0.1:8008/
     Then I receive a response code 200
 
@@ -47,6 +47,7 @@ Feature: standby cluster
     And there is a postgres1_cb.log with "on_role_change standby_leader batman1" in postgres1 data directory
     When I start postgres2 in a cluster batman1
     Then postgres2 role is the replica after 24 seconds
+    And postgres2 is replicating from postgres1 after 10 seconds
     And table foo is present on postgres2 after 20 seconds
     When I issue a GET request to http://127.0.0.1:8010/patroni
     Then I receive a response code 200

features/steps/basic_replication.py

@@ -46,11 +46,17 @@ def kill_postgres(context, name):
     return context.pctl.stop(name, kill=True, postgres=True)
 
 
+def get_wal_name(context, pg_name):
+    version = context.pctl.query(pg_name, "SHOW server_version_num").fetchone()[0]
+    return 'xlog' if int(version) / 10000 < 10 else 'wal'
+
+
 @step('I add the table {table_name:w} to {pg_name:w}')
 def add_table(context, table_name, pg_name):
     # parse the configuration file and get the port
     try:
         context.pctl.query(pg_name, "CREATE TABLE public.{0}()".format(table_name))
+        context.pctl.query(pg_name, "SELECT pg_switch_{0}()".format(get_wal_name(context, pg_name)))
     except pg.Error as e:
         assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e)
 
@@ -59,9 +65,7 @@ def add_table(context, table_name, pg_name):
 def toggle_wal_replay(context, action, pg_name):
     # pause or resume the wal replay process
     try:
-        version = context.pctl.query(pg_name, "SHOW server_version_num").fetchone()[0]
-        wal_name = 'xlog' if int(version) / 10000 < 10 else 'wal'
-        context.pctl.query(pg_name, "SELECT pg_{0}_replay_{1}()".format(wal_name, action))
+        context.pctl.query(pg_name, "SELECT pg_{0}_replay_{1}()".format(get_wal_name(context, pg_name), action))
     except pg.Error as e:
         assert False, "Error during {0} wal recovery on {1}: {2}".format(action, pg_name, e)
 

features/steps/quorum_commit.py

@@ -0,0 +1,60 @@
+import json
+import re
+import time
+
+from behave import step, then
+
+
+@step('sync key in DCS has {key:w}={value} after {time_limit:d} seconds')
+def check_sync(context, key, value, time_limit):
+    time_limit *= context.timeout_multiplier
+    max_time = time.time() + int(time_limit)
+    dcs_value = None
+    while time.time() < max_time:
+        try:
+            response = json.loads(context.dcs_ctl.query('sync'))
+            dcs_value = response.get(key)
+            if key == 'sync_standby' and set((dcs_value or '').split(',')) == set(value.split(',')):
+                return
+            elif str(dcs_value) == value:
+                return
+        except Exception:
+            pass
+        time.sleep(1)
+    assert False, "sync does not have {0}={1} (found {2}) in dcs after {3} seconds".format(key, value,
+                                                                                           dcs_value, time_limit)
+
+
+def _parse_synchronous_standby_names(value):
+    if '(' in value:
+        m = re.match(r'.*(\d+) \(([^)]+)\)', value)
+        expected_value = set(m.group(2).split())
+        expected_num = m.group(1)
+    else:
+        expected_value = set([value])
+        expected_num = '1'
+    return expected_num, expected_value
+
+
+@then('synchronous_standby_names on {name:2} is set to "{value}" after {time_limit:d} seconds')
+def check_synchronous_standby_names(context, name, value, time_limit):
+    time_limit *= context.timeout_multiplier
+    max_time = time.time() + int(time_limit)
+
+    if value == '_empty_str_':
+        value = ''
+
+    expected_num, expected_value = _parse_synchronous_standby_names(value)
+
+    ssn = None
+    while time.time() < max_time:
+        try:
+            ssn = context.pctl.query(name, "SHOW synchronous_standby_names").fetchone()[0]
+            db_num, db_value = _parse_synchronous_standby_names(ssn)
+            if expected_value == db_value and expected_num == db_num:
+                return
+        except Exception:
+            pass
+        time.sleep(1)
+    assert False, "synchronous_standby_names is not set to '{0}' (found '{1}') after {2} seconds".format(value, ssn,
+                                                                                                         time_limit)

patroni/api.py

@@ -254,6 +254,14 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
                 * HTTP status ``200``: if up and running and without ``noloadbalance`` tag.
 
+            * ``/quorum``:
+
+                * HTTP status ``200``: if up and running as a quorum synchronous standby.
+
+            * ``/read-only-quorum``:
+
+                * HTTP status ``200``: if up and running as a quorum synchronous standby or primary.
+
             * ``/synchronous`` or ``/sync``:
 
                 * HTTP status ``200``: if up and running as a synchronous standby.
@@ -334,16 +342,24 @@ class RestApiHandler(BaseHTTPRequestHandler):
             ignore_tags = True
         elif 'replica' in path:
             status_code = replica_status_code
-        elif 'read-only' in path and 'sync' not in path:
+        elif 'read-only' in path and 'sync' not in path and 'quorum' not in path:
             status_code = 200 if 200 in (primary_status_code, standby_leader_status_code) else replica_status_code
         elif 'health' in path:
             status_code = 200 if response.get('state') == 'running' else 503
         elif cluster:  # dcs is available
+            is_quorum = response.get('quorum_standby')
             is_synchronous = response.get('sync_standby')
             if path in ('/sync', '/synchronous') and is_synchronous:
                 status_code = replica_status_code
-            elif path in ('/async', '/asynchronous') and not is_synchronous:
+            elif path == '/quorum' and is_quorum:
                 status_code = replica_status_code
+            elif path in ('/async', '/asynchronous') and not is_synchronous and not is_quorum:
+                status_code = replica_status_code
+            elif path == '/read-only-quorum':
+                if 200 in (primary_status_code, standby_leader_status_code):
+                    status_code = 200
+                elif is_quorum:
+                    status_code = replica_status_code
             elif path in ('/read-only-sync', '/read-only-synchronous'):
                 if 200 in (primary_status_code, standby_leader_status_code):
                     status_code = 200
@@ -510,6 +526,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
             * ``patroni_standby_leader``: ``1`` if standby leader node, else ``0``;
             * ``patroni_replica``: ``1`` if a replica, else ``0``;
             * ``patroni_sync_standby``: ``1`` if a sync replica, else ``0``;
+            * ``patroni_quorum_standby``: ``1`` if a quorum sync replica, else ``0``;
             * ``patroni_xlog_received_location``: ``pg_wal_lsn_diff(pg_last_wal_receive_lsn(), '0/0')``;
             * ``patroni_xlog_replayed_location``: ``pg_wal_lsn_diff(pg_last_wal_replay_lsn(), '0/0)``;
             * ``patroni_xlog_replayed_timestamp``: ``pg_last_xact_replay_timestamp``;
@@ -572,10 +589,14 @@ class RestApiHandler(BaseHTTPRequestHandler):
         metrics.append("# TYPE patroni_replica gauge")
         metrics.append("patroni_replica{0} {1}".format(labels, int(postgres['role'] == 'replica')))
 
-        metrics.append("# HELP patroni_sync_standby Value is 1 if this node is a sync standby replica, 0 otherwise.")
+        metrics.append("# HELP patroni_sync_standby Value is 1 if this node is a sync standby, 0 otherwise.")
         metrics.append("# TYPE patroni_sync_standby gauge")
         metrics.append("patroni_sync_standby{0} {1}".format(labels, int(postgres.get('sync_standby', False))))
 
+        metrics.append("# HELP patroni_quorum_standby Value is 1 if this node is a quorum standby, 0 otherwise.")
+        metrics.append("# TYPE patroni_quorum_standby gauge")
+        metrics.append("patroni_quorum_standby{0} {1}".format(labels, int(postgres.get('quorum_standby', False))))
+
         metrics.append("# HELP patroni_xlog_received_location Current location of the received"
                        " Postgres transaction log, 0 if this node is not a replica.")
         metrics.append("# TYPE patroni_xlog_received_location counter")
@@ -1035,16 +1056,17 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         :returns: a string with the error message or ``None`` if good nodes are found.
         """
-        is_synchronous_mode = global_config.from_cluster(cluster).is_synchronous_mode
+        config = global_config.from_cluster(cluster)
         if leader and (not cluster.leader or cluster.leader.name != leader):
             return 'leader name does not match'
         if candidate:
-            if action == 'switchover' and is_synchronous_mode and not cluster.sync.matches(candidate):
+            if action == 'switchover' and config.is_synchronous_mode\
+                    and not config.is_quorum_commit_mode and not cluster.sync.matches(candidate):
                 return 'candidate name does not match with sync_standby'
             members = [m for m in cluster.members if m.name == candidate]
             if not members:
                 return 'candidate does not exists'
-        elif is_synchronous_mode:
+        elif config.is_synchronous_mode and not config.is_quorum_commit_mode:
             members = [m for m in cluster.members if cluster.sync.matches(m.name)]
             if not members:
                 return action + ' is not possible: can not find sync_standby'
@@ -1251,6 +1273,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
                     * ``paused``: ``pg_is_wal_replay_paused()``;
 
             * ``sync_standby``: ``True`` if replication mode is synchronous and this is a sync standby;
+            * ``quorum_standby``: ``True`` if replication mode is quorum and this is a quorum standby;
             * ``timeline``: PostgreSQL primary node timeline;
             * ``replication``: :class:`list` of :class:`dict` entries, one for each replication connection. Each entry
                 contains the following keys:
@@ -1306,7 +1329,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
             if result['role'] == 'replica' and config.is_synchronous_mode\
                     and cluster and cluster.sync.matches(postgresql.name):
-                result['sync_standby'] = True
+                result['quorum_standby' if global_config.is_quorum_commit_mode else 'sync_standby'] = True
 
             if row[1] > 0:
                 result['timeline'] = row[1]

patroni/config.py

@@ -757,7 +757,7 @@ class Config(object):
         if 'citus' in config:
             bootstrap = config.setdefault('bootstrap', {})
             dcs = bootstrap.setdefault('dcs', {})
-            dcs.setdefault('synchronous_mode', True)
+            dcs.setdefault('synchronous_mode', 'quorum')
 
         updated_fields = (
             'name',

patroni/config_generator.py

@@ -126,6 +126,7 @@ class AbstractConfigGenerator(abc.ABC):
                 'noloadbalance': False,
                 'clonefrom': True,
                 'nosync': False,
+                'nostream': False,
             }
         }
 

patroni/ctl.py

@@ -41,8 +41,12 @@ if TYPE_CHECKING:  # pragma: no cover
     from psycopg import Cursor
     from psycopg2 import cursor
 
-try:
-    from ydiff import markup_to_pager, PatchStream  # pyright: ignore [reportMissingModuleSource]
+try:  # pragma: no cover
+    from ydiff import markup_to_pager  # pyright: ignore [reportMissingModuleSource]
+    try:
+        from ydiff import PatchStream  # pyright: ignore [reportMissingModuleSource]
+    except ImportError:
+        PatchStream = iter
 except ImportError:  # pragma: no cover
     from cdiff import markup_to_pager, PatchStream  # pyright: ignore [reportMissingModuleSource]
 

patroni/dcs/__init__.py

@@ -545,11 +545,15 @@ class SyncState(NamedTuple):
     :ivar version: modification version of a synchronization key in a Configuration Store.
     :ivar leader: reference to member that was leader.
     :ivar sync_standby: synchronous standby list (comma delimited) which are last synchronized to leader.
+    :ivar quorum: if the node from :attr:`~SyncState.sync_standby` list is doing a leader race it should
+                  see at least :attr:`~SyncState.quorum` other nodes from the
+                  :attr:`~SyncState.sync_standby` + :attr:`~SyncState.leader` list.
     """
 
     version: Optional[_Version]
     leader: Optional[str]
     sync_standby: Optional[str]
+    quorum: int
 
     @staticmethod
     def from_node(version: Optional[_Version], value: Union[str, Dict[str, Any], None]) -> 'SyncState':
@@ -584,7 +588,9 @@ class SyncState(NamedTuple):
             if value and isinstance(value, str):
                 value = json.loads(value)
             assert isinstance(value, dict)
-            return SyncState(version, value.get('leader'), value.get('sync_standby'))
+            leader = value.get('leader')
+            quorum = value.get('quorum')
+            return SyncState(version, leader, value.get('sync_standby'), int(quorum) if leader and quorum else 0)
         except (AssertionError, TypeError, ValueError):
             return SyncState.empty(version)
 
@@ -596,7 +602,7 @@ class SyncState(NamedTuple):
 
         :returns: empty synchronisation state object.
         """
-        return SyncState(version, None, None)
+        return SyncState(version, None, None, 0)
 
     @property
     def is_empty(self) -> bool:
@@ -614,10 +620,17 @@ class SyncState(NamedTuple):
         return list(filter(lambda a: a, [s.strip() for s in value.split(',')]))
 
     @property
-    def members(self) -> List[str]:
+    def voters(self) -> List[str]:
         """:attr:`~SyncState.sync_standby` as list or an empty list if undefined or object considered ``empty``."""
         return self._str_to_list(self.sync_standby) if not self.is_empty and self.sync_standby else []
 
+    @property
+    def members(self) -> List[str]:
+        """:attr:`~SyncState.sync_standby` and :attr:`~SyncState.leader` as list
+           or an empty list if object considered ``empty``.
+        """
+        return [] if not self.leader else [self.leader] + self.voters
+
     def matches(self, name: Optional[str], check_leader: bool = False) -> bool:
         """Checks if node is presented in the /sync state.
 
@@ -631,7 +644,7 @@ class SyncState(NamedTuple):
                   the sync state.
 
         :Example:
-            >>> s = SyncState(1, 'foo', 'bar,zoo')
+            >>> s = SyncState(1, 'foo', 'bar,zoo', 0)
 
             >>> s.matches('foo')
             False
@@ -1039,6 +1052,8 @@ class Cluster(NamedTuple('Cluster',
         .. note::
             Permanent replication slots are only considered if ``use_slots`` configuration is enabled.
             A node that is not supposed to become a leader (*nofailover*) will not have permanent replication slots.
+            Also node with disabled streaming (*nostream*) and its cascading followers must not have permanent
+            logical slots due to lack of feedback from node to primary, which makes them unsafe to use.
 
             In a standby cluster we only support physical replication slots.
 
@@ -1054,7 +1069,7 @@ class Cluster(NamedTuple('Cluster',
         if not global_config.use_slots or tags.nofailover:
             return {}
 
-        if global_config.is_standby_cluster:
+        if global_config.is_standby_cluster or self.get_slot_name_on_primary(postgresql.name, tags) is None:
             return self.__permanent_physical_slots \
                 if postgresql.major_version >= SLOT_ADVANCE_AVAILABLE_VERSION or role == 'standby_leader' else {}
 
@@ -1069,6 +1084,10 @@ class Cluster(NamedTuple('Cluster',
         the ``replicatefrom`` destination member is currently not a member of the cluster (fallback to the
         primary), or if ``replicatefrom`` destination member happens to be the current primary.
 
+        If the ``nostream`` tag is set on the member - we should not create the replication slot for it on
+        the current primary or any other member even if ``replicatefrom`` is set, because ``nostream`` disables
+        WAL streaming.
+
         Will log an error if:
 
             * Conflicting slot names between members are found
@@ -1083,8 +1102,9 @@ class Cluster(NamedTuple('Cluster',
         if not global_config.use_slots:
             return {}
 
-        # we always want to exclude the member with our name from the list
-        members = filter(lambda m: m.name != name, self.members)
+        # we always want to exclude the member with our name from the list,
+        # also exlude members with disabled WAL streaming
+        members = filter(lambda m: m.name != name and not m.nostream, self.members)
 
         if role in ('master', 'primary', 'standby_leader'):
             members = [m for m in members if m.replicatefrom is None
@@ -1172,7 +1192,7 @@ class Cluster(NamedTuple('Cluster',
             return any(self.should_enforce_hot_standby_feedback(postgresql, m) for m in members)
         return False
 
-    def get_slot_name_on_primary(self, name: str, tags: Tags) -> str:
+    def get_slot_name_on_primary(self, name: str, tags: Tags) -> Optional[str]:
         """Get the name of physical replication slot for this node on the primary.
 
         .. note::
@@ -1186,6 +1206,8 @@ class Cluster(NamedTuple('Cluster',
 
         :returns: the slot name on the primary that is in use for physical replication on this node.
         """
+        if tags.nostream:
+            return None
         replicatefrom = self.get_member(tags.replicatefrom, False) if tags.replicatefrom else None
         return self.get_slot_name_on_primary(replicatefrom.name, replicatefrom) \
             if isinstance(replicatefrom, Member) else slot_name_from_member_name(name)
@@ -1849,18 +1871,23 @@ class AbstractDCS(abc.ABC):
         """
 
     @staticmethod
-    def sync_state(leader: Optional[str], sync_standby: Optional[Collection[str]]) -> Dict[str, Any]:
+    def sync_state(leader: Optional[str], sync_standby: Optional[Collection[str]],
+                   quorum: Optional[int]) -> Dict[str, Any]:
         """Build ``sync_state`` dictionary.
 
         :param leader: name of the leader node that manages ``/sync`` key.
         :param sync_standby: collection of currently known synchronous standby node names.
+        :param quorum: if the node from :attr:`~SyncState.sync_standby` list is doing a leader race it should
+                       see at least :attr:`~SyncState.quorum` other nodes from the
+                       :attr:`~SyncState.sync_standby` + :attr:`~SyncState.leader` list
 
         :returns: dictionary that later could be serialized to JSON or saved directly to DCS.
         """
-        return {'leader': leader, 'sync_standby': ','.join(sorted(sync_standby)) if sync_standby else None}
+        return {'leader': leader, 'quorum': quorum,
+                'sync_standby': ','.join(sorted(sync_standby)) if sync_standby else None}
 
     def write_sync_state(self, leader: Optional[str], sync_standby: Optional[Collection[str]],
-                         version: Optional[Any] = None) -> Optional[SyncState]:
+                         quorum: Optional[int], version: Optional[Any] = None) -> Optional[SyncState]:
         """Write the new synchronous state to DCS.
 
         Calls :meth:`~AbstractDCS.sync_state` to build a dictionary and then calls DCS specific
@@ -1869,10 +1896,13 @@ class AbstractDCS(abc.ABC):
         :param leader: name of the leader node that manages ``/sync`` key.
         :param sync_standby: collection of currently known synchronous standby node names.
         :param version: for conditional update of the key/object.
+        :param quorum: if the node from :attr:`~SyncState.sync_standby` list is doing a leader race it should
+                       see at least :attr:`~SyncState.quorum` other nodes from the
+                       :attr:`~SyncState.sync_standby` + :attr:`~SyncState.leader` list
 
         :returns: the new :class:`SyncState` object or ``None``.
         """
-        sync_value = self.sync_state(leader, sync_standby)
+        sync_value = self.sync_state(leader, sync_standby, quorum)
         ret = self.set_sync_state_value(json.dumps(sync_value, separators=(',', ':')), version)
         if not isinstance(ret, bool):
             return SyncState.from_node(ret, sync_value)

patroni/dcs/consul.py

@@ -577,14 +577,17 @@ class Consul(AbstractDCS):
         try:
             return retry(self._client.kv.put, self.leader_path, self._name, acquire=self._session)
         except InvalidSession:
-            logger.error('Our session disappeared from Consul. Will try to get a new one and retry attempt')
             self._session = None
-            retry.ensure_deadline(0)
+
+            if not retry.ensure_deadline(0):
+                logger.error('Our session disappeared from Consul. Deadline exceeded, giving up')
+                return False
+
+            logger.error('Our session disappeared from Consul. Will try to get a new one and retry attempt')
 
             retry(self._do_refresh_session)
 
             retry.ensure_deadline(1, ConsulError('_do_attempt_to_acquire_leader timeout'))
-
             return retry(self._client.kv.put, self.leader_path, self._name, acquire=self._session)
 
     @catch_return_false_exception

patroni/dcs/etcd3.py

@@ -198,12 +198,6 @@ def build_range_request(key: str, range_end: Union[bytes, str, None] = None) ->
     return fields
 
 
-class ReAuthenticateMode(IntEnum):
-    NOT_REQUIRED = 0
-    REQUIRED = 1
-    WITHOUT_WATCHER_RESTART = 2
-
-
 def _handle_auth_errors(func: Callable[..., Any]) -> Any:
     def wrapper(self: 'Etcd3Client', *args: Any, **kwargs: Any) -> Any:
         return self.handle_auth_errors(func, *args, **kwargs)
@@ -215,7 +209,7 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
     ERROR_CLS = Etcd3Error
 
     def __init__(self, config: Dict[str, Any], dns_resolver: DnsCachingResolver, cache_ttl: int = 300) -> None:
-        self._reauthenticate_reason = ReAuthenticateMode.NOT_REQUIRED
+        self._reauthenticate = False
         self._token = None
         self._cluster_version: Tuple[int, ...] = tuple()
         super(Etcd3Client, self).__init__({**config, 'version_prefix': '/v3beta'}, dns_resolver, cache_ttl)
@@ -294,7 +288,7 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
         fields['retry'] = retry
         return self.api_execute(self.version_prefix + method, self._MPOST, fields)
 
-    def authenticate(self, *, restart_watcher: bool = True, retry: Optional[Retry] = None) -> bool:
+    def authenticate(self, *, retry: Optional[Retry] = None) -> bool:
         if self._use_proxies and not self._cluster_version:
             kwargs = self._prepare_common_parameters(1)
             self._ensure_version_prefix(self._base_uri, **kwargs)
@@ -316,20 +310,18 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
 
     def handle_auth_errors(self: 'Etcd3Client', func: Callable[..., Any], *args: Any,
                            retry: Optional[Retry] = None, **kwargs: Any) -> Any:
+        reauthenticated = False
         exc = None
         while True:
-            if self._reauthenticate_reason:
+            if self._reauthenticate:
                 if self.username and self.password:
-                    self.authenticate(
-                        restart_watcher=self._reauthenticate_reason != ReAuthenticateMode.WITHOUT_WATCHER_RESTART,
-                        retry=retry)
-                    self._reauthenticate_reason = ReAuthenticateMode.NOT_REQUIRED
-                    if retry:
-                        retry.ensure_deadline(0)
+                    self.authenticate(retry=retry)
+                    self._reauthenticate = False
                 else:
                     msg = 'Username or password not set, authentication is not possible'
                     logger.fatal(msg)
                     raise exc or Etcd3Exception(msg)
+                reauthenticated = True
 
             try:
                 return func(self, *args, retry=retry, **kwargs)
@@ -347,11 +339,12 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
             except AuthOldRevision as e:
                 logger.error('Auth token is for old revision of auth store')
                 exc = e
-            self._reauthenticate_reason = ReAuthenticateMode.WITHOUT_WATCHER_RESTART \
-                if isinstance(exc, AuthOldRevision) else ReAuthenticateMode.REQUIRED
-            if not retry:
+            self._reauthenticate = True
+            if retry:
+                logger.error('retry = %s', retry)
+                retry.ensure_deadline(0.5, exc)
+            elif reauthenticated:
                 raise exc
-            retry.ensure_deadline(0.5, exc)
 
     @_handle_auth_errors
     def range(self, key: str, range_end: Union[bytes, str, None] = None, serializable: bool = True,
@@ -603,12 +596,6 @@ class PatroniEtcd3Client(Etcd3Client):
         super(PatroniEtcd3Client, self).set_base_uri(value)
         self._restart_watcher()
 
-    def authenticate(self, *, restart_watcher: bool = True, retry: Optional[Retry] = None) -> bool:
-        ret = super(PatroniEtcd3Client, self).authenticate(restart_watcher=restart_watcher, retry=retry)
-        if ret and restart_watcher:
-            self._restart_watcher()
-        return ret
-
     def _wait_cache(self, timeout: float) -> None:
         stop_time = time.time() + timeout
         while self._kv_cache and not self._kv_cache.is_ready():
@@ -866,14 +853,16 @@ class Etcd3(AbstractEtcd):
         try:
             return _retry(self._client.put, self.leader_path, self._name, self._lease, create_revision='0')
         except LeaseNotFound:
-            logger.error('Our lease disappeared from Etcd. Will try to get a new one and retry attempt')
             self._lease = None
-            retry.ensure_deadline(0)
+            if not retry.ensure_deadline(0):
+                logger.error('Our lease disappeared from Etcd. Deadline exceeded, giving up')
+                return False
+
+            logger.error('Our lease disappeared from Etcd. Will try to get a new one and retry attempt')
 
             _retry(self._do_refresh_lease)
 
             retry.ensure_deadline(1, Etcd3Error('_do_attempt_to_acquire_leader timeout'))
-
             return _retry(self._client.put, self.leader_path, self._name, self._lease, create_revision='0')
 
     @catch_return_false_exception

patroni/dcs/kubernetes.py

@@ -1370,15 +1370,18 @@ class Kubernetes(AbstractDCS):
         raise NotImplementedError  # pragma: no cover
 
     def write_sync_state(self, leader: Optional[str], sync_standby: Optional[Collection[str]],
-                         version: Optional[str] = None) -> Optional[SyncState]:
+                         quorum: Optional[int], version: Optional[str] = None) -> Optional[SyncState]:
         """Prepare and write annotations to $SCOPE-sync Endpoint or ConfigMap.
 
         :param leader: name of the leader node that manages /sync key
         :param sync_standby: collection of currently known synchronous standby node names
+        :param quorum: if the node from sync_standby list is doing a leader race it should
+                       see at least quorum other nodes from the sync_standby + leader list
         :param version: last known `resource_version` for conditional update of the object
         :returns: the new :class:`SyncState` object or None
         """
-        sync_state = self.sync_state(leader, sync_standby)
+        sync_state = self.sync_state(leader, sync_standby, quorum)
+        sync_state['quorum'] = str(sync_state['quorum']) if sync_state['quorum'] is not None else None
         ret = self.patch_or_create(self.sync_path, sync_state, version, False)
         if not isinstance(ret, bool):
             return SyncState.from_node(ret.metadata.resource_version, sync_state)
@@ -1390,7 +1393,7 @@ class Kubernetes(AbstractDCS):
         :param version: last known `resource_version` for conditional update of the object
         :returns: `True` if "delete" was successful
         """
-        return self.write_sync_state(None, None, version=version) is not None
+        return self.write_sync_state(None, None, None, version=version) is not None
 
     def watch(self, leader_version: Optional[str], timeout: float) -> bool:
         if self.__do_not_watch:

patroni/global_config.py

@@ -44,19 +44,20 @@ class GlobalConfig(types.ModuleType):
         """
         return bool(cluster and cluster.config and cluster.config.modify_version)
 
-    def update(self, cluster: Optional['Cluster']) -> None:
+    def update(self, cluster: Optional['Cluster'], default: Optional[Dict[str, Any]] = None) -> None:
         """Update with the new global configuration from the :class:`Cluster` object view.
 
         .. note::
-            Global configuration is updated only when configuration in the *cluster* view is valid.
-
             Update happens in-place and is executed only from the main heartbeat thread.
 
         :param cluster: the currently known cluster state from DCS.
+        :param default: default configuration, which will be used if there is no valid *cluster.config*.
         """
         # Try to protect from the case when DCS was wiped out
         if self._cluster_has_valid_config(cluster):
             self.__config = cluster.config.data  # pyright: ignore [reportOptionalMemberAccess]
+        elif default:
+            self.__config = default
 
     def from_cluster(self, cluster: Optional['Cluster']) -> 'GlobalConfig':
         """Return :class:`GlobalConfig` instance from the provided :class:`Cluster` object view.
@@ -103,10 +104,16 @@ class GlobalConfig(types.ModuleType):
         """``True`` if cluster is in maintenance mode."""
         return self.check_mode('pause')
 
+    @property
+    def is_quorum_commit_mode(self) -> bool:
+        """:returns: ``True`` if quorum commit replication is requested"""
+        return str(self.get('synchronous_mode')).lower() == 'quorum'
+
     @property
     def is_synchronous_mode(self) -> bool:
         """``True`` if synchronous replication is requested and it is not a standby cluster config."""
-        return self.check_mode('synchronous_mode') and not self.is_standby_cluster
+        return (self.check_mode('synchronous_mode') is True or self.is_quorum_commit_mode) \
+            and not self.is_standby_cluster
 
     @property
     def is_synchronous_mode_strict(self) -> bool:

patroni/ha.py

@@ -14,12 +14,13 @@ from . import global_config, psycopg
 from .__main__ import Patroni
 from .async_executor import AsyncExecutor, CriticalTask
 from .collections import CaseInsensitiveSet
-from .dcs import AbstractDCS, Cluster, Leader, Member, RemoteMember, Status, slot_name_from_member_name
+from .dcs import AbstractDCS, Cluster, Leader, Member, RemoteMember, Status, SyncState, slot_name_from_member_name
 from .exceptions import DCSError, PostgresConnectionException, PatroniFatalException
 from .postgresql.callback_executor import CallbackAction
 from .postgresql.misc import postgres_version_to_int
 from .postgresql.postmaster import PostmasterProcess
 from .postgresql.rewind import Rewind
+from .quorum import QuorumStateResolver
 from .tags import Tags
 from .utils import polling_loop, tzutc
 
@@ -161,6 +162,7 @@ class Ha(object):
         self._leader_expiry_lock = RLock()
         self._failsafe = Failsafe(patroni.dcs)
         self._was_paused = False
+        self._promote_timestamp = 0
         self._leader_timeline = None
         self.recovering = False
         self._async_response = CriticalTask()
@@ -185,6 +187,9 @@ class Ha(object):
         # used only in backoff after failing a pre_promote script
         self._released_leader_key_timestamp = 0
 
+        # Initialize global config
+        global_config.update(None, self.patroni.config.dynamic_configuration)
+
     def primary_stop_timeout(self) -> Union[int, None]:
         """:returns: "primary_stop_timeout" from the global configuration or `None` when not in synchronous mode."""
         ret = global_config.primary_stop_timeout
@@ -217,6 +222,8 @@ class Ha(object):
         """
         with self._leader_expiry_lock:
             self._leader_expiry = time.time() + self.dcs.ttl if value else 0
+            if not value:
+                self._promote_timestamp = 0
 
     def sync_mode_is_active(self) -> bool:
         """Check whether synchronous replication is requested and already active.
@@ -225,6 +232,13 @@ class Ha(object):
         """
         return self.is_synchronous_mode() and not self.cluster.sync.is_empty
 
+    def quorum_commit_mode_is_active(self) -> bool:
+        """Checks whether quorum replication is requested and already active.
+
+        :returns: ``True`` if the primary already put its name into the ``/sync`` in DCS.
+        """
+        return self.is_quorum_commit_mode() and not self.cluster.sync.is_empty
+
     def _get_failover_action_name(self) -> str:
         """Return the currently requested manual failover action name or the default ``failover``.
 
@@ -607,9 +621,12 @@ class Ha(object):
 
         :returns: the node which we should be replicating from.
         """
+        # nostream is set, the node must not use WAL streaming
+        if self.patroni.nostream:
+            return None
         # The standby leader or when there is no standby leader we want to follow
         # the remote member, except when there is no standby leader in pause.
-        if self.is_standby_cluster() \
+        elif self.is_standby_cluster() \
                 and (cluster.leader and cluster.leader.name and cluster.leader.name == self.state_handler.name
                      or cluster.is_unlocked() and not self.is_paused()):
             node_to_follow = self.get_remote_member()
@@ -691,12 +708,103 @@ class Ha(object):
         """:returns: `True` if synchronous replication is requested."""
         return global_config.is_synchronous_mode
 
+    def is_quorum_commit_mode(self) -> bool:
+        """``True`` if quorum commit replication is requested and "supported"."""
+        return global_config.is_quorum_commit_mode and self.state_handler.supports_multiple_sync
+
     def is_failsafe_mode(self) -> bool:
         """:returns: `True` if failsafe_mode is enabled in global configuration."""
         return global_config.check_mode('failsafe_mode')
 
-    def process_sync_replication(self) -> None:
-        """Process synchronous standby beahvior.
+    def _maybe_enable_synchronous_mode(self) -> Optional[SyncState]:
+        """Explicitly enable synchronous mode if not yet enabled.
+
+        We are trying to solve a corner case: synchronous mode needs to be explicitly enabled
+        by updating the ``/sync`` key with the current leader name and empty members. In opposite
+        case it will never be automatically enabled if there are no eligible candidates.
+
+        :returns: the latest version of :class:`~patroni.dcs.SyncState` object.
+        """
+        sync = self.cluster.sync
+        if sync.is_empty:
+            sync = self.dcs.write_sync_state(self.state_handler.name, None, 0, version=sync.version)
+            if sync:
+                logger.info("Enabled synchronous replication")
+            else:
+                logger.warning("Updating sync state failed")
+        return sync
+
+    def disable_synchronous_replication(self) -> None:
+        """Cleans up /sync key in DCS if synchronous replication is disabled."""
+        if not self.cluster.sync.is_empty and self.dcs.delete_sync_state(version=self.cluster.sync.version):
+            logger.info("Disabled synchronous replication")
+        self.state_handler.sync_handler.set_synchronous_standby_names(CaseInsensitiveSet())
+
+    def _process_quorum_replication(self) -> None:
+        """Process synchronous replication state when quorum commit is requested.
+
+        Synchronous standbys are registered in two places: ``postgresql.conf`` and DCS. The order of updating them must
+        keep the invariant that ``quorum + sync >= len(set(quorum pool)|set(sync pool))``. This is done using
+        :class:`QuorumStateResolver` that given a current state and set of desired synchronous nodes and replication
+        level outputs changes to DCS and synchronous replication in correct order to reach the desired state.
+        In case any of those steps causes an error we can just bail out and let next iteration rediscover the state
+        and retry necessary transitions.
+        """
+        start_time = time.time()
+
+        min_sync = global_config.min_synchronous_nodes
+        sync_wanted = global_config.synchronous_node_count
+
+        sync = self._maybe_enable_synchronous_mode()
+        if not sync or not sync.leader:
+            return
+
+        leader = sync.leader
+
+        def _check_timeout(offset: float = 0) -> bool:
+            return time.time() - start_time + offset >= self.dcs.loop_wait
+
+        while True:
+            transition = 'break'  # we need define transition value if `QuorumStateResolver` produced no changes
+            sync_state = self.state_handler.sync_handler.current_state(self.cluster)
+            for transition, leader, num, nodes in QuorumStateResolver(leader=leader,
+                                                                      quorum=sync.quorum,
+                                                                      voters=sync.voters,
+                                                                      numsync=sync_state.numsync,
+                                                                      sync=sync_state.sync,
+                                                                      numsync_confirmed=sync_state.numsync_confirmed,
+                                                                      active=sync_state.active,
+                                                                      sync_wanted=sync_wanted,
+                                                                      leader_wanted=self.state_handler.name):
+                if _check_timeout():
+                    return
+
+                if transition == 'quorum':
+                    logger.info("Setting leader to %s, quorum to %d of %d (%s)",
+                                leader, num, len(nodes), ", ".join(sorted(nodes)))
+                    sync = self.dcs.write_sync_state(leader, nodes, num, version=sync.version)
+                    if not sync:
+                        return logger.info('Synchronous replication key updated by someone else.')
+                elif transition == 'sync':
+                    logger.info("Setting synchronous replication to %d of %d (%s)",
+                                num, len(nodes), ", ".join(sorted(nodes)))
+                    # Bump up number of num nodes to meet minimum replication factor. Commits will have to wait until
+                    # we have enough nodes to meet replication target.
+                    if num < min_sync:
+                        logger.warning("Replication factor %d requested, but %d synchronous standbys available."
+                                       " Commits will be delayed.", min_sync + 1, num)
+                        num = min_sync
+                    self.state_handler.sync_handler.set_synchronous_standby_names(nodes, num)
+            if transition != 'restart' or _check_timeout(1):
+                return
+            # synchronous_standby_names was transitioned from empty to non-empty and it may take
+            # some time for nodes to become synchronous. In this case we want to restart state machine
+            # hoping that we can update /sync key earlier than in loop_wait seconds.
+            time.sleep(1)
+            self.state_handler.reset_cluster_info_state(None)
+
+    def _process_multisync_replication(self) -> None:
+        """Process synchronous replication state with one or more sync standbys.
 
         Synchronous standbys are registered in two places postgresql.conf and DCS. The order of updating them must
         be right. The invariant that should be kept is that if a node is primary and sync_standby is set in DCS,
@@ -704,59 +812,109 @@ class Ha(object):
         and then in DCS. When removing, first remove in DCS, then in postgresql.conf. This is so we only consider
         promoting standbys that were guaranteed to be replicating synchronously.
         """
-        if self.is_synchronous_mode():
-            sync = self.cluster.sync
-            if sync.is_empty:
-                # corner case: we need to explicitly enable synchronous mode by updating the
-                # ``/sync`` key with the current leader name and empty members. In opposite case
-                # it will never be automatically enabled if there are not eligible candidates.
-                sync = self.dcs.write_sync_state(self.state_handler.name, None, version=sync.version)
-                if not sync:
-                    return logger.warning("Updating sync state failed")
-                logger.info("Enabled synchronous replication")
 
-            current = CaseInsensitiveSet(sync.members)
-            picked, allow_promote = self.state_handler.sync_handler.current_state(self.cluster)
+        sync = self._maybe_enable_synchronous_mode()
+        if not sync:
+            return
 
-            if picked == current and current != allow_promote:
-                logger.warning('Inconsistent state between synchronous_standby_names = %s and /sync = %s key '
-                               'detected, updating synchronous replication key...', list(allow_promote), list(current))
-                sync = self.dcs.write_sync_state(self.state_handler.name, allow_promote, version=sync.version)
-                if not sync:
-                    return logger.warning("Updating sync state failed")
-                current = CaseInsensitiveSet(sync.members)
+        current_state = self.state_handler.sync_handler.current_state(self.cluster)
+        picked = current_state.active
+        allow_promote = current_state.sync
+        voters = CaseInsensitiveSet(sync.voters)
 
-            if picked != current:
-                # update synchronous standby list in dcs temporarily to point to common nodes in current and picked
-                sync_common = current & allow_promote
-                if sync_common != current:
-                    logger.info("Updating synchronous privilege temporarily from %s to %s",
-                                list(current), list(sync_common))
-                    sync = self.dcs.write_sync_state(self.state_handler.name, sync_common, version=sync.version)
-                    if not sync:
-                        return logger.info('Synchronous replication key updated by someone else.')
+        if picked == voters and voters != allow_promote:
+            logger.warning('Inconsistent state between synchronous_standby_names = %s and /sync = %s key '
+                           'detected, updating synchronous replication key...', list(allow_promote), list(voters))
+            sync = self.dcs.write_sync_state(self.state_handler.name, allow_promote, 0, version=sync.version)
+            if not sync:
+                return logger.warning("Updating sync state failed")
+            voters = CaseInsensitiveSet(sync.voters)
 
-                # When strict mode and no suitable replication connections put "*" to synchronous_standby_names
-                if global_config.is_synchronous_mode_strict and not picked:
-                    picked = CaseInsensitiveSet('*')
-                    logger.warning("No standbys available!")
+        if picked == voters:
+            return
 
-                # Update postgresql.conf and wait 2 secs for changes to become active
-                logger.info("Assigning synchronous standby status to %s", list(picked))
-                self.state_handler.sync_handler.set_synchronous_standby_names(picked)
+        # update synchronous standby list in dcs temporarily to point to common nodes in current and picked
+        sync_common = voters & allow_promote
+        if sync_common != voters:
+            logger.info("Updating synchronous privilege temporarily from %s to %s",
+                        list(voters), list(sync_common))
+            sync = self.dcs.write_sync_state(self.state_handler.name, sync_common, 0, version=sync.version)
+            if not sync:
+                return logger.info('Synchronous replication key updated by someone else.')
 
-                if picked and picked != CaseInsensitiveSet('*') and allow_promote != picked:
-                    # Wait for PostgreSQL to enable synchronous mode and see if we can immediately set sync_standby
-                    time.sleep(2)
-                    _, allow_promote = self.state_handler.sync_handler.current_state(self.cluster)
-                if allow_promote and allow_promote != sync_common:
-                    if not self.dcs.write_sync_state(self.state_handler.name, allow_promote, version=sync.version):
-                        return logger.info("Synchronous replication key updated by someone else")
-                    logger.info("Synchronous standby status assigned to %s", list(allow_promote))
+        # When strict mode and no suitable replication connections put "*" to synchronous_standby_names
+        if global_config.is_synchronous_mode_strict and not picked:
+            picked = CaseInsensitiveSet('*')
+            logger.warning("No standbys available!")
+
+        # Update postgresql.conf and wait 2 secs for changes to become active
+        logger.info("Assigning synchronous standby status to %s", list(picked))
+        self.state_handler.sync_handler.set_synchronous_standby_names(picked)
+
+        if picked and picked != CaseInsensitiveSet('*') and allow_promote != picked:
+            # Wait for PostgreSQL to enable synchronous mode and see if we can immediately set sync_standby
+            time.sleep(2)
+            allow_promote = self.state_handler.sync_handler.current_state(self.cluster).sync
+
+        if allow_promote and allow_promote != sync_common:
+            if self.dcs.write_sync_state(self.state_handler.name, allow_promote, 0, version=sync.version):
+                logger.info("Synchronous standby status assigned to %s", list(allow_promote))
+            else:
+                logger.info("Synchronous replication key updated by someone else")
+
+    def process_sync_replication(self) -> None:
+        """Process synchronous replication behavior on the primary."""
+        if self.is_quorum_commit_mode():
+            # The synchronous_standby_names was adjusted right before promote.
+            # After that, when postgres has become a primary, we need to reflect this change
+            # in the /sync key. Further changes of synchronous_standby_names and /sync key should
+            # be postponed for `loop_wait` seconds, to give a chance to some replicas to start streaming.
+            # In opposite case the /sync key will end up without synchronous nodes.
+            if self.state_handler.is_primary():
+                if self._promote_timestamp == 0 or time.time() - self._promote_timestamp > self.dcs.loop_wait:
+                    self._process_quorum_replication()
+                if self._promote_timestamp == 0:
+                    self._promote_timestamp = time.time()
+        elif self.is_synchronous_mode():
+            self._process_multisync_replication()
         else:
-            if not self.cluster.sync.is_empty and self.dcs.delete_sync_state(version=self.cluster.sync.version):
-                logger.info("Disabled synchronous replication")
-            self.state_handler.sync_handler.set_synchronous_standby_names(CaseInsensitiveSet())
+            self.disable_synchronous_replication()
+
+    def process_sync_replication_prepromote(self) -> bool:
+        """Handle sync replication state before promote.
+
+        If quorum replication is requested, and we can keep syncing to enough nodes satisfying the quorum invariant
+        we can promote immediately and let normal quorum resolver process handle any membership changes later.
+        Otherwise, we will just reset DCS state to ourselves and add replicas as they connect.
+
+        :returns: ``True`` if on success or ``False`` if failed to update /sync key in DCS.
+        """
+        if not self.is_synchronous_mode():
+            self.disable_synchronous_replication()
+            return True
+
+        if self.quorum_commit_mode_is_active():
+            sync = CaseInsensitiveSet(self.cluster.sync.members)
+            numsync = len(sync) - self.cluster.sync.quorum - 1
+            if self.state_handler.name not in sync:  # Node outside voters achieved quorum and got leader
+                numsync += 1
+            else:
+                sync.discard(self.state_handler.name)
+        else:
+            sync = CaseInsensitiveSet()
+            numsync = global_config.min_synchronous_nodes
+
+        if not self.is_quorum_commit_mode() or not self.state_handler.supports_multiple_sync and numsync > 1:
+            sync = CaseInsensitiveSet()
+            numsync = global_config.min_synchronous_nodes
+
+            # Just set ourselves as the authoritative source of truth for now. We don't want to wait for standbys
+            # to connect. We will try finding a synchronous standby in the next cycle.
+            if not self.dcs.write_sync_state(self.state_handler.name, None, 0, version=self.cluster.sync.version):
+                return False
+
+        self.state_handler.sync_handler.set_synchronous_standby_names(sync, numsync)
+        return True
 
     def is_sync_standby(self, cluster: Cluster) -> bool:
         """:returns: `True` if the current node is a synchronous standby."""
@@ -860,15 +1018,10 @@ class Ha(object):
             self.process_sync_replication()
             return message
         else:
-            if self.is_synchronous_mode():
-                # Just set ourselves as the authoritative source of truth for now. We don't want to wait for standbys
-                # to connect. We will try finding a synchronous standby in the next cycle.
-                if not self.dcs.write_sync_state(self.state_handler.name, None, version=self.cluster.sync.version):
-                    # Somebody else updated sync state, it may be due to us losing the lock. To be safe, postpone
-                    # promotion until next cycle. TODO: trigger immediate retry of run_cycle
-                    return 'Postponing promotion because synchronous replication state was updated by somebody else'
-                self.state_handler.sync_handler.set_synchronous_standby_names(
-                    CaseInsensitiveSet('*') if global_config.is_synchronous_mode_strict else CaseInsensitiveSet())
+            if not self.process_sync_replication_prepromote():
+                # Somebody else updated sync state, it may be due to us losing the lock. To be safe,
+                # postpone promotion until next cycle. TODO: trigger immediate retry of run_cycle.
+                return 'Postponing promotion because synchronous replication state was updated by somebody else'
             if self.state_handler.role not in ('master', 'promoted', 'primary'):
                 # reset failsafe state when promote
                 self._failsafe.set_is_active(0)
@@ -884,10 +1037,14 @@ class Ha(object):
             return promote_message
 
     def fetch_node_status(self, member: Member) -> _MemberStatus:
-        """This function perform http get request on member.api_url and fetches its status
-        :returns: `_MemberStatus` object
-        """
+        """Perform http get request on member.api_url to fetch its status.
 
+        Usually this happens during the leader race and we can't afford to wait an indefinite time
+        for a response, therefore the request timeout is hardcoded to 2 seconds, which seems to be a
+        good compromise. The node which is slow to respond is most likely unhealthy.
+
+        :returns: :class:`_MemberStatus` object
+        """
         try:
             response = self.patroni.request(member, timeout=2, retries=0)
             data = response.data.decode('utf-8')
@@ -972,18 +1129,26 @@ class Ha(object):
         return all(results)
 
     def is_lagging(self, wal_position: int) -> bool:
-        """Returns if instance with an wal should consider itself unhealthy to be promoted due to replication lag.
+        """Check if node should consider itself unhealthy to be promoted due to replication lag.
 
         :param wal_position: Current wal position.
 
-        :returns True when node is lagging
+        :returns: ``True`` when node is lagging
         """
         lag = (self.cluster.last_lsn or 0) - wal_position
         return lag > global_config.maximum_lag_on_failover
 
     def _is_healthiest_node(self, members: Collection[Member], check_replication_lag: bool = True) -> bool:
-        """This method tries to determine whether I am healthy enough to became a new leader candidate or not."""
+        """Determine whether the current node is healthy enough to become a new leader candidate.
 
+        :param members: the list of nodes to check against
+        :param check_replication_lag: whether to take the replication lag into account.
+                                      If the lag exceeds configured threshold the node disqualifies itself.
+        :returns: ``True`` if the node is eligible to become the new leader. Since this method is executed
+                  on multiple nodes independently it is possible that multiple nodes could count
+                  themselves as the healthiest because they received/replayed up to the same LSN,
+                  but this is totally fine.
+        """
         my_wal_position = self.state_handler.last_operation()
         if check_replication_lag and self.is_lagging(my_wal_position):
             logger.info('My wal position exceeds maximum replication lag')
@@ -999,8 +1164,26 @@ class Ha(object):
                 logger.info('My timeline %s is behind last known cluster timeline %s', my_timeline, cluster_timeline)
                 return False
 
-        # Prepare list of nodes to run check against
-        members = [m for m in members if m.name != self.state_handler.name and not m.nofailover and m.api_url]
+        if self.quorum_commit_mode_is_active():
+            quorum = self.cluster.sync.quorum
+            voting_set = CaseInsensitiveSet(self.cluster.sync.members)
+        else:
+            quorum = 0
+            voting_set = CaseInsensitiveSet()
+
+        # Prepare list of nodes to run check against. If quorum commit is enabled
+        # we also include members with nofailover tag if they are listed in voters.
+        members = [m for m in members if m.name != self.state_handler.name
+                   and m.api_url and (not m.nofailover or m.name in voting_set)]
+
+        # If there is a quorum active then at least one of the quorum contains latest commit. A quorum member saying
+        # their WAL position is not ahead counts as a vote saying we may become new leader. Note that a node doesn't
+        # have to be a member of the voting set to gather the necessary votes.
+
+        # Regardless of voting, if we observe a node that can become a leader and is ahead, we defer to that node.
+        # This can lead to failure to act on quorum if there is asymmetric connectivity.
+        quorum_votes = 0 if self.state_handler.name in voting_set else -1
+        nodes_ahead = 0
 
         for st in self.fetch_nodes_statuses(members):
             if st.failover_limitation() is None:
@@ -1008,22 +1191,34 @@ class Ha(object):
                     logger.warning('Primary (%s) is still alive', st.member.name)
                     return False
                 if my_wal_position < st.wal_position:
+                    nodes_ahead += 1
                     logger.info('Wal position of %s is ahead of my wal position', st.member.name)
                     # In synchronous mode the former leader might be still accessible and even be ahead of us.
                     # We should not disqualify himself from the leader race in such a situation.
                     if not self.sync_mode_is_active() or not self.cluster.sync.leader_matches(st.member.name):
                         return False
                     logger.info('Ignoring the former leader being ahead of us')
-                if my_wal_position == st.wal_position and self.patroni.failover_priority < st.failover_priority:
-                    # There's a higher priority non-lagging replica
-                    logger.info(
-                        '%s has equally tolerable WAL position and priority %s, while this node has priority %s',
-                        st.member.name,
-                        st.failover_priority,
-                        self.patroni.failover_priority,
-                    )
-                    return False
-        return True
+                elif st.wal_position > 0:  # we want to count votes only from nodes with postgres up and running!
+                    quorum_vote = st.member.name in voting_set
+                    low_priority = my_wal_position == st.wal_position \
+                        and self.patroni.failover_priority < st.failover_priority
+
+                    if low_priority and (not self.sync_mode_is_active() or quorum_vote):
+                        # There's a higher priority non-lagging replica
+                        logger.info(
+                            '%s has equally tolerable WAL position and priority %s, while this node has priority %s',
+                            st.member.name, st.failover_priority, self.patroni.failover_priority)
+                        return False
+
+                    if quorum_vote:
+                        logger.info('Got quorum vote from %s', st.member.name)
+                        quorum_votes += 1
+
+        # When not in quorum commit we just want to return `True`.
+        # In quorum commit the former leader is special and counted healthy even when there are no other nodes.
+        # Otherwise check that the number of votes exceeds the quorum field from the /sync key.
+        return not self.quorum_commit_mode_is_active() or quorum_votes >= quorum\
+            or nodes_ahead == 0 and self.cluster.sync.leader == self.state_handler.name
 
     def is_failover_possible(self, *, cluster_lsn: int = 0, exclude_failover_candidate: bool = False) -> bool:
         """Checks whether any of the cluster members is allowed to promote and is healthy enough for that.
@@ -1083,9 +1278,10 @@ class Ha(object):
                     return None
                 return False
 
-            # in synchronous mode when our name is not in the /sync key
-            # we shouldn't take any action even if the candidate is unhealthy
-            if self.is_synchronous_mode() and not self.cluster.sync.matches(self.state_handler.name, True):
+            # in synchronous mode (except quorum commit!) when our name is not in the
+            # /sync key we shouldn't take any action even if the candidate is unhealthy
+            if self.is_synchronous_mode() and not self.is_quorum_commit_mode()\
+                    and not self.cluster.sync.matches(self.state_handler.name, True):
                 return False
 
             # find specific node and check that it is healthy
@@ -1183,9 +1379,11 @@ class Ha(object):
                 all_known_members += [RemoteMember(name, {'api_url': url}) for name, url in failsafe_members.items()]
         all_known_members += self.cluster.members
 
-        # When in sync mode, only last known primary and sync standby are allowed to promote automatically.
+        # Special handling if synchronous mode was requested and activated (the leader in /sync is not empty)
         if self.sync_mode_is_active():
-            if not self.cluster.sync.matches(self.state_handler.name, True):
+            # In quorum commit mode we allow nodes outside of "voters" to take part in
+            # the leader race. They just need to get enough votes to `reach quorum + 1`.
+            if not self.is_quorum_commit_mode() and not self.cluster.sync.matches(self.state_handler.name, True):
                 return False
             # pick between synchronous candidates so we minimize unnecessary failovers/demotions
             members = {m.name: m for m in all_known_members if self.cluster.sync.matches(m.name, True)}
@@ -2078,8 +2276,11 @@ class Ha(object):
         exclude = [self.state_handler.name] + ([failover.candidate] if failover and exclude_failover_candidate else [])
 
         def is_eligible(node: Member) -> bool:
+            # If quorum commit is requested we want to check all nodes (even not voters),
+            # because they could get enough votes and reach necessary quorum + 1.
             # in synchronous mode we allow failover (not switchover!) to async node
-            if self.sync_mode_is_active() and not self.cluster.sync.matches(node.name)\
+            if self.sync_mode_is_active()\
+                    and not (self.is_quorum_commit_mode() or self.cluster.sync.matches(node.name))\
                     and not (failover and not failover.leader):
                 return False
             # Don't spend time on "nofailover" nodes checking.

patroni/postgresql/__init__.py

@@ -181,6 +181,11 @@ class Postgresql(object):
     def lsn_name(self) -> str:
         return 'lsn' if self._major_version >= 100000 else 'location'
 
+    @property
+    def supports_quorum_commit(self) -> bool:
+        """``True`` if quorum commit is supported by Postgres."""
+        return self._major_version >= 100000
+
     @property
     def supports_multiple_sync(self) -> bool:
         """:returns: `True` if Postgres version supports more than one synchronous node."""

patroni/postgresql/available_parameters/__init__.py

@@ -0,0 +1,58 @@
+import logging
+import sys
+
+from pathlib import Path
+from typing import Iterator
+
+logger = logging.getLogger(__name__)
+
+if sys.version_info < (3, 9):
+    PathLikeObj = Path
+    conf_dir = Path(__file__).parent
+else:
+    from importlib.resources import files
+
+    if sys.version_info < (3, 11):  # pragma: no cover
+        from importlib.abc import Traversable
+    else:  # pragma: no cover
+        from importlib.resources.abc import Traversable
+
+    PathLikeObj = Traversable
+    conf_dir = files(__name__)
+
+
+def get_validator_files() -> Iterator[PathLikeObj]:
+    """Recursively find YAML files from the current package directory.
+
+    :returns: an iterator of :class:`PathLikeObj` objects representing validator files.
+    """
+    return _traversable_walk(conf_dir.iterdir())
+
+
+def _traversable_walk(tvbs: Iterator[PathLikeObj]) -> Iterator[PathLikeObj]:
+    """Recursively walk through Path/Traversable objects, yielding all YAML files in deterministic order.
+
+    :param tvbs: An iterator over :class:`PathLikeObj` objects, where each object is a file or directory
+                 that potentially contains YAML files.
+
+    :yields: :class:`PathLikeObj` objects representing YAML files found during the traversal.
+    """
+    for tvb in _filter_and_sort_files(tvbs):
+        if tvb.is_file():
+            yield tvb
+        elif tvb.is_dir():
+            yield from _traversable_walk(tvb.iterdir())
+
+
+def _filter_and_sort_files(files: Iterator[PathLikeObj]) -> Iterator[PathLikeObj]:
+    """Sort files by name, and filter out non-YAML files and Python files.
+
+    :param files: A list of files and/or directories to be filtered and sorted.
+
+    :yields: filtered and sorted objects.
+    """
+    for file in sorted(files, key=lambda x: x.name):
+        if file.name.lower().endswith((".yml", ".yaml")) or file.is_dir():
+            yield file
+        elif not file.name.lower().endswith((".py", ".pyc")):
+            logger.info("Ignored a non-YAML file found under `%s` directory: `%s`.", __name__.split('.')[-1], file)

patroni/postgresql/config.py

@@ -639,8 +639,7 @@ class ConfigHandler(object):
                 # We are a standby leader and are using a replication slot. Make sure we connect to
                 # the leader of the main cluster (in case more than one host is specified in the
                 # connstr) by adding 'target_session_attrs=read-write' to primary_conninfo.
-                if is_remote_member and 'target_sesions_attrs' not in primary_conninfo and\
-                        self._postgresql.major_version >= 100000:
+                if is_remote_member and ',' in primary_conninfo['host'] and self._postgresql.major_version >= 100000:
                     primary_conninfo['target_session_attrs'] = 'read-write'
             recovery_params['primary_conninfo'] = primary_conninfo
 

patroni/postgresql/rewind.py

@@ -209,9 +209,10 @@ class Rewind(object):
         ret = member.conn_kwargs(auth)
         if not ret.get('dbname'):
             ret['dbname'] = self._postgresql.database
-        # Add target_session_attrs in case more than one hostname is specified
-        # (libpq client-side failover) making sure we hit the primary
-        if 'target_session_attrs' not in ret and self._postgresql.major_version >= 100000:
+        # Add target_session_attrs to make sure we hit the primary.
+        # It is not strictly necessary for starting from PostgreSQL v14, which made it possible
+        # to rewind from standby, but doing it from the real primary is always safer.
+        if self._postgresql.major_version >= 100000:
             ret['target_session_attrs'] = 'read-write'
         return ret
 

patroni/postgresql/sync.py

@@ -3,7 +3,7 @@ import re
 import time
 
 from copy import deepcopy
-from typing import Collection, List, NamedTuple, Tuple, TYPE_CHECKING
+from typing import Collection, List, NamedTuple, Optional, TYPE_CHECKING
 
 from .. import global_config
 from ..collections import CaseInsensitiveDict, CaseInsensitiveSet
@@ -138,7 +138,7 @@ def parse_sync_standby_names(value: str) -> _SSN:
             if len(synclist) == i + 1:  # except the last token
                 raise ValueError("Unparseable synchronous_standby_names value %r: Unexpected token %s %r at %d" %
                                  (value, a_type, a_value, a_pos))
-            elif a_type != 'comma':
+            if a_type != 'comma':
                 raise ValueError("Unparseable synchronous_standby_names value %r: ""Got token %s %r while"
                                  " expecting comma at %d" % (value, a_type, a_value, a_pos))
         elif a_type in {'ident', 'first', 'any'}:
@@ -154,6 +154,26 @@ def parse_sync_standby_names(value: str) -> _SSN:
     return _SSN(sync_type, has_star, num, members)
 
 
+class _SyncState(NamedTuple):
+    """Class representing the current synchronous state.
+
+    :ivar sync_type: possible values: ``off``, ``priority``, ``quorum``
+    :ivar numsync: how many nodes are required to be synchronous (according to ``synchronous_standby_names``).
+                   Is ``0`` if ``synchronous_standby_names`` value is invalid or contains ``*``.
+    :ivar numsync_confirmed: how many nodes are known to be synchronous according to the ``pg_stat_replication`` view.
+                             Only nodes that caught up with the :attr:`SyncHandler._primary_flush_lsn` are counted.
+    :ivar sync: collection of synchronous node names. In case of quorum commit all nodes listed
+                in ``synchronous_standby_names``, otherwise nodes that are confirmed to be synchronous according
+                to the ``pg_stat_replication`` view.
+    :ivar active: collection of node names that are streaming and have no restrictions to become synchronous.
+    """
+    sync_type: str
+    numsync: int
+    numsync_confirmed: int
+    sync: CaseInsensitiveSet
+    active: CaseInsensitiveSet
+
+
 class _Replica(NamedTuple):
     """Class representing a single replica that is eligible to be synchronous.
 
@@ -217,6 +237,8 @@ class _ReplicaList(List[_Replica]):
         # Prefer replicas that are in state ``sync`` and with higher values of ``write``/``flush``/``replay`` LSN.
         self.sort(key=lambda r: (r.sync_state, r.lsn), reverse=True)
 
+        # When checking ``maximum_lag_on_syncnode`` we want to compare with the most
+        # up-to-date replica otherwise with cluster LSN if there is only one replica.
         self.max_lsn = max(self, key=lambda x: x.lsn).lsn if len(self) > 1 else postgresql.last_operation()
 
 
@@ -278,12 +300,22 @@ END;$$""")
             # if standby name is listed in the /sync key we can count it as synchronous, otherwise
             # it becomes really synchronous when sync_state = 'sync' and it is known that it managed to catch up
             if replica.application_name not in self._ready_replicas\
-                    and replica.application_name in self._ssn_data.members\
-                    and (cluster.sync.matches(replica.application_name)
-                         or replica.sync_state == 'sync' and replica.lsn >= self._primary_flush_lsn):
-                self._ready_replicas[replica.application_name] = replica.pid
+                    and replica.application_name in self._ssn_data.members:
+                if global_config.is_quorum_commit_mode:
+                    # When quorum commit is enabled we can't check against cluster.sync because nodes
+                    # are written there when at least one of them caught up with _primary_flush_lsn.
+                    if replica.lsn >= self._primary_flush_lsn\
+                            and (replica.sync_state == 'quorum'
+                                 or (not self._postgresql.supports_quorum_commit
+                                     and replica.sync_state in ('sync', 'potential'))):
+                        self._ready_replicas[replica.application_name] = replica.pid
+                elif cluster.sync.matches(replica.application_name)\
+                        or replica.sync_state == 'sync' and replica.lsn >= self._primary_flush_lsn:
+                    # if standby name is listed in the /sync key we can count it as synchronous, otherwise it becomes
+                    # "really" synchronous when sync_state = 'sync' and we known that it managed to catch up
+                    self._ready_replicas[replica.application_name] = replica.pid
 
-    def current_state(self, cluster: Cluster) -> Tuple[CaseInsensitiveSet, CaseInsensitiveSet]:
+    def current_state(self, cluster: Cluster) -> _SyncState:
         """Find the best candidates to be the synchronous standbys.
 
         Current synchronous standby is always preferred, unless it has disconnected or does not want to be a
@@ -291,51 +323,86 @@ END;$$""")
 
         Standbys are selected based on values from the global configuration:
 
-        - `maximum_lag_on_syncnode`: would help swapping unhealthy sync replica in case if it stops
-          responding (or hung). Please set the value high enough so it won't unncessarily swap sync
-          standbys during high loads. Any value less or equal of 0 keeps the behavior backward compatible.
-          Please note that it will not also swap sync standbys in case where all replicas are hung.
-        - `synchronous_node_count`: controlls how many nodes should be set as synchronous.
+            - ``maximum_lag_on_syncnode``: would help swapping unhealthy sync replica in case it stops
+              responding (or hung). Please set the value high enough, so it won't unnecessarily swap sync
+              standbys during high loads. Any value less or equal to ``0`` keeps the behavior backwards compatible.
+              Please note that it will also not swap sync standbys when all replicas are hung.
 
-        :returns: tuple of candidates :class:`CaseInsensitiveSet` and synchronous standbys :class:`CaseInsensitiveSet`.
+            - ``synchronous_node_count``: controls how many nodes should be set as synchronous.
+
+        :param cluster: current cluster topology from DCS
+
+        :returns: current synchronous replication state as a :class:`_SyncState` object
         """
         self._handle_synchronous_standby_names_change()
 
         replica_list = _ReplicaList(self._postgresql, cluster)
         self._process_replica_readiness(cluster, replica_list)
 
+        active = CaseInsensitiveSet()
+        sync_nodes = CaseInsensitiveSet()
+        numsync_confirmed = 0
+
         sync_node_count = global_config.synchronous_node_count if self._postgresql.supports_multiple_sync else 1
         sync_node_maxlag = global_config.maximum_lag_on_syncnode
 
-        candidates = CaseInsensitiveSet()
-        sync_nodes = CaseInsensitiveSet()
         # Prefer members without nofailover tag. We are relying on the fact that sorts are guaranteed to be stable.
         for replica in sorted(replica_list, key=lambda x: x.nofailover):
             if sync_node_maxlag <= 0 or replica_list.max_lsn - replica.lsn <= sync_node_maxlag:
-                candidates.add(replica.application_name)
-                if replica.sync_state == 'sync' and replica.application_name in self._ready_replicas:
-                    sync_nodes.add(replica.application_name)
-            if len(candidates) >= sync_node_count:
-                break
+                if global_config.is_quorum_commit_mode:
+                    # We do not add nodes with `nofailover` enabled because that reduces availability.
+                    # We need to check LSN quorum only among nodes that are promotable because
+                    # there is a chance that a non-promotable node is ahead of a promotable one.
+                    if not replica.nofailover or len(active) < sync_node_count:
+                        if replica.application_name in self._ready_replicas:
+                            numsync_confirmed += 1
+                        active.add(replica.application_name)
+                else:
+                    active.add(replica.application_name)
+                    if replica.sync_state == 'sync' and replica.application_name in self._ready_replicas:
+                        sync_nodes.add(replica.application_name)
+                        numsync_confirmed += 1
+                    if len(active) >= sync_node_count:
+                        break
 
-        return candidates, sync_nodes
+        if global_config.is_quorum_commit_mode:
+            sync_nodes = CaseInsensitiveSet() if self._ssn_data.has_star else self._ssn_data.members
 
-    def set_synchronous_standby_names(self, sync: Collection[str]) -> None:
-        """Constructs and sets "synchronous_standby_names" GUC value.
+        return _SyncState(
+            self._ssn_data.sync_type,
+            0 if self._ssn_data.has_star else self._ssn_data.num,
+            numsync_confirmed,
+            sync_nodes,
+            active)
+
+    def set_synchronous_standby_names(self, sync: Collection[str], num: Optional[int] = None) -> None:
+        """Constructs and sets ``synchronous_standby_names`` GUC value.
+
+        .. note::
+            standbys in ``synchronous_standby_names`` will be sorted by name.
 
         :param sync: set of nodes to sync to
+        :param num: specifies number of nodes to sync to. The *num* is set only in case if quorum commit is enabled
         """
-        has_asterisk = '*' in sync
+        # Special case. If sync nodes set is empty but requested num of sync nodes >= 1
+        # we want to set synchronous_standby_names to '*'
+        has_asterisk = '*' in sync or num and num >= 1 and not sync
         if has_asterisk:
             sync = ['*']
         else:
-            sync = [quote_ident(x) for x in sync]
+            sync = [quote_ident(x) for x in sorted(sync)]
 
         if self._postgresql.supports_multiple_sync and len(sync) > 1:
-            sync_param = '{0} ({1})'.format(len(sync), ','.join(sync))
+            if num is None:
+                num = len(sync)
+            sync_param = ','.join(sync)
         else:
             sync_param = next(iter(sync), None)
 
+        if global_config.is_quorum_commit_mode and sync or self._postgresql.supports_multiple_sync and len(sync) > 1:
+            prefix = 'ANY ' if global_config.is_quorum_commit_mode and self._postgresql.supports_quorum_commit else ''
+            sync_param = f'{prefix}{num} ({sync_param})'
+
         if not (self._postgresql.config.set_synchronous_standby_names(sync_param)
                 and self._postgresql.state == 'running' and self._postgresql.is_primary()) or has_asterisk:
             return

patroni/postgresql/validator.py

@@ -1,11 +1,11 @@
 import abc
 from copy import deepcopy
 import logging
-import os
 import yaml
 
 from typing import Any, Dict, Iterator, List, MutableMapping, Optional, Tuple, Type, Union
 
+from .available_parameters import get_validator_files, PathLikeObj
 from ..collections import CaseInsensitiveDict, CaseInsensitiveSet
 from ..exceptions import PatroniException
 from ..utils import parse_bool, parse_int, parse_real
@@ -258,10 +258,10 @@ class InvalidGucValidatorsFile(PatroniException):
     """Raised when reading or parsing of a YAML file faces an issue."""
 
 
-def _read_postgres_gucs_validators_file(file: str) -> Dict[str, Any]:
+def _read_postgres_gucs_validators_file(file: PathLikeObj) -> Dict[str, Any]:
     """Read an YAML file and return the corresponding Python object.
 
-    :param file: path to the file to be read. It is expected to be encoded with ``UTF-8``, and to be a YAML document.
+    :param file: path-like object to read from. It is expected to be encoded with ``UTF-8``, and to be a YAML document.
 
     :returns: the YAML content parsed into a Python object. If any issue is faced while reading/parsing the file, then
         return ``None``.
@@ -270,7 +270,7 @@ def _read_postgres_gucs_validators_file(file: str) -> Dict[str, Any]:
         :class:`InvalidGucValidatorsFile`: if faces an issue while reading or parsing *file*.
     """
     try:
-        with open(file, encoding='UTF-8') as stream:
+        with file.open(encoding='UTF-8') as stream:
             return yaml.safe_load(stream)
     except Exception as exc:
         raise InvalidGucValidatorsFile(
@@ -385,21 +385,7 @@ def _load_postgres_gucs_validators() -> None:
                     version_till: null
 
     """
-    conf_dir = os.path.join(
-        os.path.dirname(os.path.abspath(__file__)),
-        'available_parameters',
-    )
-    yaml_files: List[str] = []
-
-    for root, _, files in os.walk(conf_dir):
-        for file in sorted(files):
-            full_path = os.path.join(root, file)
-            if file.lower().endswith(('.yml', '.yaml')):
-                yaml_files.append(full_path)
-            else:
-                logger.info('Ignored a non-YAML file found under `available_parameters` directory: `%s`.', full_path)
-
-    for file in yaml_files:
+    for file in get_validator_files():
         try:
             config: Dict[str, Any] = _read_postgres_gucs_validators_file(file)
         except InvalidGucValidatorsFile as exc:

patroni/quorum.py

@@ -0,0 +1,431 @@
+"""Implement state machine to manage ``synchronous_standby_names`` GUC and ``/sync`` key in DCS."""
+import logging
+
+from typing import Collection, Iterator, NamedTuple, Optional
+
+from .collections import CaseInsensitiveSet
+from .exceptions import PatroniException
+
+logger = logging.getLogger(__name__)
+
+
+class Transition(NamedTuple):
+    """Object describing transition of ``/sync`` or ``synchronous_standby_names`` to the new state.
+
+    .. note::
+        Object attributes represent the new state.
+
+    :ivar transition_type: possible values:
+
+        * ``sync`` - indicates that we needed to update ``synchronous_standby_names``.
+        * ``quorum`` - indicates that we need to update ``/sync`` key in DCS.
+        * ``restart`` - caller should stop iterating over transitions and restart :class:`QuorumStateResolver`.
+    :ivar leader: the new value of the ``leader`` field in the ``/sync`` key.
+    :ivar num: the new value of the synchronous nodes count in ``synchronous_standby_names`` or value of the ``quorum``
+               field in the ``/sync`` key for :attr:`transition_type` values ``sync`` and ``quorum`` respectively.
+    :ivar names: the new value of node names listed in ``synchronous_standby_names`` or value of ``voters``
+                 field in the ``/sync`` key  for :attr:`transition_type` values ``sync`` and ``quorum`` respectively.
+    """
+
+    transition_type: str
+    leader: str
+    num: int
+    names: CaseInsensitiveSet
+
+
+class QuorumError(PatroniException):
+    """Exception indicating that the quorum state is broken."""
+
+
+class QuorumStateResolver:
+    """Calculates a list of state transitions and yields them as :class:`Transition` named tuples.
+
+    Synchronous replication state is set in two places:
+
+    * PostgreSQL configuration sets how many and which nodes are needed for a commit to succeed, abbreviated as
+      ``numsync`` and ``sync`` set here;
+    * DCS contains information about how many and which nodes need to be interrogated to be sure to see an wal position
+      containing latest confirmed commit, abbreviated as ``quorum`` and ``voters`` set.
+
+    .. note::
+        Both of above pairs have the meaning "ANY n OF set".
+
+        The number of nodes needed for commit to succeed, ``numsync``, is also called the replication factor.
+
+    To guarantee zero transaction loss on failover we need to keep the invariant that at all times any subset of
+    nodes that can acknowledge a commit overlaps with any subset of nodes that can achieve quorum to promote a new
+    leader. Given a desired replication factor and a set of nodes able to participate in sync replication there
+    is one optimal state satisfying this condition. Given the node set ``active``, the optimal state is::
+
+        sync = voters = active
+
+        numsync = min(sync_wanted, len(active))
+
+        quorum = len(active) - numsync
+
+    We need to be able to produce a series of state changes that take the system to this desired state from any
+    other arbitrary state given arbitrary changes is node availability, configuration and interrupted transitions.
+
+    To keep the invariant the rule to follow is that when increasing ``numsync`` or ``quorum``, we need to perform the
+    increasing operation first. When decreasing either, the decreasing operation needs to be performed later. In other
+    words:
+
+    * If a user increases ``synchronous_node_count`` configuration, first we increase ``synchronous_standby_names``
+      (``numsync``), then we decrease ``quorum`` field in the ``/sync`` key;
+    * If a user decreases ``synchronous_node_count`` configuration, first we increase ``quorum`` field in the ``/sync``
+      key, then we decrease ``synchronous_standby_names`` (``numsync``).
+
+    Order of adding or removing nodes from ``sync`` and ``voters`` depends on the state of
+    ``synchronous_standby_names``.
+
+    When adding new nodes::
+
+        if ``sync`` (``synchronous_standby_names``) is empty:
+            add new nodes first to ``sync`` and then to ``voters`` when ``numsync_confirmed`` > ``0``.
+        else:
+            add new nodes first to ``voters`` and then to ``sync``.
+
+    When removing nodes::
+
+        if ``sync`` (``synchronous_standby_names``) will become empty after removal:
+            first remove nodes from ``voters`` and then from ``sync``.
+        else:
+            first remove nodes from ``sync`` and then from ``voters``.
+            Make ``voters`` empty if ``numsync_confirmed`` == ``0``.
+
+    :ivar leader: name of the leader, according to the ``/sync`` key.
+    :ivar quorum: ``quorum`` value from the ``/sync`` key, the minimal number of nodes we need see
+                  when doing the leader race.
+    :ivar voters: ``sync_standby`` value from the ``/sync`` key, set of node names we will be
+                  running the leader race against.
+    :ivar numsync: the number of synchronous nodes from the ``synchronous_standby_names``.
+    :ivar sync: set of node names listed in the ``synchronous_standby_names``.
+    :ivar numsync_confirmed: the number of nodes that are confirmed to reach "safe" LSN after they were added to the
+                  ``synchronous_standby_names``.
+    :ivar active: set of node names that are replicating from the primary (according to ``pg_stat_replication``)
+                  and are eligible to be listed in ``synchronous_standby_names``.
+    :ivar sync_wanted: desired number of synchronous nodes (``synchronous_node_count`` from the global configuration).
+    :ivar leader_wanted: the desired leader (could be different from the :attr:`leader` right after a failover).
+    """
+
+    def __init__(self, leader: str, quorum: int, voters: Collection[str],
+                 numsync: int, sync: Collection[str], numsync_confirmed: int,
+                 active: Collection[str], sync_wanted: int, leader_wanted: str) -> None:
+        """Instantiate :class:``QuorumStateResolver`` based on input parameters.
+
+        :param leader: name of the leader, according to the ``/sync`` key.
+        :param quorum: ``quorum`` value from the ``/sync`` key, the minimal number of nodes we need see
+                        when doing the leader race.
+        :param voters: ``sync_standby`` value from the ``/sync`` key, set of node names we will be
+                       running the leader race against.
+        :param numsync: the number of synchronous nodes from the ``synchronous_standby_names``.
+        :param sync: Set of node names listed in the ``synchronous_standby_names``.
+        :param numsync_confirmed: the number of nodes that are confirmed to reach "safe" LSN after
+                                  they were added to the ``synchronous_standby_names``.
+        :param active: set of node names that are replicating from the primary (according to ``pg_stat_replication``)
+                       and are eligible to be listed in ``synchronous_standby_names``.
+        :param sync_wanted: desired number of synchronous nodes
+                            (``synchronous_node_count`` from the global configuration).
+        :param leader_wanted: the desired leader (could be different from the *leader* right after a failover).
+
+        """
+        self.leader = leader
+        self.quorum = quorum
+        self.voters = CaseInsensitiveSet(voters)
+        self.numsync = min(numsync, len(sync))  # numsync can't be bigger than number of listed synchronous nodes.
+        self.sync = CaseInsensitiveSet(sync)
+        self.numsync_confirmed = numsync_confirmed
+        self.active = CaseInsensitiveSet(active)
+        self.sync_wanted = sync_wanted
+        self.leader_wanted = leader_wanted
+
+    def check_invariants(self) -> None:
+        """Checks invariant of ``synchronous_standby_names`` and ``/sync`` key in DCS.
+
+        .. seealso::
+            Check :class:`QuorumStateResolver`'s docstring for more information.
+
+        :raises:
+            :exc:`QuorumError`: in case of broken state"""
+        voters = CaseInsensitiveSet(self.voters | CaseInsensitiveSet([self.leader]))
+        sync = CaseInsensitiveSet(self.sync | CaseInsensitiveSet([self.leader_wanted]))
+
+        # We need to verify that subset of nodes that can acknowledge a commit overlaps
+        # with any subset of nodes that can achieve quorum to promote a new leader.
+        # ``+ 1`` is required because the leader is included in the set.
+        if self.voters and not (len(voters | sync) <= self.quorum + self.numsync + 1):
+            len_nodes = len(voters | sync)
+            raise QuorumError("Quorum and sync not guaranteed to overlap: "
+                              f"nodes {len_nodes} >= quorum {self.quorum} + sync {self.sync} + 1")
+        # unstable cases, we are changing synchronous_standby_names and /sync key
+        # one after another, hence one set is allowed to be a subset of another
+        if not (voters.issubset(sync) or sync.issubset(voters)):
+            voters_only = voters - sync
+            sync_only = sync - voters
+            raise QuorumError(f"Mismatched sets: voter only={voters_only} sync only={sync_only}")
+
+    def quorum_update(self, quorum: int, voters: CaseInsensitiveSet, leader: Optional[str] = None,
+                      adjust_quorum: Optional[bool] = True) -> Iterator[Transition]:
+        """Updates :attr:`quorum`, :attr:`voters` and optionally :attr:`leader` fields.
+
+        :param quorum: the new value for :attr:`quorum`, could be adjusted depending
+                       on values of :attr:`numsync_confirmed` and *adjust_quorum*.
+        :param voters: the new value for :attr:`voters`, could be adjusted if :attr:`numsync_confirmed` == ``0``.
+        :param leader: the new value for :attr:`leader`, optional.
+        :param adjust_quorum: if set to ``True`` the quorum requirement will be increased by the
+                              difference between :attr:`numsync` and :attr:`numsync_confirmed`.
+
+        :yields: the new state of the ``/sync`` key as a :class:`Transition` object.
+
+        :raises:
+            :exc:`QuorumError` in case of invalid data or if the invariant after transition could not be satisfied.
+        """
+        if quorum < 0:
+            raise QuorumError(f'Quorum {quorum} < 0 of ({voters})')
+        if quorum > 0 and quorum >= len(voters):
+            raise QuorumError(f'Quorum {quorum} >= N of ({voters})')
+
+        old_leader = self.leader
+        if leader is not None:  # Change of leader was requested
+            self.leader = leader
+        elif self.numsync_confirmed == 0:
+            # If there are no nodes that known to caught up with the primary we want to reset quorum/voters in /sync key
+            quorum = 0
+            voters = CaseInsensitiveSet()
+        elif adjust_quorum:
+            # It could be that the number of nodes that are known to catch up with the primary is below desired numsync.
+            # We want to increase quorum to guarantee that the sync node will be found during the leader race.
+            quorum += max(self.numsync - self.numsync_confirmed, 0)
+
+        if (self.leader, quorum, voters) == (old_leader, self.quorum, self.voters):
+            if self.voters:
+                return
+            # If transition produces no change of leader/quorum/voters we want to give a hint to
+            # the caller to fetch the new state from the database and restart QuorumStateResolver.
+            yield Transition('restart', self.leader, self.quorum, self.voters)
+
+        self.quorum = quorum
+        self.voters = voters
+        self.check_invariants()
+        logger.debug('quorum %s %s %s', self.leader, self.quorum, self.voters)
+        yield Transition('quorum', self.leader, self.quorum, self.voters)
+
+    def sync_update(self, numsync: int, sync: CaseInsensitiveSet) -> Iterator[Transition]:
+        """Updates :attr:`numsync` and :attr:`sync` fields.
+
+        :param numsync: the new value for :attr:`numsync`.
+        :param sync: the new value for :attr:`sync`:
+
+        :yields: the new state of ``synchronous_standby_names`` as a :class:`Transition` object.
+
+        :raises:
+            :exc:`QuorumError` in case of invalid data or if invariant after transition could not be satisfied
+        """
+        if numsync < 0:
+            raise QuorumError(f'Sync {numsync} < 0 of ({sync})')
+        if numsync > len(sync):
+            raise QuorumError(f'Sync {numsync} > N of ({sync})')
+
+        self.numsync = numsync
+        self.sync = sync
+        self.check_invariants()
+        logger.debug('sync %s %s %s', self.leader, self.numsync, self.sync)
+        yield Transition('sync', self.leader, self.numsync, self.sync)
+
+    def __iter__(self) -> Iterator[Transition]:
+        """Iterate over the transitions produced by :meth:`_generate_transitions`.
+
+        .. note::
+            Merge two transitions of the same type to a single one.
+
+            This is always safe because skipping the first transition is equivalent
+            to no one observing the intermediate state.
+
+        :yields: transitions as :class:`Transition` objects.
+        """
+        transitions = list(self._generate_transitions())
+        for cur_transition, next_transition in zip(transitions, transitions[1:] + [None]):
+            if isinstance(next_transition, Transition) \
+                    and cur_transition.transition_type == next_transition.transition_type:
+                continue
+            yield cur_transition
+            if cur_transition.transition_type == 'restart':
+                break
+
+    def __handle_non_steady_cases(self) -> Iterator[Transition]:
+        """Handle cases when set of transitions produced on previous run was interrupted.
+
+        :yields: transitions as :class:`Transition` objects.
+        """
+        if self.sync < self.voters:
+            logger.debug("Case 1: synchronous_standby_names %s is a subset of DCS state %s", self.sync, self.voters)
+            # Case 1: voters is superset of sync nodes. In the middle of changing voters (quorum).
+            # Evict  dead nodes from voters that are not being synced.
+            remove_from_voters = self.voters - (self.sync | self.active)
+            if remove_from_voters:
+                yield from self.quorum_update(
+                    quorum=len(self.voters) - len(remove_from_voters) - self.numsync,
+                    voters=CaseInsensitiveSet(self.voters - remove_from_voters),
+                    adjust_quorum=not (self.sync - self.active))
+            # Start syncing to nodes that are in voters and alive
+            add_to_sync = (self.voters & self.active) - self.sync
+            if add_to_sync:
+                yield from self.sync_update(self.numsync, CaseInsensitiveSet(self.sync | add_to_sync))
+        elif self.sync > self.voters:
+            logger.debug("Case 2: synchronous_standby_names %s is a superset of DCS state %s", self.sync, self.voters)
+            # Case 2: sync is superset of voters nodes. In the middle of changing replication factor (sync).
+            # Add to voters nodes that are already synced and active
+            add_to_voters = (self.sync - self.voters) & self.active
+            if add_to_voters:
+                voters = CaseInsensitiveSet(self.voters | add_to_voters)
+                yield from self.quorum_update(len(voters) - self.numsync, voters)
+            # Remove from sync nodes that are dead
+            remove_from_sync = self.sync - self.voters
+            if remove_from_sync:
+                yield from self.sync_update(
+                    numsync=min(self.numsync, len(self.sync) - len(remove_from_sync)),
+                    sync=CaseInsensitiveSet(self.sync - remove_from_sync))
+
+        # After handling these two cases voters and sync must match.
+        assert self.voters == self.sync
+
+        safety_margin = self.quorum + min(self.numsync, self.numsync_confirmed) - len(self.voters | self.sync)
+        if safety_margin > 0:  # In the middle of changing replication factor.
+            if self.numsync > self.sync_wanted:
+                numsync = max(self.sync_wanted, len(self.voters) - self.quorum)
+                logger.debug('Case 3: replication factor %d is bigger than needed %d', self.numsync, numsync)
+                yield from self.sync_update(numsync, self.sync)
+            else:
+                quorum = len(self.sync) - self.numsync
+                logger.debug('Case 4: quorum %d is bigger than needed %d', self.quorum, quorum)
+                yield from self.quorum_update(quorum, self.voters)
+        else:
+            safety_margin = self.quorum + self.numsync - len(self.voters | self.sync)
+            if self.numsync == self.sync_wanted and safety_margin > 0 and self.numsync > self.numsync_confirmed:
+                yield from self.quorum_update(len(self.sync) - self.numsync, self.voters)
+
+    def __remove_gone_nodes(self) -> Iterator[Transition]:
+        """Remove inactive nodes from ``synchronous_standby_names`` and from ``/sync`` key.
+
+        :yields: transitions as :class:`Transition` objects.
+        """
+        to_remove = self.sync - self.active
+        if to_remove and self.sync == to_remove:
+            logger.debug("Removing nodes: %s", to_remove)
+            yield from self.quorum_update(0, CaseInsensitiveSet(), adjust_quorum=False)
+            yield from self.sync_update(0, CaseInsensitiveSet())
+        elif to_remove:
+            logger.debug("Removing nodes: %s", to_remove)
+            can_reduce_quorum_by = self.quorum
+            # If we can reduce quorum size try to do so first
+            if can_reduce_quorum_by:
+                # Pick nodes to remove by sorted order to provide deterministic behavior for tests
+                remove = CaseInsensitiveSet(sorted(to_remove, reverse=True)[:can_reduce_quorum_by])
+                sync = CaseInsensitiveSet(self.sync - remove)
+                # when removing nodes from sync we can safely increase numsync if requested
+                numsync = min(self.sync_wanted, len(sync)) if self.sync_wanted > self.numsync else self.numsync
+                yield from self.sync_update(numsync, sync)
+                voters = CaseInsensitiveSet(self.voters - remove)
+                to_remove &= self.sync
+                yield from self.quorum_update(len(voters) - self.numsync, voters,
+                                              adjust_quorum=not to_remove)
+            if to_remove:
+                assert self.quorum == 0
+                numsync = self.numsync - len(to_remove)
+                sync = CaseInsensitiveSet(self.sync - to_remove)
+                voters = CaseInsensitiveSet(self.voters - to_remove)
+                sync_decrease = numsync - min(self.sync_wanted, len(sync))
+                quorum = min(sync_decrease, len(voters) - 1) if sync_decrease else 0
+                yield from self.quorum_update(quorum, voters, adjust_quorum=False)
+                yield from self.sync_update(numsync, sync)
+
+    def __add_new_nodes(self) -> Iterator[Transition]:
+        """Add new active nodes to ``synchronous_standby_names`` and to ``/sync`` key.
+
+        :yields: transitions as :class:`Transition` objects.
+        """
+        to_add = self.active - self.sync
+        if to_add:
+            # First get to requested replication factor
+            logger.debug("Adding nodes: %s", to_add)
+            sync_wanted = min(self.sync_wanted, len(self.sync | to_add))
+            increase_numsync_by = sync_wanted - self.numsync
+            if increase_numsync_by > 0:
+                if self.sync:
+                    add = CaseInsensitiveSet(sorted(to_add)[:increase_numsync_by])
+                    increase_numsync_by = len(add)
+                else:  # there is only the leader
+                    add = to_add  # and it is safe to add all nodes at once if sync is empty
+                yield from self.sync_update(self.numsync + increase_numsync_by, CaseInsensitiveSet(self.sync | add))
+                voters = CaseInsensitiveSet(self.voters | add)
+                yield from self.quorum_update(len(voters) - sync_wanted, voters)
+                to_add -= self.sync
+            if to_add:
+                voters = CaseInsensitiveSet(self.voters | to_add)
+                yield from self.quorum_update(len(voters) - sync_wanted, voters,
+                                              adjust_quorum=sync_wanted > self.numsync_confirmed)
+                yield from self.sync_update(sync_wanted, CaseInsensitiveSet(self.sync | to_add))
+
+    def __handle_replication_factor_change(self) -> Iterator[Transition]:
+        """Handle change of the replication factor (:attr:`sync_wanted`, aka ``synchronous_node_count``).
+
+        :yields: transitions as :class:`Transition` objects.
+        """
+        # Apply requested replication factor change
+        sync_increase = min(self.sync_wanted, len(self.sync)) - self.numsync
+        if sync_increase > 0:
+            # Increase replication factor
+            logger.debug("Increasing replication factor to %s", self.numsync + sync_increase)
+            yield from self.sync_update(self.numsync + sync_increase, self.sync)
+            yield from self.quorum_update(len(self.voters) - self.numsync, self.voters)
+        elif sync_increase < 0:
+            # Reduce replication factor
+            logger.debug("Reducing replication factor to %s", self.numsync + sync_increase)
+            if self.quorum - sync_increase < len(self.voters):
+                yield from self.quorum_update(len(self.voters) - self.numsync - sync_increase, self.voters,
+                                              adjust_quorum=self.sync_wanted > self.numsync_confirmed)
+            yield from self.sync_update(self.numsync + sync_increase, self.sync)
+
+    def _generate_transitions(self) -> Iterator[Transition]:
+        """Produce a set of changes to safely transition from the current state to the desired.
+
+        :yields: transitions as :class:`Transition` objects.
+        """
+        logger.debug("Quorum state: leader %s quorum %s, voters %s, numsync %s, sync %s, "
+                     "numsync_confirmed %s, active %s, sync_wanted %s leader_wanted %s",
+                     self.leader, self.quorum, self.voters, self.numsync, self.sync,
+                     self.numsync_confirmed, self.active, self.sync_wanted, self.leader_wanted)
+        try:
+            if self.leader_wanted != self.leader:  # failover
+                voters = (self.voters - CaseInsensitiveSet([self.leader_wanted])) | CaseInsensitiveSet([self.leader])
+                if not self.sync:
+                    # If sync is empty we need to update synchronous_standby_names first
+                    numsync = len(voters) - self.quorum
+                    yield from self.sync_update(numsync, CaseInsensitiveSet(voters))
+                # If leader changed we need to add the old leader to quorum (voters)
+                yield from self.quorum_update(self.quorum, CaseInsensitiveSet(voters), self.leader_wanted)
+                # right after promote there could be no replication connections yet
+                if not self.sync & self.active:
+                    return  # give another loop_wait seconds for replicas to reconnect before removing them from quorum
+            else:
+                self.check_invariants()
+        except QuorumError as e:
+            logger.warning('%s', e)
+            yield from self.quorum_update(len(self.sync) - self.numsync, self.sync)
+
+        assert self.leader == self.leader_wanted
+
+        # numsync_confirmed could be 0 after restart/failover, we will calculate it from quorum
+        if self.numsync_confirmed == 0 and self.sync & self.active:
+            self.numsync_confirmed = min(len(self.sync & self.active), len(self.voters) - self.quorum)
+            logger.debug('numsync_confirmed=0, adjusting it to %d', self.numsync_confirmed)
+
+        yield from self.__handle_non_steady_cases()
+
+        # We are in a steady state point. Find if desired state is different and act accordingly.
+
+        yield from self.__remove_gone_nodes()
+
+        yield from self.__add_new_nodes()
+
+        yield from self.__handle_replication_factor_change()

patroni/scripts/barman/__init__.py

@@ -0,0 +1 @@
+"""Create :mod:`patroni.scripts.barman`."""

patroni/scripts/barman/cli.py

@@ -0,0 +1,240 @@
+#!/usr/bin/env python
+
+"""Perform operations on Barman through ``pg-backup-api``.
+
+The actual operations are implemented by separate modules. This module only
+builds the CLI that makes an interface with the actual commands.
+
+.. note::
+    See :class:ExitCode` for possible exit codes of this main script.
+"""
+
+from argparse import ArgumentParser
+from enum import IntEnum
+import logging
+import sys
+
+from .config_switch import run_barman_config_switch
+from .recover import run_barman_recover
+from .utils import ApiNotOk, PgBackupApi, set_up_logging
+
+
+class ExitCode(IntEnum):
+    """Possible exit codes of this script.
+
+    :cvar NO_COMMAND: if no sub-command of ``patroni_barman`` application has
+        been selected by the user.
+    :cvar API_NOT_OK: ``pg-backup-api`` status is not ``OK``.
+    """
+
+    NO_COMMAND = -1
+    API_NOT_OK = -2
+
+
+def main() -> None:
+    """Entry point of ``patroni_barman`` application.
+
+    Implements the parser for the application and for its sub-commands.
+
+    The script exit code may be one of:
+
+    * :attr:`ExitCode.NO_COMMAND`: if no sub-command was specified in the
+        ``patroni_barman`` call;
+    * :attr:`ExitCode.API_NOT_OK`: if ``pg-backup-api`` is not correctly up and
+        running;
+    * Value returned by :func:`~patroni.scripts.barman.config_switch.run_barman_config_switch`,
+        if running ``patroni_barman config-switch``;
+    * Value returned by :func:`~patroni.scripts.barman.recover.run_barman_recover`,
+        if running ``patroni_barman recover``.
+
+    The called sub-command is expected to exit execution once finished using
+    its own set of exit codes.
+    """
+    parser = ArgumentParser(
+        description=(
+            "Wrapper application for pg-backup-api. Communicate with the API "
+            "running at the given URL to perform remote Barman operations."
+        ),
+    )
+    parser.add_argument(
+        "--api-url",
+        type=str,
+        required=True,
+        help="URL to reach the pg-backup-api, e.g. 'http://localhost:7480'",
+        dest="api_url",
+    )
+    parser.add_argument(
+        "--cert-file",
+        type=str,
+        required=False,
+        help="Certificate to authenticate against the API, if required.",
+        dest="cert_file",
+    )
+    parser.add_argument(
+        "--key-file",
+        type=str,
+        required=False,
+        help="Certificate key to authenticate against the API, if required.",
+        dest="key_file",
+    )
+    parser.add_argument(
+        "--retry-wait",
+        type=int,
+        required=False,
+        default=2,
+        help="How long in seconds to wait before retrying a failed "
+             "pg-backup-api request (default: '%(default)s')",
+        dest="retry_wait",
+    )
+    parser.add_argument(
+        "--max-retries",
+        type=int,
+        required=False,
+        default=5,
+        help="Maximum number of retries when receiving malformed responses "
+             "from the pg-backup-api (default: '%(default)s')",
+        dest="max_retries",
+    )
+    parser.add_argument(
+        "--log-file",
+        type=str,
+        required=False,
+        help="File where to log messages produced by this application, if any.",
+        dest="log_file",
+    )
+
+    subparsers = parser.add_subparsers(title="Sub-commands")
+
+    recover_parser = subparsers.add_parser(
+        "recover",
+        help="Remote 'barman recover'",
+        description="Restore a Barman backup of a given Barman server"
+    )
+    recover_parser.add_argument(
+        "--barman-server",
+        type=str,
+        required=True,
+        help="Name of the Barman server from which to restore the backup.",
+        dest="barman_server",
+    )
+    recover_parser.add_argument(
+        "--backup-id",
+        type=str,
+        required=False,
+        default="latest",
+        help="ID of the Barman backup to be restored. You can use any value "
+             "supported by 'barman recover' command "
+             "(default: '%(default)s')",
+        dest="backup_id",
+    )
+    recover_parser.add_argument(
+        "--ssh-command",
+        type=str,
+        required=True,
+        help="Value to be passed as '--remote-ssh-command' to 'barman recover'.",
+        dest="ssh_command",
+    )
+    recover_parser.add_argument(
+        "--data-directory",
+        "--datadir",
+        type=str,
+        required=True,
+        help="Destination path where to restore the barman backup in the "
+             "local host.",
+        dest="data_directory",
+    )
+    recover_parser.add_argument(
+        "--loop-wait",
+        type=int,
+        required=False,
+        default=10,
+        help="How long to wait before checking again the status of the "
+             "recovery process, in seconds. Use higher values if your "
+             "recovery is expected to take long (default: '%(default)s')",
+        dest="loop_wait",
+    )
+    recover_parser.set_defaults(func=run_barman_recover)
+
+    config_switch_parser = subparsers.add_parser(
+        "config-switch",
+        help="Remote 'barman config-switch'",
+        description="Switch the configuration of a given Barman server. "
+                    "Intended to be used as a 'on_role_change' callback."
+    )
+    config_switch_parser.add_argument(
+        "action",
+        type=str,
+        choices=["on_role_change"],
+        help="Name of the callback (automatically filled by Patroni)",
+    )
+    config_switch_parser.add_argument(
+        "role",
+        type=str,
+        choices=["master", "primary", "promoted", "standby_leader", "replica",
+                 "demoted"],
+        help="Name of the new role of this node (automatically filled by "
+             "Patroni)",
+    )
+    config_switch_parser.add_argument(
+        "cluster",
+        type=str,
+        help="Name of the Patroni cluster involved in the callback "
+             "(automatically filled by Patroni)",
+    )
+    config_switch_parser.add_argument(
+        "--barman-server",
+        type=str,
+        required=True,
+        help="Name of the Barman server which config is to be switched.",
+        dest="barman_server",
+    )
+    group = config_switch_parser.add_mutually_exclusive_group(required=True)
+    group.add_argument(
+        "--barman-model",
+        type=str,
+        help="Name of the Barman config model to be applied to the server.",
+        dest="barman_model",
+    )
+    group.add_argument(
+        "--reset",
+        action="store_true",
+        help="Unapply the currently active model for the server, if any.",
+        dest="reset",
+    )
+    config_switch_parser.add_argument(
+        "--switch-when",
+        type=str,
+        required=True,
+        default="promoted",
+        choices=["promoted", "demoted", "always"],
+        help="Controls under which circumstances the 'on_role_change' callback "
+             "should actually switch config in Barman. 'promoted' means the "
+             "'role' is either 'master', 'primary' or 'promoted'. 'demoted' "
+             "means the 'role' is either 'replica' or 'demoted' "
+             "(default: '%(default)s')",
+        dest="switch_when",
+    )
+    config_switch_parser.set_defaults(func=run_barman_config_switch)
+
+    args, _ = parser.parse_known_args()
+
+    set_up_logging(args.log_file)
+
+    if not hasattr(args, "func"):
+        parser.print_help()
+        sys.exit(ExitCode.NO_COMMAND)
+
+    api = None
+
+    try:
+        api = PgBackupApi(args.api_url, args.cert_file, args.key_file,
+                          args.retry_wait, args.max_retries)
+    except ApiNotOk as exc:
+        logging.error("pg-backup-api is not working: %r", exc)
+        sys.exit(ExitCode.API_NOT_OK)
+
+    sys.exit(args.func(api, args))
+
+
+if __name__ == "__main__":
+    main()

patroni/scripts/barman/config_switch.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+
+"""Implements ``patroni_barman config-switch`` sub-command.
+
+Apply a Barman configuration model through ``pg-backup-api``.
+
+This sub-command is specially useful as a ``on_role_change`` callback to change
+Barman configuration in response to failovers and switchovers. Check the output
+of ``--help`` to understand the parameters supported by the sub-command.
+
+It requires that you have previously configured a Barman server and Barman
+config models, and that you have ``pg-backup-api`` configured and running in
+the same host as Barman.
+
+Refer to :class:`ExitCode` for possible exit codes of this sub-command.
+"""
+from argparse import Namespace
+from enum import IntEnum
+import logging
+import time
+from typing import Optional, TYPE_CHECKING
+
+from .utils import OperationStatus, RetriesExceeded
+
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .utils import PgBackupApi
+
+
+class ExitCode(IntEnum):
+    """Possible exit codes of this script.
+
+    :cvar CONFIG_SWITCH_DONE: config switch was successfully performed.
+    :cvar CONFIG_SWITCH_SKIPPED: if the execution was skipped because of not
+        matching user expectations.
+    :cvar CONFIG_SWITCH_FAILED: config switch faced an issue.
+    :cvar HTTP_ERROR: an error has occurred while communicating with
+        ``pg-backup-api``
+    :cvar INVALID_ARGS: an invalid set of arguments has been given to the
+        operation.
+    """
+
+    CONFIG_SWITCH_DONE = 0
+    CONFIG_SWITCH_SKIPPED = 1
+    CONFIG_SWITCH_FAILED = 2
+    HTTP_ERROR = 3
+    INVALID_ARGS = 4
+
+
+def _should_skip_switch(args: Namespace) -> bool:
+    """Check if we should skip the config switch operation.
+
+    :param args: arguments received from the command-line of
+        ``patroni_barman config-switch`` command.
+
+    :returns: if the operation should be skipped.
+    """
+    if args.switch_when == "promoted":
+        return args.role not in {"master", "primary", "promoted"}
+    if args.switch_when == "demoted":
+        return args.role not in {"replica", "demoted"}
+    return False
+
+
+def _switch_config(api: "PgBackupApi", barman_server: str,
+                   barman_model: Optional[str], reset: Optional[bool]) -> int:
+    """Switch configuration of Barman server through ``pg-backup-api``.
+
+    .. note::
+        If requests to ``pg-backup-api`` fail recurrently or we face HTTP
+        errors, then exit with :attr:`ExitCode.HTTP_ERROR`.
+
+    :param api: a :class:`PgBackupApi` instance to handle communication with
+        the API.
+    :param barman_server: name of the Barman server which config is to be
+        switched.
+    :param barman_model: name of the Barman model to be applied to the server,
+        if any.
+    :param reset: ``True`` if you would like to unapply the currently active
+        model for the server, if any.
+
+    :returns: the return code to be used when exiting the ``patroni_barman``
+        application. Refer to :class:`ExitCode`.
+    """
+    operation_id = None
+
+    try:
+        operation_id = api.create_config_switch_operation(
+            barman_server,
+            barman_model,
+            reset,
+        )
+    except RetriesExceeded as exc:
+        logging.error("An issue was faced while trying to create a config "
+                      "switch operation: %r", exc)
+        return ExitCode.HTTP_ERROR
+
+    logging.info("Created the config switch operation with ID %s",
+                 operation_id)
+
+    status = None
+
+    while True:
+        try:
+            status = api.get_operation_status(barman_server, operation_id)
+        except RetriesExceeded:
+            logging.error("Maximum number of retries exceeded, exiting.")
+            return ExitCode.HTTP_ERROR
+
+        if status != OperationStatus.IN_PROGRESS:
+            break
+
+        logging.info("Config switch operation %s is still in progress",
+                     operation_id)
+        time.sleep(5)
+
+    if status == OperationStatus.DONE:
+        logging.info("Config switch operation finished successfully.")
+        return ExitCode.CONFIG_SWITCH_DONE
+    else:
+        logging.error("Config switch operation failed.")
+        return ExitCode.CONFIG_SWITCH_FAILED
+
+
+def run_barman_config_switch(api: "PgBackupApi", args: Namespace) -> int:
+    """Run a remote ``barman config-switch`` through the ``pg-backup-api``.
+
+    :param api: a :class:`PgBackupApi` instance to handle communication with
+        the API.
+    :param args: arguments received from the command-line of
+        ``patroni_barman config-switch`` command.
+
+    :returns: the return code to be used when exiting the ``patroni_barman``
+        application. Refer to :class:`ExitCode`.
+    """
+    if _should_skip_switch(args):
+        logging.info("Config switch operation was skipped (role=%s, "
+                     "switch_when=%s).", args.role, args.switch_when)
+        return ExitCode.CONFIG_SWITCH_SKIPPED
+
+    if not bool(args.barman_model) ^ bool(args.reset):
+        logging.error("One, and only one among 'barman_model' ('%s') and "
+                      "'reset' ('%s') should be given", args.barman_model, args.reset)
+        return ExitCode.INVALID_ARGS
+
+    return _switch_config(api, args.barman_server, args.barman_model, args.reset)

patroni/scripts/barman/recover.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+
+"""Implements ``patroni_barman recover`` sub-command.
+
+Restore a Barman backup to the local node through ``pg-backup-api``.
+
+This sub-command can be used both as a custom bootstrap method, and as a custom
+create replica method. Check the output of ``--help`` to understand the
+parameters supported by the sub-command. ``--datadir`` is a special parameter
+and it is automatically filled by Patroni in both cases.
+
+It requires that you have previously configured a Barman server, and that you
+have ``pg-backup-api`` configured and running in the same host as Barman.
+
+Refer to :class:`ExitCode` for possible exit codes of this sub-command.
+"""
+from argparse import Namespace
+from enum import IntEnum
+import logging
+import time
+from typing import TYPE_CHECKING
+
+from .utils import OperationStatus, RetriesExceeded
+
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .utils import PgBackupApi
+
+
+class ExitCode(IntEnum):
+    """Possible exit codes of this script.
+
+    :cvar RECOVERY_DONE: backup was successfully restored.
+    :cvar RECOVERY_FAILED: recovery of the backup faced an issue.
+    :cvar HTTP_ERROR: an error has occurred while communicating with
+        ``pg-backup-api``
+    """
+
+    RECOVERY_DONE = 0
+    RECOVERY_FAILED = 1
+    HTTP_ERROR = 2
+
+
+def _restore_backup(api: "PgBackupApi", barman_server: str, backup_id: str,
+                    ssh_command: str, data_directory: str,
+                    loop_wait: int) -> int:
+    """Restore the configured Barman backup through ``pg-backup-api``.
+
+    .. note::
+        If requests to ``pg-backup-api`` fail recurrently or we face HTTP
+        errors, then exit with :attr:`ExitCode.HTTP_ERROR`.
+
+    :param api: a :class:`PgBackupApi` instance to handle communication with
+        the API.
+    :param barman_server: name of the Barman server which backup is to be
+        restored.
+    :param backup_id: ID of the backup from the Barman server.
+    :param ssh_command: SSH command to connect from the Barman host to the
+        target host.
+    :param data_directory: path to the Postgres data directory where to restore
+        the backup in.
+    :param loop_wait: how long in seconds to wait before checking again the
+        status of the recovery process. Higher values are useful for backups
+        that are expected to take longer to restore.
+
+    :returns: the return code to be used when exiting the ``patroni_barman``
+        application. Refer to :class:`ExitCode`.
+    """
+    operation_id = None
+
+    try:
+        operation_id = api.create_recovery_operation(
+            barman_server,
+            backup_id,
+            ssh_command,
+            data_directory,
+        )
+    except RetriesExceeded as exc:
+        logging.error("An issue was faced while trying to create a recovery "
+                      "operation: %r", exc)
+        return ExitCode.HTTP_ERROR
+
+    logging.info("Created the recovery operation with ID %s", operation_id)
+
+    status = None
+
+    while True:
+        try:
+            status = api.get_operation_status(barman_server, operation_id)
+        except RetriesExceeded:
+            logging.error("Maximum number of retries exceeded, exiting.")
+            return ExitCode.HTTP_ERROR
+
+        if status != OperationStatus.IN_PROGRESS:
+            break
+
+        logging.info("Recovery operation %s is still in progress",
+                     operation_id)
+        time.sleep(loop_wait)
+
+    if status == OperationStatus.DONE:
+        logging.info("Recovery operation finished successfully.")
+        return ExitCode.RECOVERY_DONE
+    else:
+        logging.error("Recovery operation failed.")
+        return ExitCode.RECOVERY_FAILED
+
+
+def run_barman_recover(api: "PgBackupApi", args: Namespace) -> int:
+    """Run a remote ``barman recover`` through the ``pg-backup-api``.
+
+    :param api: a :class:`PgBackupApi` instance to handle communication with
+        the API.
+    :param args: arguments received from the command-line of
+        ``patroni_barman recover`` command.
+
+    :returns: the return code to be used when exiting the ``patroni_barman``
+        application. Refer to :class:`ExitCode`.
+    """
+    return _restore_backup(api, args.barman_server, args.backup_id,
+                           args.ssh_command, args.data_directory,
+                           args.loop_wait)

patroni/scripts/barman/utils.py

@@ -0,0 +1,308 @@
+#!/usr/bin/env python
+
+"""Utilitary stuff to be used by Barman related scripts."""
+
+from enum import IntEnum
+import json
+import logging
+from typing import Any, Callable, Dict, Optional, Tuple, Type, Union
+import time
+from urllib.parse import urljoin
+
+from urllib3 import PoolManager
+from urllib3.exceptions import MaxRetryError
+from urllib3.response import HTTPResponse
+
+
+class RetriesExceeded(Exception):
+    """Maximum number of retries exceeded."""
+
+
+def retry(exceptions: Union[Type[Exception], Tuple[Type[Exception], ...]]) \
+        -> Any:
+    """Retry an operation n times if expected *exceptions* are faced.
+
+    .. note::
+        Should be used as a decorator of a class' method as it expects the
+        first argument to be a class instance.
+
+        The class which method is going to be decorated should contain a couple
+        attributes:
+
+        * ``max_retries``: maximum retry attempts before failing;
+        * ``retry_wait``: how long in seconds to wait before retrying.
+
+    :param exceptions: exceptions that could trigger a retry attempt.
+
+    :raises:
+        :exc:`RetriesExceeded`: if the maximum number of attempts has been
+            exhausted.
+    """
+    def decorator(func: Callable[..., Any]) -> Any:
+        def inner_func(instance: object, *args: Any, **kwargs: Any) -> Any:
+            times: int = getattr(instance, "max_retries")
+            retry_wait: int = getattr(instance, "retry_wait")
+            method_name = f"{instance.__class__.__name__}.{func.__name__}"
+
+            attempt = 1
+
+            while attempt <= times:
+                try:
+                    return func(instance, *args, **kwargs)
+                except exceptions as exc:
+                    logging.warning("Attempt %d of %d on method %s failed "
+                                    "with %r.",
+                                    attempt, times, method_name, exc)
+                    attempt += 1
+
+                time.sleep(retry_wait)
+
+            raise RetriesExceeded("Maximum number of retries exceeded for "
+                                  f"method {method_name}.")
+        return inner_func
+    return decorator
+
+
+def set_up_logging(log_file: Optional[str] = None) -> None:
+    """Set up logging to file, if *log_file* is given, otherwise to console.
+
+    :param log_file: file where to log messages, if any.
+    """
+    logging.basicConfig(filename=log_file, level=logging.INFO,
+                        format="%(asctime)s %(levelname)s: %(message)s")
+
+
+class OperationStatus(IntEnum):
+    """Possible status of ``pg-backup-api`` operations.
+
+    :cvar IN_PROGRESS: the operation is still ongoing.
+    :cvar FAILED: the operation failed.
+    :cvar DONE: the operation finished successfully.
+    """
+
+    IN_PROGRESS = 0
+    FAILED = 1
+    DONE = 2
+
+
+class ApiNotOk(Exception):
+    """The ``pg-backup-api`` is not currently up and running."""
+
+
+class PgBackupApi:
+    """Facilities for communicating with the ``pg-backup-api``.
+
+    :ivar api_url: base URL to reach the ``pg-backup-api``.
+    :ivar cert_file: certificate to authenticate against the ``pg-backup-api``,
+        if required.
+    :ivar key_file: certificate key to authenticate against the
+        ``pg-backup-api``, if required.
+    :ivar retry_wait: how long in seconds to wait before retrying a failed
+        request to the ``pg-backup-api``.
+    :ivar max_retries: maximum number of retries when ``pg-backup-api`` returns
+        malformed responses.
+    :ivar http: a HTTP pool manager for performing web requests.
+    """
+
+    def __init__(self, api_url: str, cert_file: Optional[str],
+                 key_file: Optional[str], retry_wait: int,
+                 max_retries: int) -> None:
+        """Create a new instance of :class:`BarmanRecover`.
+
+        Make sure the ``pg-backup-api`` is reachable and running fine.
+
+        .. note::
+            When using any method which send requests to the API, be aware that
+            they might raise :exc:`RetriesExceeded` upon HTTP request errors.
+
+            Similarly, when instantiating this class you may face an
+            :exc:`ApiNotOk`, if the API is down or returns a bogus status.
+
+        :param api_url: base URL to reach the ``pg-backup-api``.
+        :param cert_file: certificate to authenticate against the
+            ``pg-backup-api``, if required.
+        :param key_file: certificate key to authenticate against the
+            ``pg-backup-api``, if required.
+        :param retry_wait: how long in seconds to wait before retrying a failed
+            request to the ``pg-backup-api``.
+        :param max_retries: maximum number of retries when ``pg-backup-api``
+            returns malformed responses.
+        """
+        self.api_url = api_url
+        self.cert_file = cert_file
+        self.key_file = key_file
+        self.retry_wait = retry_wait
+        self.max_retries = max_retries
+        self._http = PoolManager(cert_file=cert_file, key_file=key_file)
+        self._ensure_api_ok()
+
+    def _build_full_url(self, url_path: str) -> str:
+        """Build the full URL by concatenating *url_path* with the base URL.
+
+        :param url_path: path to be accessed in the ``pg-backup-api``.
+
+        :returns: the full URL after concatenating.
+        """
+        return urljoin(self.api_url, url_path)
+
+    @staticmethod
+    def _deserialize_response(response: HTTPResponse) -> Any:
+        """Retrieve body from *response* as a deserialized JSON object.
+
+        :param response: response from which JSON body will be deserialized.
+
+        :returns: the deserialized JSON body.
+        """
+        return json.loads(response.data.decode("utf-8"))
+
+    @staticmethod
+    def _serialize_request(body: Any) -> Any:
+        """Serialize a request body.
+
+        :param body: content of the request body to be serialized.
+
+        :returns: the serialized request body.
+        """
+        return json.dumps(body).encode("utf-8")
+
+    def _get_request(self, url_path: str) -> Any:
+        """Perform a ``GET`` request to *url_path*.
+
+        :param url_path: URL to perform the ``GET`` request against.
+
+        :returns: the deserialized response body.
+
+        :raises:
+            :exc:`RetriesExceeded`: raised from the corresponding :mod:`urllib3`
+                exception.
+        """
+        url = self._build_full_url(url_path)
+        response = None
+
+        try:
+            response = self._http.request("GET", url)
+        except MaxRetryError as exc:
+            msg = f"Failed to perform a GET request to {url}"
+            raise RetriesExceeded(msg) from exc
+
+        return self._deserialize_response(response)
+
+    def _post_request(self, url_path: str, body: Any) -> Any:
+        """Perform a ``POST`` request to *url_path* serializing *body* as JSON.
+
+        :param url_path: URL to perform the ``POST`` request against.
+        :param body: the body to be serialized as JSON and sent in the request.
+
+        :returns: the deserialized response body.
+
+        :raises:
+            :exc:`RetriesExceeded`: raised from the corresponding :mod:`urllib3`
+                exception.
+        """
+        body = self._serialize_request(body)
+
+        url = self._build_full_url(url_path)
+        response = None
+
+        try:
+            response = self._http.request("POST",
+                                          url,
+                                          body=body,
+                                          headers={
+                                              "Content-Type": "application/json"
+                                          })
+        except MaxRetryError as exc:
+            msg = f"Failed to perform a POST request to {url} with {body}"
+            raise RetriesExceeded(msg) from exc
+
+        return self._deserialize_response(response)
+
+    def _ensure_api_ok(self) -> None:
+        """Ensure ``pg-backup-api`` is reachable and ``OK``.
+
+        :raises:
+            :exc:`ApiNotOk`: if ``pg-backup-api`` status is not ``OK``.
+        """
+        response = self._get_request("status")
+
+        if response != "OK":
+            msg = (
+                "pg-backup-api is currently not up and running at "
+                f"{self.api_url}: {response}"
+            )
+
+            raise ApiNotOk(msg)
+
+    @retry(KeyError)
+    def get_operation_status(self, barman_server: str,
+                             operation_id: str) -> OperationStatus:
+        """Get status of the operation which ID is *operation_id*.
+
+        :param barman_server: name of the Barman server related with the
+            operation.
+        :param operation_id: ID of the operation to be checked.
+
+        :returns: the status of the operation.
+        """
+        response = self._get_request(
+            f"servers/{barman_server}/operations/{operation_id}",
+        )
+
+        status = response["status"]
+        return OperationStatus[status]
+
+    @retry(KeyError)
+    def create_recovery_operation(self, barman_server: str, backup_id: str,
+                                  ssh_command: str, data_directory: str) -> str:
+        """Create a recovery operation on the ``pg-backup-api``.
+
+        :param barman_server: name of the Barman server which backup is to be
+            restored.
+        :param backup_id: ID of the backup from the Barman server.
+        :param ssh_command: SSH command to connect from the Barman host to the
+            target host.
+        :param data_directory: path to the Postgres data directory where to
+            restore the backup at.
+
+        :returns: the ID of the recovery operation that has been created.
+        """
+        response = self._post_request(
+            f"servers/{barman_server}/operations",
+            {
+                "type": "recovery",
+                "backup_id": backup_id,
+                "remote_ssh_command": ssh_command,
+                "destination_directory": data_directory,
+            },
+        )
+
+        return response["operation_id"]
+
+    @retry(KeyError)
+    def create_config_switch_operation(self, barman_server: str,
+                                       barman_model: Optional[str],
+                                       reset: Optional[bool]) -> str:
+        """Create a config switch operation on the ``pg-backup-api``.
+
+        :param barman_server: name of the Barman server which config is to be
+            switched.
+        :param barman_model: name of the Barman model to be applied to the
+            server, if any.
+        :param reset: ``True`` if you would like to unapply the currently active
+            model for the server, if any.
+
+        :returns: the ID of the config switch operation that has been created.
+        """
+        body: Dict[str, Any] = {"type": "config_switch"}
+
+        if barman_model:
+            body["model_name"] = barman_model
+        elif reset:
+            body["reset"] = reset
+
+        response = self._post_request(
+            f"servers/{barman_server}/operations",
+            body,
+        )
+
+        return response["operation_id"]

patroni/scripts/barman_recover.py

@@ -1,468 +0,0 @@
-#!/usr/bin/env python
-
-"""Restore a Barman backup to the local node through ``pg-backup-api``.
-
-This script can be used both as a custom bootstrap method, and as a custom
-create replica method. Check the output of ``--help`` to understand the
-parameters supported by the script. ``--datadir`` is a special parameter and it
-is automatically filled by Patroni in both cases.
-
-It requires that you have previously configured a Barman server, and that you
-have ``pg-backup-api`` configured and running in the same host as Barman.
-
-Refer to :class:`ExitCode` for possible exit codes of this script.
-"""
-from argparse import ArgumentParser
-from enum import IntEnum
-import json
-import logging
-import sys
-import time
-from typing import Any, Callable, Optional, Tuple, Type, Union
-from urllib.parse import urljoin
-from urllib3 import PoolManager
-from urllib3.exceptions import MaxRetryError
-from urllib3.response import HTTPResponse
-
-
-class ExitCode(IntEnum):
-    """Possible exit codes of this script.
-
-    :cvar RECOVERY_DONE: backup was successfully restored.
-    :cvar RECOVERY_FAILED: recovery of the backup faced an issue.
-    :cvar API_NOT_OK: ``pg-backup-api`` status is not ``OK``.
-    :cvar HTTP_REQUEST_ERROR: an error has occurred during a request to the
-        ``pg-backup-api``.
-    :cvar HTTP_RESPONSE_MALFORMED: ``pg-backup-api`` returned a bogus response.
-    """
-
-    RECOVERY_DONE = 0
-    RECOVERY_FAILED = 1
-    API_NOT_OK = 2
-    HTTP_REQUEST_ERROR = 3
-    HTTP_RESPONSE_MALFORMED = 4
-
-
-class RetriesExceeded(Exception):
-    """Maximum number of retries exceeded."""
-
-
-def retry(exceptions: Union[Type[Exception], Tuple[Type[Exception], ...]]) \
-        -> Any:
-    """Retry an operation n times if expected *exceptions* are faced.
-
-    .. note::
-        Should be used as a decorator of a class' method as it expects the
-        first argument to be a class instance.
-
-        The class which method is going to be decorated should contain a couple
-        attributes:
-
-        * ``max_retries``: maximum retry attempts before failing;
-        * ``retry_wait``: how long to wait before retrying.
-
-    :param exceptions: exceptions that could trigger a retry attempt.
-
-    :raises:
-        :exc:`RetriesExceeded`: if the maximum number of attempts has been
-            exhausted.
-    """
-    def decorator(func: Callable[..., Any]) -> Any:
-        def inner_func(instance: object, *args: Any, **kwargs: Any) -> Any:
-            times: int = getattr(instance, "max_retries")
-            retry_wait: int = getattr(instance, "retry_wait")
-            method_name = f"{instance.__class__.__name__}.{func.__name__}"
-
-            attempt = 1
-
-            while attempt <= times:
-                try:
-                    return func(instance, *args, **kwargs)
-                except exceptions as exc:
-                    logging.warning("Attempt %d of %d on method %s failed "
-                                    "with %r.",
-                                    attempt, times, method_name, exc)
-                    attempt += 1
-
-                time.sleep(retry_wait)
-
-            raise RetriesExceeded("Maximum number of retries exceeded for "
-                                  f"method {method_name}.")
-        return inner_func
-    return decorator
-
-
-class BarmanRecover:
-    """Facilities for performing a remote ``barman recover`` operation.
-
-    You should instantiate this class, which will take care of configuring the
-    operation accordingly. When you want to start the operation, you should
-    call :meth:`restore_backup`. At any point of interaction with this class,
-    you may face a :func:`sys.exit` call. Refer to :class:`ExitCode` for a view
-    on the possible exit codes.
-
-    :ivar api_url: base URL to reach the ``pg-backup-api``.
-    :ivar cert_file: certificate to authenticate against the
-        ``pg-backup-api``, if required.
-    :ivar key_file: certificate key to authenticate against the
-        ``pg-backup-api``, if required.
-    :ivar barman_server: name of the Barman server which backup is to be
-        restored.
-    :ivar backup_id: ID of the backup from the Barman server.
-    :ivar ssh_command: SSH command to connect from the Barman host to the
-        local host.
-    :ivar data_directory: path to the Postgres data directory where to
-        restore the backup at.
-    :ivar loop_wait: how long to wait before checking again the status of the
-        recovery process. Higher values are useful for backups that are
-        expected to take long to restore.
-    :ivar retry_wait: how long to wait before retrying a failed request to the
-        ``pg-backup-api``.
-    :ivar max_retries: maximum number of retries when ``pg-backup-api`` returns
-        malformed responses.
-    :ivar http: a HTTP pool manager for performing web requests.
-    """
-
-    def __init__(self, api_url: str, barman_server: str, backup_id: str,
-                 ssh_command: str, data_directory: str, loop_wait: int,
-                 retry_wait: int, max_retries: int,
-                 cert_file: Optional[str] = None,
-                 key_file: Optional[str] = None) -> None:
-        """Create a new instance of :class:`BarmanRecover`.
-
-        Make sure the ``pg-backup-api`` is reachable and running fine.
-
-        :param api_url: base URL to reach the ``pg-backup-api``.
-        :param barman_server: name of the Barman server which backup is to be
-            restored.
-        :param backup_id: ID of the backup from the Barman server.
-        :param ssh_command: SSH command to connect from the Barman host to the
-            local host.
-        :param data_directory: path to the Postgres data directory where to
-            restore the backup at.
-        :param loop_wait: how long to wait before checking again the status of
-            the recovery process. Higher values are useful for backups that are
-            expected to take long to restore.
-        :param retry_wait: how long to wait before retrying a failed request to
-            the ``pg-backup-api``.
-        :param max_retries: maximum number of retries when ``pg-backup-api``
-            returns malformed responses.
-        :param cert_file: certificate to authenticate against the
-            ``pg-backup-api``, if required.
-        :param key_file: certificate key to authenticate against the
-            ``pg-backup-api``, if required.
-        """
-        self.api_url = api_url
-        self.cert_file = cert_file
-        self.key_file = key_file
-        self.barman_server = barman_server
-        self.backup_id = backup_id
-        self.ssh_command = ssh_command
-        self.data_directory = data_directory
-        self.loop_wait = loop_wait
-        self.retry_wait = retry_wait
-        self.max_retries = max_retries
-        self.http = PoolManager(cert_file=cert_file, key_file=key_file)
-        self._ensure_api_ok()
-
-    def _build_full_url(self, url_path: str) -> str:
-        """Build the full URL by concatenating *url_path* with the base URL.
-
-        :param url_path: path to be accessed in the ``pg-backup-api``.
-
-        :returns: the full URL after concatenating.
-        """
-        return urljoin(self.api_url, url_path)
-
-    @staticmethod
-    def _deserialize_response(response: HTTPResponse) -> Any:
-        """Retrieve body from *response* as a deserialized JSON object.
-
-        :param response: response from which JSON body will be deserialized.
-
-        :returns: the deserialized JSON body.
-        """
-        return json.loads(response.data.decode("utf-8"))
-
-    @staticmethod
-    def _serialize_request(body: Any) -> Any:
-        """Serialize a request body.
-
-        :param body: content of the request body to be serialized.
-
-        :returns: the serialized request body.
-        """
-        return json.dumps(body).encode("utf-8")
-
-    def _get_request(self, url_path: str) -> Any:
-        """Perform a ``GET`` request to *url_path*.
-
-        .. note::
-            If a :exc:`MaxRetryError` is faced while performing the request,
-            then exit with :attr:`ExitCode.HTTP_REQUEST_ERROR`
-
-        :param url_path: URL to perform the ``GET`` request against.
-
-        :returns: the deserialized response body.
-        """
-        response = None
-
-        try:
-            response = self.http.request("GET", self._build_full_url(url_path))
-        except MaxRetryError as exc:
-            logging.critical("An error occurred while performing an HTTP GET "
-                             "request: %r", exc)
-            sys.exit(ExitCode.HTTP_REQUEST_ERROR)
-
-        return self._deserialize_response(response)
-
-    def _post_request(self, url_path: str, body: Any) -> Any:
-        """Perform a ``POST`` request to *url_path* serializing *body* as JSON.
-
-        .. note::
-            If a :exc:`MaxRetryError` is faced while performing the request,
-            then exit with :attr:`ExitCode.HTTP_REQUEST_ERROR`
-
-        :param url_path: URL to perform the ``POST`` request against.
-        :param body: the body to be serialized as JSON and sent in the request.
-
-        :returns: the deserialized response body.
-        """
-        body = self._serialize_request(body)
-
-        response = None
-
-        try:
-            response = self.http.request("POST",
-                                         self._build_full_url(url_path),
-                                         body=body,
-                                         headers={
-                                             "Content-Type": "application/json"
-                                         })
-        except MaxRetryError as exc:
-            logging.critical("An error occurred while performing an HTTP POST "
-                             "request: %r", exc)
-            sys.exit(ExitCode.HTTP_REQUEST_ERROR)
-
-        return self._deserialize_response(response)
-
-    def _ensure_api_ok(self) -> None:
-        """Ensure ``pg-backup-api`` is reachable and ``OK``.
-
-        .. note::
-            If ``pg-backup-api`` status is not ``OK``, then exit with
-            :attr:`ExitCode.API_NOT_OK`.
-        """
-        response = self._get_request("status")
-
-        if response != "OK":
-            logging.critical("pg-backup-api is not working: %s", response)
-            sys.exit(ExitCode.API_NOT_OK)
-
-    @retry(KeyError)
-    def _create_recovery_operation(self) -> str:
-        """Create a recovery operation on the ``pg-backup-api``.
-
-        :returns: the ID of the recovery operation that has been created.
-        """
-        response = self._post_request(
-            f"servers/{self.barman_server}/operations",
-            {
-                "type": "recovery",
-                "backup_id": self.backup_id,
-                "remote_ssh_command": self.ssh_command,
-                "destination_directory": self.data_directory,
-            },
-        )
-
-        return response["operation_id"]
-
-    @retry(KeyError)
-    def _get_recovery_operation_status(self, operation_id: str) -> str:
-        """Get status of the recovery operation *operation_id*.
-
-        :param operation_id: ID of the recovery operation to be checked.
-
-        :returns: the status of the recovery operation.
-        """
-        response = self._get_request(
-            f"servers/{self.barman_server}/operations/{operation_id}",
-        )
-
-        return response["status"]
-
-    def restore_backup(self) -> bool:
-        """Restore the configured Barman backup through ``pg-backup-api``.
-
-        .. note::
-            If recovery API request returns a malformed response, then exit with
-            :attr:`ExitCode.HTTP_RESPONSE_MALFORMED`.
-
-        :returns: ``True`` if it was successfully recovered, ``False``
-            otherwise.
-        """
-        operation_id = None
-
-        try:
-            operation_id = self._create_recovery_operation()
-        except RetriesExceeded:
-            logging.critical("Maximum number of retries exceeded, exiting.")
-            sys.exit(ExitCode.HTTP_RESPONSE_MALFORMED)
-
-        logging.info("Created the recovery operation with ID %s", operation_id)
-
-        status = None
-
-        while True:
-            try:
-                status = self._get_recovery_operation_status(operation_id)
-            except RetriesExceeded:
-                logging.critical("Maximum number of retries exceeded, "
-                                 "exiting.")
-                sys.exit(ExitCode.HTTP_RESPONSE_MALFORMED)
-
-            if status != "IN_PROGRESS":
-                break
-
-            logging.info("Recovery operation %s is still in progress",
-                         operation_id)
-            time.sleep(self.loop_wait)
-
-        return status == "DONE"
-
-
-def set_up_logging(log_file: Optional[str] = None) -> None:
-    """Set up logging to file, if *log_file* is given, otherwise to console.
-
-    :param log_file: file where to log messages, if any.
-    """
-    logging.basicConfig(filename=log_file, level=logging.INFO,
-                        format="%(asctime)s %(levelname)s: %(message)s")
-
-
-def main() -> None:
-    """Entry point of this script.
-
-    Parse the command-line arguments and recover a Barman backup through
-    ``pg-backup-api`` to the local host.
-    """
-    parser = ArgumentParser(
-        epilog=(
-            "Wrapper script for ``pg-backup-api``. Communicate with the API "
-            "running  at ``--api-url`` to restore a ``--backup-id`` Barman "
-            "backup of the server ``--barman-server``."
-        ),
-    )
-    parser.add_argument(
-        "--api-url",
-        type=str,
-        required=True,
-        help="URL to reach the ``pg-backup-api``, e.g. "
-             "``http://localhost:7480``",
-        dest="api_url",
-    )
-    parser.add_argument(
-        "--cert-file",
-        type=str,
-        required=False,
-        help="Certificate to authenticate against the API, if required.",
-        dest="cert_file",
-    )
-    parser.add_argument(
-        "--key-file",
-        type=str,
-        required=False,
-        help="Certificate key to authenticate against the API, if required.",
-        dest="key_file",
-    )
-    parser.add_argument(
-        "--barman-server",
-        type=str,
-        required=True,
-        help="Name of the Barman server from which to restore the backup.",
-        dest="barman_server",
-    )
-    parser.add_argument(
-        "--backup-id",
-        type=str,
-        required=False,
-        default="latest",
-        help="ID of the Barman backup to be restored. You can use any value "
-             "supported by ``barman recover`` command "
-             "(default: ``%(default)s``)",
-        dest="backup_id",
-    )
-    parser.add_argument(
-        "--ssh-command",
-        type=str,
-        required=True,
-        help="Value to be passed as ``--remote-ssh-command`` to "
-             "``barman recover``.",
-        dest="ssh_command",
-    )
-    parser.add_argument(
-        "--data-directory",
-        "--datadir",
-        type=str,
-        required=True,
-        help="Destination path where to restore the barman backup in the "
-             "local host.",
-        dest="data_directory",
-    )
-    parser.add_argument(
-        "--log-file",
-        type=str,
-        required=False,
-        help="File where to log messages produced by this script, if any.",
-        dest="log_file",
-    )
-    parser.add_argument(
-        "--loop-wait",
-        type=int,
-        required=False,
-        default=10,
-        help="How long to wait before checking again the status of the "
-             "recovery process, in seconds. Use higher values if your "
-             "recovery is expected to take long (default: ``%(default)s``)",
-        dest="loop_wait",
-    )
-    parser.add_argument(
-        "--retry-wait",
-        type=int,
-        required=False,
-        default=2,
-        help="How long to wait before retrying a failed ``pg-backup-api`` "
-             "request (default: ``%(default)s``)",
-        dest="retry_wait",
-    )
-    parser.add_argument(
-        "--max-retries",
-        type=int,
-        required=False,
-        default=5,
-        help="Maximum number of retries when receiving malformed responses "
-             "from the ``pg-backup-api`` (default: ``%(default)s``)",
-        dest="max_retries",
-    )
-    args, _ = parser.parse_known_args()
-
-    set_up_logging(args.log_file)
-
-    barman_recover = BarmanRecover(args.api_url, args.barman_server,
-                                   args.backup_id, args.ssh_command,
-                                   args.data_directory, args.loop_wait,
-                                   args.retry_wait, args.max_retries,
-                                   args.cert_file, args.key_file)
-
-    successful = barman_recover.restore_backup()
-
-    if successful:
-        logging.info("Recovery operation finished successfully.")
-        sys.exit(ExitCode.RECOVERY_DONE)
-    else:
-        logging.critical("Recovery operation failed.")
-        sys.exit(ExitCode.RECOVERY_FAILED)
-
-
-if __name__ == "__main__":
-    main()

patroni/tags.py

@@ -3,13 +3,16 @@ import abc
 
 from typing import Any, Dict, Optional
 
-from patroni.utils import parse_int
+from patroni.utils import parse_int, parse_bool
 
 
 class Tags(abc.ABC):
     """An abstract class that encapsulates all the ``tags`` logic.
 
     Child classes that want to use provided facilities must implement ``tags`` abstract property.
+
+    .. note::
+        Due to backward-compatibility reasons, old tags may have a less strict type conversion than new ones.
     """
 
     @staticmethod
@@ -20,7 +23,7 @@ class Tags(abc.ABC):
 
         .. note::
             A custom tag is any tag added to the configuration ``tags`` section that is not one of ``clonefrom``,
-            ``nofailover``, ``noloadbalance`` or ``nosync``.
+            ``nofailover``, ``noloadbalance``,``nosync`` or ``nostream``.
 
             For most of the Patroni predefined tags, the returning object will only contain them if they are enabled as
             they all are boolean values that default to disabled.
@@ -31,7 +34,7 @@ class Tags(abc.ABC):
             tag value.
         """
         return {tag: value for tag, value in tags.items()
-                if any((tag not in ('clonefrom', 'nofailover', 'noloadbalance', 'nosync'),
+                if any((tag not in ('clonefrom', 'nofailover', 'noloadbalance', 'nosync', 'nostream'),
                         value,
                         tag == 'nofailover' and 'failover_priority' in tags))}
 
@@ -89,3 +92,8 @@ class Tags(abc.ABC):
     def replicatefrom(self) -> Optional[str]:
         """Value of ``replicatefrom`` tag, if any."""
         return self.tags.get('replicatefrom')
+
+    @property
+    def nostream(self) -> bool:
+        """``True`` if ``nostream`` is ``True``, else ``False``."""
+        return parse_bool(self.tags.get('nostream')) or False

patroni/utils.py

@@ -716,7 +716,7 @@ class Retry(object):
         return self._cur_stoptime or 0
 
     def ensure_deadline(self, timeout: float, raise_ex: Optional[Exception] = None) -> bool:
-        """Calculates, sets, and checks the remaining deadline time.
+        """Calculates and checks the remaining deadline time.
 
         :param timeout: if the *deadline* is smaller than the provided *timeout* value raise *raise_ex* exception.
         :param raise_ex: the exception object that will be raised if the *deadline* is smaller than provided *timeout*.
@@ -727,8 +727,7 @@ class Retry(object):
         :raises:
             :class:`Exception`: *raise_ex* if calculated deadline is smaller than provided *timeout*.
         """
-        self.deadline = self.stoptime - time.time()
-        if self.deadline < timeout:
+        if self.stoptime - time.time() < timeout:
             if raise_ex:
                 raise raise_ex
             return False
@@ -923,7 +922,7 @@ def cluster_as_json(cluster: 'Cluster') -> Dict[str, Any]:
         * ``members``: list of members in the cluster. Each value is a :class:`dict` that may have the following keys:
 
             * ``name``: the name of the host (unique in the cluster). The ``members`` list is sorted by this key;
-            * ``role``: ``leader``, ``standby_leader``, ``sync_standby``, or ``replica``;
+            * ``role``: ``leader``, ``standby_leader``, ``sync_standby``, ``quorum_standby``, or ``replica``;
             * ``state``: ``stopping``, ``stopped``, ``stop failed``, ``crashed``, ``running``, ``starting``,
                 ``start failed``, ``restarting``, ``restart failed``, ``initializing new cluster``, ``initdb failed``,
                 ``running custom bootstrap script``, ``custom bootstrap failed``, or ``creating replica``;
@@ -950,11 +949,12 @@ def cluster_as_json(cluster: 'Cluster') -> Dict[str, Any]:
     cluster_lsn = cluster.last_lsn or 0
 
     ret: Dict[str, Any] = {'members': []}
+    sync_role = 'quorum_standby' if config.is_quorum_commit_mode else 'sync_standby'
     for m in cluster.members:
         if m.name == leader_name:
             role = 'standby_leader' if config.is_standby_cluster else 'leader'
         elif cluster.sync.matches(m.name):
-            role = 'sync_standby'
+            role = sync_role
         else:
             role = 'replica'
 

patroni/validator.py

@@ -1172,6 +1172,7 @@ schema = Schema({
         Optional("clonefrom"): bool,
         Optional("noloadbalance"): bool,
         Optional("replicatefrom"): str,
-        Optional("nosync"): bool
+        Optional("nosync"): bool,
+        Optional("nostream"): bool
     }
 })

postgres0.yml

@@ -136,3 +136,4 @@ tags:
     noloadbalance: false
     clonefrom: false
     nosync: false
+    nostream: false

setup.py

@@ -55,7 +55,7 @@ CONSOLE_SCRIPTS = ['patroni = patroni.__main__:main',
                    'patroni_raft_controller = patroni.raft_controller:main',
                    "patroni_wale_restore = patroni.scripts.wale_restore:main",
                    "patroni_aws = patroni.scripts.aws:main",
-                   "patroni_barman_recover = patroni.scripts.barman_recover:main"]
+                   "patroni_barman = patroni.scripts.barman.cli:main"]
 
 
 class _Command(Command):

tests/test_api.py

@@ -205,7 +205,6 @@ class TestRestApiHandler(unittest.TestCase):
     def test_do_GET(self):
         MockPostgresql.pending_restart_reason = {'max_connections': get_param_diff('200', '100')}
         MockPatroni.dcs.cluster.last_lsn = 20
-        MockPatroni.dcs.cluster.sync.members = [MockPostgresql.name]
         with patch.object(global_config.__class__, 'is_synchronous_mode', PropertyMock(return_value=True)):
             MockRestApiServer(RestApiHandler, 'GET /replica')
         MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M')
@@ -223,12 +222,16 @@ class TestRestApiHandler(unittest.TestCase):
                           Mock(return_value={'role': 'replica', 'sync_standby': True})):
             MockRestApiServer(RestApiHandler, 'GET /synchronous')
             MockRestApiServer(RestApiHandler, 'GET /read-only-sync')
+        with patch.object(RestApiHandler, 'get_postgresql_status',
+                          Mock(return_value={'role': 'replica', 'quorum_standby': True})):
+            MockRestApiServer(RestApiHandler, 'GET /quorum')
+            MockRestApiServer(RestApiHandler, 'GET /read-only-quorum')
         with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'replica'})):
-            MockPatroni.dcs.cluster.sync.members = []
             MockRestApiServer(RestApiHandler, 'GET /asynchronous')
         with patch.object(MockHa, 'is_leader', Mock(return_value=True)):
             MockRestApiServer(RestApiHandler, 'GET /replica')
             MockRestApiServer(RestApiHandler, 'GET /read-only-sync')
+            MockRestApiServer(RestApiHandler, 'GET /read-only-quorum')
             with patch.object(global_config.__class__, 'is_standby_cluster', Mock(return_value=True)):
                 MockRestApiServer(RestApiHandler, 'GET /standby_leader')
         MockPatroni.dcs.cluster = None

tests/test_barman.py

@@ -0,0 +1,765 @@
+import logging
+import mock
+from mock import MagicMock, Mock, patch
+import unittest
+from urllib3.exceptions import MaxRetryError
+
+from patroni.scripts.barman.cli import main
+from patroni.scripts.barman.config_switch import (ExitCode as BarmanConfigSwitchExitCode, _should_skip_switch,
+                                                  _switch_config, run_barman_config_switch)
+from patroni.scripts.barman.recover import ExitCode as BarmanRecoverExitCode, _restore_backup, run_barman_recover
+from patroni.scripts.barman.utils import ApiNotOk, OperationStatus, PgBackupApi, RetriesExceeded, set_up_logging
+
+
+API_URL = "http://localhost:7480"
+BARMAN_SERVER = "my_server"
+BARMAN_MODEL = "my_model"
+BACKUP_ID = "backup_id"
+SSH_COMMAND = "ssh postgres@localhost"
+DATA_DIRECTORY = "/path/to/pgdata"
+LOOP_WAIT = 10
+RETRY_WAIT = 2
+MAX_RETRIES = 5
+
+
+# stuff from patroni.scripts.barman.utils
+
+@patch("logging.basicConfig")
+def test_set_up_logging(mock_log_config):
+    log_file = "/path/to/some/file.log"
+    set_up_logging(log_file)
+    mock_log_config.assert_called_once_with(filename=log_file, level=logging.INFO,
+                                            format="%(asctime)s %(levelname)s: %(message)s")
+
+
+class TestPgBackupApi(unittest.TestCase):
+
+    @patch.object(PgBackupApi, "_ensure_api_ok", Mock())
+    @patch("patroni.scripts.barman.utils.PoolManager", MagicMock())
+    def setUp(self):
+        self.api = PgBackupApi(API_URL, None, None, RETRY_WAIT, MAX_RETRIES)
+        # Reset the mock as the same instance is used across tests
+        self.api._http.request.reset_mock()
+        self.api._http.request.side_effect = None
+
+    def test__build_full_url(self):
+        self.assertEqual(self.api._build_full_url("/some/path"), f"{API_URL}/some/path")
+
+    @patch("json.loads")
+    def test__deserialize_response(self, mock_json_loads):
+        mock_response = MagicMock()
+        self.assertIsNotNone(self.api._deserialize_response(mock_response))
+        mock_json_loads.assert_called_once_with(mock_response.data.decode("utf-8"))
+
+    @patch("json.dumps")
+    def test__serialize_request(self, mock_json_dumps):
+        body = "some_body"
+        ret = self.api._serialize_request(body)
+        self.assertIsNotNone(ret)
+        mock_json_dumps.assert_called_once_with(body)
+        mock_json_dumps.return_value.encode.assert_called_once_with("utf-8")
+
+    @patch.object(PgBackupApi, "_deserialize_response", Mock(return_value="test"))
+    def test__get_request(self):
+        mock_request = self.api._http.request
+
+        # with no error
+        self.assertEqual(self.api._get_request("/some/path"), "test")
+        mock_request.assert_called_once_with("GET", f"{API_URL}/some/path")
+
+        # with MaxRetryError
+        http_error = MaxRetryError(self.api._http, f"{API_URL}/some/path")
+        mock_request.side_effect = http_error
+
+        with self.assertRaises(RetriesExceeded) as exc:
+            self.assertIsNone(self.api._get_request("/some/path"))
+
+        self.assertEqual(
+            str(exc.exception),
+            "Failed to perform a GET request to http://localhost:7480/some/path"
+        )
+
+    @patch.object(PgBackupApi, "_deserialize_response", Mock(return_value="test"))
+    @patch.object(PgBackupApi, "_serialize_request")
+    def test__post_request(self, mock_serialize):
+        mock_request = self.api._http.request
+
+        # with no error
+        self.assertEqual(self.api._post_request("/some/path", "some body"), "test")
+        mock_serialize.assert_called_once_with("some body")
+        mock_request.assert_called_once_with("POST", f"{API_URL}/some/path", body=mock_serialize.return_value,
+                                             headers={"Content-Type": "application/json"})
+
+        # with HTTPError
+        http_error = MaxRetryError(self.api._http, f"{API_URL}/some/path")
+        mock_request.side_effect = http_error
+
+        with self.assertRaises(RetriesExceeded) as exc:
+            self.assertIsNone(self.api._post_request("/some/path", "some body"))
+
+        self.assertEqual(
+            str(exc.exception),
+            f"Failed to perform a POST request to http://localhost:7480/some/path with {mock_serialize.return_value}"
+        )
+
+    @patch.object(PgBackupApi, "_get_request")
+    def test__ensure_api_ok(self, mock_get_request):
+        # API ok
+        mock_get_request.return_value = "OK"
+        self.assertIsNone(self.api._ensure_api_ok())
+
+        # API not ok
+        mock_get_request.return_value = "random"
+
+        with self.assertRaises(ApiNotOk) as exc:
+            self.assertIsNone(self.api._ensure_api_ok())
+
+        self.assertEqual(
+            str(exc.exception),
+            "pg-backup-api is currently not up and running at http://localhost:7480: random",
+        )
+
+    @patch("patroni.scripts.barman.utils.OperationStatus")
+    @patch("logging.warning")
+    @patch("time.sleep")
+    @patch.object(PgBackupApi, "_get_request")
+    def test_get_operation_status(self, mock_get_request, mock_sleep, mock_logging, mock_op_status):
+        # well formed response
+        mock_get_request.return_value = {"status": "some status"}
+        mock_op_status.__getitem__.return_value = "SOME_STATUS"
+        self.assertEqual(self.api.get_operation_status(BARMAN_SERVER, "some_id"), "SOME_STATUS")
+        mock_get_request.assert_called_once_with(f"servers/{BARMAN_SERVER}/operations/some_id")
+        mock_sleep.assert_not_called()
+        mock_logging.assert_not_called()
+        mock_op_status.__getitem__.assert_called_once_with("some status")
+
+        # malformed response
+        mock_get_request.return_value = {"statuss": "some status"}
+
+        with self.assertRaises(RetriesExceeded) as exc:
+            self.api.get_operation_status(BARMAN_SERVER, "some_id")
+
+        self.assertEqual(str(exc.exception),
+                         "Maximum number of retries exceeded for method PgBackupApi.get_operation_status.")
+
+        self.assertEqual(mock_sleep.call_count, self.api.max_retries)
+        mock_sleep.assert_has_calls([mock.call(self.api.retry_wait)] * self.api.max_retries)
+
+        self.assertEqual(mock_logging.call_count, self.api.max_retries)
+        for i in range(mock_logging.call_count):
+            call_args = mock_logging.call_args_list[i][0]
+            self.assertEqual(len(call_args), 5)
+            self.assertEqual(call_args[0], "Attempt %d of %d on method %s failed with %r.")
+            self.assertEqual(call_args[1], i + 1)
+            self.assertEqual(call_args[2], self.api.max_retries)
+            self.assertEqual(call_args[3], "PgBackupApi.get_operation_status")
+            self.assertIsInstance(call_args[4], KeyError)
+            self.assertEqual(call_args[4].args, ('status',))
+
+    @patch("logging.warning")
+    @patch("time.sleep")
+    @patch.object(PgBackupApi, "_post_request")
+    def test_create_recovery_operation(self, mock_post_request, mock_sleep, mock_logging):
+        # well formed response
+        mock_post_request.return_value = {"operation_id": "some_id"}
+        self.assertEqual(
+            self.api.create_recovery_operation(BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY),
+            "some_id",
+        )
+        mock_sleep.assert_not_called()
+        mock_logging.assert_not_called()
+        mock_post_request.assert_called_once_with(
+            f"servers/{BARMAN_SERVER}/operations",
+            {
+                "type": "recovery",
+                "backup_id": BACKUP_ID,
+                "remote_ssh_command": SSH_COMMAND,
+                "destination_directory": DATA_DIRECTORY,
+            }
+        )
+
+        # malformed response
+        mock_post_request.return_value = {"operation_idd": "some_id"}
+
+        with self.assertRaises(RetriesExceeded) as exc:
+            self.api.create_recovery_operation(BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY)
+
+        self.assertEqual(str(exc.exception),
+                         "Maximum number of retries exceeded for method PgBackupApi.create_recovery_operation.")
+
+        self.assertEqual(mock_sleep.call_count, self.api.max_retries)
+
+        mock_sleep.assert_has_calls([mock.call(self.api.retry_wait)] * self.api.max_retries)
+
+        self.assertEqual(mock_logging.call_count, self.api.max_retries)
+        for i in range(mock_logging.call_count):
+            call_args = mock_logging.call_args_list[i][0]
+            self.assertEqual(len(call_args), 5)
+            self.assertEqual(call_args[0], "Attempt %d of %d on method %s failed with %r.")
+            self.assertEqual(call_args[1], i + 1)
+            self.assertEqual(call_args[2], self.api.max_retries)
+            self.assertEqual(call_args[3], "PgBackupApi.create_recovery_operation")
+            self.assertIsInstance(call_args[4], KeyError)
+            self.assertEqual(call_args[4].args, ('operation_id',))
+
+    @patch("logging.warning")
+    @patch("time.sleep")
+    @patch.object(PgBackupApi, "_post_request")
+    def test_create_config_switch_operation(self, mock_post_request, mock_sleep, mock_logging):
+        # well formed response -- sample 1
+        mock_post_request.return_value = {"operation_id": "some_id"}
+        self.assertEqual(
+            self.api.create_config_switch_operation(BARMAN_SERVER, BARMAN_MODEL, None),
+            "some_id",
+        )
+        mock_sleep.assert_not_called()
+        mock_logging.assert_not_called()
+        mock_post_request.assert_called_once_with(
+            f"servers/{BARMAN_SERVER}/operations",
+            {
+                "type": "config_switch",
+                "model_name": BARMAN_MODEL,
+            }
+        )
+
+        # well formed response -- sample 2
+        mock_post_request.reset_mock()
+
+        self.assertEqual(
+            self.api.create_config_switch_operation(BARMAN_SERVER, None, True),
+            "some_id",
+        )
+        mock_sleep.assert_not_called()
+        mock_logging.assert_not_called()
+        mock_post_request.assert_called_once_with(
+            f"servers/{BARMAN_SERVER}/operations",
+            {
+                "type": "config_switch",
+                "reset": True,
+            }
+        )
+
+        # malformed response
+        mock_post_request.return_value = {"operation_idd": "some_id"}
+
+        with self.assertRaises(RetriesExceeded) as exc:
+            self.api.create_config_switch_operation(BARMAN_SERVER, BARMAN_MODEL, None)
+
+        self.assertEqual(str(exc.exception),
+                         "Maximum number of retries exceeded for method PgBackupApi.create_config_switch_operation.")
+
+        self.assertEqual(mock_sleep.call_count, self.api.max_retries)
+
+        mock_sleep.assert_has_calls([mock.call(self.api.retry_wait)] * self.api.max_retries)
+
+        self.assertEqual(mock_logging.call_count, self.api.max_retries)
+        for i in range(mock_logging.call_count):
+            call_args = mock_logging.call_args_list[i][0]
+            self.assertEqual(len(call_args), 5)
+            self.assertEqual(call_args[0], "Attempt %d of %d on method %s failed with %r.")
+            self.assertEqual(call_args[1], i + 1)
+            self.assertEqual(call_args[2], self.api.max_retries)
+            self.assertEqual(call_args[3], "PgBackupApi.create_config_switch_operation")
+            self.assertIsInstance(call_args[4], KeyError)
+            self.assertEqual(call_args[4].args, ('operation_id',))
+
+
+# stuff from patroni.scripts.barman.recover
+
+
+class TestBarmanRecover(unittest.TestCase):
+
+    def setUp(self):
+        self.api = MagicMock()
+        # Reset the mock as the same instance is used across tests
+        self.api._http.request.reset_mock()
+        self.api._http.request.side_effect = None
+
+    @patch("time.sleep")
+    @patch("logging.info")
+    @patch("logging.error")
+    def test__restore_backup(self, mock_log_error, mock_log_info, mock_sleep):
+        mock_create_op = self.api.create_recovery_operation
+        mock_get_status = self.api.get_operation_status
+
+        # successful fast restore
+        mock_create_op.return_value = "some_id"
+        mock_get_status.return_value = OperationStatus.DONE
+
+        self.assertEqual(
+            _restore_backup(self.api, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT),
+            BarmanRecoverExitCode.RECOVERY_DONE,
+        )
+
+        mock_create_op.assert_called_once_with(BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY)
+        mock_get_status.assert_called_once_with(BARMAN_SERVER, "some_id")
+        mock_log_info.assert_has_calls([
+            mock.call("Created the recovery operation with ID %s", "some_id"),
+            mock.call("Recovery operation finished successfully."),
+        ])
+        mock_log_error.assert_not_called()
+        mock_sleep.assert_not_called()
+
+        # successful slow restore
+        mock_create_op.reset_mock()
+        mock_get_status.reset_mock()
+        mock_log_info.reset_mock()
+        mock_get_status.side_effect = [OperationStatus.IN_PROGRESS] * 20 + [OperationStatus.DONE]
+
+        self.assertEqual(
+            _restore_backup(self.api, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT),
+            BarmanRecoverExitCode.RECOVERY_DONE,
+        )
+
+        mock_create_op.assert_called_once()
+
+        self.assertEqual(mock_get_status.call_count, 21)
+        mock_get_status.assert_has_calls([mock.call(BARMAN_SERVER, "some_id")] * 21)
+
+        self.assertEqual(mock_log_info.call_count, 22)
+        mock_log_info.assert_has_calls([mock.call("Created the recovery operation with ID %s", "some_id")]
+                                       + [mock.call("Recovery operation %s is still in progress", "some_id")] * 20
+                                       + [mock.call("Recovery operation finished successfully.")])
+
+        mock_log_error.assert_not_called()
+
+        self.assertEqual(mock_sleep.call_count, 20)
+        mock_sleep.assert_has_calls([mock.call(LOOP_WAIT)] * 20)
+
+        # failed fast restore
+        mock_create_op.reset_mock()
+        mock_get_status.reset_mock()
+        mock_log_info.reset_mock()
+        mock_sleep.reset_mock()
+        mock_get_status.side_effect = None
+        mock_get_status.return_value = OperationStatus.FAILED
+
+        self.assertEqual(
+            _restore_backup(self.api, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT),
+            BarmanRecoverExitCode.RECOVERY_FAILED,
+        )
+
+        mock_create_op.assert_called_once()
+        mock_get_status.assert_called_once_with(BARMAN_SERVER, "some_id")
+        mock_log_info.assert_has_calls([
+            mock.call("Created the recovery operation with ID %s", "some_id"),
+        ])
+        mock_log_error.assert_has_calls([
+            mock.call("Recovery operation failed."),
+        ])
+        mock_sleep.assert_not_called()
+
+        # failed slow restore
+        mock_create_op.reset_mock()
+        mock_get_status.reset_mock()
+        mock_log_info.reset_mock()
+        mock_log_error.reset_mock()
+        mock_sleep.reset_mock()
+        mock_get_status.side_effect = [OperationStatus.IN_PROGRESS] * 20 + [OperationStatus.FAILED]
+
+        self.assertEqual(
+            _restore_backup(self.api, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT),
+            BarmanRecoverExitCode.RECOVERY_FAILED,
+        )
+
+        mock_create_op.assert_called_once()
+
+        self.assertEqual(mock_get_status.call_count, 21)
+        mock_get_status.assert_has_calls([mock.call(BARMAN_SERVER, "some_id")] * 21)
+
+        self.assertEqual(mock_log_info.call_count, 21)
+        mock_log_info.assert_has_calls([mock.call("Created the recovery operation with ID %s", "some_id")]
+                                       + [mock.call("Recovery operation %s is still in progress", "some_id")] * 20)
+
+        mock_log_error.assert_has_calls([
+            mock.call("Recovery operation failed."),
+        ])
+
+        self.assertEqual(mock_sleep.call_count, 20)
+        mock_sleep.assert_has_calls([mock.call(LOOP_WAIT)] * 20)
+
+        # create retries exceeded
+        mock_log_info.reset_mock()
+        mock_log_error.reset_mock()
+        mock_sleep.reset_mock()
+        mock_create_op.side_effect = RetriesExceeded()
+        mock_get_status.side_effect = None
+
+        self.assertEqual(
+            _restore_backup(self.api, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT),
+            BarmanRecoverExitCode.HTTP_ERROR,
+        )
+
+        mock_log_info.assert_not_called()
+        mock_log_error.assert_called_once_with("An issue was faced while trying to create a recovery operation: %r",
+                                               mock_create_op.side_effect)
+        mock_sleep.assert_not_called()
+
+        # get status retries exceeded
+        mock_create_op.reset_mock()
+        mock_create_op.side_effect = None
+        mock_log_error.reset_mock()
+        mock_log_info.reset_mock()
+        mock_get_status.side_effect = RetriesExceeded
+
+        self.assertEqual(
+            _restore_backup(self.api, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT),
+            BarmanRecoverExitCode.HTTP_ERROR,
+        )
+
+        mock_log_info.assert_called_once_with("Created the recovery operation with ID %s", "some_id")
+        mock_log_error.assert_called_once_with("Maximum number of retries exceeded, exiting.")
+        mock_sleep.assert_not_called()
+
+
+class TestBarmanRecoverCli(unittest.TestCase):
+
+    @patch("patroni.scripts.barman.recover._restore_backup")
+    def test_run_barman_recover(self, mock_rb):
+        api = MagicMock()
+        args = MagicMock()
+
+        # successful execution
+        mock_rb.return_value = BarmanRecoverExitCode.RECOVERY_DONE
+
+        self.assertEqual(
+            run_barman_recover(api, args),
+            BarmanRecoverExitCode.RECOVERY_DONE,
+        )
+
+        mock_rb.assert_called_once_with(api, args.barman_server, args.backup_id,
+                                        args.ssh_command, args.data_directory,
+                                        args.loop_wait)
+
+        # failed execution
+        mock_rb.reset_mock()
+
+        mock_rb.return_value = BarmanRecoverExitCode.RECOVERY_FAILED
+
+        self.assertEqual(
+            run_barman_recover(api, args),
+            BarmanRecoverExitCode.RECOVERY_FAILED,
+        )
+
+        mock_rb.assert_called_once_with(api, args.barman_server, args.backup_id,
+                                        args.ssh_command, args.data_directory,
+                                        args.loop_wait)
+
+
+# stuff from patroni.scripts.barman.config_switch
+
+
+class TestBarmanConfigSwitch(unittest.TestCase):
+
+    def setUp(self):
+        self.api = MagicMock()
+        # Reset the mock as the same instance is used across tests
+        self.api._http.request.reset_mock()
+        self.api._http.request.side_effect = None
+
+    @patch("time.sleep")
+    @patch("logging.info")
+    @patch("logging.error")
+    def test__switch_config(self, mock_log_error, mock_log_info, mock_sleep):
+        mock_create_op = self.api.create_config_switch_operation
+        mock_get_status = self.api.get_operation_status
+
+        # successful fast config-switch
+        mock_create_op.return_value = "some_id"
+        mock_get_status.return_value = OperationStatus.DONE
+
+        self.assertEqual(
+            _switch_config(self.api, BARMAN_SERVER, BARMAN_MODEL, None),
+            BarmanConfigSwitchExitCode.CONFIG_SWITCH_DONE,
+        )
+
+        mock_create_op.assert_called_once_with(BARMAN_SERVER, BARMAN_MODEL, None)
+        mock_get_status.assert_called_once_with(BARMAN_SERVER, "some_id")
+        mock_log_info.assert_has_calls([
+            mock.call("Created the config switch operation with ID %s", "some_id"),
+            mock.call("Config switch operation finished successfully."),
+        ])
+        mock_log_error.assert_not_called()
+        mock_sleep.assert_not_called()
+
+        # successful slow config-switch
+        mock_create_op.reset_mock()
+        mock_get_status.reset_mock()
+        mock_log_info.reset_mock()
+        mock_get_status.side_effect = [OperationStatus.IN_PROGRESS] * 20 + [OperationStatus.DONE]
+
+        self.assertEqual(
+            _switch_config(self.api, BARMAN_SERVER, BARMAN_MODEL, None),
+            BarmanConfigSwitchExitCode.CONFIG_SWITCH_DONE,
+        )
+
+        mock_create_op.assert_called_once_with(BARMAN_SERVER, BARMAN_MODEL, None)
+
+        self.assertEqual(mock_get_status.call_count, 21)
+        mock_get_status.assert_has_calls([mock.call(BARMAN_SERVER, "some_id")] * 21)
+
+        self.assertEqual(mock_log_info.call_count, 22)
+        mock_log_info.assert_has_calls([mock.call("Created the config switch operation with ID %s", "some_id")]
+                                       + [mock.call("Config switch operation %s is still in progress", "some_id")] * 20
+                                       + [mock.call("Config switch operation finished successfully.")])
+
+        mock_log_error.assert_not_called()
+
+        self.assertEqual(mock_sleep.call_count, 20)
+        mock_sleep.assert_has_calls([mock.call(5)] * 20)
+
+        # failed fast config-switch
+        mock_create_op.reset_mock()
+        mock_get_status.reset_mock()
+        mock_log_info.reset_mock()
+        mock_sleep.reset_mock()
+        mock_get_status.side_effect = None
+        mock_get_status.return_value = OperationStatus.FAILED
+
+        self.assertEqual(
+            _switch_config(self.api, BARMAN_SERVER, BARMAN_MODEL, None),
+            BarmanConfigSwitchExitCode.CONFIG_SWITCH_FAILED,
+        )
+
+        mock_create_op.assert_called_once()
+        mock_get_status.assert_called_once_with(BARMAN_SERVER, "some_id")
+        mock_log_info.assert_called_once_with("Created the config switch operation with ID %s", "some_id")
+        mock_log_error.assert_called_once_with("Config switch operation failed.")
+        mock_sleep.assert_not_called()
+
+        # failed slow config-switch
+        mock_create_op.reset_mock()
+        mock_get_status.reset_mock()
+        mock_log_info.reset_mock()
+        mock_log_error.reset_mock()
+        mock_sleep.reset_mock()
+        mock_get_status.side_effect = [OperationStatus.IN_PROGRESS] * 20 + [OperationStatus.FAILED]
+
+        self.assertEqual(
+            _switch_config(self.api, BARMAN_SERVER, BARMAN_MODEL, None),
+            BarmanConfigSwitchExitCode.CONFIG_SWITCH_FAILED,
+        )
+
+        mock_create_op.assert_called_once()
+
+        self.assertEqual(mock_get_status.call_count, 21)
+        mock_get_status.assert_has_calls([mock.call(BARMAN_SERVER, "some_id")] * 21)
+
+        self.assertEqual(mock_log_info.call_count, 21)
+        mock_log_info.assert_has_calls([mock.call("Created the config switch operation with ID %s", "some_id")]
+                                       + [mock.call("Config switch operation %s is still in progress", "some_id")] * 20)
+
+        mock_log_error.assert_called_once_with("Config switch operation failed.")
+
+        self.assertEqual(mock_sleep.call_count, 20)
+        mock_sleep.assert_has_calls([mock.call(5)] * 20)
+
+        # create retries exceeded
+        mock_log_info.reset_mock()
+        mock_log_error.reset_mock()
+        mock_sleep.reset_mock()
+        mock_create_op.side_effect = RetriesExceeded()
+        mock_get_status.side_effect = None
+
+        self.assertEqual(
+            _switch_config(self.api, BARMAN_SERVER, BARMAN_MODEL, None),
+            BarmanConfigSwitchExitCode.HTTP_ERROR,
+        )
+
+        mock_log_info.assert_not_called()
+        mock_log_error.assert_called_once_with("An issue was faced while trying to create a config switch operation: "
+                                               "%r",
+                                               mock_create_op.side_effect)
+        mock_sleep.assert_not_called()
+
+        # get status retries exceeded
+        mock_create_op.reset_mock()
+        mock_create_op.side_effect = None
+        mock_log_error.reset_mock()
+        mock_get_status.side_effect = RetriesExceeded
+
+        self.assertEqual(
+            _switch_config(self.api, BARMAN_SERVER, BARMAN_MODEL, None),
+            BarmanConfigSwitchExitCode.HTTP_ERROR,
+        )
+
+        mock_log_info.assert_called_once_with("Created the config switch operation with ID %s", "some_id")
+        mock_log_error.assert_called_once_with("Maximum number of retries exceeded, exiting.")
+        mock_sleep.assert_not_called()
+
+
+class TestBarmanConfigSwitchCli(unittest.TestCase):
+
+    def test__should_skip_switch(self):
+        args = MagicMock()
+
+        for role, switch_when, expected in [
+            ("master", "promoted", False),
+            ("master", "demoted", True),
+            ("master", "always", False),
+
+            ("primary", "promoted", False),
+            ("primary", "demoted", True),
+            ("primary", "always", False),
+
+            ("promoted", "promoted", False),
+            ("promoted", "demoted", True),
+            ("promoted", "always", False),
+
+            ("standby_leader", "promoted", True),
+            ("standby_leader", "demoted", True),
+            ("standby_leader", "always", False),
+
+            ("replica", "promoted", True),
+            ("replica", "demoted", False),
+            ("replica", "always", False),
+
+            ("demoted", "promoted", True),
+            ("demoted", "demoted", False),
+            ("demoted", "always", False),
+        ]:
+            args.role = role
+            args.switch_when = switch_when
+            self.assertEqual(_should_skip_switch(args), expected)
+
+    @patch("patroni.scripts.barman.config_switch._should_skip_switch")
+    @patch("patroni.scripts.barman.config_switch._switch_config")
+    @patch("logging.error")
+    @patch("logging.info")
+    def test_run_barman_config_switch(self, mock_log_info, mock_log_error, mock_sc, mock_skip):
+        api = MagicMock()
+        args = MagicMock()
+        args.reset = None
+
+        # successful execution
+        mock_skip.return_value = False
+        mock_sc.return_value = BarmanConfigSwitchExitCode.CONFIG_SWITCH_DONE
+
+        self.assertEqual(
+            run_barman_config_switch(api, args),
+            BarmanConfigSwitchExitCode.CONFIG_SWITCH_DONE,
+        )
+
+        mock_sc.assert_called_once_with(api, args.barman_server, args.barman_model,
+                                        args.reset)
+
+        # failed execution
+        mock_sc.reset_mock()
+
+        mock_sc.return_value = BarmanConfigSwitchExitCode.CONFIG_SWITCH_FAILED
+
+        self.assertEqual(
+            run_barman_config_switch(api, args),
+            BarmanConfigSwitchExitCode.CONFIG_SWITCH_FAILED,
+        )
+
+        mock_sc.assert_called_once_with(api, args.barman_server, args.barman_model,
+                                        args.reset)
+
+        # skipped execution
+        mock_sc.reset_mock()
+        mock_skip.return_value = True
+
+        self.assertEqual(
+            run_barman_config_switch(api, args),
+            BarmanConfigSwitchExitCode.CONFIG_SWITCH_SKIPPED
+        )
+
+        mock_sc.assert_not_called()
+        mock_log_info.assert_called_once_with("Config switch operation was skipped (role=%s, "
+                                              "switch_when=%s).", args.role, args.switch_when)
+        mock_log_error.assert_not_called()
+
+        # invalid args -- sample 1
+        mock_skip.return_value = False
+        args = MagicMock()
+        args.barman_server = BARMAN_SERVER
+        args.barman_model = BARMAN_MODEL
+        args.reset = True
+
+        self.assertEqual(
+            run_barman_config_switch(api, args),
+            BarmanConfigSwitchExitCode.INVALID_ARGS,
+        )
+
+        mock_log_error.assert_called_once_with("One, and only one among 'barman_model' ('%s') and 'reset' "
+                                               "('%s') should be given", BARMAN_MODEL, True)
+        api.assert_not_called()
+
+        # invalid args -- sample 2
+        args = MagicMock()
+        args.barman_server = BARMAN_SERVER
+        args.barman_model = None
+        args.reset = None
+
+        mock_log_error.reset_mock()
+        api.reset_mock()
+
+        self.assertEqual(
+            run_barman_config_switch(api, args),
+            BarmanConfigSwitchExitCode.INVALID_ARGS,
+        )
+
+        mock_log_error.assert_called_once_with("One, and only one among 'barman_model' ('%s') and 'reset' "
+                                               "('%s') should be given", None, None)
+        api.assert_not_called()
+
+
+# stuff from patroni.scripts.barman.cli
+
+
+class TestMain(unittest.TestCase):
+
+    @patch("patroni.scripts.barman.cli.PgBackupApi")
+    @patch("patroni.scripts.barman.cli.set_up_logging")
+    @patch("patroni.scripts.barman.cli.ArgumentParser")
+    def test_main(self, mock_arg_parse, mock_set_up_log, mock_api):
+        # sub-command specified
+        args = MagicMock()
+        args.func.return_value = 0
+        mock_arg_parse.return_value.parse_known_args.return_value = (args, None)
+
+        with self.assertRaises(SystemExit) as exc:
+            main()
+
+        mock_arg_parse.assert_called_once()
+        mock_set_up_log.assert_called_once_with(args.log_file)
+        mock_api.assert_called_once_with(args.api_url, args.cert_file,
+                                         args.key_file, args.retry_wait,
+                                         args.max_retries)
+        mock_arg_parse.return_value.print_help.assert_not_called()
+        args.func.assert_called_once_with(mock_api.return_value, args)
+        self.assertEqual(exc.exception.code, 0)
+
+        # Issue in the API
+        mock_arg_parse.reset_mock()
+        mock_set_up_log.reset_mock()
+        mock_api.reset_mock()
+        mock_api.side_effect = ApiNotOk()
+
+        with self.assertRaises(SystemExit) as exc:
+            main()
+
+        mock_arg_parse.assert_called_once()
+        mock_set_up_log.assert_called_once_with(args.log_file)
+        mock_api.assert_called_once_with(args.api_url, args.cert_file,
+                                         args.key_file, args.retry_wait,
+                                         args.max_retries)
+        mock_arg_parse.return_value.print_help.assert_not_called()
+        self.assertEqual(exc.exception.code, -2)
+
+        # sub-command not specified
+        mock_arg_parse.reset_mock()
+        mock_set_up_log.reset_mock()
+        mock_api.reset_mock()
+        delattr(args, "func")
+        mock_api.side_effect = None
+
+        with self.assertRaises(SystemExit) as exc:
+            main()
+
+        mock_arg_parse.assert_called_once()
+        mock_set_up_log.assert_called_once_with(args.log_file)
+        mock_api.assert_not_called()
+        mock_arg_parse.return_value.print_help.assert_called_once_with()
+        self.assertEqual(exc.exception.code, -1)

tests/test_barman_recover.py

@@ -1,366 +0,0 @@
-import logging
-import mock
-from mock import MagicMock, Mock, patch
-import unittest
-from urllib3.exceptions import MaxRetryError
-
-from patroni.scripts.barman_recover import BarmanRecover, ExitCode, RetriesExceeded, main, set_up_logging
-
-
-API_URL = "http://localhost:7480"
-BARMAN_SERVER = "my_server"
-BACKUP_ID = "backup_id"
-SSH_COMMAND = "ssh postgres@localhost"
-DATA_DIRECTORY = "/path/to/pgdata"
-LOOP_WAIT = 10
-RETRY_WAIT = 2
-MAX_RETRIES = 5
-
-
-class TestBarmanRecover(unittest.TestCase):
-
-    @patch.object(BarmanRecover, "_ensure_api_ok", Mock())
-    @patch("patroni.scripts.barman_recover.PoolManager", MagicMock())
-    def setUp(self):
-        self.br = BarmanRecover(API_URL, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT, RETRY_WAIT,
-                                MAX_RETRIES)
-        # Reset the mock as the same instance is used across tests
-        self.br.http.request.reset_mock()
-        self.br.http.request.side_effect = None
-
-    def test__build_full_url(self):
-        self.assertEqual(self.br._build_full_url("/some/path"), f"{API_URL}/some/path")
-
-    @patch("json.loads")
-    def test__deserialize_response(self, mock_json_loads):
-        mock_response = MagicMock()
-        self.assertIsNotNone(self.br._deserialize_response(mock_response))
-        mock_json_loads.assert_called_once_with(mock_response.data.decode("utf-8"))
-
-    @patch("json.dumps")
-    def test__serialize_request(self, mock_json_dumps):
-        body = "some_body"
-        ret = self.br._serialize_request(body)
-        self.assertIsNotNone(ret)
-        mock_json_dumps.assert_called_once_with(body)
-        mock_json_dumps.return_value.encode.assert_called_once_with("utf-8")
-
-    @patch.object(BarmanRecover, "_deserialize_response", Mock(return_value="test"))
-    @patch("logging.critical")
-    def test__get_request(self, mock_logging):
-        mock_request = self.br.http.request
-
-        # with no error
-        self.assertEqual(self.br._get_request("/some/path"), "test")
-        mock_request.assert_called_once_with("GET", f"{API_URL}/some/path")
-
-        # with MaxRetryError
-        http_error = MaxRetryError(self.br.http, f"{API_URL}/some/path")
-        mock_request.side_effect = http_error
-
-        with self.assertRaises(SystemExit) as exc:
-            self.assertIsNone(self.br._get_request("/some/path"))
-
-        mock_logging.assert_called_once_with("An error occurred while performing an HTTP GET request: %r", http_error)
-        self.assertEqual(exc.exception.code, ExitCode.HTTP_REQUEST_ERROR)
-
-        # with Exception
-        mock_logging.reset_mock()
-        mock_request.side_effect = Exception("Some error.")
-
-        with patch("sys.exit") as mock_sys:
-            with self.assertRaises(Exception):
-                self.assertIsNone(self.br._get_request("/some/path"))
-
-            mock_logging.assert_not_called()
-            mock_sys.assert_not_called()
-
-    @patch.object(BarmanRecover, "_deserialize_response", Mock(return_value="test"))
-    @patch("logging.critical")
-    @patch.object(BarmanRecover, "_serialize_request")
-    def test__post_request(self, mock_serialize, mock_logging):
-        mock_request = self.br.http.request
-
-        # with no error
-        self.assertEqual(self.br._post_request("/some/path", "some body"), "test")
-        mock_serialize.assert_called_once_with("some body")
-        mock_request.assert_called_once_with("POST", f"{API_URL}/some/path", body=mock_serialize.return_value,
-                                             headers={"Content-Type": "application/json"})
-
-        # with HTTPError
-        http_error = MaxRetryError(self.br.http, f"{API_URL}/some/path")
-        mock_request.side_effect = http_error
-
-        with self.assertRaises(SystemExit) as exc:
-            self.assertIsNone(self.br._post_request("/some/path", "some body"))
-
-        mock_logging.assert_called_once_with("An error occurred while performing an HTTP POST request: %r", http_error)
-        self.assertEqual(exc.exception.code, ExitCode.HTTP_REQUEST_ERROR)
-
-        # with Exception
-        mock_logging.reset_mock()
-        mock_request.side_effect = Exception("Some error.")
-
-        with patch("sys.exit") as mock_sys:
-            with self.assertRaises(Exception):
-                self.br._post_request("/some/path", "some body")
-
-            mock_logging.assert_not_called()
-            mock_sys.assert_not_called()
-
-    @patch("logging.critical")
-    @patch.object(BarmanRecover, "_get_request")
-    def test__ensure_api_ok(self, mock_get_request, mock_logging):
-        # API ok
-        mock_get_request.return_value = "OK"
-
-        with patch("sys.exit") as mock_sys:
-            self.assertIsNone(self.br._ensure_api_ok())
-            mock_logging.assert_not_called()
-            mock_sys.assert_not_called()
-
-        # API not ok
-        mock_get_request.return_value = "random"
-
-        with self.assertRaises(SystemExit) as exc:
-            self.assertIsNone(self.br._ensure_api_ok())
-
-        mock_logging.assert_called_once_with("pg-backup-api is not working: %s", "random")
-        self.assertEqual(exc.exception.code, ExitCode.API_NOT_OK)
-
-    @patch("logging.warning")
-    @patch("time.sleep")
-    @patch.object(BarmanRecover, "_post_request")
-    def test__create_recovery_operation(self, mock_post_request, mock_sleep, mock_logging):
-        # well formed response
-        mock_post_request.return_value = {"operation_id": "some_id"}
-        self.assertEqual(self.br._create_recovery_operation(), "some_id")
-        mock_sleep.assert_not_called()
-        mock_logging.assert_not_called()
-        mock_post_request.assert_called_once_with(
-            f"servers/{BARMAN_SERVER}/operations",
-            {
-                "type": "recovery",
-                "backup_id": BACKUP_ID,
-                "remote_ssh_command": SSH_COMMAND,
-                "destination_directory": DATA_DIRECTORY,
-            }
-        )
-
-        # malformed response
-        mock_post_request.return_value = {"operation_idd": "some_id"}
-
-        with self.assertRaises(RetriesExceeded) as exc:
-            self.br._create_recovery_operation()
-
-        self.assertEqual(str(exc.exception),
-                         "Maximum number of retries exceeded for method BarmanRecover._create_recovery_operation.")
-
-        self.assertEqual(mock_sleep.call_count, self.br.max_retries)
-
-        mock_sleep.assert_has_calls([mock.call(self.br.retry_wait)] * self.br.max_retries)
-
-        self.assertEqual(mock_logging.call_count, self.br.max_retries)
-        for i in range(mock_logging.call_count):
-            call_args = mock_logging.call_args_list[i][0]
-            self.assertEqual(len(call_args), 5)
-            self.assertEqual(call_args[0], "Attempt %d of %d on method %s failed with %r.")
-            self.assertEqual(call_args[1], i + 1)
-            self.assertEqual(call_args[2], self.br.max_retries)
-            self.assertEqual(call_args[3], "BarmanRecover._create_recovery_operation")
-            self.assertIsInstance(call_args[4], KeyError)
-            self.assertEqual(call_args[4].args, ('operation_id',))
-
-    @patch("logging.warning")
-    @patch("time.sleep")
-    @patch.object(BarmanRecover, "_get_request")
-    def test__get_recovery_operation_status(self, mock_get_request, mock_sleep, mock_logging):
-        # well formed response
-        mock_get_request.return_value = {"status": "some status"}
-        self.assertEqual(self.br._get_recovery_operation_status("some_id"), "some status")
-        mock_get_request.assert_called_once_with(f"servers/{BARMAN_SERVER}/operations/some_id")
-        mock_sleep.assert_not_called()
-        mock_logging.assert_not_called()
-
-        # malformed response
-        mock_get_request.return_value = {"statuss": "some status"}
-
-        with self.assertRaises(RetriesExceeded) as exc:
-            self.br._get_recovery_operation_status("some_id")
-
-        self.assertEqual(str(exc.exception),
-                         "Maximum number of retries exceeded for method BarmanRecover._get_recovery_operation_status.")
-
-        self.assertEqual(mock_sleep.call_count, self.br.max_retries)
-        mock_sleep.assert_has_calls([mock.call(self.br.retry_wait)] * self.br.max_retries)
-
-        self.assertEqual(mock_logging.call_count, self.br.max_retries)
-        for i in range(mock_logging.call_count):
-            call_args = mock_logging.call_args_list[i][0]
-            self.assertEqual(len(call_args), 5)
-            self.assertEqual(call_args[0], "Attempt %d of %d on method %s failed with %r.")
-            self.assertEqual(call_args[1], i + 1)
-            self.assertEqual(call_args[2], self.br.max_retries)
-            self.assertEqual(call_args[3], "BarmanRecover._get_recovery_operation_status")
-            self.assertIsInstance(call_args[4], KeyError)
-            self.assertEqual(call_args[4].args, ('status',))
-
-    @patch.object(BarmanRecover, "_get_recovery_operation_status")
-    @patch("time.sleep")
-    @patch("logging.info")
-    @patch("logging.critical")
-    @patch.object(BarmanRecover, "_create_recovery_operation")
-    def test_restore_backup(self, mock_create_op, mock_log_critical, mock_log_info, mock_sleep, mock_get_status):
-        # successful fast restore
-        mock_create_op.return_value = "some_id"
-        mock_get_status.return_value = "DONE"
-
-        self.assertTrue(self.br.restore_backup())
-
-        mock_create_op.assert_called_once()
-        mock_get_status.assert_called_once_with("some_id")
-        mock_log_info.assert_called_once_with("Created the recovery operation with ID %s", "some_id")
-        mock_log_critical.assert_not_called()
-        mock_sleep.assert_not_called()
-
-        # successful slow restore
-        mock_create_op.reset_mock()
-        mock_get_status.reset_mock()
-        mock_log_info.reset_mock()
-        mock_get_status.side_effect = ["IN_PROGRESS"] * 20 + ["DONE"]
-
-        self.assertTrue(self.br.restore_backup())
-
-        mock_create_op.assert_called_once()
-
-        self.assertEqual(mock_get_status.call_count, 21)
-        mock_get_status.assert_has_calls([mock.call("some_id")] * 21)
-
-        self.assertEqual(mock_log_info.call_count, 21)
-        mock_log_info.assert_has_calls([mock.call("Created the recovery operation with ID %s", "some_id")]
-                                       + [mock.call("Recovery operation %s is still in progress", "some_id")] * 20)
-
-        mock_log_critical.assert_not_called()
-
-        self.assertEqual(mock_sleep.call_count, 20)
-        mock_sleep.assert_has_calls([mock.call(LOOP_WAIT)] * 20)
-
-        # failed fast restore
-        mock_create_op.reset_mock()
-        mock_get_status.reset_mock()
-        mock_log_info.reset_mock()
-        mock_sleep.reset_mock()
-        mock_get_status.side_effect = None
-        mock_get_status.return_value = "FAILED"
-
-        self.assertFalse(self.br.restore_backup())
-
-        mock_create_op.assert_called_once()
-        mock_get_status.assert_called_once_with("some_id")
-        mock_log_info.assert_called_once_with("Created the recovery operation with ID %s", "some_id")
-        mock_log_critical.assert_not_called()
-        mock_sleep.assert_not_called()
-
-        # failed slow restore
-        mock_create_op.reset_mock()
-        mock_get_status.reset_mock()
-        mock_log_info.reset_mock()
-        mock_sleep.reset_mock()
-        mock_get_status.side_effect = ["IN_PROGRESS"] * 20 + ["FAILED"]
-
-        self.assertFalse(self.br.restore_backup())
-
-        mock_create_op.assert_called_once()
-
-        self.assertEqual(mock_get_status.call_count, 21)
-        mock_get_status.assert_has_calls([mock.call("some_id")] * 21)
-
-        self.assertEqual(mock_log_info.call_count, 21)
-        mock_log_info.assert_has_calls([mock.call("Created the recovery operation with ID %s", "some_id")]
-                                       + [mock.call("Recovery operation %s is still in progress", "some_id")] * 20)
-
-        mock_log_critical.assert_not_called()
-
-        self.assertEqual(mock_sleep.call_count, 20)
-        mock_sleep.assert_has_calls([mock.call(LOOP_WAIT)] * 20)
-
-        # create retries exceeded
-        mock_log_info.reset_mock()
-        mock_sleep.reset_mock()
-        mock_create_op.side_effect = RetriesExceeded
-        mock_get_status.side_effect = None
-
-        with self.assertRaises(SystemExit) as exc:
-            self.assertIsNone(self.br.restore_backup())
-
-        self.assertEqual(exc.exception.code, ExitCode.HTTP_RESPONSE_MALFORMED)
-        mock_log_info.assert_not_called()
-        mock_log_critical.assert_called_once_with("Maximum number of retries exceeded, exiting.")
-        mock_sleep.assert_not_called()
-
-        # get status retries exceeded
-        mock_create_op.reset_mock()
-        mock_create_op.side_effect = None
-        mock_log_critical.reset_mock()
-        mock_log_info.reset_mock()
-        mock_get_status.side_effect = RetriesExceeded
-
-        with self.assertRaises(SystemExit) as exc:
-            self.assertIsNone(self.br.restore_backup())
-
-        self.assertEqual(exc.exception.code, ExitCode.HTTP_RESPONSE_MALFORMED)
-        mock_log_info.assert_called_once_with("Created the recovery operation with ID %s", "some_id")
-        mock_log_critical.assert_called_once_with("Maximum number of retries exceeded, exiting.")
-        mock_sleep.assert_not_called()
-
-
-class TestMain(unittest.TestCase):
-
-    @patch("logging.basicConfig")
-    def test_set_up_logging(self, mock_log_config):
-        log_file = "/path/to/some/file.log"
-        set_up_logging(log_file)
-        mock_log_config.assert_called_once_with(filename=log_file, level=logging.INFO,
-                                                format="%(asctime)s %(levelname)s: %(message)s")
-
-    @patch("logging.critical")
-    @patch("logging.info")
-    @patch("patroni.scripts.barman_recover.set_up_logging")
-    @patch("patroni.scripts.barman_recover.BarmanRecover")
-    @patch("patroni.scripts.barman_recover.ArgumentParser")
-    def test_main(self, mock_arg_parse, mock_br, mock_set_up_log, mock_log_info, mock_log_critical):
-        # successful restore
-        args = MagicMock()
-        mock_arg_parse.return_value.parse_known_args.return_value = (args, None)
-        mock_br.return_value.restore_backup.return_value = True
-
-        with self.assertRaises(SystemExit) as exc:
-            main()
-
-        mock_arg_parse.assert_called_once()
-        mock_set_up_log.assert_called_once_with(args.log_file)
-        mock_br.assert_called_once_with(args.api_url, args.barman_server, args.backup_id, args.ssh_command,
-                                        args.data_directory, args.loop_wait, args.retry_wait, args.max_retries,
-                                        args.cert_file, args.key_file)
-        mock_log_info.assert_called_once_with("Recovery operation finished successfully.")
-        mock_log_critical.assert_not_called()
-        self.assertEqual(exc.exception.code, ExitCode.RECOVERY_DONE)
-
-        # failed restore
-        mock_arg_parse.reset_mock()
-        mock_set_up_log.reset_mock()
-        mock_br.reset_mock()
-        mock_log_info.reset_mock()
-        mock_br.return_value.restore_backup.return_value = False
-
-        with self.assertRaises(SystemExit) as exc:
-            main()
-
-        mock_arg_parse.assert_called_once()
-        mock_set_up_log.assert_called_once_with(args.log_file)
-        mock_br.assert_called_once_with(args.api_url, args.barman_server, args.backup_id, args.ssh_command,
-                                        args.data_directory, args.loop_wait, args.retry_wait, args.max_retries,
-                                        args.cert_file, args.key_file)
-        mock_log_info.assert_not_called()
-        mock_log_critical.assert_called_once_with("Recovery operation failed.")
-        self.assertEqual(exc.exception.code, ExitCode.RECOVERY_FAILED)

tests/test_config_generator.py

@@ -142,6 +142,7 @@ class TestGenerateConfig(unittest.TestCase):
                 'noloadbalance': False,
                 'clonefrom': True,
                 'nosync': False,
+                'nostream': False
             }
         }
         patch_config(self.config, conf)

tests/test_consul.py

@@ -160,10 +160,8 @@ class TestConsul(unittest.TestCase):
         self.c.set_ttl(20)
         self.c._do_refresh_session = Mock()
         self.assertFalse(self.c.take_leader())
-        with patch('time.time', Mock(side_effect=[0, 100])):
-            self.assertRaises(ConsulError, self.c.take_leader)
-        with patch('time.time', Mock(side_effect=[0, 0, 0, 0, 0, 0, 100])):
-            self.assertRaises(ConsulError, self.c.take_leader)
+        with patch('time.time', Mock(side_effect=[0, 0, 0, 100, 100])):
+            self.assertFalse(self.c.take_leader())
 
     @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
     def test_set_failover_value(self):

tests/test_etcd.py

@@ -344,7 +344,7 @@ class TestEtcd(unittest.TestCase):
         self.assertTrue(self.etcd.watch(None, 1))
 
     def test_sync_state(self):
-        self.assertIsNone(self.etcd.write_sync_state('leader', None))
+        self.assertIsNone(self.etcd.write_sync_state('leader', None, 0))
         self.assertFalse(self.etcd.delete_sync_state())
 
     def test_set_history_value(self):

tests/test_etcd3.py

@@ -7,7 +7,7 @@ from mock import Mock, PropertyMock, patch
 from patroni.dcs import get_dcs
 from patroni.dcs.etcd import DnsCachingResolver
 from patroni.dcs.etcd3 import PatroniEtcd3Client, Cluster, Etcd3, Etcd3Client, \
-    Etcd3Error, Etcd3ClientError, ReAuthenticateMode, RetryFailedError, InvalidAuthToken, Unavailable, \
+    Etcd3Error, Etcd3ClientError, RetryFailedError, InvalidAuthToken, Unavailable, \
     Unknown, UnsupportedEtcdVersion, UserEmpty, AuthFailed, AuthOldRevision, base64_encode
 from patroni.postgresql.mpp import get_mpp
 from threading import Thread
@@ -166,12 +166,14 @@ class TestPatroniEtcd3Client(BaseTestEtcd3):
             retry = self.etcd3._retry.copy()
             with patch('time.time', Mock(side_effect=[0, 10, 20, 30, 40])):
                 self.assertRaises(InvalidAuthToken, retry, self.client.deleteprefix, 'foo', retry=retry)
+            with patch('time.time', Mock(side_effect=[0, 10])):
+                self.assertRaises(InvalidAuthToken, self.client.deleteprefix, 'foo')
             self.client.username = None
-            self.client._reauthenticate_reason = ReAuthenticateMode.NOT_REQUIRED
+            self.client._reauthenticate = False
             retry = self.etcd3._retry.copy()
             self.assertRaises(InvalidAuthToken, retry, self.client.deleteprefix, 'foo', retry=retry)
             mock_urlopen.return_value.content = '{"code":3,"error":"etcdserver: revision of auth store is old"}'
-            self.client._reauthenticate_reason = ReAuthenticateMode.NOT_REQUIRED
+            self.client._reauthenticate = False
             self.assertRaises(AuthOldRevision, retry, self.client.deleteprefix, 'foo', retry=retry)
 
     def test__handle_server_response(self):
@@ -271,8 +273,8 @@ class TestEtcd3(BaseTestEtcd3):
 
     def test_attempt_to_acquire_leader(self):
         self.assertFalse(self.etcd3.attempt_to_acquire_leader())
-        with patch('time.time', Mock(side_effect=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 200])):
-            self.assertRaises(Etcd3Error, self.etcd3.attempt_to_acquire_leader)
+        with patch('time.time', Mock(side_effect=[0, 0, 0, 0, 0, 100, 200])):
+            self.assertFalse(self.etcd3.attempt_to_acquire_leader())
         with patch('time.time', Mock(side_effect=[0, 100, 200, 300, 400])):
             self.assertRaises(Etcd3Error, self.etcd3.attempt_to_acquire_leader)
         with patch.object(PatroniEtcd3Client, 'put', Mock(return_value=False)):

tests/test_ha.py

@@ -18,6 +18,7 @@ from patroni.postgresql.config import ConfigHandler
 from patroni.postgresql.postmaster import PostmasterProcess
 from patroni.postgresql.rewind import Rewind
 from patroni.postgresql.slots import SlotsHandler
+from patroni.postgresql.sync import _SyncState
 from patroni.utils import tzutc
 from patroni.watchdog import Watchdog
 
@@ -63,7 +64,7 @@ def get_cluster_initialized_without_leader(leader=False, failover=None, sync=Non
                                  'tags': {'clonefrom': True},
                                  'scheduled_restart': {'schedule': "2100-01-01 10:53:07.560445+00:00",
                                                        'postgres_version': '99.0.0'}})
-    syncstate = SyncState(0 if sync else None, sync and sync[0], sync and sync[1])
+    syncstate = SyncState(0 if sync else None, sync and sync[0], sync and sync[1], 0)
     failsafe = {m.name: m.api_url for m in (m1, m2)} if failsafe else None
     return get_cluster(SYSID, leader, [m1, m2], failover, syncstate, cluster_config, failsafe)
 
@@ -151,6 +152,7 @@ zookeeper:
         self.api.connection_string = 'http://127.0.0.1:8008'
         self.clonefrom = None
         self.nosync = False
+        self.nostream = False
         self.scheduled_restart = {'schedule': future_restart_time,
                                   'postmaster_start_time': str(postmaster_start_time)}
         self.watchdog = Watchdog(self.config)
@@ -206,6 +208,7 @@ class TestHa(PostgresInit):
     @patch('patroni.dcs.dcs_modules', Mock(return_value=['patroni.dcs.etcd']))
     @patch.object(etcd.Client, 'read', etcd_read)
     @patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379']))
+    @patch.object(Config, '_load_cache', Mock())
     def setUp(self):
         super(TestHa, self).setUp()
         self.p.set_state('running')
@@ -255,8 +258,6 @@ class TestHa(PostgresInit):
         self.p.data_directory_empty = true
         self.ha.cluster = get_cluster_not_initialized_without_leader(
             cluster_config=ClusterConfig(1, {"standby_cluster": {"port": 5432}}, 1))
-        global_config.update(self.ha.cluster)
-        self.ha.cluster = get_cluster_not_initialized_without_leader(cluster_config=ClusterConfig(0, {}, 0))
         self.assertEqual(self.ha.run_cycle(), 'trying to bootstrap a new standby leader')
 
     def test_bootstrap_waiting_for_standby_leader(self):
@@ -322,7 +323,6 @@ class TestHa(PostgresInit):
             self.ha.state_handler.cancellable._process = Mock()
             self.ha._crash_recovery_started -= 600
             self.ha.cluster.config.data.update({'maximum_lag_on_failover': 10})
-            global_config.update(self.ha.cluster)
             self.assertEqual(self.ha.run_cycle(), 'terminated crash recovery because of startup timeout')
 
     @patch.object(Rewind, 'ensure_clean_shutdown', Mock())
@@ -474,6 +474,11 @@ class TestHa(PostgresInit):
         self.p.is_primary = false
         self.assertEqual(self.ha.run_cycle(), 'not promoting because failed to update leader lock in DCS')
 
+    def test_get_node_to_follow_nostream(self):
+        self.ha.patroni.nostream = True
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.assertEqual(self.ha._get_node_to_follow(self.ha.cluster), None)
+
     @patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
     def test_follow(self):
         self.p.is_primary = false
@@ -770,7 +775,6 @@ class TestHa(PostgresInit):
         with patch('patroni.ha.logger.info') as mock_info:
             self.ha.fetch_node_status = get_node_status(wal_position=1)
             self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5})
-            global_config.update(self.ha.cluster)
             self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
             self.assertEqual(mock_info.call_args_list[0][0], ('Member %s exceeds maximum replication lag', 'leader'))
 
@@ -1276,7 +1280,6 @@ class TestHa(PostgresInit):
         self.p.is_running = false
         self.ha.cluster = get_cluster_initialized_with_leader(sync=(self.p.name, 'other'))
         self.ha.cluster.config.data.update({'synchronous_mode': True, 'primary_start_timeout': 0})
-        global_config.update(self.ha.cluster)
         self.ha.has_lock = true
         self.ha.update_lock = true
         self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
@@ -1302,7 +1305,7 @@ class TestHa(PostgresInit):
         self.ha.demote('immediate')
         follow.assert_called_once_with(None)
 
-    def test_process_sync_replication(self):
+    def test__process_multisync_replication(self):
         self.ha.has_lock = true
         mock_set_sync = self.p.sync_handler.set_synchronous_standby_names = Mock()
         self.p.name = 'leader'
@@ -1327,8 +1330,9 @@ class TestHa(PostgresInit):
         self.ha.is_synchronous_mode = true
 
         # Test sync standby not touched when picking the same node
-        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(['other']),
-                                                               CaseInsensitiveSet(['other'])))
+        self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 1, 1,
+                                                                         CaseInsensitiveSet(['other']),
+                                                                         CaseInsensitiveSet(['other'])))
         self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
         self.ha.run_cycle()
         mock_set_sync.assert_not_called()
@@ -1336,14 +1340,16 @@ class TestHa(PostgresInit):
         mock_set_sync.reset_mock()
 
         # Test sync standby is replaced when switching standbys
-        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(['other2']), CaseInsensitiveSet()))
+        self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 0, 0, CaseInsensitiveSet(),
+                                                                         CaseInsensitiveSet(['other2'])))
         self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
         self.ha.run_cycle()
         mock_set_sync.assert_called_once_with(CaseInsensitiveSet(['other2']))
 
         # Test sync standby is replaced when new standby is joined
-        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(['other2', 'other3']),
-                                                               CaseInsensitiveSet(['other2'])))
+        self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 1, 1,
+                                                                         CaseInsensitiveSet(['other2']),
+                                                                         CaseInsensitiveSet(['other2', 'other3'])))
         self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
         self.ha.run_cycle()
         self.assertEqual(mock_set_sync.call_args_list[0][0], (CaseInsensitiveSet(['other2']),))
@@ -1360,8 +1366,9 @@ class TestHa(PostgresInit):
         self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
         self.ha.dcs.get_cluster = Mock(return_value=get_cluster_initialized_with_leader(sync=('leader', 'other')))
         # self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
-        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(['other2']),
-                                                               CaseInsensitiveSet(['other2'])))
+        self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 1, 1,
+                                                                         CaseInsensitiveSet(['other2']),
+                                                                         CaseInsensitiveSet(['other2'])))
         self.ha.run_cycle()
         self.assertEqual(self.ha.dcs.write_sync_state.call_count, 2)
 
@@ -1383,10 +1390,10 @@ class TestHa(PostgresInit):
 
         # Test sync set to '*' when synchronous_mode_strict is enabled
         mock_set_sync.reset_mock()
-        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(), CaseInsensitiveSet()))
-        self.ha.cluster.config.data['synchronous_mode_strict'] = True
-        global_config.update(self.ha.cluster)
-        self.ha.run_cycle()
+        self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 0, 0, CaseInsensitiveSet(),
+                                                                         CaseInsensitiveSet()))
+        with patch.object(global_config.__class__, 'is_synchronous_mode_strict', PropertyMock(return_value=True)):
+            self.ha.run_cycle()
         mock_set_sync.assert_called_once_with(CaseInsensitiveSet('*'))
 
     def test_sync_replication_become_primary(self):
@@ -1402,8 +1409,8 @@ class TestHa(PostgresInit):
 
         # When we just became primary nobody is sync
         self.assertEqual(self.ha.enforce_primary_role('msg', 'promote msg'), 'promote msg')
-        mock_set_sync.assert_called_once_with(CaseInsensitiveSet())
-        mock_write_sync.assert_called_once_with('leader', None, version=0)
+        mock_set_sync.assert_called_once_with(CaseInsensitiveSet(), 0)
+        mock_write_sync.assert_called_once_with('leader', None, 0, version=0)
 
         mock_set_sync.reset_mock()
 
@@ -1441,7 +1448,7 @@ class TestHa(PostgresInit):
         mock_acquire.assert_called_once()
         mock_follow.assert_not_called()
         mock_promote.assert_called_once()
-        mock_write_sync.assert_called_once_with('other', None, version=0)
+        mock_write_sync.assert_called_once_with('other', None, 0, version=0)
 
     def test_disable_sync_when_restarting(self):
         self.ha.is_synchronous_mode = true
@@ -1483,7 +1490,8 @@ class TestHa(PostgresInit):
         self.ha.is_synchronous_mode = true
         self.ha.has_lock = true
         self.p.name = 'leader'
-        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(), CaseInsensitiveSet()))
+        self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 0, 0,
+                                                                         CaseInsensitiveSet(), CaseInsensitiveSet()))
         self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
         with patch('patroni.ha.logger.info') as mock_logger:
             self.ha.run_cycle()
@@ -1499,7 +1507,8 @@ class TestHa(PostgresInit):
         self.ha.has_lock = true
         self.p.name = 'leader'
         self.ha.cluster = get_cluster_initialized_without_leader(sync=('leader', 'a'))
-        self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet('a'), CaseInsensitiveSet()))
+        self.p.sync_handler.current_state = Mock(return_value=_SyncState('priority', 0, 0,
+                                                                         CaseInsensitiveSet(), CaseInsensitiveSet('a')))
         self.ha.dcs.write_sync_state = Mock(return_value=SyncState.empty())
         mock_set_sync = self.p.sync_handler.set_synchronous_standby_names = Mock()
         with patch('patroni.ha.logger.warning') as mock_logger:
@@ -1669,3 +1678,113 @@ class TestHa(PostgresInit):
             mock_logger.assert_called()
             self.assertTrue(mock_logger.call_args[0][0].startswith('Request to %s coordinator leader'))
             self.assertEqual(mock_logger.call_args[0][1], 'Citus')
+
+    @patch.object(global_config.__class__, 'is_synchronous_mode', PropertyMock(return_value=True))
+    @patch.object(global_config.__class__, 'is_quorum_commit_mode', PropertyMock(return_value=True))
+    def test_process_sync_replication_prepromote(self):
+        self.p._major_version = 90500
+        self.ha.cluster = get_cluster_initialized_without_leader(sync=('other', self.p.name + ',foo'))
+        self.p.is_primary = false
+        self.p.set_role('replica')
+        mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=None)
+        # Postgres 9.5, write_sync_state to DCS failed
+        self.assertEqual(self.ha.run_cycle(),
+                         'Postponing promotion because synchronous replication state was updated by somebody else')
+        self.assertEqual(self.ha.dcs.write_sync_state.call_count, 1)
+        self.assertEqual(mock_write_sync.call_args_list[0][0], (self.p.name, None, 0))
+        self.assertEqual(mock_write_sync.call_args_list[0][1], {'version': 0})
+
+        mock_set_sync = self.p.config.set_synchronous_standby_names = Mock()
+        mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=True)
+        # Postgres 9.5, our name is written to leader of the /sync key, while voters list and ssn is empty
+        self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
+        self.assertEqual(self.ha.dcs.write_sync_state.call_count, 1)
+        self.assertEqual(mock_write_sync.call_args_list[0][0], (self.p.name, None, 0))
+        self.assertEqual(mock_write_sync.call_args_list[0][1], {'version': 0})
+        self.assertEqual(mock_set_sync.call_count, 1)
+        self.assertEqual(mock_set_sync.call_args_list[0][0], (None,))
+
+        self.p._major_version = 90600
+        mock_set_sync.reset_mock()
+        mock_write_sync.reset_mock()
+        self.p.set_role('replica')
+        # Postgres 9.6, with quorum commit we avoid updating /sync key and put some nodes to ssn
+        self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
+        self.assertEqual(mock_write_sync.call_count, 0)
+        self.assertEqual(mock_set_sync.call_count, 1)
+        self.assertEqual(mock_set_sync.call_args_list[0][0], ('2 (foo,other)',))
+
+        self.p._major_version = 150000
+        mock_set_sync.reset_mock()
+        self.p.set_role('replica')
+        self.p.name = 'nonsync'
+        self.ha.fetch_node_status = get_node_status()
+        # Postgres 15, with quorum commit. Non-sync node promoted we avoid updating /sync key and put some nodes to ssn
+        self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
+        self.assertEqual(mock_write_sync.call_count, 0)
+        self.assertEqual(mock_set_sync.call_count, 1)
+        self.assertEqual(mock_set_sync.call_args_list[0][0], ('ANY 3 (foo,other,postgresql0)',))
+
+    @patch.object(global_config.__class__, 'is_synchronous_mode', PropertyMock(return_value=True))
+    @patch.object(global_config.__class__, 'is_quorum_commit_mode', PropertyMock(return_value=True))
+    def test__process_quorum_replication(self):
+        self.p._major_version = 150000
+        self.ha.has_lock = true
+        mock_set_sync = self.p.config.set_synchronous_standby_names = Mock()
+        self.p.name = 'leader'
+
+        mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=None)
+        # Test /sync key is attempted to set and failed when missing or invalid
+        self.p.sync_handler.current_state = Mock(return_value=_SyncState('quorum', 1, 1, CaseInsensitiveSet(['other']),
+                                                                         CaseInsensitiveSet(['other'])))
+        self.ha.run_cycle()
+        self.assertEqual(mock_write_sync.call_count, 1)
+        self.assertEqual(mock_write_sync.call_args_list[0][0], (self.p.name, None, 0))
+        self.assertEqual(mock_write_sync.call_args_list[0][1], {'version': None})
+        self.assertEqual(mock_set_sync.call_count, 0)
+
+        self.ha._promote_timestamp = 1
+        mock_write_sync = self.ha.dcs.write_sync_state = Mock(side_effect=[SyncState(None, self.p.name, None, 0), None])
+        # Test /sync key is attempted to set and succeed when missing or invalid
+        with patch.object(SyncState, 'is_empty', Mock(side_effect=[True, False])):
+            self.ha.run_cycle()
+        self.assertEqual(mock_write_sync.call_count, 2)
+        self.assertEqual(mock_write_sync.call_args_list[0][0], (self.p.name, None, 0))
+        self.assertEqual(mock_write_sync.call_args_list[0][1], {'version': None})
+        self.assertEqual(mock_write_sync.call_args_list[1][0], (self.p.name, CaseInsensitiveSet(['other']), 0))
+        self.assertEqual(mock_write_sync.call_args_list[1][1], {'version': None})
+        self.assertEqual(mock_set_sync.call_count, 0)
+
+        self.p.sync_handler.current_state = Mock(side_effect=[_SyncState('quorum', 1, 0, CaseInsensitiveSet(['foo']),
+                                                                         CaseInsensitiveSet(['other'])),
+                                                              _SyncState('quorum', 1, 1, CaseInsensitiveSet(['foo']),
+                                                                         CaseInsensitiveSet(['foo']))])
+        mock_write_sync = self.ha.dcs.write_sync_state = Mock(return_value=SyncState(1, 'leader', 'foo', 0))
+        self.ha.cluster = get_cluster_initialized_with_leader(sync=('leader', 'foo'))
+        # Test the sync node is removed from voters, added to ssn
+        with patch.object(Postgresql, 'synchronous_standby_names', Mock(return_value='other')), \
+                patch('time.sleep', Mock()):
+            self.ha.run_cycle()
+        self.assertEqual(mock_write_sync.call_count, 1)
+        self.assertEqual(mock_write_sync.call_args_list[0][0], (self.p.name, CaseInsensitiveSet(), 0))
+        self.assertEqual(mock_write_sync.call_args_list[0][1], {'version': 0})
+        self.assertEqual(mock_set_sync.call_count, 1)
+        self.assertEqual(mock_set_sync.call_args_list[0][0], ('ANY 1 (other)',))
+
+        # Test ANY 1 (*) when synchronous_mode_strict and no nodes available
+        self.p.sync_handler.current_state = Mock(return_value=_SyncState('quorum', 1, 0,
+                                                                         CaseInsensitiveSet(['other', 'foo']),
+                                                                         CaseInsensitiveSet()))
+        mock_write_sync.reset_mock()
+        mock_set_sync.reset_mock()
+        with patch.object(global_config.__class__, 'is_synchronous_mode_strict', PropertyMock(return_value=True)):
+            self.ha.run_cycle()
+        self.assertEqual(mock_write_sync.call_count, 1)
+        self.assertEqual(mock_write_sync.call_args_list[0][0], (self.p.name, CaseInsensitiveSet(), 0))
+        self.assertEqual(mock_write_sync.call_args_list[0][1], {'version': 0})
+        self.assertEqual(mock_set_sync.call_count, 1)
+        self.assertEqual(mock_set_sync.call_args_list[0][0], ('ANY 1 (*)',))
+
+        # Test that _process_quorum_replication doesn't take longer than loop_wait
+        with patch('time.time', Mock(side_effect=[30, 60, 90, 120])):
+            self.ha.process_sync_replication()

tests/test_kubernetes.py

@@ -437,7 +437,7 @@ class TestKubernetesEndpoints(BaseTestKubernetes):
 
     @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_endpoints', mock_namespaced_kind, create=True)
     def test_write_sync_state(self):
-        self.assertIsNotNone(self.k.write_sync_state('a', ['b'], 1))
+        self.assertIsNotNone(self.k.write_sync_state('a', ['b'], 0, 1))
 
     @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_pod', mock_namespaced_kind, create=True)
     @patch.object(k8s_client.CoreV1Api, 'create_namespaced_endpoints', mock_namespaced_kind, create=True)

tests/test_patroni.py

@@ -249,6 +249,16 @@ class TestPatroni(unittest.TestCase):
         self.p.tags['nosync'] = None
         self.assertFalse(self.p.nosync)
 
+    def test_nostream(self):
+        self.p.tags['nostream'] = 'True'
+        self.assertTrue(self.p.nostream)
+        self.p.tags['nostream'] = 'None'
+        self.assertFalse(self.p.nostream)
+        self.p.tags['nostream'] = 'foo'
+        self.assertFalse(self.p.nostream)
+        self.p.tags['nostream'] = ''
+        self.assertFalse(self.p.nostream)
+
     @patch.object(Thread, 'join', Mock())
     def test_shutdown(self):
         self.p.api.shutdown = Mock(side_effect=Exception)

tests/test_postgresql.py

@@ -7,6 +7,7 @@ import time
 
 from copy import deepcopy
 from mock import Mock, MagicMock, PropertyMock, patch, mock_open
+from pathlib import Path
 
 import patroni.psycopg as psycopg
 
@@ -363,7 +364,7 @@ class TestPostgresql(BaseTestPostgresql):
     @patch.object(Postgresql, 'start', Mock())
     def test_follow(self):
         self.p.call_nowait(CallbackAction.ON_START)
-        m = RemoteMember('1', {'restore_command': '2', 'primary_slot_name': 'foo', 'conn_kwargs': {'host': 'bar'}})
+        m = RemoteMember('1', {'restore_command': '2', 'primary_slot_name': 'foo', 'conn_kwargs': {'host': 'foo,bar'}})
         self.p.follow(m)
         with patch.object(Postgresql, 'ensure_major_version_is_known', Mock(return_value=False)):
             self.assertIsNone(self.p.follow(m))
@@ -1064,7 +1065,7 @@ class TestPostgresql(BaseTestPostgresql):
     def test__read_postgres_gucs_validators_file(self):
         # raise exception
         with self.assertRaises(InvalidGucValidatorsFile) as exc:
-            _read_postgres_gucs_validators_file('random_file.yaml')
+            _read_postgres_gucs_validators_file(Path('random_file.yaml'))
         self.assertEqual(
             str(exc.exception),
             "Unexpected issue while reading parameters file `random_file.yaml`: `[Errno 2] No such file or directory: "
@@ -1073,17 +1074,32 @@ class TestPostgresql(BaseTestPostgresql):
 
     def test__load_postgres_gucs_validators(self):
         # log messages
-        with patch('os.walk', Mock(return_value=iter([('.', [], ['file.txt', 'random.yaml'])]))), \
-             patch('patroni.postgresql.validator.logger.info') as mock_info, \
+        file1_attrs = {'is_file.return_value': True, 'is_dir.return_value': False}
+        file1_mock = MagicMock(**file1_attrs)
+        file1_mock.name = '__init__.py'
+        file2_attrs = {'is_file.return_value': False, 'is_dir.return_value': True, 'iterdir.return_value': []}
+        file2_mock = MagicMock(**file2_attrs)
+        file2_mock.name = '__pycache__'
+        file3_attrs = {'is_file.return_value': True, 'is_dir.return_value': False}
+        file3_mock = MagicMock(**file3_attrs)
+        file3_mock.name = file3_mock.__str__.return_value = 'random.yaml'
+        file3_mock.open.side_effect = FileNotFoundError('[Errno 2] No such file or directory: random.yaml')
+        file4_attrs = {'is_file.return_value': True, 'is_dir.return_value': False}
+        file4_mock = MagicMock(**file4_attrs)
+        file4_mock.name = 'file.txt'
+        dir_attrs = {'name': 'available_parameters', 'is_file.return_value': False, 'is_dir.return_value': True}
+        dir_mock = MagicMock(**dir_attrs)
+        dir_mock.iterdir.return_value = [file1_mock, file2_mock, file3_mock, file4_mock]
+        with patch('patroni.postgresql.available_parameters.conf_dir', dir_mock), \
+             patch('patroni.postgresql.available_parameters.logger.info') as mock_info, \
              patch('patroni.postgresql.validator.logger.warning') as mock_warning:
             _load_postgres_gucs_validators()
-            mock_info.assert_called_once_with('Ignored a non-YAML file found under `available_parameters` directory: '
-                                              '`%s`.', os.path.join('.', 'file.txt'))
+            mock_info.assert_called_once_with('Ignored a non-YAML file found under `%s` '
+                                              'directory: `%s`.', 'available_parameters', file4_mock)
             mock_warning.assert_called_once()
             self.assertIn(
-                "Unexpected issue while reading parameters file `{0}`: `[Errno 2] No such file or "
-                "directory:".format(os.path.join('.', 'random.yaml')),
-                mock_warning.call_args[0][0]
+                "Unexpected issue while reading parameters file `random.yaml`: `[Errno 2] No such file or "
+                "directory:", mock_warning.call_args[0][0]
             )
 
 

tests/test_quorum.py

@@ -0,0 +1,473 @@
+import unittest
+
+from typing import List, Set, Tuple
+
+from patroni.quorum import QuorumStateResolver, QuorumError
+
+
+class QuorumTest(unittest.TestCase):
+
+    def check_state_transitions(self, leader: str, quorum: int, voters: Set[str], numsync: int, sync: Set[str],
+                                numsync_confirmed: int, active: Set[str], sync_wanted: int, leader_wanted: str,
+                                expected: List[Tuple[str, str, int, Set[str]]]) -> None:
+        kwargs = {
+            'leader': leader, 'quorum': quorum, 'voters': voters,
+            'numsync': numsync, 'sync': sync, 'numsync_confirmed': numsync_confirmed,
+            'active': active, 'sync_wanted': sync_wanted, 'leader_wanted': leader_wanted
+        }
+        result = list(QuorumStateResolver(**kwargs))
+        self.assertEqual(result, expected)
+
+        # also check interrupted transitions
+        if len(result) > 0 and result[0][0] != 'restart' and kwargs['leader'] == result[0][1]:
+            if result[0][0] == 'sync':
+                kwargs.update(numsync=result[0][2], sync=result[0][3])
+            else:
+                kwargs.update(leader=result[0][1], quorum=result[0][2], voters=result[0][3])
+            kwargs['expected'] = expected[1:]
+            self.check_state_transitions(**kwargs)
+
+    def test_1111(self):
+        leader = 'a'
+
+        # Add node
+        self.check_state_transitions(leader=leader, quorum=0, voters=set(),
+                                     numsync=0, sync=set(), numsync_confirmed=0, active=set('b'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            ('sync', leader, 1, set('b')),
+            ('restart', leader, 0, set()),
+        ])
+        self.check_state_transitions(leader=leader, quorum=0, voters=set(),
+                                     numsync=1, sync=set('b'), numsync_confirmed=1, active=set('b'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            ('quorum', leader, 0, set('b'))
+        ])
+
+        self.check_state_transitions(leader=leader, quorum=0, voters=set(),
+                                     numsync=0, sync=set(), numsync_confirmed=0, active=set('bcde'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            ('sync', leader, 2, set('bcde')),
+            ('restart', leader, 0, set()),
+        ])
+        self.check_state_transitions(leader=leader, quorum=0, voters=set(),
+                                     numsync=2, sync=set('bcde'), numsync_confirmed=1, active=set('bcde'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            ('quorum', leader, 3, set('bcde')),
+        ])
+
+    def test_1222(self):
+        """2 node cluster"""
+        leader = 'a'
+
+        # Active set matches state
+        self.check_state_transitions(leader=leader, quorum=0, voters=set('b'),
+                                     numsync=1, sync=set('b'), numsync_confirmed=1, active=set('b'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[])
+
+        # Add node by increasing quorum
+        self.check_state_transitions(leader=leader, quorum=0, voters=set('b'),
+                                     numsync=1, sync=set('b'), numsync_confirmed=1, active=set('BC'),
+                                     sync_wanted=1, leader_wanted=leader, expected=[
+            ('quorum', leader, 1, set('bC')),
+            ('sync', leader, 1, set('bC')),
+        ])
+
+        # Add node by increasing sync
+        self.check_state_transitions(leader=leader, quorum=0, voters=set('b'),
+                                     numsync=1, sync=set('b'), numsync_confirmed=1, active=set('bc'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            ('sync', leader, 2, set('bc')),
+            ('quorum', leader, 1, set('bc')),
+        ])
+        # Reduce quorum after added node caught up
+        self.check_state_transitions(leader=leader, quorum=1, voters=set('bc'),
+                                     numsync=2, sync=set('bc'), numsync_confirmed=2, active=set('bc'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            ('quorum', leader, 0, set('bc')),
+        ])
+
+        # Add multiple nodes by increasing both sync and quorum
+        self.check_state_transitions(leader=leader, quorum=0, voters=set('b'),
+                                     numsync=1, sync=set('b'), numsync_confirmed=1, active=set('BCdE'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            ('sync', leader, 2, set('bC')),
+            ('quorum', leader, 3, set('bCdE')),
+            ('sync', leader, 2, set('bCdE')),
+        ])
+        # Reduce quorum after added nodes caught up
+        self.check_state_transitions(leader=leader, quorum=3, voters=set('bcde'),
+                                     numsync=2, sync=set('bcde'), numsync_confirmed=3, active=set('bcde'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            ('quorum', leader, 2, set('bcde')),
+        ])
+
+        # Primary is alone
+        self.check_state_transitions(leader=leader, quorum=0, voters=set('b'),
+                                     numsync=1, sync=set('b'), numsync_confirmed=0, active=set(),
+                                     sync_wanted=1, leader_wanted=leader, expected=[
+            ('quorum', leader, 0, set()),
+            ('sync', leader, 0, set()),
+        ])
+
+        # Swap out sync replica
+        self.check_state_transitions(leader=leader, quorum=0, voters=set('b'),
+                                     numsync=1, sync=set('b'), numsync_confirmed=0, active=set('c'),
+                                     sync_wanted=1, leader_wanted=leader, expected=[
+            ('quorum', leader, 0, set()),
+            ('sync', leader, 1, set('c')),
+            ('restart', leader, 0, set()),
+        ])
+        # Update quorum when added node caught up
+        self.check_state_transitions(leader=leader, quorum=0, voters=set(),
+                                     numsync=1, sync=set('c'), numsync_confirmed=1, active=set('c'),
+                                     sync_wanted=1, leader_wanted=leader, expected=[
+            ('quorum', leader, 0, set('c')),
+        ])
+
+    def test_1233(self):
+        """Interrupted transition from 2 node cluster to 3 node fully sync cluster"""
+        leader = 'a'
+
+        # Node c went away, transition back to 2 node cluster
+        self.check_state_transitions(leader=leader, quorum=0, voters=set('b'),
+                                     numsync=2, sync=set('bc'), numsync_confirmed=1, active=set('b'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            ('sync', leader, 1, set('b')),
+        ])
+
+        # Node c is available transition to larger quorum set, but not yet caught up.
+        self.check_state_transitions(leader=leader, quorum=0, voters=set('b'),
+                                     numsync=2, sync=set('bc'), numsync_confirmed=1, active=set('bc'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            ('quorum', leader, 1, set('bc')),
+        ])
+
+        # Add in a new node at the same time, but node c didn't caught up yet
+        self.check_state_transitions(leader=leader, quorum=0, voters=set('b'),
+                                     numsync=2, sync=set('bc'), numsync_confirmed=1, active=set('bcd'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            ('quorum', leader, 2, set('bcd')),
+            ('sync', leader, 2, set('bcd')),
+        ])
+        # All sync nodes caught up, reduce quorum
+        self.check_state_transitions(leader=leader, quorum=2, voters=set('bcd'),
+                                     numsync=2, sync=set('bcd'), numsync_confirmed=3, active=set('bcd'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            ('quorum', leader, 1, set('bcd')),
+        ])
+
+        # Change replication factor at the same time
+        self.check_state_transitions(leader=leader, quorum=0, voters=set('b'),
+                                     numsync=2, sync=set('bc'), numsync_confirmed=1, active=set('bc'),
+                                     sync_wanted=1, leader_wanted=leader, expected=[
+            ('quorum', leader, 1, set('bc')),
+            ('sync', leader, 1, set('bc')),
+        ])
+
+    def test_2322(self):
+        """Interrupted transition from 2 node cluster to 3 node cluster with replication factor 2"""
+        leader = 'a'
+
+        # Node c went away, transition back to 2 node cluster
+        self.check_state_transitions(leader=leader, quorum=1, voters=set('bc'),
+                                     numsync=1, sync=set('b'), numsync_confirmed=1, active=set('b'),
+                                     sync_wanted=1, leader_wanted=leader, expected=[
+            ('quorum', leader, 0, set('b')),
+        ])
+
+        # Node c is available transition to larger quorum set.
+        self.check_state_transitions(leader=leader, quorum=1, voters=set('bc'),
+                                     numsync=1, sync=set('b'), numsync_confirmed=1, active=set('bc'),
+                                     sync_wanted=1, leader_wanted=leader, expected=[
+            ('sync', leader, 1, set('bc')),
+        ])
+
+        # Add in a new node at the same time
+        self.check_state_transitions(leader=leader, quorum=1, voters=set('bc'),
+                                     numsync=1, sync=set('b'), numsync_confirmed=1, active=set('bcd'),
+                                     sync_wanted=1, leader_wanted=leader, expected=[
+            ('sync', leader, 1, set('bc')),
+            ('quorum', leader, 2, set('bcd')),
+            ('sync', leader, 1, set('bcd')),
+        ])
+
+        # Convert to a fully synced cluster
+        self.check_state_transitions(leader=leader, quorum=1, voters=set('bc'),
+                                     numsync=1, sync=set('b'), numsync_confirmed=1, active=set('bc'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            ('sync', leader, 2, set('bc')),
+        ])
+        # Reduce quorum after all nodes caught up
+        self.check_state_transitions(leader=leader, quorum=1, voters=set('bc'),
+                                     numsync=2, sync=set('bc'), numsync_confirmed=2, active=set('bc'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            ('quorum', leader, 0, set('bc')),
+        ])
+
+    def test_3535(self):
+        leader = 'a'
+
+        # remove nodes
+        self.check_state_transitions(leader=leader, quorum=2, voters=set('bcde'),
+                                     numsync=2, sync=set('bcde'), numsync_confirmed=2, active=set('bc'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            ('sync', leader, 2, set('bc')),
+            ('quorum', leader, 0, set('bc')),
+        ])
+        self.check_state_transitions(leader=leader, quorum=2, voters=set('bcde'),
+                                     numsync=2, sync=set('bcde'), numsync_confirmed=3, active=set('bcd'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            ('sync', leader, 2, set('bcd')),
+            ('quorum', leader, 1, set('bcd')),
+        ])
+
+        # remove nodes and decrease sync
+        self.check_state_transitions(leader=leader, quorum=2, voters=set('bcde'),
+                                     numsync=2, sync=set('bcde'), numsync_confirmed=2, active=set('bc'),
+                                     sync_wanted=1, leader_wanted=leader, expected=[
+            ('sync', leader, 2, set('bc')),
+            ('quorum', leader, 1, set('bc')),
+            ('sync', leader, 1, set('bc')),
+        ])
+        self.check_state_transitions(leader=leader, quorum=1, voters=set('bcde'),
+                                     numsync=3, sync=set('bcde'), numsync_confirmed=2, active=set('bc'),
+                                     sync_wanted=1, leader_wanted=leader, expected=[
+            ('sync', leader, 3, set('bcd')),
+            ('quorum', leader, 1, set('bc')),
+            ('sync', leader, 1, set('bc')),
+        ])
+
+        # Increase replication factor and decrease quorum
+        self.check_state_transitions(leader=leader, quorum=2, voters=set('bcde'),
+                                     numsync=2, sync=set('bcde'), numsync_confirmed=2, active=set('bcde'),
+                                     sync_wanted=3, leader_wanted=leader, expected=[
+            ('sync', leader, 3, set('bcde')),
+        ])
+        # decrease quorum after more nodes caught up
+        self.check_state_transitions(leader=leader, quorum=2, voters=set('bcde'),
+                                     numsync=3, sync=set('bcde'), numsync_confirmed=3, active=set('bcde'),
+                                     sync_wanted=3, leader_wanted=leader, expected=[
+            ('quorum', leader, 1, set('bcde')),
+        ])
+
+        # Add node with decreasing sync and increasing quorum
+        self.check_state_transitions(leader=leader, quorum=2, voters=set('bcde'),
+                                     numsync=2, sync=set('bcde'), numsync_confirmed=2, active=set('bcdef'),
+                                     sync_wanted=1, leader_wanted=leader, expected=[
+            # increase quorum by 2, 1 for added node and another for reduced sync
+            ('quorum', leader, 4, set('bcdef')),
+            # now reduce replication factor to requested value
+            ('sync', leader, 1, set('bcdef')),
+        ])
+
+        # Remove node with increasing sync and decreasing quorum
+        self.check_state_transitions(leader=leader, quorum=2, voters=set('bcde'),
+                                     numsync=2, sync=set('bcde'), numsync_confirmed=2, active=set('bcd'),
+                                     sync_wanted=3, leader_wanted=leader, expected=[
+            # node e removed from sync wth replication factor increase
+            ('sync', leader, 3, set('bcd')),
+            # node e removed from voters with quorum decrease
+            ('quorum', leader, 1, set('bcd')),
+        ])
+
+    def test_remove_nosync_node(self):
+        leader = 'a'
+        self.check_state_transitions(leader=leader, quorum=0, voters=set('bc'),
+                                     numsync=2, sync=set('bc'), numsync_confirmed=1, active=set('b'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            ('quorum', leader, 0, set('b')),
+            ('sync', leader, 1, set('b'))
+        ])
+
+    def test_swap_sync_node(self):
+        leader = 'a'
+        self.check_state_transitions(leader=leader, quorum=0, voters=set('bc'),
+                                     numsync=2, sync=set('bc'), numsync_confirmed=1, active=set('bd'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            ('quorum', leader, 0, set('b')),
+            ('sync', leader, 2, set('bd')),
+            ('quorum', leader, 1, set('bd'))
+        ])
+
+    def test_promotion(self):
+        # Beginning stat: 'a' in the primary, 1 of bcd in sync
+        # a fails, c gets quorum votes and promotes
+        self.check_state_transitions(leader='a', quorum=2, voters=set('bcd'),
+                                     numsync=0, sync=set(), numsync_confirmed=0, active=set(),
+                                     sync_wanted=1, leader_wanted='c', expected=[
+            ('sync', 'a', 1, set('abd')),    # set a and b to sync
+            ('quorum', 'c', 2, set('abd')),  # set c as a leader and move a to voters
+            #  and stop because there are no active nodes
+        ])
+
+        # next loop, b managed to reconnect
+        self.check_state_transitions(leader='c', quorum=2, voters=set('abd'),
+                                     numsync=1, sync=set('abd'), numsync_confirmed=0, active=set('b'),
+                                     sync_wanted=1, leader_wanted='c', expected=[
+            ('sync', 'c', 1, set('b')),    # remove a from sync as inactive
+            ('quorum', 'c', 0, set('b')),  # remove a from voters and reduce quorum
+        ])
+
+        # alternative reality: next loop, no one reconnected
+        self.check_state_transitions(leader='c', quorum=2, voters=set('abd'),
+                                     numsync=1, sync=set('abd'), numsync_confirmed=0, active=set(),
+                                     sync_wanted=1, leader_wanted='c', expected=[
+            ('quorum', 'c', 0, set()),
+            ('sync', 'c', 0, set()),
+        ])
+
+    def test_nonsync_promotion(self):
+        # Beginning state: 1 of bc in sync. e.g. (a primary, ssn = ANY 1 (b c))
+        # a fails, d sees b and c, knows that it is in sync and decides to promote.
+        # We include in sync state former primary increasing replication factor
+        # and let situation resolve. Node d ssn=ANY 1 (b c)
+        leader = 'd'
+        self.check_state_transitions(leader='a', quorum=1, voters=set('bc'),
+                                     numsync=0, sync=set(), numsync_confirmed=0, active=set(),
+                                     sync_wanted=1, leader_wanted=leader, expected=[
+            # Set a, b, and c to sync and increase replication factor
+            ('sync', 'a', 2, set('abc')),
+            # Set ourselves as the leader and move the old leader to voters
+            ('quorum', leader, 1, set('abc')),
+            # and stop because there are no active nodes
+        ])
+        # next loop, b and c managed to reconnect
+        self.check_state_transitions(leader=leader, quorum=1, voters=set('abc'),
+                                     numsync=2, sync=set('abc'), numsync_confirmed=0, active=set('bc'),
+                                     sync_wanted=1, leader_wanted=leader, expected=[
+            ('sync', leader, 2, set('bc')),     # Remove a from being synced to.
+            ('quorum', leader, 1, set('bc')),   # Remove a from quorum
+            ('sync', leader, 1, set('bc')),     # Can now reduce replication factor back
+        ])
+        # alternative reality: next loop, no one reconnected
+        self.check_state_transitions(leader=leader, quorum=1, voters=set('abc'),
+                                     numsync=2, sync=set('abc'), numsync_confirmed=0, active=set(),
+                                     sync_wanted=1, leader_wanted=leader, expected=[
+            ('quorum', leader, 0, set()),
+            ('sync', leader, 0, set()),
+        ])
+
+    def test_invalid_states(self):
+        leader = 'a'
+
+        # Main invariant is not satisfied, system is in an unsafe state
+        resolver = QuorumStateResolver(leader=leader, quorum=0, voters=set('bc'),
+                                       numsync=1, sync=set('bc'), numsync_confirmed=1,
+                                       active=set('bc'), sync_wanted=1, leader_wanted=leader)
+        self.assertRaises(QuorumError, resolver.check_invariants)
+        self.assertEqual(list(resolver), [
+            ('quorum', leader, 1, set('bc'))
+        ])
+
+        # Quorum and sync states mismatched, somebody other than Patroni modified system state
+        resolver = QuorumStateResolver(leader=leader, quorum=1, voters=set('bc'),
+                                       numsync=2, sync=set('bd'), numsync_confirmed=1,
+                                       active=set('bd'), sync_wanted=1, leader_wanted=leader)
+        self.assertRaises(QuorumError, resolver.check_invariants)
+        self.assertEqual(list(resolver), [
+            ('quorum', leader, 1, set('bd')),
+            ('sync', leader, 1, set('bd')),
+        ])
+        self.assertTrue(repr(resolver.sync).startswith('<CaseInsensitiveSet'))
+
+    def test_sync_high_quorum_low_safety_margin_high(self):
+        leader = 'a'
+
+        self.check_state_transitions(leader=leader, quorum=2, voters=set('bcdef'),
+                                     numsync=4, sync=set('bcdef'), numsync_confirmed=3, active=set('bcdef'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            ('quorum', leader, 3, set('bcdef')),  # Adjust quorum requirements
+            ('sync', leader, 2, set('bcdef')),    # Reduce synchronization
+        ])
+
+    def test_quorum_update(self):
+        resolver = QuorumStateResolver(leader='a', quorum=1, voters=set('bc'), numsync=1, sync=set('bc'),
+                                       numsync_confirmed=1, active=set('bc'), sync_wanted=1, leader_wanted='a')
+        self.assertRaises(QuorumError, list, resolver.quorum_update(-1, set()))
+        self.assertRaises(QuorumError, list, resolver.quorum_update(1, set()))
+
+    def test_sync_update(self):
+        resolver = QuorumStateResolver(leader='a', quorum=1, voters=set('bc'), numsync=1, sync=set('bc'),
+                                       numsync_confirmed=1, active=set('bc'), sync_wanted=1, leader_wanted='a')
+        self.assertRaises(QuorumError, list, resolver.sync_update(-1, set()))
+        self.assertRaises(QuorumError, list, resolver.sync_update(1, set()))
+
+    def test_remove_nodes_with_decreasing_sync(self):
+        leader = 'a'
+
+        # Remove node with decreasing sync
+        self.check_state_transitions(leader=leader, quorum=1, voters=set('bcdef'),
+                                     numsync=4, sync=set('bcdef'), numsync_confirmed=2, active=set('bcd'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            # node f removed from sync
+            ('sync', leader, 4, set('bcde')),
+            # nodes e and f removed from voters with quorum decrease
+            ('quorum', leader, 1, set('bcd')),
+            # node e removed from sync with replication factor decrease
+            ('sync', leader, 2, set('bcd')),
+        ])
+
+        # Interrupted state, and node g joined
+        self.check_state_transitions(leader=leader, quorum=1, voters=set('bcdef'),
+                                     numsync=4, sync=set('bcde'), numsync_confirmed=2, active=set('bcdg'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            # remove nodes e and f from voters
+            ('quorum', leader, 1, set('bcd')),
+            # remove node e from sync and reduce replication factor
+            ('sync', leader, 3, set('bcd')),
+            # add node g to voters with quorum increase
+            ('quorum', leader, 2, set('bcdg')),
+            # add node g to sync and reduce replication factor
+            ('sync', leader, 2, set('bcdg')),
+        ])
+
+        # node f returned
+        self.check_state_transitions(leader=leader, quorum=1, voters=set('bcdef'),
+                                     numsync=4, sync=set('bcde'), numsync_confirmed=2, active=set('bcdf'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            # replace node e with f in sync
+            ('sync', leader, 4, set('bcdf')),
+            # remove nodes e from voters with quorum decrease
+            ('quorum', leader, 2, set('bcdf')),
+            # reduce replication factor as it was requested
+            ('sync', leader, 2, set('bcdf')),
+        ])
+
+        # node e returned
+        self.check_state_transitions(leader=leader, quorum=1, voters=set('bcdef'),
+                                     numsync=4, sync=set('bcde'), numsync_confirmed=2, active=set('bcde'),
+                                     sync_wanted=2, leader_wanted=leader, expected=[
+            # remove nodes f from voters with quorum decrease
+            ('quorum', leader, 2, set('bcde')),
+            # reduce replication factor as it was requested
+            ('sync', leader, 2, set('bcde')),
+        ])
+
+        # node b is also lost
+        self.check_state_transitions(leader=leader, quorum=1, voters=set('bcdef'),
+                                     numsync=4, sync=set('bcde'), numsync_confirmed=2, active=set('cd'),
+                                     sync_wanted=1, leader_wanted=leader, expected=[
+            # remove nodes b, e, and f from voters
+            ('quorum', leader, 1, set('cd')),
+            # remove nodes b and e from sync with replication factor decrease
+            ('sync', leader, 1, set('cd')),
+        ])
+
+    def test_empty_ssn(self):
+        # Beginning stat: 'a' in the primary, 1 of bc in sync
+        # a fails, c gets quorum votes and promotes
+        self.check_state_transitions(leader='a', quorum=1, voters=set('bc'),
+                                     numsync=1, sync=set(), numsync_confirmed=0, active=set(),
+                                     sync_wanted=1, leader_wanted='c', expected=[
+            ('sync', 'a', 1, set('ab')),    # remove a from sync as inactive
+            ('quorum', 'c', 1, set('ab')),  # set c as a leader and move a to voters
+            #  and stop because there are no active nodes
+        ])
+
+        # next loop, b managed to reconnect
+        self.check_state_transitions(leader='c', quorum=1, voters=set('ab'),
+                                     numsync=1, sync=set('ab'), numsync_confirmed=0, active=set('b'),
+                                     sync_wanted=1, leader_wanted='c', expected=[
+            ('sync', 'c', 1, set('b')),    # remove a from sync as inactive
+            ('quorum', 'c', 0, set('b')),  # remove a from voters and reduce quorum
+        ])

tests/test_raft.py

@@ -140,8 +140,8 @@ class TestRaft(unittest.TestCase):
         self.assertTrue(raft.initialize())
         self.assertTrue(raft.cancel_initialization())
         self.assertTrue(raft.set_config_value('{}'))
-        self.assertTrue(raft.write_sync_state('foo', 'bar'))
-        self.assertFalse(raft.write_sync_state('foo', 'bar', 1))
+        self.assertTrue(raft.write_sync_state('foo', 'bar', 0))
+        self.assertFalse(raft.write_sync_state('foo', 'bar', 0, 1))
         raft._mpp = get_mpp({'citus': {'group': 1, 'database': 'postgres'}})
         self.assertTrue(raft.manual_failover('foo', 'bar'))
         raft._mpp = get_mpp({'citus': {'group': 0, 'database': 'postgres'}})

tests/test_rewind.py

@@ -98,7 +98,8 @@ class TestRewind(BaseTestPostgresql):
             self.r.rewind_or_reinitialize_needed_and_possible(self.leader)
 
     @patch.object(CancellableSubprocess, 'call', mock_cancellable_call)
-    @patch.object(Postgresql, 'checkpoint', side_effect=['', '1'],)
+    @patch.object(Postgresql, 'get_guc_value', Mock(return_value=''))
+    @patch.object(Postgresql, 'checkpoint', side_effect=['', '1'])
     @patch.object(Postgresql, 'stop', Mock(return_value=False))
     @patch.object(Postgresql, 'start', Mock())
     def test_execute(self, mock_checkpoint):

tests/test_slots.py

@@ -124,6 +124,68 @@ class TestSlotsHandler(BaseTestPostgresql):
                   "confirmed_flush_lsn": 12345, "catalog_xmin": 105}])]
             self.assertEqual(self.p.slots(), {})
 
+    def test_nostream_slot_processing(self):
+        config = ClusterConfig(
+            1, {'slots': {'foo': {'type': 'logical', 'database': 'a', 'plugin': 'b'}, 'bar': {'type': 'physical'}}}, 1)
+        nostream_node = Member(0, 'test-2', 28, {
+            'state': 'running', 'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
+            'tags': {'nostream': 'True'}
+        })
+        cascade_node = Member(0, 'test-3', 28, {
+            'state': 'running', 'conn_url': 'postgres://replicator:[email protected]:5436/postgres',
+            'tags': {'replicatefrom': 'test-2'}
+        })
+        stream_node = Member(0, 'test-4', 28, {
+            'state': 'running', 'conn_url': 'postgres://replicator:[email protected]:5436/postgres'})
+        cluster = Cluster(
+            True, config, self.leader, Status.empty(),
+            [self.leadermem, nostream_node, cascade_node, stream_node], None, SyncState.empty(), None, None)
+        global_config.update(cluster)
+
+        # sanity for primary
+        self.p.name = self.leadermem.name
+        self.assertEqual(
+            cluster._get_permanent_slots(self.p, self.leadermem, 'primary'),
+            {'foo': {'type': 'logical', 'database': 'a', 'plugin': 'b'}, 'bar': {'type': 'physical'}})
+        self.assertEqual(
+            cluster._get_members_slots(self.p.name, 'primary'),
+            {'test_4': {'type': 'physical'}})
+
+        # nostream node must not have slot on primary
+        self.p.name = nostream_node.name
+        # permanent logical slots are not allowed on nostream node
+        self.assertEqual(
+            cluster._get_permanent_slots(self.p, nostream_node, 'replica'),
+            {'bar': {'type': 'physical'}})
+        self.assertEqual(
+            cluster.get_slot_name_on_primary(self.p.name, nostream_node),
+            None)
+
+        # check cascade member-slot existence on nostream node
+        self.assertEqual(
+            cluster._get_members_slots(nostream_node.name, 'replica'),
+            {'test_3': {'type': 'physical'}})
+
+        # cascade also does not entitled to have logical slot on itself ...
+        self.p.name = cascade_node.name
+        self.assertEqual(
+            cluster._get_permanent_slots(self.p, cascade_node, 'replica'),
+            {'bar': {'type': 'physical'}})
+        # ... and member-slot on primary
+        self.assertEqual(
+            cluster.get_slot_name_on_primary(self.p.name, cascade_node),
+            None)
+
+        # simple replica must have every permanent slot ...
+        self.p.name = stream_node.name
+        self.assertEqual(
+            cluster._get_permanent_slots(self.p, stream_node, 'replica'),
+            {'foo': {'type': 'logical', 'database': 'a', 'plugin': 'b'}, 'bar': {'type': 'physical'}})
+        # ... and member-slot on primary
+        self.assertEqual(
+            cluster.get_slot_name_on_primary(self.p.name, stream_node),
+            'test_4')
+
     @patch.object(Postgresql, 'is_primary', Mock(return_value=False))
     def test__ensure_logical_slots_replica(self):
         self.p.set_role('replica')

tests/test_sync.py

@@ -1,10 +1,10 @@
 import os
 
-from mock import Mock, patch, PropertyMock
+from mock import Mock, patch
 
 from patroni import global_config
 from patroni.collections import CaseInsensitiveSet
-from patroni.dcs import Cluster, SyncState
+from patroni.dcs import Cluster, ClusterConfig, Status, SyncState
 from patroni.postgresql import Postgresql
 
 from . import BaseTestPostgresql, psycopg_connect, mock_available_gucs
@@ -13,7 +13,6 @@ from . import BaseTestPostgresql, psycopg_connect, mock_available_gucs
 @patch('subprocess.call', Mock(return_value=0))
 @patch('patroni.psycopg.connect', psycopg_connect)
 @patch.object(Postgresql, 'available_gucs', mock_available_gucs)
-@patch.object(global_config.__class__, 'is_synchronous_mode', PropertyMock(return_value=True))
 class TestSync(BaseTestPostgresql):
 
     @patch('subprocess.call', Mock(return_value=0))
@@ -26,12 +25,13 @@ class TestSync(BaseTestPostgresql):
         super(TestSync, self).setUp()
         self.p.config.write_postgresql_conf()
         self.s = self.p.sync_handler
+        config = ClusterConfig(1, {'synchronous_mode': True}, 1)
+        self.cluster = Cluster(True, config, self.leader, Status.empty(), [self.me, self.other, self.leadermem],
+                               None, SyncState(0, self.me.name, self.leadermem.name, 0), None, None, None)
+        global_config.update(self.cluster)
 
     @patch.object(Postgresql, 'last_operation', Mock(return_value=1))
     def test_pick_sync_standby(self):
-        cluster = Cluster(True, None, self.leader, 0, [self.me, self.other, self.leadermem], None,
-                          SyncState(0, self.me.name, self.leadermem.name), None, None, None)
-
         pg_stat_replication = [
             {'pid': 100, 'application_name': self.leadermem.name, 'sync_state': 'sync', 'flush_lsn': 1},
             {'pid': 101, 'application_name': self.me.name, 'sync_state': 'async', 'flush_lsn': 2},
@@ -40,34 +40,55 @@ class TestSync(BaseTestPostgresql):
         # sync node is a bit behind of async, but we prefer it anyway
         with patch.object(Postgresql, "_cluster_info_state_get", side_effect=[self.leadermem.name,
                                                                               'on', pg_stat_replication]):
-            self.assertEqual(self.s.current_state(cluster), (CaseInsensitiveSet([self.leadermem.name]),
-                                                             CaseInsensitiveSet([self.leadermem.name])))
+            self.assertEqual(self.s.current_state(self.cluster), ('priority', 1, 1,
+                                                                  CaseInsensitiveSet([self.leadermem.name]),
+                                                                  CaseInsensitiveSet([self.leadermem.name])))
 
         # prefer node with sync_state='potential', even if it is slightly behind of async
         pg_stat_replication[0]['sync_state'] = 'potential'
         for r in pg_stat_replication:
             r['write_lsn'] = r.pop('flush_lsn')
         with patch.object(Postgresql, "_cluster_info_state_get", side_effect=['', 'remote_write', pg_stat_replication]):
-            self.assertEqual(self.s.current_state(cluster), (CaseInsensitiveSet([self.leadermem.name]),
-                                                             CaseInsensitiveSet()))
+            self.assertEqual(self.s.current_state(self.cluster), ('off', 0, 0, CaseInsensitiveSet(),
+                                                                  CaseInsensitiveSet([self.leadermem.name])))
 
         # when there are no sync or potential candidates we pick async with the minimal replication lag
         for i, r in enumerate(pg_stat_replication):
             r.update(replay_lsn=3 - i, application_name=r['application_name'].upper())
         missing = pg_stat_replication.pop(0)
         with patch.object(Postgresql, "_cluster_info_state_get", side_effect=['', 'remote_apply', pg_stat_replication]):
-            self.assertEqual(self.s.current_state(cluster), (CaseInsensitiveSet([self.me.name]), CaseInsensitiveSet()))
+            self.assertEqual(self.s.current_state(self.cluster), ('off', 0, 0, CaseInsensitiveSet(),
+                                                                  CaseInsensitiveSet([self.me.name])))
 
         # unknown sync node is ignored
         missing.update(application_name='missing', sync_state='sync')
         pg_stat_replication.insert(0, missing)
         with patch.object(Postgresql, "_cluster_info_state_get", side_effect=['', 'remote_apply', pg_stat_replication]):
-            self.assertEqual(self.s.current_state(cluster), (CaseInsensitiveSet([self.me.name]), CaseInsensitiveSet()))
+            self.assertEqual(self.s.current_state(self.cluster), ('off', 0, 0, CaseInsensitiveSet(),
+                                                                  CaseInsensitiveSet([self.me.name])))
 
         # invalid synchronous_standby_names and empty pg_stat_replication
         with patch.object(Postgresql, "_cluster_info_state_get", side_effect=['a b', 'remote_apply', None]):
             self.p._major_version = 90400
-            self.assertEqual(self.s.current_state(cluster), (CaseInsensitiveSet(), CaseInsensitiveSet()))
+            self.assertEqual(self.s.current_state(self.cluster), ('off', 0, 0, CaseInsensitiveSet(),
+                                                                  CaseInsensitiveSet()))
+
+    @patch.object(Postgresql, 'last_operation', Mock(return_value=1))
+    def test_current_state_quorum(self):
+        self.cluster.config.data['synchronous_mode'] = 'quorum'
+        global_config.update(self.cluster)
+
+        pg_stat_replication = [
+            {'pid': 100, 'application_name': self.leadermem.name, 'sync_state': 'quorum', 'flush_lsn': 1},
+            {'pid': 101, 'application_name': self.other.name, 'sync_state': 'quorum', 'flush_lsn': 2}]
+
+        # sync node is a bit behind of async, but we prefer it anyway
+        with patch.object(Postgresql, "_cluster_info_state_get",
+                          side_effect=['ANY 1 ({0},"{1}")'.format(self.leadermem.name, self.other.name),
+                                       'on', pg_stat_replication]):
+            self.assertEqual(self.s.current_state(self.cluster),
+                             ('quorum', 1, 2, CaseInsensitiveSet([self.other.name, self.leadermem.name]),
+                              CaseInsensitiveSet([self.leadermem.name, self.other.name])))
 
     def test_set_sync_standby(self):
         def value_in_conf():
@@ -86,6 +107,7 @@ class TestSync(BaseTestPostgresql):
         mock_reload.assert_not_called()
         self.assertEqual(value_in_conf(), "synchronous_standby_names = 'n1'")
 
+        mock_reload.reset_mock()
         self.s.set_synchronous_standby_names(CaseInsensitiveSet(['n1', 'n2']))
         mock_reload.assert_called()
         self.assertEqual(value_in_conf(), "synchronous_standby_names = '2 (n1,n2)'")
@@ -99,3 +121,36 @@ class TestSync(BaseTestPostgresql):
         self.s.set_synchronous_standby_names(CaseInsensitiveSet('*'))
         mock_reload.assert_called()
         self.assertEqual(value_in_conf(), "synchronous_standby_names = '*'")
+
+        self.cluster.config.data['synchronous_mode'] = 'quorum'
+        global_config.update(self.cluster)
+        mock_reload.reset_mock()
+        self.s.set_synchronous_standby_names([], 1)
+        mock_reload.assert_called()
+        self.assertEqual(value_in_conf(), "synchronous_standby_names = 'ANY 1 (*)'")
+
+        mock_reload.reset_mock()
+        self.s.set_synchronous_standby_names(['a', 'b'], 1)
+        mock_reload.assert_called()
+        self.assertEqual(value_in_conf(), "synchronous_standby_names = 'ANY 1 (a,b)'")
+
+        mock_reload.reset_mock()
+        self.s.set_synchronous_standby_names(['a', 'b'], 3)
+        mock_reload.assert_called()
+        self.assertEqual(value_in_conf(), "synchronous_standby_names = 'ANY 3 (a,b)'")
+
+        self.p._major_version = 90601
+        mock_reload.reset_mock()
+        self.s.set_synchronous_standby_names([], 1)
+        mock_reload.assert_called()
+        self.assertEqual(value_in_conf(), "synchronous_standby_names = '1 (*)'")
+
+        mock_reload.reset_mock()
+        self.s.set_synchronous_standby_names(['a', 'b'], 1)
+        mock_reload.assert_called()
+        self.assertEqual(value_in_conf(), "synchronous_standby_names = '1 (a,b)'")
+
+        mock_reload.reset_mock()
+        self.s.set_synchronous_standby_names(['a', 'b'], 3)
+        mock_reload.assert_called()
+        self.assertEqual(value_in_conf(), "synchronous_standby_names = '3 (a,b)'")

tests/test_validator.py

@@ -103,7 +103,8 @@ config = {
         "nofailover": False,
         "clonefrom": False,
         "noloadbalance": False,
-        "nosync": False
+        "nosync": False,
+        "nostream": False
     }
 }