Merge branch 'release/v3.3.0' of github.com:zalando/patroni into feature/citus-secondaries

Most of them are related to partially unknown types of values from empty
dict or list. To solve it for the empty dict we use `EMPTY_DICT` object of
newly introduced `_FrozenDict` class.

.github/workflows/install_deps.py

@@ -110,7 +110,7 @@ def install_etcd():
 
 
 def install_postgres():
-    version = os.environ.get('PGVERSION', '15.1-1')
+    version = os.environ.get('PGVERSION', '16.1-1')
     platform = {'darwin': 'osx', 'win32': 'windows-x64', 'cygwin': 'windows-x64'}[sys.platform]
     if platform == 'osx':
         return subprocess.call(['brew', 'install', 'expect', 'postgresql@{0}'.format(version.split('.')[0])])

.github/workflows/mapping.py

@@ -1 +1 @@
-versions = {'etcd': '9.6', 'etcd3': '16', 'consul': '13', 'exhibitor': '12', 'raft': '11', 'kubernetes': '15'}
+versions = {'etcd': '9.6', 'etcd3': '16', 'consul': '13', 'exhibitor': '12', 'raft': '14', 'kubernetes': '15'}

.github/workflows/run_tests.py

@@ -30,7 +30,7 @@ def main():
         unbuffer = ['timeout', '900', 'unbuffer']
     else:
         if sys.platform == 'darwin':
-            version = os.environ.get('PGVERSION', '15.1-1')
+            version = os.environ.get('PGVERSION', '16.1-1')
             path = '/usr/local/opt/postgresql@{0}/bin:.'.format(version.split('.')[0])
             unbuffer = ['unbuffer']
         else:

.github/workflows/tests.yaml

@@ -85,7 +85,7 @@ jobs:
     env:
       DCS: ${{ matrix.dcs }}
       ETCDVERSION: 3.4.23
-      PGVERSION: 15.1-1  # for windows and macos
+      PGVERSION: 16.1-1  # for windows and macos
     strategy:
       fail-fast: false
       matrix:
@@ -174,7 +174,7 @@ jobs:
 
     - uses: jakebailey/pyright-action@v1
       with:
-        version: 1.1.333
+        version: 1.1.356
 
   docs:
     runs-on: ubuntu-latest

.gitignore

@@ -27,7 +27,7 @@ lib64
 pip-log.txt
 
 # Unit test / coverage reports
-.coverage
+.coverage*
 .tox
 nosetests.xml
 coverage.xml
@@ -35,6 +35,7 @@ htmlcov
 junit.xml
 features/output*
 dummy
+result.json
 
 # Translations
 *.mo

Dockerfile

@@ -1,6 +1,6 @@
 ## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine
 ## It has all the necessary components to play/debug with a single node appliance, running etcd
-ARG PG_MAJOR=15
+ARG PG_MAJOR=16
 ARG COMPRESS=false
 ARG PGHOME=/home/postgres
 ARG PGDATA=$PGHOME/data
@@ -94,9 +94,9 @@ RUN set -ex \
         /usr/share/locale/??_?? \
         /usr/share/postgresql/*/man \
         /usr/share/postgresql-common/pg_wrapper \
-        /usr/share/vim/vim80/doc \
-        /usr/share/vim/vim80/lang \
-        /usr/share/vim/vim80/tutor \
+        /usr/share/vim/vim*/doc \
+        /usr/share/vim/vim*/lang \
+        /usr/share/vim/vim*/tutor \
 #        /var/lib/dpkg/info/* \
     && find /usr/bin -xtype l -delete \
     && find /var/log -type f -exec truncate --size 0 {} \; \
@@ -125,6 +125,8 @@ RUN if [ "$COMPRESS" = "true" ]; then \
         && /bin/busybox sh -c "(find $save_dirs -not -type d && cat /exclude /exclude && echo exclude) | sort | uniq -u | xargs /bin/busybox rm" \
         && /bin/busybox --install -s \
         && /bin/busybox sh -c "find $save_dirs -type d -depth -exec rmdir -p {} \; 2> /dev/null"; \
+    else \
+        /bin/busybox --install -s; \
     fi
 
 FROM scratch
@@ -143,6 +145,7 @@ ARG PGBIN=/usr/lib/postgresql/$PG_MAJOR/bin
 
 ENV LC_ALL=$LC_ALL LANG=$LANG EDITOR=/usr/bin/editor
 ENV PGDATA=$PGDATA PATH=$PATH:$PGBIN
+ENV ETCDCTL_API=3
 
 COPY patroni /patroni/
 COPY extras/confd/conf.d/haproxy.toml /etc/confd/conf.d/

Dockerfile.citus

@@ -1,6 +1,6 @@
 ## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine
 ## It has all the necessary components to play/debug with a single node appliance, running etcd
-ARG PG_MAJOR=15
+ARG PG_MAJOR=16
 ARG COMPRESS=false
 ARG PGHOME=/home/postgres
 ARG PGDATA=$PGHOME/data
@@ -40,7 +40,7 @@ RUN set -ex \
         echo "deb [signed-by=/etc/apt/trusted.gpg.d/citusdata_community.gpg] https://packagecloud.io/citusdata/community/debian/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/citusdata_community.list \
         && curl -sL https://packagecloud.io/citusdata/community/gpgkey | gpg --dearmor > /etc/apt/trusted.gpg.d/citusdata_community.gpg \
         && apt-get update -y \
-        && apt-get -y install postgresql-$PG_MAJOR-citus-11.3; \
+        && apt-get -y install postgresql-$PG_MAJOR-citus-12.1; \
     fi \
 \
     # Cleanup all locales but en_US.UTF-8
@@ -113,9 +113,9 @@ RUN set -ex \
         /usr/share/locale/??_?? \
         /usr/share/postgresql/*/man \
         /usr/share/postgresql-common/pg_wrapper \
-        /usr/share/vim/vim80/doc \
-        /usr/share/vim/vim80/lang \
-        /usr/share/vim/vim80/tutor \
+        /usr/share/vim/vim*/doc \
+        /usr/share/vim/vim*/lang \
+        /usr/share/vim/vim*/tutor \
 #        /var/lib/dpkg/info/* \
     && find /usr/bin -xtype l -delete \
     && find /var/log -type f -exec truncate --size 0 {} \; \
@@ -164,6 +164,7 @@ ARG PGBIN=/usr/lib/postgresql/$PG_MAJOR/bin
 
 ENV LC_ALL=$LC_ALL LANG=$LANG EDITOR=/usr/bin/editor
 ENV PGDATA=$PGDATA PATH=$PATH:$PGBIN
+ENV ETCDCTL_API=3
 
 COPY patroni /patroni/
 COPY extras/confd/conf.d/haproxy.toml /etc/confd/conf.d/

README.rst

@@ -151,7 +151,7 @@ run:
 YAML Configuration
 ==================
 
-Go `here <https://github.com/zalando/patroni/blob/master/docs/SETTINGS.rst>`__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
+Go `here <https://github.com/zalando/patroni/blob/master/docs/dynamic_configuration.rst>`__ for comprehensive information about settings for etcd, consul, and ZooKeeper. And for an example, see `postgres0.yml <https://github.com/zalando/patroni/blob/master/postgres0.yml>`__.
 
 =========================
 Environment Configuration

docker-compose-citus.yml

@@ -19,7 +19,6 @@ services:
         image: ${PATRONI_TEST_IMAGE:-patroni-citus}
         networks: [ demo ]
         environment:
-            ETCDCTL_API: 3
             ETCD_LISTEN_PEER_URLS: http://0.0.0.0:2380
             ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379
             ETCD_INITIAL_CLUSTER: etcd1=http://etcd1:2380,etcd2=http://etcd2:2380,etcd3=http://etcd3:2380
@@ -28,19 +27,19 @@ services:
             ETCD_UNSUPPORTED_ARCH: arm64
         container_name: demo-etcd1
         hostname: etcd1
-        command: etcd -name etcd1 -initial-advertise-peer-urls http://etcd1:2380
+        command: etcd --name etcd1 --initial-advertise-peer-urls http://etcd1:2380
 
     etcd2:
         <<: *etcd
         container_name: demo-etcd2
         hostname: etcd2
-        command: etcd -name etcd2 -initial-advertise-peer-urls http://etcd2:2380
+        command: etcd --name etcd2 --initial-advertise-peer-urls http://etcd2:2380
 
     etcd3:
         <<: *etcd
         container_name: demo-etcd3
         hostname: etcd3
-        command: etcd -name etcd3 -initial-advertise-peer-urls http://etcd3:2380
+        command: etcd --name etcd3 --initial-advertise-peer-urls http://etcd3:2380
 
     haproxy:
         image: ${PATRONI_TEST_IMAGE:-patroni-citus}
@@ -53,7 +52,6 @@ services:
             - "5001:5001"  # Load-balancing across workers primaries
         command: haproxy
         environment: &haproxy_env
-            ETCDCTL_API: 3
             ETCDCTL_ENDPOINTS: http://etcd1:2379,http://etcd2:2379,http://etcd3:2379
             PATRONI_ETCD3_HOSTS: "'etcd1:2379','etcd2:2379','etcd3:2379'"
             PATRONI_SCOPE: demo

docker-compose.yml

@@ -25,19 +25,19 @@ services:
             ETCD_UNSUPPORTED_ARCH: arm64
         container_name: demo-etcd1
         hostname: etcd1
-        command: etcd -name etcd1 -initial-advertise-peer-urls http://etcd1:2380
+        command: etcd --name etcd1 --initial-advertise-peer-urls http://etcd1:2380
 
     etcd2:
         <<: *etcd
         container_name: demo-etcd2
         hostname: etcd2
-        command: etcd -name etcd2 -initial-advertise-peer-urls http://etcd2:2380
+        command: etcd --name etcd2 --initial-advertise-peer-urls http://etcd2:2380
 
     etcd3:
         <<: *etcd
         container_name: demo-etcd3
         hostname: etcd3
-        command: etcd -name etcd3 -initial-advertise-peer-urls http://etcd3:2380
+        command: etcd --name etcd3 --initial-advertise-peer-urls http://etcd3:2380
 
     haproxy:
         image: ${PATRONI_TEST_IMAGE:-patroni}

docker/README.md

@@ -19,102 +19,97 @@ The haproxy listens on ports 5000 (connects to the primary) and 5001 (does load-
 
 Example session:
 
-    $ docker-compose up -d
-    Creating demo-haproxy ...
-    Creating demo-patroni2 ...
-    Creating demo-patroni1 ...
-    Creating demo-patroni3 ...
-    Creating demo-etcd2 ...
-    Creating demo-etcd1 ...
-    Creating demo-etcd3 ...
-    Creating demo-haproxy
-    Creating demo-patroni2
-    Creating demo-patroni1
-    Creating demo-patroni3
-    Creating demo-etcd1
-    Creating demo-etcd2
-    Creating demo-etcd2 ... done
+    $ docker compose up -d
+    ✔ Network patroni_demo     Created
+    ✔ Container demo-etcd1     Started
+    ✔ Container demo-haproxy   Started
+    ✔ Container demo-patroni1  Started
+    ✔ Container demo-patroni2  Started
+    ✔ Container demo-patroni3  Started
+    ✔ Container demo-etcd2     Started
+    ✔ Container demo-etcd3     Started
 
     $ docker ps
-    CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                              NAMES
-    5b7a90b4cfbf        patroni             "/bin/sh /entrypoint…"   29 seconds ago      Up 27 seconds                                          demo-etcd2
-    e30eea5222f2        patroni             "/bin/sh /entrypoint…"   29 seconds ago      Up 27 seconds                                          demo-etcd1
-    83bcf3cb208f        patroni             "/bin/sh /entrypoint…"   29 seconds ago      Up 27 seconds                                          demo-etcd3
-    922532c56e7d        patroni             "/bin/sh /entrypoint…"   29 seconds ago      Up 28 seconds                                          demo-patroni3
-    14f875e445f3        patroni             "/bin/sh /entrypoint…"   29 seconds ago      Up 28 seconds                                          demo-patroni2
-    110d1073b383        patroni             "/bin/sh /entrypoint…"   29 seconds ago      Up 28 seconds                                          demo-patroni1
-    5af5e6e36028        patroni             "/bin/sh /entrypoint…"   29 seconds ago      Up 28 seconds       0.0.0.0:5000-5001->5000-5001/tcp   demo-haproxy
+    CONTAINER ID   IMAGE     COMMAND                  CREATED          STATUS          PORTS                                                           NAMES
+    a37bcec56726   patroni   "/bin/sh /entrypoint…"   15 minutes ago   Up 15 minutes                                                                   demo-etcd3
+    034ab73868a8   patroni   "/bin/sh /entrypoint…"   15 minutes ago   Up 15 minutes                                                                   demo-patroni2
+    03837736f710   patroni   "/bin/sh /entrypoint…"   15 minutes ago   Up 15 minutes                                                                   demo-patroni3
+    22815c3d85b3   patroni   "/bin/sh /entrypoint…"   15 minutes ago   Up 15 minutes                                                                   demo-etcd2
+    814b4304d132   patroni   "/bin/sh /entrypoint…"   15 minutes ago   Up 15 minutes   0.0.0.0:5000-5001->5000-5001/tcp, :::5000-5001->5000-5001/tcp   demo-haproxy
+    6375b0ba2d0a   patroni   "/bin/sh /entrypoint…"   15 minutes ago   Up 15 minutes                                                                   demo-patroni1
+    aef8bf3ee91f   patroni   "/bin/sh /entrypoint…"   15 minutes ago   Up 15 minutes                                                                   demo-etcd1
 
     $ docker logs demo-patroni1
-    2019-02-20 08:19:32,714 INFO: Failed to import patroni.dcs.consul
-    2019-02-20 08:19:32,737 INFO: Selected new etcd server http://etcd3:2379
-    2019-02-20 08:19:35,140 INFO: Lock owner: None; I am patroni1
-    2019-02-20 08:19:35,174 INFO: trying to bootstrap a new cluster
+    2023-11-21 09:04:33,547 INFO: Selected new etcd server http://172.29.0.3:2379
+    2023-11-21 09:04:33,605 INFO: Lock owner: None; I am patroni1
+    2023-11-21 09:04:33,693 INFO: trying to bootstrap a new cluster
     ...
-    2019-02-20 08:19:39,310 INFO: postmaster pid=37
-    2019-02-20 08:19:39.314 UTC [37] LOG:  listening on IPv4 address "0.0.0.0", port 5432
-    2019-02-20 08:19:39.321 UTC [37] LOG:  listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432"
-    2019-02-20 08:19:39.353 UTC [39] LOG:  database system was shut down at 2019-02-20 08:19:36 UTC
-    2019-02-20 08:19:39.354 UTC [40] FATAL:  the database system is starting up
-    localhost:5432 - rejecting connections
-    2019-02-20 08:19:39.369 UTC [37] LOG:  database system is ready to accept connections
+    2023-11-21 09:04:34.920 UTC [43] LOG:  starting PostgreSQL 15.5 (Debian 15.5-1.pgdg120+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14) 12.2.0, 64-bit
+    2023-11-21 09:04:34.921 UTC [43] LOG:  listening on IPv4 address "0.0.0.0", port 5432
+    2023-11-21 09:04:34,922 INFO: postmaster pid=43
+    2023-11-21 09:04:34.922 UTC [43] LOG:  listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432"
+    2023-11-21 09:04:34.925 UTC [47] LOG:  database system was shut down at 2023-11-21 09:04:34 UTC
+    2023-11-21 09:04:34.928 UTC [43] LOG:  database system is ready to accept connections
     localhost:5432 - accepting connections
-    2019-02-20 08:19:39,383 INFO: establishing a new patroni connection to the postgres cluster
-    2019-02-20 08:19:39,408 INFO: running post_bootstrap
-    2019-02-20 08:19:39,432 WARNING: Could not activate Linux watchdog device: "Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog'"
-    2019-02-20 08:19:39,515 INFO: initialized a new cluster
-    2019-02-20 08:19:49,424 INFO: Lock owner: patroni1; I am patroni1
-    2019-02-20 08:19:49,447 INFO: Lock owner: patroni1; I am patroni1
-    2019-02-20 08:19:49,480 INFO: no action.  i am the leader with the lock
-    2019-02-20 08:19:59,422 INFO: Lock owner: patroni1; I am patroni1
+    localhost:5432 - accepting connections
+    2023-11-21 09:04:34,938 INFO: establishing a new patroni heartbeat connection to postgres
+    2023-11-21 09:04:34,992 INFO: running post_bootstrap
+    2023-11-21 09:04:35,004 WARNING: User creation via "bootstrap.users" will be removed in v4.0.0
+    2023-11-21 09:04:35,009 WARNING: Could not activate Linux watchdog device: Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog'
+    2023-11-21 09:04:35,189 INFO: initialized a new cluster
+    2023-11-21 09:04:35,328 INFO: no action. I am (patroni1), the leader with the lock
+    2023-11-21 09:04:43,824 INFO: establishing a new patroni restapi connection to postgres
+    2023-11-21 09:04:45,322 INFO: no action. I am (patroni1), the leader with the lock
+    2023-11-21 09:04:55,320 INFO: no action. I am (patroni1), the leader with the lock
+    ...
 
     $ docker exec -ti demo-patroni1 bash
     postgres@patroni1:~$ patronictl list
-    +---------+----------+------------+--------+---------+----+-----------+
-    | Cluster |  Member  |    Host    |  Role  |  State  | TL | Lag in MB |
-    +---------+----------+------------+--------+---------+----+-----------+
-    |   demo  | patroni1 | 172.22.0.3 | Leader | running |  1 |         0 |
-    |   demo  | patroni2 | 172.22.0.7 |        | running |  1 |         0 |
-    |   demo  | patroni3 | 172.22.0.4 |        | running |  1 |         0 |
-    +---------+----------+------------+--------+---------+----+-----------+
+    + Cluster: demo (7303838734793224214) --------+----+-----------+
+    | Member   | Host       | Role    | State     | TL | Lag in MB |
+    +----------+------------+---------+-----------+----+-----------+
+    | patroni1 | 172.29.0.2 | Leader  | running   |  1 |           |
+    | patroni2 | 172.29.0.6 | Replica | streaming |  1 |         0 |
+    | patroni3 | 172.29.0.5 | Replica | streaming |  1 |         0 |
+    +----------+------------+---------+-----------+----+-----------+
 
     postgres@patroni1:~$ etcdctl get --keys-only --prefix /service/demo
     /service/demo/config
     /service/demo/initialize
     /service/demo/leader
-    /service/demo/members/
     /service/demo/members/patroni1
     /service/demo/members/patroni2
     /service/demo/members/patroni3
-    /service/demo/optime/
-    /service/demo/optime/leader
+    /service/demo/status
 
     postgres@patroni1:~$ etcdctl member list
-    1bab629f01fa9065: name=etcd3 peerURLs=http://etcd3:2380 clientURLs=http://etcd3:2379 isLeader=false
-    8ecb6af518d241cc: name=etcd2 peerURLs=http://etcd2:2380 clientURLs=http://etcd2:2379 isLeader=true
-    b2e169fcb8a34028: name=etcd1 peerURLs=http://etcd1:2380 clientURLs=http://etcd1:2379 isLeader=false
+    2bf3e2ceda5d5960, started, etcd2, http://etcd2:2380, http://172.29.0.3:2379
+    55b3264e129c7005, started, etcd3, http://etcd3:2380, http://172.29.0.7:2379
+    acce7233f8ec127e, started, etcd1, http://etcd1:2380, http://172.29.0.8:2379
+
+
     postgres@patroni1:~$ exit
 
     $ docker exec -ti demo-haproxy bash
     postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -W
     Password: postgres
-    psql (11.2 (Ubuntu 11.2-1.pgdg18.04+1), server 10.7 (Debian 10.7-1.pgdg90+1))
+    psql (15.5 (Debian 15.5-1.pgdg120+1))
     Type "help" for help.
 
-    localhost/postgres=# select pg_is_in_recovery();
+    postgres=# SELECT pg_is_in_recovery();
      pg_is_in_recovery
     ───────────────────
      f
     (1 row)
 
-    localhost/postgres=# \q
+    postgres=# \q
 
-    $postgres@haproxy:~ psql -h localhost -p 5001 -U postgres -W
+    postgres@haproxy:~$ psql -h localhost -p 5001 -U postgres -W
     Password: postgres
-    psql (11.2 (Ubuntu 11.2-1.pgdg18.04+1), server 10.7 (Debian 10.7-1.pgdg90+1))
+    psql (15.5 (Debian 15.5-1.pgdg120+1))
     Type "help" for help.
 
-    localhost/postgres=# select pg_is_in_recovery();
+    postgres=# SELECT pg_is_in_recovery();
      pg_is_in_recovery
     ───────────────────
      t
@@ -127,81 +122,86 @@ The haproxy listens on ports 5000 (connects to the coordinator primary) and 5001
 
 Example session:
 
-    $ docker-compose -f docker-compose-citus.yml up -d
-    Creating demo-work2-1 ... done
-    Creating demo-work1-1 ... done
-    Creating demo-etcd2   ... done
-    Creating demo-etcd1   ... done
-    Creating demo-coord3  ... done
-    Creating demo-etcd3   ... done
-    Creating demo-coord1  ... done
-    Creating demo-haproxy ... done
-    Creating demo-work2-2 ... done
-    Creating demo-coord2  ... done
-    Creating demo-work1-2 ... done
+    $ docker compose -f docker-compose-citus.yml up -d
+    ✔ Network patroni_demo    Created
+    ✔ Container demo-coord2   Started
+    ✔ Container demo-work2-2  Started
+    ✔ Container demo-etcd1    Started
+    ✔ Container demo-haproxy  Started
+    ✔ Container demo-work1-1  Started
+    ✔ Container demo-work2-1  Started
+    ✔ Container demo-work1-2  Started
+    ✔ Container demo-coord1   Started
+    ✔ Container demo-etcd3    Started
+    ✔ Container demo-coord3   Started
+    ✔ Container demo-etcd2    Started
+
 
     $ docker ps
-    CONTAINER ID   IMAGE                  COMMAND                  CREATED         STATUS         PORTS                              NAMES
-    852d8885a612   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-coord3
-    cdd692f947ab   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-work1-2
-    9f4e340b36da   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-etcd3
-    d69c129a960a   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds                                      demo-etcd1
-    c5849689b8cd   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds                                      demo-coord1
-    c9d72bd6217d   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-work2-1
-    24b1b43efa05   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds                                      demo-coord2
-    cb0cc2b4ca0a   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 3 seconds                                      demo-work2-2
-    9796c6b8aad5   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 5 seconds                                      demo-work1-1
-    8baccd74dcae   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds                                      demo-etcd2
-    353ec62a0187   patroni-citus          "/bin/sh /entrypoint…"   6 seconds ago   Up 4 seconds   0.0.0.0:5000-5001->5000-5001/tcp   demo-haproxy
+    CONTAINER ID   IMAGE           COMMAND                  CREATED          STATUS          PORTS                                                           NAMES
+    79c95492fac9   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-etcd3
+    77eb82d0f0c1   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-work2-1
+    03dacd7267ef   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-etcd1
+    db9206c66f85   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-etcd2
+    9a0fef7b7dd4   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-work1-2
+    f06b031d99dc   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-work2-2
+    f7c58545f314   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-coord2
+    383f9e7e188a   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-work1-1
+    f02e96dcc9d6   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-coord3
+    6945834b7056   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes                                                                   demo-coord1
+    b96ca42f785d   patroni-citus   "/bin/sh /entrypoint…"   11 minutes ago   Up 11 minutes   0.0.0.0:5000-5001->5000-5001/tcp, :::5000-5001->5000-5001/tcp   demo-haproxy
+
 
     $ docker logs demo-coord1
-    2023-01-05 15:09:31,295 INFO: Selected new etcd server http://172.27.0.4:2379
-    2023-01-05 15:09:31,388 INFO: Lock owner: None; I am coord1
-    2023-01-05 15:09:31,501 INFO: trying to bootstrap a new cluster
+    2023-11-21 09:36:14,293 INFO: Selected new etcd server http://172.30.0.4:2379
+    2023-11-21 09:36:14,390 INFO: Lock owner: None; I am coord1
+    2023-11-21 09:36:14,478 INFO: trying to bootstrap a new cluster
     ...
-    2023-01-05 15:09:45,096 INFO: postmaster pid=39
+    2023-11-21 09:36:16,475 INFO: postmaster pid=52
     localhost:5432 - no response
-    2023-01-05 15:09:45.137 UTC [39] LOG:  starting PostgreSQL 15.1 (Debian 15.1-1.pgdg110+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 10.2.1-6) 10.2.1 20210110, 64-bit
-    2023-01-05 15:09:45.137 UTC [39] LOG:  listening on IPv4 address "0.0.0.0", port 5432
-    2023-01-05 15:09:45.152 UTC [39] LOG:  listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432"
-    2023-01-05 15:09:45.177 UTC [43] LOG:  database system was shut down at 2023-01-05 15:09:32 UTC
-    2023-01-05 15:09:45.193 UTC [39] LOG:  database system is ready to accept connections
+    2023-11-21 09:36:16.495 UTC [52] LOG:  starting PostgreSQL 15.5 (Debian 15.5-1.pgdg120+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14) 12.2.0, 64-bit
+    2023-11-21 09:36:16.495 UTC [52] LOG:  listening on IPv4 address "0.0.0.0", port 5432
+    2023-11-21 09:36:16.496 UTC [52] LOG:  listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432"
+    2023-11-21 09:36:16.498 UTC [56] LOG:  database system was shut down at 2023-11-21 09:36:15 UTC
+    2023-11-21 09:36:16.501 UTC [52] LOG:  database system is ready to accept connections
     localhost:5432 - accepting connections
     localhost:5432 - accepting connections
-    2023-01-05 15:09:46,139 INFO: establishing a new patroni connection to the postgres cluster
-    2023-01-05 15:09:46,208 INFO: running post_bootstrap
-    2023-01-05 15:09:47.209 UTC [55] LOG:  starting maintenance daemon on database 16386 user 10
-    2023-01-05 15:09:47.209 UTC [55] CONTEXT:  Citus maintenance daemon for database 16386 user 10
-    2023-01-05 15:09:47,215 WARNING: Could not activate Linux watchdog device: "Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog'"
-    2023-01-05 15:09:47.446 UTC [41] LOG:  checkpoint starting: immediate force wait
-    2023-01-05 15:09:47,466 INFO: initialized a new cluster
-    2023-01-05 15:09:47,594 DEBUG: query(SELECT nodeid, groupid, nodename, nodeport, noderole FROM pg_catalog.pg_dist_node WHERE noderole = 'primary', ())
-    2023-01-05 15:09:47,594 INFO: establishing a new patroni connection to the postgres cluster
-    2023-01-05 15:09:47,467 INFO: Lock owner: coord1; I am coord1
-    2023-01-05 15:09:47,613 DEBUG: query(SELECT pg_catalog.citus_set_coordinator_host(%s, %s, 'primary', 'default'), ('172.27.0.6', 5432))
-    2023-01-05 15:09:47,924 INFO: no action. I am (coord1), the leader with the lock
-    2023-01-05 15:09:51.282 UTC [41] LOG:  checkpoint complete: wrote 1086 buffers (53.0%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.029 s, sync=3.746 s, total=3.837 s; sync files=280, longest=0.028 s, average=0.014 s; distance=8965 kB, estimate=8965 kB
-    2023-01-05 15:09:51.283 UTC [41] LOG:  checkpoint starting: immediate force wait
-    2023-01-05 15:09:51.495 UTC [41] LOG:  checkpoint complete: wrote 18 buffers (0.9%); 0 WAL file(s) added, 0 removed, 0 recycled; write=0.044 s, sync=0.091 s, total=0.212 s; sync files=15, longest=0.015 s, average=0.007 s; distance=67 kB, estimate=8076 kB
-    2023-01-05 15:09:57,467 INFO: Lock owner: coord1; I am coord1
-    2023-01-05 15:09:57,569 INFO: Assigning synchronous standby status to ['coord3']
+    2023-11-21 09:36:17,509 INFO: establishing a new patroni heartbeat connection to postgres
+    2023-11-21 09:36:17,569 INFO: running post_bootstrap
+    2023-11-21 09:36:17,593 WARNING: User creation via "bootstrap.users" will be removed in v4.0.0
+    2023-11-21 09:36:17,783 INFO: establishing a new patroni restapi connection to postgres
+    2023-11-21 09:36:17,969 WARNING: Could not activate Linux watchdog device: Can't open watchdog device: [Errno 2] No such file or directory: '/dev/watchdog'
+    2023-11-21 09:36:17.969 UTC [70] LOG:  starting maintenance daemon on database 16386 user 10
+    2023-11-21 09:36:17.969 UTC [70] CONTEXT:  Citus maintenance daemon for database 16386 user 10
+    2023-11-21 09:36:18.159 UTC [54] LOG:  checkpoint starting: immediate force wait
+    2023-11-21 09:36:18,162 INFO: initialized a new cluster
+    2023-11-21 09:36:18,164 INFO: Lock owner: coord1; I am coord1
+    2023-11-21 09:36:18,297 INFO: Enabled synchronous replication
+    2023-11-21 09:36:18,298 DEBUG: Adding the new task: PgDistNode(nodeid=None,group=0,host=172.30.0.3,port=5432,event=after_promote)
+    2023-11-21 09:36:18,298 DEBUG: Adding the new task: PgDistNode(nodeid=None,group=1,host=172.30.0.7,port=5432,event=after_promote)
+    2023-11-21 09:36:18,298 DEBUG: Adding the new task: PgDistNode(nodeid=None,group=2,host=172.30.0.8,port=5432,event=after_promote)
+    2023-11-21 09:36:18,299 DEBUG: query(SELECT nodeid, groupid, nodename, nodeport, noderole FROM pg_catalog.pg_dist_node WHERE noderole = 'primary', ())
+    2023-11-21 09:36:18,299 INFO: establishing a new patroni citus connection to postgres
+    2023-11-21 09:36:18,323 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.30.0.7', 5432, 1))
+    2023-11-21 09:36:18,361 INFO: no action. I am (coord1), the leader with the lock
+    2023-11-21 09:36:18,393 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.30.0.8', 5432, 2))
+    2023-11-21 09:36:28,164 INFO: Lock owner: coord1; I am coord1
+    2023-11-21 09:36:28,251 INFO: Assigning synchronous standby status to ['coord3']
     server signaled
-    2023-01-05 15:09:57.574 UTC [39] LOG:  received SIGHUP, reloading configuration files
-    2023-01-05 15:09:57.580 UTC [39] LOG:  parameter "synchronous_standby_names" changed to "coord3"
-    2023-01-05 15:09:59,637 INFO: Synchronous standby status assigned to ['coord3']
-    2023-01-05 15:09:59,638 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.27.0.2', 5432, 1))
-    2023-01-05 15:09:59.690 UTC [67] LOG:  standby "coord3" is now a synchronous standby with priority 1
-    2023-01-05 15:09:59.690 UTC [67] STATEMENT:  START_REPLICATION SLOT "coord3" 0/3000000 TIMELINE 1
-    2023-01-05 15:09:59,694 INFO: no action. I am (coord1), the leader with the lock
-    2023-01-05 15:09:59,704 DEBUG: query(SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default'), ('172.27.0.8', 5432, 2))
-    2023-01-05 15:10:07,625 INFO: no action. I am (coord1), the leader with the lock
-    2023-01-05 15:10:17,579 INFO: no action. I am (coord1), the leader with the lock
+    2023-11-21 09:36:28.435 UTC [52] LOG:  received SIGHUP, reloading configuration files
+    2023-11-21 09:36:28.436 UTC [52] LOG:  parameter "synchronous_standby_names" changed to "coord3"
+    2023-11-21 09:36:28.641 UTC [83] LOG:  standby "coord3" is now a synchronous standby with priority 1
+    2023-11-21 09:36:28.641 UTC [83] STATEMENT:  START_REPLICATION SLOT "coord3" 0/3000000 TIMELINE 1
+    2023-11-21 09:36:30,582 INFO: Synchronous standby status assigned to ['coord3']
+    2023-11-21 09:36:30,626 INFO: no action. I am (coord1), the leader with the lock
+    2023-11-21 09:36:38,250 INFO: no action. I am (coord1), the leader with the lock
+    ...
 
     $ docker exec -ti demo-haproxy bash
     postgres@haproxy:~$ etcdctl member list
-    1bab629f01fa9065, started, etcd3, http://etcd3:2380, http://172.27.0.10:2379
-    8ecb6af518d241cc, started, etcd2, http://etcd2:2380, http://172.27.0.4:2379
-    b2e169fcb8a34028, started, etcd1, http://etcd1:2380, http://172.27.0.7:2379
+    2b28411e74c0c281, started, etcd3, http://etcd3:2380, http://172.30.0.4:2379
+    6c70137d27cfa6c1, started, etcd2, http://etcd2:2380, http://172.30.0.5:2379
+    a28f9a70ebf21304, started, etcd1, http://etcd1:2380, http://172.30.0.6:2379
 
     postgres@haproxy:~$ etcdctl get --keys-only --prefix /service/demo
     /service/demo/0/config
@@ -229,7 +229,7 @@ Example session:
 
     postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -d citus
     Password for user postgres: postgres
-    psql (15.1 (Debian 15.1-1.pgdg110+1))
+    psql (15.5 (Debian 15.5-1.pgdg120+1))
     SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, compression: off)
     Type "help" for help.
 
@@ -240,67 +240,67 @@ Example session:
     (1 row)
 
     citus=# table pg_dist_node;
-     nodeid | groupid |  nodename  | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards
+     nodeid | groupid |  nodename  | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards 
     --------+---------+------------+----------+----------+-------------+----------+----------+-------------+----------------+------------------
-          1 |       0 | 172.27.0.6 |     5432 | default  | t           | t        | primary  | default     | t              | f
-          2 |       1 | 172.27.0.2 |     5432 | default  | t           | t        | primary  | default     | t              | t
-          3 |       2 | 172.27.0.8 |     5432 | default  | t           | t        | primary  | default     | t              | t
+          1 |       0 | 172.30.0.3 |     5432 | default  | t           | t        | primary  | default     | t              | f
+          2 |       1 | 172.30.0.7 |     5432 | default  | t           | t        | primary  | default     | t              | t
+          3 |       2 | 172.30.0.8 |     5432 | default  | t           | t        | primary  | default     | t              | t
     (3 rows)
 
     citus=# \q
 
     postgres@haproxy:~$ patronictl list
-    + Citus cluster: demo ----------+--------------+---------+----+-----------+
-    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
-    +-------+---------+-------------+--------------+---------+----+-----------+
-    |     0 | coord1  | 172.27.0.6  | Leader       | running |  1 |           |
-    |     0 | coord2  | 172.27.0.5  | Replica      | running |  1 |         0 |
-    |     0 | coord3  | 172.27.0.9  | Sync Standby | running |  1 |         0 |
-    |     1 | work1-1 | 172.27.0.2  | Leader       | running |  1 |           |
-    |     1 | work1-2 | 172.27.0.12 | Sync Standby | running |  1 |         0 |
-    |     2 | work2-1 | 172.27.0.11 | Sync Standby | running |  1 |         0 |
-    |     2 | work2-2 | 172.27.0.8  | Leader       | running |  1 |           |
-    +-------+---------+-------------+--------------+---------+----+-----------+
+    + Citus cluster: demo ----------+--------------+-----------+----+-----------+
+    | Group | Member  | Host        | Role         | State     | TL | Lag in MB |
+    +-------+---------+-------------+--------------+-----------+----+-----------+
+    |     0 | coord1  | 172.30.0.3  | Leader       | running   |  1 |           |
+    |     0 | coord2  | 172.30.0.12 | Replica      | streaming |  1 |         0 |
+    |     0 | coord3  | 172.30.0.2  | Sync Standby | streaming |  1 |         0 |
+    |     1 | work1-1 | 172.30.0.7  | Leader       | running   |  1 |           |
+    |     1 | work1-2 | 172.30.0.10 | Sync Standby | streaming |  1 |         0 |
+    |     2 | work2-1 | 172.30.0.8  | Leader       | running   |  1 |           |
+    |     2 | work2-2 | 172.30.0.11 | Sync Standby | streaming |  1 |         0 |
+    +-------+---------+-------------+--------------+-----------+----+-----------+
+
 
     postgres@haproxy:~$ patronictl switchover --group 2 --force
     Current cluster topology
-    + Citus cluster: demo (group: 2, 7185185529556963355) +-----------+
-    | Member  | Host        | Role         | State   | TL | Lag in MB |
-    +---------+-------------+--------------+---------+----+-----------+
-    | work2-1 | 172.27.0.11 | Sync Standby | running |  1 |         0 |
-    | work2-2 | 172.27.0.8  | Leader       | running |  1 |           |
-    +---------+-------------+--------------+---------+----+-----------+
-    2023-01-05 15:29:29.54204 Successfully switched over to "work2-1"
-    + Citus cluster: demo (group: 2, 7185185529556963355) -------+
+    + Citus cluster: demo (group: 2, 7303846899271086103) --+-----------+
+    | Member  | Host        | Role         | State     | TL | Lag in MB |
+    +---------+-------------+--------------+-----------+----+-----------+
+    | work2-1 | 172.30.0.8  | Leader       | running   |  1 |           |
+    | work2-2 | 172.30.0.11 | Sync Standby | streaming |  1 |         0 |
+    +---------+-------------+--------------+-----------+----+-----------+
+    2023-11-21 09:44:15.83849 Successfully switched over to "work2-2"
+    + Citus cluster: demo (group: 2, 7303846899271086103) -------+
     | Member  | Host        | Role    | State   | TL | Lag in MB |
     +---------+-------------+---------+---------+----+-----------+
-    | work2-1 | 172.27.0.11 | Leader  | running |  1 |           |
-    | work2-2 | 172.27.0.8  | Replica | stopped |    |   unknown |
+    | work2-1 | 172.30.0.8  | Replica | stopped |    |   unknown |
+    | work2-2 | 172.30.0.11 | Leader  | running |  1 |           |
     +---------+-------------+---------+---------+----+-----------+
 
     postgres@haproxy:~$ patronictl list
-    + Citus cluster: demo ----------+--------------+---------+----+-----------+
-    | Group | Member  | Host        | Role         | State   | TL | Lag in MB |
-    +-------+---------+-------------+--------------+---------+----+-----------+
-    |     0 | coord1  | 172.27.0.6  | Leader       | running |  1 |           |
-    |     0 | coord2  | 172.27.0.5  | Replica      | running |  1 |         0 |
-    |     0 | coord3  | 172.27.0.9  | Sync Standby | running |  1 |         0 |
-    |     1 | work1-1 | 172.27.0.2  | Leader       | running |  1 |           |
-    |     1 | work1-2 | 172.27.0.12 | Sync Standby | running |  1 |         0 |
-    |     2 | work2-1 | 172.27.0.11 | Leader       | running |  2 |           |
-    |     2 | work2-2 | 172.27.0.8  | Sync Standby | running |  2 |         0 |
-    +-------+---------+-------------+--------------+---------+----+-----------+
+    + Citus cluster: demo ----------+--------------+-----------+----+-----------+
+    | Group | Member  | Host        | Role         | State     | TL | Lag in MB |
+    +-------+---------+-------------+--------------+-----------+----+-----------+
+    |     0 | coord1  | 172.30.0.3  | Leader       | running   |  1 |           |
+    |     0 | coord2  | 172.30.0.12 | Replica      | streaming |  1 |         0 |
+    |     0 | coord3  | 172.30.0.2  | Sync Standby | streaming |  1 |         0 |
+    |     1 | work1-1 | 172.30.0.7  | Leader       | running   |  1 |           |
+    |     1 | work1-2 | 172.30.0.10 | Sync Standby | streaming |  1 |         0 |
+    |     2 | work2-1 | 172.30.0.8  | Sync Standby | streaming |  2 |         0 |
+    |     2 | work2-2 | 172.30.0.11 | Leader       | running   |  2 |           |
+    +-------+---------+-------------+--------------+-----------+----+-----------+
 
     postgres@haproxy:~$ psql -h localhost -p 5000 -U postgres -d citus
-    Password for user postgres: postgres
-    psql (15.1 (Debian 15.1-1.pgdg110+1))
+    psql (15.5 (Debian 15.5-1.pgdg120+1))
     SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, compression: off)
     Type "help" for help.
 
     citus=# table pg_dist_node;
-     nodeid | groupid |  nodename   | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards
+     nodeid | groupid |  nodename   | nodeport | noderack | hasmetadata | isactive | noderole | nodecluster | metadatasynced | shouldhaveshards 
     --------+---------+-------------+----------+----------+-------------+----------+----------+-------------+----------------+------------------
-          1 |       0 | 172.27.0.6  |     5432 | default  | t           | t        | primary  | default     | t              | f
-          3 |       2 | 172.27.0.11 |     5432 | default  | t           | t        | primary  | default     | t              | t
-          2 |       1 | 172.27.0.2  |     5432 | default  | t           | t        | primary  | default     | t              | t
+          1 |       0 | 172.30.0.3  |     5432 | default  | t           | t        | primary  | default     | t              | f
+          3 |       2 | 172.30.0.11 |     5432 | default  | t           | t        | primary  | default     | t              | t
+          2 |       1 | 172.30.0.7  |     5432 | default  | t           | t        | primary  | default     | t              | t
     (3 rows)

docker/entrypoint.sh

@@ -13,6 +13,8 @@ readonly PATRONI_NAMESPACE="${PATRONI_NAMESPACE%/}"
 DOCKER_IP=$(hostname --ip-address)
 readonly DOCKER_IP
 
+export DUMB_INIT_SETSID=0
+
 case "$1" in
     haproxy)
         haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D
@@ -72,4 +74,4 @@ export PATRONI_SUPERUSER_SSLKEY="${PATRONI_SUPERUSER_SSLKEY:-$PGSSLKEY}"
 export PATRONI_SUPERUSER_SSLCERT="${PATRONI_SUPERUSER_SSLCERT:-$PGSSLCERT}"
 export PATRONI_SUPERUSER_SSLROOTCERT="${PATRONI_SUPERUSER_SSLROOTCERT:-$PGSSLROOTCERT}"
 
-exec python3 /patroni.py postgres0.yml
+exec dumb-init python3 /patroni.py postgres0.yml

docs/ENVIRONMENT.rst

@@ -14,10 +14,18 @@ Global/Universal
 
 Log
 ---
+-  **PATRONI\_LOG\_TYPE**: sets the format of logs. Can be either **plain** or **json**. To use **json** format, you must have the :ref:`jsonlogger <extras>` installed. The default value is **plain**.
 -  **PATRONI\_LOG\_LEVEL**: sets the general logging level. Default value is **INFO** (see `the docs for Python logging <https://docs.python.org/3.6/library/logging.html#levels>`_)
 -  **PATRONI\_LOG\_TRACEBACK\_LEVEL**: sets the level where tracebacks will be visible. Default value is **ERROR**. Set it to **DEBUG** if you want to see tracebacks only if you enable **PATRONI\_LOG\_LEVEL=DEBUG**.
--  **PATRONI\_LOG\_FORMAT**: sets the log formatting string. Default value is **%(asctime)s %(levelname)s: %(message)s** (see `the LogRecord attributes <https://docs.python.org/3.6/library/logging.html#logrecord-attributes>`_)
+-  **PATRONI\_LOG\_FORMAT**: sets the log formatting string. If the log type is **plain**, the log format should be a string.
+   Refer to `the LogRecord attributes <https://docs.python.org/3.6/library/logging.html#logrecord-attributes>`_ for
+   available attributes. If the log type is **json**, the log format can be a list in addition to a string. Each list
+   item should correspond to LogRecord attributes. Be cautious that only the field name is required, and the **%(**
+   and **)** should be omitted. If you wish to print a log field with a different key name, use a dictionary where
+   the dictionary key is the log field, and the value is the name of the field you want to be printed in the log.
+   Default value is **%(asctime)s %(levelname)s: %(message)s**
 -  **PATRONI\_LOG\_DATEFORMAT**: sets the datetime formatting string. (see the `formatTime() documentation <https://docs.python.org/3.6/library/logging.html#logging.Formatter.formatTime>`_)
+-  **PATRONI\_LOG\_STATIC\_FIELDS**: add additional fields to the log. This option is only available when the log type is set to **json**. Example ``PATRONI_LOG_STATIC_FIELDS="{app: patroni}"``
 -  **PATRONI\_LOG\_MAX\_QUEUE\_SIZE**: Patroni is using two-step logging. Log records are written into the in-memory queue and there is a separate thread which pulls them from the queue and writes to stderr or file. The maximum size of the internal queue is limited by default by **1000** records, which is enough to keep logs for the past 1h20m.
 -  **PATRONI\_LOG\_DIR**: Directory to write application logs to. The directory must exist and be writable by the user executing Patroni. If you set this env variable, the application will retain 4 25MB logs by default. You can tune those retention values with `PATRONI_LOG_FILE_NUM` and `PATRONI_LOG_FILE_SIZE` (see below).
 -  **PATRONI\_LOG\_FILE\_NUM**: The number of application logs to retain.
@@ -85,6 +93,7 @@ ZooKeeper
 -  **PATRONI\_ZOOKEEPER\_KEY\_PASSWORD**: (optional) The client key password.
 -  **PATRONI\_ZOOKEEPER\_VERIFY**: (optional) Whether to verify certificate or not. Defaults to ``true``.
 -  **PATRONI\_ZOOKEEPER\_SET\_ACLS**: (optional) If set, configure Kazoo to apply a default ACL to each ZNode that it creates. ACLs will assume 'x509' schema and should be specified as a dictionary with the principal as the key and one or more permissions as a list in the value.  Permissions may be one of ``CREATE``, ``READ``, ``WRITE``, ``DELETE`` or ``ADMIN``.  For example, ``set_acls: {CN=principal1: [CREATE, READ], CN=principal2: [ALL]}``.
+-  **PATRONI\_ZOOKEEPER\_AUTH\_DATA**: (optional) Authentication credentials to use for the connection. Should be a dictionary in the form that `scheme` is the key and `credential` is the value. Defaults to empty dictionary.
 
 .. note::
     It is required to install ``kazoo>=2.6.0`` to support SSL.

docs/contributing_guidelines.rst

@@ -3,11 +3,15 @@
 Contributing guidelines
 =======================
 
+.. _chatting:
+
 Chatting
 --------
 
 If you have a question, looking for an interactive troubleshooting help or want to chat with other Patroni users, join us on channel `#patroni <https://postgresteam.slack.com/archives/C9XPYG92A>`__ in the `PostgreSQL Slack <https://pgtreats.info/slack-invite>`__.
 
+.. _reporting_bugs:
+
 Reporting bugs
 --------------
 

docs/dynamic_configuration.rst

@@ -108,3 +108,7 @@ Note: if cluster topology is static (fixed number of nodes that never change the
 .. warning::
    Permanent replication slots are synchronized only from the ``primary``/``standby_leader`` to replica nodes. That means, applications are supposed to be using them only from the leader node. Using them on replica nodes will cause indefinite growth of ``pg_wal`` on all other nodes in the cluster.
    An exception to that rule are permanent physical slots that match the Patroni member names, if you happen to configure any. Those will be synchronized among all nodes as they are used for replication among them.
+
+
+.. warning::
+   Setting ``nostream`` tag on standby disables copying and synchronization of permanent logical replication slots on the node itself and all its cascading replicas if any.

docs/faq.rst

@@ -0,0 +1,329 @@
+.. _faq:
+
+FAQ
+===
+
+In this section you will find answers for the most frequently asked questions about Patroni.
+Each sub-section attempts to focus on different kinds of questions.
+
+We hope that this helps you to clarify most of your questions.
+If you still have further concerns or find yourself facing an unexpected issue, please refer to :ref:`chatting` and :ref:`reporting_bugs` for instructions on how to get help or report issues.
+
+Comparison with other HA solutions
+----------------------------------
+
+Why does Patroni require a separate cluster of DCS nodes while other solutions like ``repmgr`` do not?
+    There are different ways of implementing HA solutions, each of them with their pros and cons.
+
+    Software like ``repmgr`` performs communication among the nodes to decide when actions should be taken.
+
+    Patroni on the other hand relies on the state stored in the DCS. The DCS acts as a source of truth for Patroni to decide what it should do.
+
+    While having a separate DCS cluster can make you bloat your architecture, this approach also makes it less likely for split-brain scenarios to happen in your Postgres cluster.
+
+What is the difference between Patroni and other HA solutions in regards to Postgres management?
+    Patroni does not just manage the high availability of the Postgres cluster but also manages Postgres itself.
+
+    If Postgres nodes do not exist yet, it takes care of bootstrapping the primary and the standby nodes, and also manages Postgres configuration of the nodes. If the Postgres nodes already exist, Patroni will take over management of the cluster.
+
+    Besides the above, Patroni also has self-healing capabilities. In other words, if a primary node fails, Patroni will not only fail over to a replica, but also attempt to rejoin the former primary as a replica of the new primary. Similarly, if a replica fails, Patroni will attempt to rejoin that replica.
+
+    That is way we call Patroni as a "template for HA solutions". It goes further than just managing physical replication: it manages Postgres as a whole.
+
+DCS
+---
+
+Can I use the same ``etcd`` cluster to store data from two or more Patroni clusters?
+    Yes, you can!
+
+    Information about a Patroni cluster is stored in the DCS under a path prefixed with the ``namespace`` and ``scope`` Patroni settings.
+
+    As long as you do not have conflicting namespace and scope across different Patroni clusters, you should be able to use the same DCS cluster to store information from multiple Patroni clusters.
+
+What occurs if I attempt to use the same combination of ``namespace`` and ``scope`` for different Patroni clusters that point to the same DCS cluster?
+    The second Patroni cluster that attempts to use the same ``namespace`` and ``scope`` will not be able to manage Postgres because it will find information related with that same combination in the DCS, but with an incompatible Postgres system identifier.
+    The mismatch on the system identifier causes Patroni to abort the management of the second cluster, as it assumes that refers to a different cluster and that the user has misconfigured Patroni.
+
+    Make sure to use different ``namespace`` / ``scope`` when dealing with different Patroni clusters that share the same DCS cluster.
+
+What occurs if I lose my DCS cluster?
+    The DCS is used to store basically status and the dynamic configuration of the Patroni cluster.
+
+    They very first consequence is that all the Patroni clusters that rely on that DCS will go to read-only mode -- unless :ref:`dcs_failsafe_mode` is enabled.
+
+What should I do if I lose my DCS cluster?
+    There are three possible outcomes upon losing your DCS cluster:
+
+    1. The DCS cluster is fully recovered: this requires no action from the Patroni side. Once the DCS cluster is recovered, Patroni should be able to recover too;
+    2. The DCS cluster is re-created in place, and the endpoints remain the same. No changes are required on the Patroni side;
+    3. A new DCS cluster is created with different endpoints. You will need to update the DCS endpoints in the Patroni configuration of each Patroni node.
+
+    If you face scenario ``2.`` or ``3.`` Patroni will take care of creating the status information again based on the current status of the cluster, and recreate the dynamic configuration on the DCS based on a backup file named ``patroni.dynamic.json`` which is stored inside the Postgres data directory of each member of the Patroni cluster.
+
+What occurs if I lose majority in my DCS cluster?
+    The DCS will become unresponsive, which will cause Patroni to demote the current read/write Postgres node.
+
+    Remember: Patroni relies on the state of the DCS to take actions on the cluster.
+
+    You can use the :ref:`dcs_failsafe_mode` to alleviate that situation.
+
+patronictl
+----------
+
+Do I need to run :ref:`patronictl` in the Patroni host?
+    No, you do not need to do that.
+
+    Running :ref:`patronictl` in the Patroni host is handy if you have access to the Patroni host because you can use the very same configuration file from the ``patroni`` agent for the :ref:`patronictl` application.
+
+    However, :ref:`patronictl` is basically a client and it can be executed from remote machines. You just need to provide it with enough configuration so it can reach the DCS and the REST API of the Patroni member(s).
+
+Why did the information from one of my Patroni members disappear from the output of :ref:`patronictl_list` command?
+    Information shown by :ref:`patronictl_list` is based on the contents of the DCS.
+
+    If information about a member disappeared from the DCS it is very likely that the Patroni agent on that node is not running anymore, or it is not able to communicate with the DCS.
+
+    As the member is not able to update the information, the information eventually expires from the DCS, and consequently the member is not shown anymore in the output of :ref:`patronictl_list`.
+
+Why is the information about one of my Patroni members not up-to-date in the output of :ref:`patronictl_list` command?
+    Information shown by :ref:`patronictl_list` is based on the contents of the DCS.
+
+    By default, that information is updated by Patroni roughly every ``loop_wait`` seconds.
+    In other words, even if everything is normally functional you may still see a "delay" of up to ``loop_wait`` seconds in the information stored in the DCS.
+
+    Be aware that that is not a rule, though. Some operations performed by Patroni cause it to immediately update the DCS information.
+
+Configuration
+-------------
+
+What is the difference between dynamic configuration and local configuration?
+    Dynamic configuration (or global configuration) is the configuration stored in the DCS, and which is applied to all members of the Patroni cluster.
+    This is primarily where you should store your configuration.
+
+    Settings that are specific to a node, or settings that you would like to overwrite the global configuration with, you should set only on the desired Patroni member as a local configuration.
+    That local configuration can be specified either through the configuration file or through environment variables.
+
+    See more in :ref:`patroni_configuration`.
+
+What are the types of configuration in Patroni, and what is the precedence?
+    The types are:
+
+    * Dynamic configuration: applied to all members;
+    * Local configuration: applied to the local member, overrides dynamic configuration;
+    * Environment configuration: applied to the local member, overrides both dynamic and local configuration.
+
+    **Note:** some Postgres GUCs can only be set globally, i.e., through dynamic configuration. Besides that, there are GUCs which Patroni enforces a hard-coded value.
+
+    See more in :ref:`patroni_configuration`.
+
+Is there any facility to help me create my Patroni configuration file?
+    Yes, there is.
+
+    You can use ``patroni --generate-sample-config`` or ``patroni --generate-config`` commands to generate a sample Patroni configuration or a Patroni configuration based on an existing Postgres instance, respectively.
+
+    Please refer to :ref:`generate_sample_config` and :ref:`generate_config` for more details.
+
+I changed my parameters under ``bootstrap.dcs`` configuration but Patroni is not applying the changes to the cluster members. What is wrong?
+    The values configured under ``bootstrap.dcs`` are only used when bootstrapping a fresh cluster. Those values will be written to the DCS during the bootstrap.
+
+    After the bootstrap phase finishes, you will only be able to change the dynamic configuration through the DCS.
+
+    Refer to the next question for more details.
+
+How can I change my dynamic configuration?
+    You need to change the configuration in the DCS. That is accomplished either through:
+
+    * :ref:`patronictl_edit_config`; or
+    * A ``PATCH`` request to :ref:`config_endpoint`.
+
+How can I change my local configuration?
+    You need to change the configuration file of the corresponding Patroni member and signal the Patroni agent with ``SIHGUP``. You can do that using either of these approaches:
+
+    * Send a ``POST`` request to the REST API :ref:`reload_endpoint`; or
+    * Run :ref:`patronictl_reload`; or
+    * Locally signal the Patroni process with ``SIGHUP``:
+
+        * If you started Patroni through systemd, you can use the command ``systemctl reload PATRONI_UNIT.service``, ``PATRONI_UNIT`` being the name of the Patroni service; or
+        * If you started Patroni through other means, you will need to identify the ``patroni`` process and run ``kill -s HUP PID``, ``PID`` being the process ID of the ``patroni`` process.
+
+    **Note:** there are cases where a reload through the :ref:`patronictl_reload` may not work:
+
+    * Expired REST API certificates: you can mitigate that by using the ``-k`` option of the :ref:`patronictl`;
+    * Wrong credentials: for example when changing ``restapi`` or ``ctl`` credentials in the configuration file, and using that same configuration file for Patroni and :ref:`patronictl`.
+
+How can I change my environment configuration?
+    The environment configuration is only read by Patroni during startup.
+
+    With that in mind, if you change the environment configuration you will need to restart the corresponding Patroni agent.
+
+    Take care to not cause a failover in the cluster! You might be interested in checking :ref:`patronictl_pause`.
+
+What occurs if I change a Postgres GUC that requires a reload?
+    When you change the dynamic or the local configuration as explained in the previous questions, Patroni will take care of reloading the Postgres configuration for you.
+
+What occurs if I change a Postgres GUC that requires a restart?
+    Patroni will mark the affected members with a flag of ``pending restart``.
+
+    It is up to you to determine when and how to restart the members. That can be accomplished either through:
+
+    * :ref:`patronictl_restart`; or
+    * A ``POST`` request to :ref:`restart_endpoint`.
+
+    **Note:** some Postgres GUCs require a special management in terms of the order for restarting the Postgres nodes. Refer to :ref:`shared_memory_gucs` for more details.
+
+What is the difference between ``etcd`` and ``etcd3`` in Patroni configuration?
+    ``etcd`` uses the API version 2 of ``etcd``, while ``etcd3`` uses the API version 3 of ``etcd``.
+
+    Be aware that information stored by the API version 2 is not manageable by API version 3 and vice-versa.
+
+    We recommend that you configure ``etcd3`` instead of ``etcd`` because:
+
+    * API version 2 is disabled by default from Etcd v3.4 onward;
+    * API version 2 will be completely removed on Etcd v3.6.
+
+I have ``use_slots`` enabled in my Patroni configuration, but when a cluster member goes offline for some time, the replication slot used by that member is dropped on the upstream node. What can I do to avoid that issue?
+    You can configure a permanent physical replication slot for the members.
+
+    Since Patroni ``3.2.0`` it is now possible to have member slots as permanent slots managed by Patroni.
+
+    Patroni will create the permanent physical slots on all nodes, and make sure to not remove the slots, as well as to advance the slots' LSN on all nodes according to the LSN that has been consumed by the member.
+
+    Later, if you decide to remove the corresponding member, it's **your responsability** to adjust the permanent slots configuration, otherwise Patroni will keep the slots around forever.
+
+    **Note:** on Patroni older than ``3.2.0`` you could still have member slots configured as permanent physical slots, however they would be managed only on the current leader. That is, in case of failover/switchover these slots would be created on the new leader, but that wouldn't guarantee that it had all WAL segments for the absent node.
+
+    **Note:** even with Patroni ``3.2.0`` there might be a small race condition. In the very beginning, when the slot is created on the replica it could be ahead of the same slot on the leader and in case if nobody is consuming the slot there is still a chance that some files could be missing after failover. With that in mind, it is recommended that you configure continuous archiving, which makes it possible to restore required WALs or perform PITR.
+
+What is the difference between ``loop_wait``, ``retry_timeout`` and ``ttl``?
+    Patroni performs what we call a HA cycle from time to time. On each HA cycle it takes care of performing a series of checks on the cluster to determine its healthiness, and depending on the status it may take actions, like failing over to a standby.
+
+    ``loop_wait`` determines for how long, in seconds, Patroni should sleep before performing a new cycle of HA checks.
+
+    ``retry_timeout`` sets the timeout for retry operations on the DCS and on Postgres. For example: if the DCS is unresponsive for more than ``retry_timeout`` seconds, Patroni might demote the primary node as a security action.
+
+    ``ttl`` sets the lease time on the ``leader`` lock in the DCS. If the current leader of the cluster is not able to renew the lease during its HA cycles for longer than ``ttl``, then the lease will expire and that will trigger a ``leader race`` in the cluster.
+
+    **Note:** when modifying these settings, please keep in mind that Patroni enforces the rule and minimal values described in :ref:`dynamic_configuration` section of the docs.
+
+Postgres management
+-------------------
+
+Can I change Postgres GUCs directly in Postgres configuration?
+    You can, but you should avoid that.
+
+    Postgres configuration is managed by Patroni, and attempts to edit the configuration files may end up being frustrated by Patroni as it may eventually overwrite them.
+
+    There are a few options available to overcome the management performed by Patroni:
+
+    * Change Postgres GUCs through ``$PGDATA/postgresql.base.conf``; or
+    * Define a ``postgresql.custom_conf`` which will be used instead of ``postgresql.base.conf`` so you can manage that externally; or
+    * Change GUCs using ``ALTER SYSTEM`` / ``ALTER DATABASE`` / ``ALTER USER``.
+
+    You can find more information about that in the section :ref:`important_configuration_rules`.
+
+    In any case we recommend that you manage all the Postgres configuration through Patroni. That will centralize the management and make it easier to debug Patroni when needed.
+
+Can I restart Postgres nodes directly?
+    No, you should **not** attempt to manage Postgres directly!
+
+    Any attempt of bouncing the Postgres server without Patroni can lead your cluster to face failovers.
+
+    If you need to manage the Postgres server, do that through the ways exposed by Patroni.
+
+Is Patroni able to take over management of an already existing Postgres cluster?
+    Yes, it can!
+
+    Please refer to :ref:`existing_data` for detailed instructions.
+
+How does Patroni manage Postgres?
+    Patroni takes care of bringing Postgres up and down by running the Postgres binaries, like ``pg_ctl`` and ``postgres``.
+
+    With that in mind you **MUST** disable any other sources that could manage the Postgres clusters, like the systemd units, e.g. ``postgresql.service``. Only Patroni should be able to start, stop and promote Postgres instances in the cluster. Not doing so may result in split-brain scenarios. For example: if the node running as a primary failed and the unit ``postgresql.service`` is enabled, it may bring Postgres back up and cause a split-brain.
+
+Concepts and requirements
+-------------------------
+
+Which are the applications that make part of Patroni?
+    Patroni basically ships a couple applications:
+
+    * ``patroni``: This is the Patroni agent, which takes care of managing a Postgres node;
+    * ``patronictl``: This is a command-line utility used to interact with a Patroni cluster (perform switchovers, restarts, changes in the configuration, etc.). Please find more information in :ref:`patronictl`.
+
+What is a ``standby cluster`` in Patroni?
+    It is a cluster that does not have any primary Postgres node running, i.e., there is no read/write member in the cluster.
+
+    These kinds of clusters exist to replicate data from another cluster and are usually useful when you want to replicate data across data centers.
+
+    There will be a leader in the cluster which will be a standby in charge of replicating changes from a remote Postgres node.
+    Then, there will be a set of standbys configured with cascading replication from such leader member.
+
+    **Note:** the standby cluster doesn't know anything about the source cluster which it is replicating from -- it can even use ``restore_command`` instead of WAL streaming, and may use an absolutely independent DCS cluster.
+
+    Refer to :ref:`standby_cluster` for more details.
+
+What is a ``leader`` in Patroni?
+    A ``leader`` in Patroni is like a coordinator of the cluster.
+
+    In a regular Patroni cluster, the ``leader`` will be the read/write node.
+
+    In a standby Patroni cluster, the ``leader`` (AKA ``standby leader``) will be in charge of replicating from a remote Postgres node, and cascading those changes to the other members of the standby cluster.
+
+Does Patroni require a minimum number of Postgres nodes in the cluster?
+    No, you can run Patroni with any number of Postgres nodes.
+
+    Remember: Patroni is decoupled from the DCS.
+
+What does ``pause`` mean in Patroni?
+    Pause is an operation exposed by Patroni so the user can ask Patroni to step back in regards to Postgres management.
+
+    That is mainly useful when you want to perform maintenance on the cluster, and would like to avoid that Patroni takes decisions related with HA, like failing over to a standby when you stop the primary.
+
+    You can find more information about that in :ref:`pause`.
+
+Automatic failover
+------------------
+
+How does the automatic failover mechanism of Patroni work?
+    Patroni automatic failover is based on what we call ``leader race``.
+
+    Patroni stores the cluster's status in the DCS, among them a ``leader`` lock which holds the name of the Patroni member which is the current ``leader`` of the cluster.
+
+    That ``leader`` lock has a time-to-live associated with it. If the leader node fails to update the lease of the ``leader`` lock in time, the key will eventually expire from the DCS.
+
+    When the ``leader`` lock expires, it triggers what Patroni calls a ``leader race``: all nodes start performing checks to determine if they are the best candidates for taking over the ``leader`` role.
+    Some of these checks include calls to the REST API of all other Patroni members.
+
+    All Patroni members that find themselves as the best candidate for taking over the ``leader`` lock will attempt to do so.
+    The first Patroni member that is able to take the ``leader`` lock will promote itself to a read/write node (or ``standby leader``), and the others will be configured to follow it.
+
+Can I temporarily disable automatic failover in the Patroni cluster?
+    Yes, you can!
+
+    You can achieve that by temporarily pausing the cluster.
+    This is typically useful for performing maintenance.
+
+    When you want to resume the automatic failover of the cluster, you just need to unpause it.
+
+    You can find more information about that in :ref:`pause`.
+
+Bootstrapping and standbys creation
+-----------------------------------
+
+How does Patroni create a primary Postgres node? What about a standby Postgres node?
+    By default Patroni will use ``initdb`` to bootstrap a fresh cluster, and ``pg_basebackup`` to create standby nodes from a copy of the ``leader`` member.
+
+    You can customize that behavior by writing your custom bootstrap methods, and your custom replica creation methods.
+
+    Custom methods are usually useful when you want to restore backups created by backup tools like pgBackRest or Barman, for example.
+
+    For detailed information please refer to :ref:`custom_bootstrap` and :ref:`custom_replica_creation`.
+
+Monitoring
+----------
+
+How can I monitor my Patroni cluster?
+    Patroni exposes a couple handy endpoints in its :ref:`rest_api`:
+
+    * ``/metrics``: exposes monitoring metrics in a format that can be consumed by Prometheus;
+    * ``/patroni``: exposes the status of the cluster in a JSON format. The information shown here is very similar to what is shown by the ``/metrics`` endpoint.
+
+    You can use those endpoints to implement monitoring checks.

docs/index.rst

@@ -28,14 +28,17 @@ Currently supported PostgreSQL versions: 9.3 to 16.
    patronictl
    replica_bootstrap
    replication_modes
+   standby_cluster
    watchdog
    pause
    dcs_failsafe_mode
    kubernetes
    citus
    existing_data
+   tools_integration
    security
    ha_multi_dc
+   faq
    releases
    CONTRIBUTING
 

docs/installation.rst

@@ -60,6 +60,8 @@ raft
     `pysyncobj` module in order to use python Raft implementation as DCS
 aws
     `boto3` in order to use AWS callbacks
+jsonlogger
+    `python-json-logger` module in order to enable :ref:`logging <log_settings>` in json format
 all
     all of the above (except psycopg family)
 psycopg

docs/patroni_configuration.rst

@@ -30,6 +30,7 @@ There are 3 types of Patroni configuration:
 	It is possible to set/override some of the "Local" configuration parameters with environment variables.
 	Environment configuration is very useful when you are running in a dynamic environment and you don't know some of the parameters in advance (for example it's not possible to know your external IP address when you are running inside ``docker``).
 
+.. _important_configuration_rules:
 
 Important rules
 ---------------
@@ -90,6 +91,7 @@ The parameters would be applied in the following order (run-time are given the h
 
 This allows configuration for all the nodes (2), configuration for a specific node using ``ALTER SYSTEM`` (3) and ensures that parameters essential to the running of Patroni are enforced (4), as well as leaves room for configuration tools that manage `postgresql.conf` directly without involving Patroni (1).
 
+.. _shared_memory_gucs:
 
 PostgreSQL parameters that touch shared memory
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -156,6 +158,7 @@ Patroni provides command-line interfaces for a Patroni :ref:`local configuration
 - Create a Patroni configuration file for the locally running PostgreSQL instance (e.g. as a preparation step for the :ref:`Patroni integration <existing_data>`);
 - Validate a given Patroni configuration file.
 
+.. _generate_sample_config:
 
 Sample Patroni configuration
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -183,6 +186,7 @@ Parameters
 
 ``configfile`` - full path to the configuration file used to store the result. If not provided, the result is sent to ``stdout``.
 
+.. _generate_config:
 
 Patroni configuration for a running instance
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

docs/releases.rst

@@ -3,6 +3,155 @@
 Release notes
 =============
 
+Version 3.3.0
+-------------
+
+**New features**
+
+- Add ability to pass ``auth_data`` to Zookeeper client (Aras Mumcuyan)
+
+  It allows to specify the authentication credentials to use for the connection.
+
+- Add a contrib script for ``Barman`` integration (Israel Barth Rubio)
+
+  Provide an application ``patroni_barman`` that allows to perform ``Barman`` operations remotely and can be used as a custom bootstrap/custom replica method or as an ``on_role_change`` callback. Please check :ref:`here <tools_integration>` for more information.
+
+- Support JSON log format (alisalemmi)
+
+  Apart from ``plain``, Patroni now also supports ``json`` log format. Requires ``python-json-logger`` library to be installed.
+
+- Show ``pending_restart_reason`` information (Polina Bungina)
+
+  Provide extended information about the PostgreSQL parameters that caused ``pending_restart`` flag to be set. Both ``patronictl list`` and ``/patroni`` REST API endpoint now show the parameters names and their "diff" as ``pending_restart_reason``.
+
+- Implement ``nostream`` tag (Grigory Smolkin)
+
+  If ``nostream`` tag is set to ``true``, the node will not use replication protocol to stream WAL but instead rely on archive recovery (if ``restore_command`` is configured). It also disables copying and synchronization of permanent logical replication slots on the node itself and all its cascading replicas.
+
+
+**Improvements**
+
+- Implement validation of the log section (Alexander Kukushkin)
+
+  Until now validator was not checking the correctness of the logging configuration provided.
+
+- Improve logging for PostgreSQL parameters change (Polina Bungina)
+
+  Convert old values to a human-readable format and log information about the ``pg_controldata`` vs Patroni global configuration mismatch.
+
+
+**Bugfixes**
+
+- Properly filter out not allowed ``pg_basebackup`` options (Israel Barth Rubio)
+
+  Due to a bug, Patroni was not properly filtering out the not allowed options configured for the ``basebackup`` replica bootstrap method, when provided in the ``- setting: value`` format.
+
+- Fix ``etcd3`` authentication error handling (Alexander Kukushkin)
+
+  Always retry one time on ``etcd3`` authentication error if authentication was not done right before executing the request. Also, do not restart watchers on reauthentication.
+
+- Improve logic of the validator files discovery (Waynerv)
+
+  Use ``importlib`` library to discover the files with available configuration parameters when possible (for Python 3.9+). This implementation is more stable and doesn't break the Patroni distributions based on ``zip`` archives.
+
+- Use ``target_session_attrs`` only when multiple hosts are specified in the ``standby_cluster`` section (Alexander Kukushkin)
+
+  ``target_session_attrs=read-write`` is now added to the ``primary_conninfo`` on the standby leader node only when ``standby_cluster.host`` section contains multiple hosts separated by commas.
+
+- Add compatibility code for ``ydiff`` library version 1.3+ (Alexander Kukushkin)
+
+.. warning::
+    All older Partoni versions are not compatible with ``ydiff`` 1.3+. Please upgrade Patroni, use ``ydiff`` version <1.3, or install ``cdiff``.
+
+
+Version 3.2.2
+-------------
+
+**Bugfixes**
+
+- Don't let replica restore initialize key when DCS was wiped (Alexander Kukushkin)
+
+  It was happening in the method where Patroni was supposed to take over a standalone PG cluster.
+
+- Use consistent read when fetching just updated sync key from Consul (Alexander Kukushkin)
+
+  Consul doesn't provide any interface to immediately get ``ModifyIndex`` for the key that we just updated, therefore we have to perform an explicit read operation. Since stale reads are allowed by default, we sometimes used to get an outdated version of the key.
+
+- Reload Postgres config if a parameter that requires restart was reset to the original value (Polina Bungina)
+
+  Previously Patroni wasn't updating the config, but only resetting the ``pending_restart``.
+
+- Fix erroneous inverted logic of the confirmation prompt message when doing a failover to an async candidate in synchronous mode (Polina Bungina)
+
+  The problem existed only in ``patronictl``.
+
+- Exclude leader from failover candidates in ``patronictl`` (Polina Bungina)
+
+  If the cluster is healthy, failing over to an existing leader is no-op.
+
+- Create Citus database and extension idempotently (Alexander Kukushkin, Zhao Junwang)
+
+  It will allow to create them in the ``post_bootstrap`` script in case if there is a need to add some more dependencies to the Citus database.
+
+- Don't filter our contradictory ``nofailover`` tag (Polina Bungina)
+
+  The configuration ``{nofailover: false, failover_priority: 0}`` set on a node didn't allow it to participate in the race, while it should, because ``nofailover`` tag should take precedence.
+
+- Fixed PyInstaller frozen issue (Sophia Ruan)
+
+  The ``freeze_support()`` was called after ``argparse`` and as a result, Patroni wasn't able to start Postgres.
+
+- Fixed bug in the config generator for ``patronictl`` and ``Citus`` configuration (Israel Barth Rubio)
+
+  It prevented ``patronictl`` and ``Citus`` configuration parameters set via environment variables from being written into the generated config.
+
+- Restore recovery GUCs and some Patroni-managed parameters when joining a running standby (Alexander Kukushkin)
+
+  Patroni was failing to restart Postgres v12 onwards with an error about missing ``port`` in one of the internal structures.
+
+- Fixes around ``pending_restart`` flag (Polina Bungina)
+
+  Don't expose ``pending_restart`` when in custom bootstrap with ``recovery_target_action = promote`` or when someone changed ``hot_standby`` or ``wal_log_hints`` using for example ``ALTER SYSTEM``.
+
+
+Version 3.2.1
+-------------
+
+**Bugfixes**
+
+- Limit accepted values for ``--format`` argument in ``patronictl`` (Alexander Kukushkin)
+
+  It used to accept any arbitrary string and produce no output if the value wasn't recognized.
+
+- Verify that replica nodes received checkpoint LSN on shutdown before releasing the leader key (Alexander Kukushkin)
+
+  Previously in some cases, we were using LSN of the SWITCH record that is followed by CHECKPOINT (if archiving mode is enabled). As a result the former primary sometimes had to do ``pg_rewind``, but there would be no data loss involved.
+
+- Do a real HTTP request when performing node name uniqueness check (Alexander Kukushkin)
+
+  When running Patroni in containers it is possible that the traffic is routed using ``docker-proxy``, which listens on the port and accepts incoming connections. It was causing false positives.
+
+- Fixed Citus support with Etcd v2 (Alexander Kukushkin)
+
+  Patroni was failing to deploy a new Citus cluster with Etcd v2.
+
+- Fixed ``pg_rewind`` behavior with Postgres v16+ (Alexander Kukushkin)
+
+  The error message format of ``pg_waldump`` changed in v16 which caused ``pg_rewind`` to be called by Patroni even when it was not necessary.
+
+- Fixed bug with custom bootstrap (Alexander Kukushkin)
+
+  Patroni was falsely applying ``--command`` argument, which is a bootstrap command itself.
+
+- Fixed the issue with REST API health check endpoints (Sophia Ruan)
+
+  There were chances that after Postgres restart it could return ``unknown`` state for Postgres because connections were not properly closed.
+
+- Cache ``postgres --describe-config`` output results (Waynerv)
+
+  They are used to figure out which GUCs are available to validate PostgreSQL configuration and we don't expect this list to change while Patroni is running.
+
+
 Version 3.2.0
 -------------
 

docs/replica_bootstrap.rst

@@ -1,3 +1,5 @@
+.. _replica_imaging_and_bootstrap:
+
 Replica imaging and bootstrap
 =============================
 
@@ -71,6 +73,21 @@ Makes the configured ``command`` to be called additionally with ``--arg1=value1
 
  .. note:: Bootstrap methods are neither chained, nor fallen-back to the default one in case the primary one fails
 
+As an example, you are able to bootstrap a fresh Patroni cluster from a Barman backup with a configuration like this:
+
+.. code:: YAML
+
+    bootstrap:
+        method: barman
+        barman:
+            keep_existing_recovery_conf: true
+            command: patroni_barman --api-url https://barman-host:7480 recover
+            barman-server: my_server
+            ssh-command: ssh postgres@patroni-host
+
+.. note::
+    ``patroni_barman recover`` requires that you have both Barman and ``pg-backup-api`` configured in the Barman host, so it can execute a remote ``barman recover`` through the backup API.
+    The above example uses a subset of the available parameters. You can get more information running ``patroni_barman recover --help``.
 
 .. _custom_replica_creation:
 
@@ -125,6 +142,24 @@ example: pgbackrest
         basebackup:
             max-rate: '100M'
 
+example: Barman
+
+.. code:: YAML
+
+    postgresql:
+        create_replica_methods:
+            - barman
+            - basebackup
+        barman:
+            command: patroni_barman --api-url https://barman-host:7480 recover
+            barman-server: my_server
+            ssh-command: ssh postgres@patroni-host
+        basebackup:
+            max-rate: '100M'
+
+.. note::
+    ``patroni_barman recover`` requires that you have both Barman and ``pg-backup-api`` configured in the Barman host, so it can execute a remote ``barman recover`` through the backup API.
+    The above example uses a subset of the available parameters. You can get more information running ``patroni_barman recover --help``.
 
 The ``create_replica_methods`` defines available replica creation methods and the order of executing them. Patroni will
 stop on the first one that returns 0. Each method should define a separate section in the configuration file, listing the command
@@ -184,59 +219,3 @@ and
             - waldir: /pg-wal-mount/external-waldir
 
 If all replica creation methods fail, Patroni will try again all methods in order during the next event loop cycle.
-
-.. _standby_cluster:
-
-Standby cluster
----------------
-
-Another available option is to run a "standby cluster", that contains only of
-standby nodes replicating from some remote node. This type of clusters has:
-
-* "standby leader", that behaves pretty much like a regular cluster leader,
-  except it replicates from a remote node.
-
-* cascade replicas, that are replicating from standby leader.
-
-Standby leader holds and updates a leader lock in DCS. If the leader lock
-expires, cascade replicas will perform an election to choose another leader
-from the standbys.
-
-There is no further relationship between the standby cluster and the primary
-cluster it replicates from, in particular, they must not share the same DCS
-scope if they use the same DCS. They do not know anything else from each other
-apart from replication information. Also, the standby cluster is not being
-displayed in :ref:`patronictl_list` or :ref:`patronictl_topology` output on the
-primary cluster.
-
-For the sake of flexibility, you can specify methods of creating a replica and
-recovery WAL records when a cluster is in the "standby mode" by providing
-`create_replica_methods` key in `standby_cluster` section. It is distinct from
-creating replicas, when cluster is detached and functions as a normal cluster,
-which is controlled by `create_replica_methods` in `postgresql` section. Both
-"standby" and "normal" `create_replica_methods` reference  keys in `postgresql`
-section.
-
-To configure such cluster you need to specify the section ``standby_cluster``
-in a patroni configuration:
-
-.. code:: YAML
-
-    bootstrap:
-        dcs:
-            standby_cluster:
-                host: 1.2.3.4
-                port: 5432
-                primary_slot_name: patroni
-                create_replica_methods:
-                - basebackup
-
-Note, that these options will be applied only once during cluster bootstrap,
-and the only way to change them afterwards is through DCS.
-
-Patroni expects to find `postgresql.conf` or `postgresql.conf.backup` in PGDATA
-of the remote primary and will not start if it does not find it after a
-basebackup. If the remote primary keeps its `postgresql.conf` elsewhere, it is
-your responsibility to copy it to PGDATA.
-
-If you use replication slots on the standby cluster, you must also create the corresponding replication slot on the primary cluster.  It will not be done automatically by the standby cluster implementation.  You can use Patroni's permanent replication slots feature on the primary cluster to maintain a replication slot with the same name as ``primary_slot_name``, or its default value if ``primary_slot_name`` is not provided.

docs/replication_modes.rst

@@ -53,7 +53,7 @@ are available. As a downside, the primary is not be available for writes
 blocking all client write requests until at least one synchronous replica comes
 up.
 
-You can ensure that a standby never becomes the synchronous standby by setting ``nosync`` tag to true. This is recommended to set for standbys that are behind slow network connections and would cause performance degradation when becoming a synchronous standby.
+You can ensure that a standby never becomes the synchronous standby by setting ``nosync`` tag to true. This is recommended to set for standbys that are behind slow network connections and would cause performance degradation when becoming a synchronous standby. Setting tag ``nostream`` to true will also have the same effect.
 
 Synchronous mode can be switched on and off via Patroni REST interface. See :ref:`dynamic configuration <dynamic_configuration>` for instructions.
 

docs/rest_api.rst

@@ -426,6 +426,7 @@ Cluster status endpoints
       ]
     ]
 
+.. _config_endpoint:
 
 Config endpoint
 ---------------
@@ -666,6 +667,7 @@ There are a couple of checks that a member of a cluster should pass to be able t
 	- its lag exceeds the maximum replication lag allowed;
 	- it has the timeline number smaller than the last known cluster timeline.
 
+.. _restart_endpoint:
 
 Restart endpoint
 ----------------
@@ -682,6 +684,7 @@ Restart endpoint
 
 ``POST /restart`` and ``DELETE /restart`` endpoints are used by :ref:`patronictl_restart` and :ref:`patronictl flush cluster-name restart <patronictl_flush_parameters>` respectively.
 
+.. _reload_endpoint:
 
 Reload endpoint
 ---------------

docs/standby_cluster.rst

@@ -0,0 +1,78 @@
+.. _standby_cluster:
+
+Standby cluster
+---------------
+
+Patroni also support running cascading replication to a remote datacenter
+(region) using a feature that is called "standby cluster". This type of
+clusters has:
+
+* "standby leader", that behaves pretty much like a regular cluster leader,
+  except it replicates from a remote node.
+
+* cascade replicas, that are replicating from standby leader.
+
+Standby leader holds and updates a leader lock in DCS. If the leader lock
+expires, cascade replicas will perform an election to choose another leader
+from the standbys.
+
+There is no further relationship between the standby cluster and the primary
+cluster it replicates from, in particular, they must not share the same DCS
+scope if they use the same DCS. They do not know anything else from each other
+apart from replication information. Also, the standby cluster is not being
+displayed in :ref:`patronictl_list` or :ref:`patronictl_topology` output on the
+primary cluster.
+
+For the sake of flexibility, you can specify methods of creating a replica and
+recovery WAL records when a cluster is in the "standby mode" by providing
+:ref:`create_replica_methods <custom_replica_creation>` key in
+`standby_cluster` section. It is distinct from creating replicas, when cluster
+is detached and functions as a normal cluster, which is controlled by
+`create_replica_methods` in `postgresql` section. Both "standby" and "normal"
+`create_replica_methods` reference  keys in `postgresql` section.
+
+To configure such cluster you need to specify the section ``standby_cluster``
+in a patroni configuration:
+
+.. code:: YAML
+
+    bootstrap:
+        dcs:
+            standby_cluster:
+                host: 1.2.3.4
+                port: 5432
+                primary_slot_name: patroni
+                create_replica_methods:
+                - basebackup
+
+Note, that these options will be applied only once during cluster bootstrap,
+and the only way to change them afterwards is through DCS.
+
+Patroni expects to find `postgresql.conf` or `postgresql.conf.backup` in PGDATA
+of the remote primary and will not start if it does not find it after a
+basebackup. If the remote primary keeps its `postgresql.conf` elsewhere, it is
+your responsibility to copy it to PGDATA.
+
+If you use replication slots on the standby cluster, you must also create the
+corresponding replication slot on the primary cluster.  It will not be done
+automatically by the standby cluster implementation.  You can use Patroni's
+permanent replication slots feature on the primary cluster to maintain a
+replication slot with the same name as ``primary_slot_name``, or its default
+value if ``primary_slot_name`` is not provided.
+
+In case the remote site doesn't provide a single endpoint that connects to a
+primary, one could list all hosts of the source cluster in the
+``standby_cluster.host`` section.  When ``standby_cluster.host`` contains
+multiple hosts separated by commas, Patroni will:
+
+* add ``target_session_attrs=read-write`` to the ``primary_conninfo`` on the
+  standby leader node.
+* use ``target_session_attrs=read-write`` when trying to determine whether we
+  need to run ``pg_rewind`` or when executing ``pg_rewind`` on all nodes of the
+  standby cluster.
+  
+There is also a possibility to replicate the standby cluster from another
+standby cluster or from a standby member of the primary cluster: for that, you
+need to define a single host in the ``standby_cluster.host`` section. However,
+you need to beware that in this case ``pg_rewind`` will fail to execute on the
+standby cluster.

docs/tools_integration.rst

@@ -0,0 +1,64 @@
+.. _tools_integration:
+
+Integration with other tools
+============================
+
+Patroni is able to integrate with other tools in your stack. In this section you
+will find a list of examples, which although not an exhaustive list, might
+provide you with ideas on how Patroni can integrate with other tools.
+
+Barman
+------
+
+Patroni delivers an application named ``patroni_barman`` which has logic to
+communicate with ``pg-backup-api``, so you are able to perform Barman operations
+remotely.
+
+This application currently has a couple of sub-commands: ``recover`` and
+``config-switch``.
+
+patroni_barman recover
+^^^^^^^^^^^^^^^^^^^^^^
+
+The ``recover`` sub-command can be used as a custom bootstrap or custom replica
+creation method. You can find more information about that in
+:ref:`replica_imaging_and_bootstrap`.
+
+patroni_barman config-switch
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The ``config-switch`` sub-command is designed to be used as an ``on_role_change``
+callback in Patroni. As an example, assume you are streaming WALs from your
+current primary to your Barman host. In the event of a failover in the cluster
+you might want to start streaming WALs from the new primary. You can accomplish
+this by using ``patroni_barman config-switch`` as the ``on_role_change`` callback.
+
+.. note::
+    That sub-command relies on the ``barman config-switch`` command, which is in
+    charge of overriding the configuration of a Barman server by applying a
+    pre-defined model on top of it. This command is available since Barman 3.10.
+    Please consult the Barman documentation for more details.
+
+This is an example of how you can configure Patroni to apply a configuration
+model in case this Patroni node is promoted to primary:
+
+.. code:: YAML
+
+    postgresql:
+        callbacks:
+            on_role_change: >
+                patroni_barman
+                    --api-url YOUR_API_URL
+                    config-switch
+                    --barman-server YOUR_BARMAN_SERVER_NAME
+                    --barman-model YOUR_BARMAN_MODEL_NAME
+                    --switch-when promoted
+
+.. note::
+    ``patroni_barman config-switch`` requires that you have both Barman and
+    ``pg-backup-api`` configured in the Barman host, so it can execute a remote
+    ``barman config-switch`` through the backup API. Also, it requires that you
+    have pre-configured Barman models to be applied. The above example uses a
+    subset of the available parameters. You can get more information running
+    ``patroni_barman config-switch --help``, and by consulting the Barman
+    documentation.

docs/yaml_configuration.rst

@@ -11,12 +11,22 @@ Global/Universal
 -  **namespace**: path within the configuration store where Patroni will keep information about the cluster. Default value: "/service"
 -  **scope**: cluster name
 
+.. _log_settings:
+
 Log
 ---
+-  **type**: sets the format of logs. Can be either **plain** or **json**. To use **json** format, you must have the :ref:`jsonlogger <extras>` installed. The default value is **plain**.
 -  **level**: sets the general logging level. Default value is **INFO** (see `the docs for Python logging <https://docs.python.org/3.6/library/logging.html#levels>`_)
 -  **traceback\_level**: sets the level where tracebacks will be visible. Default value is **ERROR**. Set it to **DEBUG** if you want to see tracebacks only if you enable **log.level=DEBUG**.
--  **format**: sets the log formatting string. Default value is **%(asctime)s %(levelname)s: %(message)s** (see `the LogRecord attributes <https://docs.python.org/3.6/library/logging.html#logrecord-attributes>`_)
+-  **format**: sets the log formatting string. If the log type is **plain**, the log format should be a string. Refer to
+   `the LogRecord attributes <https://docs.python.org/3.6/library/logging.html#logrecord-attributes>`_ for
+   available attributes. If the log type is **json**, the log format can be a list in addition to a string. Each list
+   item should correspond to LogRecord attributes. Be cautious that only the field name is required, and the **%(**
+   and **)** should be omitted. If you wish to print a log field with a different key name, use a dictionary where
+   the dictionary key is the log field, and the value is the name of the field you want to be printed in the log.
+   Default value is **%(asctime)s %(levelname)s: %(message)s**
 -  **dateformat**: sets the datetime formatting string. (see the `formatTime() documentation <https://docs.python.org/3.6/library/logging.html#logging.Formatter.formatTime>`_)
+-  **static_fields**: add additional fields to the log. This option is only available when the log type is set to **json**.
 -  **max\_queue\_size**: Patroni is using two-step logging. Log records are written into the in-memory queue and there is a separate thread which pulls them from the queue and writes to stderr or file. The maximum size of the internal queue is limited by default by **1000** records, which is enough to keep logs for the past 1h20m.
 -  **dir**: Directory to write application logs to. The directory must exist and be writable by the user executing Patroni. If you set this value, the application will retain 4 25MB logs by default. You can tune those retention values with `file_num` and `file_size` (see below).
 -  **file\_num**: The number of application logs to retain.
@@ -26,6 +36,20 @@ Log
    -  **patroni.postmaster: WARNING**
    -  **urllib3: DEBUG**
 
+Here is an example of how to config patroni to log in json format.
+
+.. code:: YAML
+
+   log:
+      type: json
+      format:
+         - message
+         - module
+         - asctime: '@timestamp'
+         - levelname: level
+      static_fields:
+         app: patroni
+
 .. _bootstrap_settings:
 
 Bootstrap configuration
@@ -133,6 +157,7 @@ ZooKeeper
 -  **key_password**: (optional) The client key password.
 -  **verify**: (optional) Whether to verify certificate or not. Defaults to ``true``.
 -  **set_acls**: (optional) If set, configure Kazoo to apply a default ACL to each ZNode that it creates. ACLs will assume 'x509' schema and should be specified as a dictionary with the principal as the key and one or more permissions as a list in the value.  Permissions may be one of ``CREATE``, ``READ``, ``WRITE``, ``DELETE`` or ``ADMIN``.  For example, ``set_acls: {CN=principal1: [CREATE, READ], CN=principal2: [ALL]}``.
+-  **auth_data**: (optional) Authentication credentials to use for the connection. Should be a dictionary in the form that `scheme` is the key and `credential` is the value. Defaults to empty dictionary.
 
 .. note::
     It is required to install ``kazoo>=2.6.0`` to support SSL.
@@ -374,6 +399,7 @@ Tags
 -  **nosync**: ``true`` or ``false``. If set to ``true`` the node will never be selected as a synchronous replica.
 -  **nofailover**: ``true`` or ``false``, controls whether this node is allowed to participate in the leader race and become a leader. Defaults to ``false``, meaning this node _can_ participate in leader races. 
 -  **failover_priority**: integer, controls the priority that this node should have during failover. Nodes with higher priority will be preferred over lower priority nodes if they received/replayed the same amount of WAL. However, nodes with higher values of receive/replay LSN are preferred regardless of their priority. If the ``failover_priority`` is 0 or negative - such node is not allowed to participate in the leader race and to become a leader (similar to ``nofailover: true``).
+-  **nostream**: ``true`` or ``false``. If set to ``true`` the node will not use replication protocol to stream WAL. It will rely instead on archive recovery (if ``restore_command`` is configured) and ``pg_wal``/``pg_xlog`` polling. It also disables copying and synchronization of permanent logical replication slots on the node itself and all its cascading replicas. Setting this tag on primary node has no effect.
 
 .. warning::
    Provide only one of ``nofailover`` or ``failover_priority``. Providing ``nofailover: true`` is the same as ``failover_priority: 0``, and providing ``nofailover: false`` will give the node priority 1. 

features/citus.feature

@@ -11,7 +11,9 @@ Feature: citus
     Then replication works from postgres0 to postgres1 after 15 seconds
     Then replication works from postgres2 to postgres3 after 15 seconds
     And postgres0 is registered in the postgres0 as the primary in group 0 after 5 seconds
+    And postgres1 is registered in the postgres0 as the secondary in group 0 after 5 seconds
     And postgres2 is registered in the postgres0 as the primary in group 1 after 5 seconds
+    And postgres3 is registered in the postgres0 as the secondary in group 1 after 5 seconds
 
   Scenario: coordinator failover updates pg_dist_node
     Given I run patronictl.py failover batman --group 0 --candidate postgres1 --force
@@ -19,11 +21,13 @@ Feature: citus
     And "members/postgres0" key in a group 0 in DCS has state=running after 15 seconds
     And replication works from postgres1 to postgres0 after 15 seconds
     And postgres1 is registered in the postgres2 as the primary in group 0 after 5 seconds
+    And postgres0 is registered in the postgres2 as the secondary in group 0 after 15 seconds
     And "sync" key in a group 0 in DCS has sync_standby=postgres0 after 15 seconds
     When I run patronictl.py switchover batman --group 0 --candidate postgres0 --force
     Then postgres0 role is the primary after 10 seconds
     And replication works from postgres0 to postgres1 after 15 seconds
     And postgres0 is registered in the postgres2 as the primary in group 0 after 5 seconds
+    And postgres1 is registered in the postgres2 as the secondary in group 0 after 15 seconds
     And "sync" key in a group 0 in DCS has sync_standby=postgres1 after 15 seconds
 
   Scenario: worker switchover doesn't break client queries on the coordinator
@@ -35,6 +39,7 @@ Feature: citus
     And "members/postgres2" key in a group 1 in DCS has state=running after 15 seconds
     And replication works from postgres3 to postgres2 after 15 seconds
     And postgres3 is registered in the postgres0 as the primary in group 1 after 5 seconds
+    And postgres2 is registered in the postgres0 as the secondary in group 1 after 15 seconds
     And "sync" key in a group 1 in DCS has sync_standby=postgres2 after 15 seconds
     And a thread is still alive
     When I run patronictl.py switchover batman --group 1 --force
@@ -42,6 +47,7 @@ Feature: citus
     And postgres2 role is the primary after 10 seconds
     And replication works from postgres2 to postgres3 after 15 seconds
     And postgres2 is registered in the postgres0 as the primary in group 1 after 5 seconds
+    And postgres3 is registered in the postgres0 as the secondary in group 1 after 15 seconds
     And "sync" key in a group 1 in DCS has sync_standby=postgres3 after 15 seconds
     And a thread is still alive
     When I stop a thread
@@ -55,6 +61,7 @@ Feature: citus
     And postgres2 role is the primary after 10 seconds
     And replication works from postgres2 to postgres3 after 15 seconds
     And postgres2 is registered in the postgres0 as the primary in group 1 after 5 seconds
+    And postgres3 is registered in the postgres0 as the secondary in group 1 after 15 seconds
     And a thread is still alive
     When I stop a thread
     Then a distributed table on postgres0 has expected rows

features/environment.py

@@ -256,10 +256,18 @@ class PatroniController(AbstractController):
                         'parameters': {
                             'wal_keep_segments': 100,
                             'archive_mode': 'on',
-                            'archive_command': (PatroniPoolController.ARCHIVE_RESTORE_SCRIPT
-                                                + ' --mode archive '
-                                                + '--dirname {} --filename %f --pathname %p').format(
-                                                    os.path.join(self._work_directory, 'data', 'wal_archive'))
+                            'archive_command':
+                                (PatroniPoolController.ARCHIVE_RESTORE_SCRIPT
+                                 + ' --mode archive '
+                                 + '--dirname {} --filename %f --pathname %p').format(
+                                     os.path.join(self._work_directory, 'data',
+                                                  f'wal_archive{str(self._citus_group or "")}')).replace('\\', '/'),
+                            'restore_command':
+                                (PatroniPoolController.ARCHIVE_RESTORE_SCRIPT
+                                 + ' --mode restore '
+                                 + '--dirname {} --filename %f --pathname %p').format(
+                                     os.path.join(self._work_directory, 'data',
+                                                  f'wal_archive{str(self._citus_group or "")}')).replace('\\', '/')
                         }
                     }
                 }
@@ -928,11 +936,6 @@ class PatroniPoolController(object):
         custom_config = {
             'scope': cluster_name,
             'postgresql': {
-                'recovery_conf': {
-                    'restore_command': (self.ARCHIVE_RESTORE_SCRIPT + ' --mode restore '
-                                        + '--dirname {} --filename %f --pathname %p')
-                    .format(os.path.join(self.patroni_path, 'data', 'wal_archive').replace('\\', '/'))
-                },
                 'create_replica_methods': ['no_leader_bootstrap'],
                 'no_leader_bootstrap': self.backup_restore_config({'no_leader': '1'})
             }
@@ -1073,6 +1076,8 @@ def before_all(context):
     context.keyfile = os.path.join(context.pctl.output_dir, 'patroni.key')
     context.certfile = os.path.join(context.pctl.output_dir, 'patroni.crt')
     try:
+        if sys.platform == 'darwin' and 'GITHUB_ACTIONS' in os.environ:
+            raise Exception
         with open(os.devnull, 'w') as null:
             ret = subprocess.call(['openssl', 'req', '-nodes', '-new', '-x509', '-subj', '/CN=batman.patroni',
                                    '-addext', 'subjectAltName=IP:127.0.0.1', '-keyout', context.keyfile,

features/nostream_node.feature

@@ -0,0 +1,18 @@
+Feature: nostream node
+
+Scenario: check nostream node is recovering from archive
+    When I start postgres0
+    And I configure and start postgres1 with a tag nostream true
+    Then "members/postgres1" key in DCS has replication_state=in archive recovery after 10 seconds
+    And replication works from postgres0 to postgres1 after 30 seconds
+
+@slot-advance
+Scenario: check permanent logical replication slots are not copied
+    When I issue a PATCH request to http://127.0.0.1:8008/config with {"postgresql": {"parameters": {"wal_level": "logical"}}, "slots":{"test_logical":{"type":"logical","database":"postgres","plugin":"test_decoding"}}}
+    Then I receive a response code 200
+    When I run patronictl.py restart batman postgres0 --force
+    Then postgres0 has a logical replication slot named test_logical with the test_decoding plugin after 10 seconds
+    When I configure and start postgres2 with a tag replicatefrom postgres1
+    Then "members/postgres2" key in DCS has replication_state=streaming after 10 seconds 
+    And postgres1 does not have a replication slot named test_logical
+    And postgres2 does not have a replication slot named test_logical

features/priority_failover.feature

@@ -6,10 +6,9 @@ Feature: priority replication
     And I configure and start postgres1 with a tag failover_priority 0
     Then replication works from postgres0 to postgres1 after 20 seconds
     When I shut down postgres0
-    And I sleep for 5 seconds
-    Then postgres1 role is the secondary after 10 seconds
     And there is one of ["following a different leader because I am not allowed to promote"] INFO in the postgres1 patroni log after 5 seconds
-    Given I start postgres0
+    Then postgres1 role is the secondary after 10 seconds
+    When I start postgres0
     Then postgres0 role is the primary after 10 seconds
 
   Scenario: check higher failover priority is respected
@@ -18,6 +17,23 @@ Feature: priority replication
     Then replication works from postgres0 to postgres2 after 20 seconds
     And replication works from postgres0 to postgres3 after 20 seconds
     When I shut down postgres0
-    And I sleep for 5 seconds
     Then postgres3 role is the primary after 10 seconds
     And there is one of ["postgres3 has equally tolerable WAL position and priority 2, while this node has priority 1","Wal position of postgres3 is ahead of my wal position"] INFO in the postgres2 patroni log after 5 seconds
+
+ Scenario: check conflicting configuration handling
+    When I set nofailover tag in postgres2 config
+    And I issue an empty POST request to http://127.0.0.1:8010/reload
+    Then I receive a response code 202
+    And there is one of ["Conflicting configuration between nofailover: True and failover_priority: 1. Defaulting to nofailover: True"] WARNING in the postgres2 patroni log after 5 seconds
+    And "members/postgres2" key in DCS has tags={'failover_priority': '1', 'nofailover': True} after 10 seconds
+    When I issue a POST request to http://127.0.0.1:8010/failover with {"candidate": "postgres2"}
+    Then I receive a response code 412
+    And I receive a response text "failover is not possible: no good candidates have been found"
+    When I reset nofailover tag in postgres1 config
+    And I issue an empty POST request to http://127.0.0.1:8009/reload
+    Then I receive a response code 202
+    And there is one of ["Conflicting configuration between nofailover: False and failover_priority: 0. Defaulting to nofailover: False"] WARNING in the postgres1 patroni log after 5 seconds
+    And "members/postgres1" key in DCS has tags={'failover_priority': '0', 'nofailover': False} after 10 seconds
+    And I issue a POST request to http://127.0.0.1:8009/failover with {"candidate": "postgres1"}
+    Then I receive a response code 200
+    And postgres1 role is the primary after 10 seconds

features/standby_cluster.feature

@@ -26,7 +26,7 @@ Feature: standby cluster
   Scenario: Detach exiting node from the cluster
     When I shut down postgres1
     Then postgres0 is a leader after 10 seconds
-    And "members/postgres0" key in DCS has role=master after 3 seconds
+    And "members/postgres0" key in DCS has role=master after 5 seconds
     When I issue a GET request to http://127.0.0.1:8008/
     Then I receive a response code 200
 
@@ -47,6 +47,7 @@ Feature: standby cluster
     And there is a postgres1_cb.log with "on_role_change standby_leader batman1" in postgres1 data directory
     When I start postgres2 in a cluster batman1
     Then postgres2 role is the replica after 24 seconds
+    And postgres2 is replicating from postgres1 after 10 seconds
     And table foo is present on postgres2 after 20 seconds
     When I issue a GET request to http://127.0.0.1:8010/patroni
     Then I receive a response code 200

features/steps/basic_replication.py

@@ -46,11 +46,17 @@ def kill_postgres(context, name):
     return context.pctl.stop(name, kill=True, postgres=True)
 
 
+def get_wal_name(context, pg_name):
+    version = context.pctl.query(pg_name, "SHOW server_version_num").fetchone()[0]
+    return 'xlog' if int(version) / 10000 < 10 else 'wal'
+
+
 @step('I add the table {table_name:w} to {pg_name:w}')
 def add_table(context, table_name, pg_name):
     # parse the configuration file and get the port
     try:
         context.pctl.query(pg_name, "CREATE TABLE public.{0}()".format(table_name))
+        context.pctl.query(pg_name, "SELECT pg_switch_{0}()".format(get_wal_name(context, pg_name)))
     except pg.Error as e:
         assert False, "Error creating table {0} on {1}: {2}".format(table_name, pg_name, e)
 
@@ -59,9 +65,7 @@ def add_table(context, table_name, pg_name):
 def toggle_wal_replay(context, action, pg_name):
     # pause or resume the wal replay process
     try:
-        version = context.pctl.query(pg_name, "SHOW server_version_num").fetchone()[0]
-        wal_name = 'xlog' if int(version) / 10000 < 10 else 'wal'
-        context.pctl.query(pg_name, "SELECT pg_{0}_replay_{1}()".format(wal_name, action))
+        context.pctl.query(pg_name, "SELECT pg_{0}_replay_{1}()".format(get_wal_name(context, pg_name), action))
     except pg.Error as e:
         assert False, "Error during {0} wal recovery on {1}: {2}".format(action, pg_name, e)
 
@@ -114,7 +118,7 @@ def replication_works(context, primary, replica, time_limit):
     """.format(str(time()).replace('.', '_').replace(',', '_'), primary, replica, time_limit))
 
 
-@then('there is one of {message_list} {level:w} in the {node} patroni log after {timeout:d} seconds')
+@step('there is one of {message_list} {level:w} in the {node} patroni log after {timeout:d} seconds')
 def check_patroni_log(context, message_list, level, node, timeout):
     timeout *= context.timeout_multiplier
     message_list = json.loads(message_list)
@@ -123,6 +127,6 @@ def check_patroni_log(context, message_list, level, node, timeout):
         messsages_of_level = context.pctl.read_patroni_log(node, level)
         if any(any(message in line for line in messsages_of_level) for message in message_list):
             break
-        time.sleep(1)
+        sleep(1)
     else:
         assert False, f"There were none of {message_list} {level} in the {node} patroni log after {timeout} seconds"

features/steps/citus.py

@@ -131,5 +131,5 @@ def check_transaction(context, name, time_limit):
 
 @step("a transaction finishes in {timeout:d} seconds")
 def check_transaction_timeout(context, timeout):
-    assert (datetime.now(tzutc) - context.xact_start).seconds > timeout, \
+    assert (datetime.now(tzutc) - context.xact_start).seconds >= timeout, \
         "a transaction finished earlier than in {0} seconds".format(timeout)

features/steps/patroni_api.py

@@ -128,6 +128,12 @@ def scheduled_restart(context, url, in_seconds, data):
     context.execute_steps(u"""Given I issue a POST request to {0}/restart with {1}""".format(url, json.dumps(data)))
 
 
+@step('I {action:w} {tag:w} tag in {pg_name:w} config')
+def add_bool_tag_to_config(context, action, tag, pg_name):
+    value = action == 'set'
+    context.pctl.add_tag_to_config(pg_name, tag, value)
+
+
 @step('I add tag {tag:w} {value:w} to {pg_name:w} config')
 def add_tag_to_config(context, tag, value, pg_name):
     context.pctl.add_tag_to_config(pg_name, tag, value)

kubernetes/Dockerfile

@@ -1,4 +1,4 @@
-FROM postgres:15
+FROM postgres:16
 LABEL maintainer="Alexander Kukushkin <[email protected]>"
 
 RUN export DEBIAN_FRONTEND=noninteractive \

kubernetes/Dockerfile.citus

@@ -1,4 +1,4 @@
-FROM postgres:15
+FROM postgres:16
 LABEL maintainer="Alexander Kukushkin <[email protected]>"
 
 RUN export DEBIAN_FRONTEND=noninteractive \
@@ -11,7 +11,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \
     ## Make sure we have a en_US.UTF-8 locale available
     && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
     && if [ $(dpkg --print-architecture) = 'arm64' ]; then \
-        apt-get install -y postgresql-server-dev-15 \
+        apt-get install -y postgresql-server-dev-16 \
                            gcc make autoconf \
                            libc6-dev flex libcurl4-gnutls-dev \
                            libicu-dev libkrb5-dev liblz4-dev \
@@ -24,7 +24,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \
         echo "deb [signed-by=/etc/apt/trusted.gpg.d/citusdata_community.gpg] https://packagecloud.io/citusdata/community/debian/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/citusdata_community.list \
         && curl -sL https://packagecloud.io/citusdata/community/gpgkey | gpg --dearmor > /etc/apt/trusted.gpg.d/citusdata_community.gpg \
         && apt-get update -y \
-        && apt-get -y install postgresql-15-citus-12.0; \
+        && apt-get -y install postgresql-16-citus-12.1; \
     fi \
     && pip3 install --break-system-packages setuptools \
     && pip3 install --break-system-packages 'git+https://github.com/zalando/patroni.git#egg=patroni[kubernetes]' \
@@ -38,7 +38,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \
     && chmod 664 /etc/passwd \
     # Clean up
     && apt-get remove -y git python3-pip python3-wheel \
-               postgresql-server-dev-15 gcc make autoconf \
+               postgresql-server-dev-16 gcc make autoconf \
                libc6-dev flex libicu-dev libkrb5-dev liblz4-dev \
                libpam0g-dev libreadline-dev libselinux1-dev libssl-dev libxslt1-dev libzstd-dev uuid-dev \
     && apt-get autoremove -y \

patroni/__main__.py

@@ -68,7 +68,7 @@ class Patroni(AbstractPatroniDaemon, Tags):
         self.watchdog = Watchdog(self.config)
         self.load_dynamic_configuration()
 
-        self.postgresql = Postgresql(self.config['postgresql'])
+        self.postgresql = Postgresql(self.config['postgresql'], self.dcs.mpp)
         self.api = RestApiServer(self, self.config['restapi'])
         self.ha = Ha(self)
 
@@ -107,8 +107,6 @@ class Patroni(AbstractPatroniDaemon, Tags):
 
     def ensure_unique_name(self) -> None:
         """A helper method to prevent splitbrain from operator naming error."""
-        from urllib.parse import urlparse
-        from urllib3.connection import HTTPConnection
         from patroni.dcs import Member
 
         cluster = self.dcs.get_cluster()
@@ -118,14 +116,14 @@ class Patroni(AbstractPatroniDaemon, Tags):
         if not isinstance(member, Member):
             return
         try:
-            parts = urlparse(member.api_url)
-            if isinstance(parts.hostname, str):
-                connection = HTTPConnection(parts.hostname, port=parts.port or 80, timeout=3)
-                connection.connect()
-                logger.fatal("Can't start; there is already a node named '%s' running", self.config['name'])
-                sys.exit(1)
+            # Silence annoying WARNING: Retrying (...) messages when Patroni is quickly restarted.
+            # At this moment we don't have custom log levels configured and hence shouldn't lose anything useful.
+            self.logger.update_loggers({'urllib3.connectionpool': 'ERROR'})
+            _ = self.request(member, endpoint="/liveness", timeout=3)
+            logger.fatal("Can't start; there is already a node named '%s' running", self.config['name'])
+            sys.exit(1)
         except Exception:
-            return
+            self.logger.update_loggers({})
 
     def _get_tags(self) -> Dict[str, Any]:
         """Get tags configured for this node, if any.
@@ -231,11 +229,6 @@ def patroni_main(configfile: str) -> None:
 
     :param configfile: path to Patroni configuration file.
     """
-    from multiprocessing import freeze_support
-
-    # Windows executables created by PyInstaller are frozen, thus we need to enable frozen support for
-    # :mod:`multiprocessing` to avoid :class:`RuntimeError` exceptions.
-    freeze_support()
     abstract_main(Patroni, configfile)
 
 
@@ -337,6 +330,12 @@ def main() -> None:
         ``patroni`` daemon as another process. In that case relevant signals received by the main process and forwarded
         to ``patroni`` daemon process.
     """
+    from multiprocessing import freeze_support
+
+    # Executables created by PyInstaller are frozen, thus we need to enable frozen support for
+    # :mod:`multiprocessing` to avoid :class:`RuntimeError` exceptions.
+    freeze_support()
+
     check_psycopg()
 
     args = process_arguments()

patroni/api.py

@@ -26,7 +26,7 @@ from urllib.parse import urlparse, parse_qs
 
 from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, TYPE_CHECKING, Union
 
-from . import psycopg
+from . import global_config, psycopg
 from .__main__ import Patroni
 from .dcs import Cluster
 from .exceptions import PostgresConnectionException, PostgresException
@@ -37,7 +37,7 @@ from .utils import deep_compare, enable_keepalive, parse_bool, patch_config, Ret
 logger = logging.getLogger(__name__)
 
 
-def check_access(func: Callable[['RestApiHandler'], None]) -> Callable[..., None]:
+def check_access(func: Callable[..., None]) -> Callable[..., None]:
     """Check the source ip, authorization header, or client certificates.
 
     .. note::
@@ -180,6 +180,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
             * ``tags``: tags that were set through Patroni configuration merged with dynamically applied tags;
             * ``database_system_identifier``: ``Database system identifier`` from ``pg_controldata`` output;
             * ``pending_restart``: ``True`` if PostgreSQL is pending to be restarted;
+            * ``pending_restart_reason``: dictionary where each key is the parameter that caused "pending restart" flag
+                to be set and the value is a dictionary with the old and the new value.
             * ``scheduled_restart``: a dictionary with a single key ``schedule``, which is the timestamp for the
                 scheduled restart;
             * ``watchdog_failed``: ``True`` if watchdog device is unhealthy;
@@ -196,8 +198,9 @@ class RestApiHandler(BaseHTTPRequestHandler):
             response['tags'] = tags
         if patroni.postgresql.sysid:
             response['database_system_identifier'] = patroni.postgresql.sysid
-        if patroni.postgresql.pending_restart:
+        if patroni.postgresql.pending_restart_reason:
             response['pending_restart'] = True
+            response['pending_restart_reason'] = dict(patroni.postgresql.pending_restart_reason)
         response['patroni'] = {
             'version': patroni.version,
             'scope': patroni.postgresql.scope,
@@ -290,7 +293,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         patroni = self.server.patroni
         cluster = patroni.dcs.cluster
-        global_config = patroni.config.get_global_config(cluster)
+        config = global_config.from_cluster(cluster)
 
         leader_optime = cluster and cluster.last_lsn or 0
         replayed_location = response.get('xlog', {}).get('replayed_location', 0)
@@ -308,7 +311,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
             standby_leader_status_code = 200 if response.get('role') == 'standby_leader' else 503
         elif patroni.ha.is_leader():
             leader_status_code = 200
-            if global_config.is_standby_cluster:
+            if config.is_standby_cluster:
                 primary_status_code = replica_status_code = 503
                 standby_leader_status_code = 200 if response.get('role') in ('replica', 'standby_leader') else 503
             else:
@@ -452,9 +455,8 @@ class RestApiHandler(BaseHTTPRequestHandler):
         HTTP status ``200`` and the JSON representation of the cluster topology.
         """
         cluster = self.server.patroni.dcs.get_cluster()
-        global_config = self.server.patroni.config.get_global_config(cluster)
 
-        response = cluster_as_json(cluster, global_config)
+        response = cluster_as_json(cluster)
         response['scope'] = self.server.patroni.postgresql.scope
         self._write_json_response(200, response)
 
@@ -635,7 +637,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         metrics.append("# HELP patroni_pending_restart Value is 1 if the node needs a restart, 0 otherwise.")
         metrics.append("# TYPE patroni_pending_restart gauge")
         metrics.append("patroni_pending_restart{0} {1}"
-                       .format(labels, int(patroni.postgresql.pending_restart)))
+                       .format(labels, int(bool(patroni.postgresql.pending_restart_reason))))
 
         metrics.append("# HELP patroni_is_paused Value is 1 if auto failover is disabled, 0 otherwise.")
         metrics.append("# TYPE patroni_is_paused gauge")
@@ -864,7 +866,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         if request:
             logger.debug("received restart request: {0}".format(request))
 
-        if self.server.patroni.config.get_global_config(cluster).is_paused and 'schedule' in request:
+        if global_config.from_cluster(cluster).is_paused and 'schedule' in request:
             self.write_response(status_code, "Can't schedule restart in the paused state")
             return
 
@@ -1033,7 +1035,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
 
         :returns: a string with the error message or ``None`` if good nodes are found.
         """
-        is_synchronous_mode = self.server.patroni.config.get_global_config(cluster).is_synchronous_mode
+        is_synchronous_mode = global_config.from_cluster(cluster).is_synchronous_mode
         if leader and (not cluster.leader or cluster.leader.name != leader):
             return 'leader name does not match'
         if candidate:
@@ -1091,7 +1093,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         candidate = request.get('candidate') or request.get('member')
         scheduled_at = request.get('scheduled_at')
         cluster = self.server.patroni.dcs.get_cluster()
-        global_config = self.server.patroni.config.get_global_config(cluster)
+        config = global_config.from_cluster(cluster)
 
         logger.info("received %s request with leader=%s candidate=%s scheduled_at=%s",
                     action, leader, candidate, scheduled_at)
@@ -1104,12 +1106,12 @@ class RestApiHandler(BaseHTTPRequestHandler):
         if not data and scheduled_at:
             if action == 'failover':
                 data = "Failover can't be scheduled"
-            elif global_config.is_paused:
+            elif config.is_paused:
                 data = "Can't schedule switchover in the paused state"
             else:
                 (status_code, data, scheduled_at) = self.parse_schedule(scheduled_at, action)
 
-        if not data and global_config.is_paused and not candidate:
+        if not data and config.is_paused and not candidate:
             data = 'Switchover is possible only to a specific candidate in a paused state'
 
         if action == 'failover' and leader:
@@ -1154,8 +1156,16 @@ class RestApiHandler(BaseHTTPRequestHandler):
     def do_POST_citus(self) -> None:
         """Handle a ``POST`` request to ``/citus`` path.
 
-        Call :func:`~patroni.postgresql.CitusHandler.handle_event` to handle the request, then write a response with
-        HTTP status code ``200``.
+        .. note::
+            We keep this entrypoint for backward compatibility and simply dispatch the request to :meth:`do_POST_mpp`.
+        """
+        self.do_POST_mpp()
+
+    def do_POST_mpp(self) -> None:
+        """Handle a ``POST`` request to ``/mpp`` path.
+
+        Call :func:`~patroni.postgresql.mpp.AbstractMPPHandler.handle_event` to handle the request,
+        then write a response with HTTP status code ``200``.
 
         .. note::
             If unable to parse the request body, then the request is silently discarded.
@@ -1165,9 +1175,9 @@ class RestApiHandler(BaseHTTPRequestHandler):
             return
 
         patroni = self.server.patroni
-        if patroni.postgresql.citus_handler.is_coordinator() and patroni.ha.is_leader():
+        if patroni.postgresql.mpp_handler.is_coordinator() and patroni.ha.is_leader():
             cluster = patroni.dcs.get_cluster()
-            patroni.postgresql.citus_handler.handle_event(cluster, request)
+            patroni.postgresql.mpp_handler.handle_event(cluster, request)
         self.write_response(200, 'OK')
 
     def parse_request(self) -> bool:
@@ -1260,7 +1270,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
         """
         postgresql = self.server.patroni.postgresql
         cluster = self.server.patroni.dcs.cluster
-        global_config = self.server.patroni.config.get_global_config(cluster)
+        config = global_config.from_cluster(cluster)
         try:
 
             if postgresql.state not in ('running', 'restarting', 'starting'):
@@ -1291,10 +1301,10 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 })
             }
 
-            if result['role'] == 'replica' and global_config.is_standby_cluster:
+            if result['role'] == 'replica' and config.is_standby_cluster:
                 result['role'] = postgresql.role
 
-            if result['role'] == 'replica' and global_config.is_synchronous_mode\
+            if result['role'] == 'replica' and config.is_synchronous_mode\
                     and cluster and cluster.sync.matches(postgresql.name):
                 result['sync_standby'] = True
 
@@ -1319,7 +1329,7 @@ class RestApiHandler(BaseHTTPRequestHandler):
                 state = 'unknown'
             result: Dict[str, Any] = {'state': state, 'role': postgresql.role}
 
-        if global_config.is_paused:
+        if config.is_paused:
             result['pause'] = True
         if not cluster or cluster.is_unlocked():
             result['cluster_unlocked'] = True

patroni/collections.py

@@ -1,9 +1,10 @@
 """Patroni custom object types somewhat like :mod:`collections` module.
 
-Provides a case insensitive :class:`dict` and :class:`set` object types.
+Provides a case insensitive :class:`dict` and :class:`set` object types, and `EMPTY_DICT` frozen dictionary object.
 """
 from collections import OrderedDict
-from typing import Any, Collection, Dict, Iterator, KeysView, MutableMapping, MutableSet, Optional
+from copy import deepcopy
+from typing import Any, Collection, Dict, Iterator, KeysView, Mapping, MutableMapping, MutableSet, Optional
 
 
 class CaseInsensitiveSet(MutableSet[str]):
@@ -48,7 +49,7 @@ class CaseInsensitiveSet(MutableSet[str]):
         """
         return str(set(self._values.values()))
 
-    def __contains__(self, value: str) -> bool:
+    def __contains__(self, value: object) -> bool:
         """Check if set contains *value*.
 
         The check is performed case-insensitively.
@@ -57,7 +58,7 @@ class CaseInsensitiveSet(MutableSet[str]):
 
         :returns: ``True`` if *value* is already in the set, ``False`` otherwise.
         """
-        return value.lower() in self._values
+        return isinstance(value, str) and value.lower() in self._values
 
     def __iter__(self) -> Iterator[str]:
         """Iterate over the values in this set.
@@ -207,3 +208,47 @@ class CaseInsensitiveDict(MutableMapping[str, Any]):
             "<CaseInsensitiveDict{'A': 'B', 'c': 'd'} at ..."
         """
         return '<{0}{1} at {2:x}>'.format(type(self).__name__, dict(self.items()), id(self))
+
+
+class _FrozenDict(Mapping[str, Any]):
+    """Frozen dictionary object."""
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """Create a new instance of :class:`_FrozenDict` with given data."""
+        self.__values: Dict[str, Any] = dict(*args, **kwargs)
+
+    def __iter__(self) -> Iterator[str]:
+        """Iterate over keys of this dict.
+
+        :yields: each key present in the dict. Yields each key with its last case that has been stored.
+        """
+        return iter(self.__values)
+
+    def __len__(self) -> int:
+        """Get the length of this dict.
+
+        :returns: number of keys in the dict.
+
+        :Example:
+
+        >>> len(_FrozenDict())
+        0
+        """
+        return len(self.__values)
+
+    def __getitem__(self, key: str) -> Any:
+        """Get the value corresponding to *key*.
+
+        :returns: value corresponding to *key*.
+        """
+        return self.__values[key]
+
+    def copy(self) -> Dict[str, Any]:
+        """Create a copy of this dict.
+
+        :return: a new dict object with the same keys and values of this dict.
+        """
+        return deepcopy(self.__values)
+
+
+EMPTY_DICT = _FrozenDict()

patroni/config.py

@@ -1,4 +1,5 @@
 """Facilities related to Patroni configuration."""
+import re
 import json
 import logging
 import os
@@ -11,8 +12,8 @@ from copy import deepcopy
 from typing import Any, Callable, Collection, Dict, List, Optional, Union, TYPE_CHECKING
 
 from . import PATRONI_ENV_PREFIX
-from .collections import CaseInsensitiveDict
-from .dcs import ClusterConfig, Cluster
+from .collections import CaseInsensitiveDict, EMPTY_DICT
+from .dcs import ClusterConfig
 from .exceptions import ConfigParseError
 from .file_perm import pg_perm
 from .postgresql.config import ConfigHandler
@@ -54,154 +55,6 @@ def default_validator(conf: Dict[str, Any]) -> List[str]:
     return []
 
 
-class GlobalConfig(object):
-    """A class that wraps global configuration and provides convenient methods to access/check values.
-
-    It is instantiated either by calling :func:`get_global_config` or :meth:`Config.get_global_config`, which picks
-    either a configuration from provided :class:`Cluster` object (the most up-to-date) or from the
-    local cache if :class:`ClusterConfig` is not initialized or doesn't have a valid config.
-    """
-
-    def __init__(self, config: Dict[str, Any]) -> None:
-        """Initialize :class:`GlobalConfig` object with given *config*.
-
-        :param config: current configuration either from
-                       :class:`ClusterConfig` or from :func:`Config.dynamic_configuration`.
-        """
-        self.__config = config
-
-    def get(self, name: str) -> Any:
-        """Gets global configuration value by *name*.
-
-        :param name: parameter name.
-
-        :returns: configuration value or ``None`` if it is missing.
-        """
-        return self.__config.get(name)
-
-    def check_mode(self, mode: str) -> bool:
-        """Checks whether the certain parameter is enabled.
-
-        :param mode: parameter name, e.g. ``synchronous_mode``, ``failsafe_mode``, ``pause``, ``check_timeline``, and
-            so on.
-
-        :returns: ``True`` if parameter *mode* is enabled in the global configuration.
-        """
-        return bool(parse_bool(self.__config.get(mode)))
-
-    @property
-    def is_paused(self) -> bool:
-        """``True`` if cluster is in maintenance mode."""
-        return self.check_mode('pause')
-
-    @property
-    def is_synchronous_mode(self) -> bool:
-        """``True`` if synchronous replication is requested and it is not a standby cluster config."""
-        return self.check_mode('synchronous_mode') and not self.is_standby_cluster
-
-    @property
-    def is_synchronous_mode_strict(self) -> bool:
-        """``True`` if at least one synchronous node is required."""
-        return self.check_mode('synchronous_mode_strict')
-
-    def get_standby_cluster_config(self) -> Union[Dict[str, Any], Any]:
-        """Get ``standby_cluster`` configuration.
-
-        :returns: a copy of ``standby_cluster`` configuration.
-        """
-        return deepcopy(self.get('standby_cluster'))
-
-    @property
-    def is_standby_cluster(self) -> bool:
-        """``True`` if global configuration has a valid ``standby_cluster`` section."""
-        config = self.get_standby_cluster_config()
-        return isinstance(config, dict) and\
-            bool(config.get('host') or config.get('port') or config.get('restore_command'))
-
-    def get_int(self, name: str, default: int = 0) -> int:
-        """Gets current value of *name* from the global configuration and try to return it as :class:`int`.
-
-        :param name: name of the parameter.
-        :param default: default value if *name* is not in the configuration or invalid.
-
-        :returns: currently configured value of *name* from the global configuration or *default* if it is not set or
-            invalid.
-        """
-        ret = parse_int(self.get(name))
-        return default if ret is None else ret
-
-    @property
-    def min_synchronous_nodes(self) -> int:
-        """The minimal number of synchronous nodes based on whether ``synchronous_mode_strict`` is enabled or not."""
-        return 1 if self.is_synchronous_mode_strict else 0
-
-    @property
-    def synchronous_node_count(self) -> int:
-        """Currently configured value of ``synchronous_node_count`` from the global configuration.
-
-        Assume ``1`` if it is not set or invalid.
-        """
-        return max(self.get_int('synchronous_node_count', 1), self.min_synchronous_nodes)
-
-    @property
-    def maximum_lag_on_failover(self) -> int:
-        """Currently configured value of ``maximum_lag_on_failover`` from the global configuration.
-
-        Assume ``1048576`` if it is not set or invalid.
-        """
-        return self.get_int('maximum_lag_on_failover', 1048576)
-
-    @property
-    def maximum_lag_on_syncnode(self) -> int:
-        """Currently configured value of ``maximum_lag_on_syncnode`` from the global configuration.
-
-        Assume ``-1`` if it is not set or invalid.
-        """
-        return self.get_int('maximum_lag_on_syncnode', -1)
-
-    @property
-    def primary_start_timeout(self) -> int:
-        """Currently configured value of ``primary_start_timeout`` from the global configuration.
-
-        Assume ``300`` if it is not set or invalid.
-
-        .. note::
-            ``master_start_timeout`` is still supported to keep backward compatibility.
-        """
-        default = 300
-        return self.get_int('primary_start_timeout', default)\
-            if 'primary_start_timeout' in self.__config else self.get_int('master_start_timeout', default)
-
-    @property
-    def primary_stop_timeout(self) -> int:
-        """Currently configured value of ``primary_stop_timeout`` from the global configuration.
-
-        Assume ``0`` if it is not set or invalid.
-
-        .. note::
-            ``master_stop_timeout`` is still supported to keep backward compatibility.
-        """
-        default = 0
-        return self.get_int('primary_stop_timeout', default)\
-            if 'primary_stop_timeout' in self.__config else self.get_int('master_stop_timeout', default)
-
-
-def get_global_config(cluster: Optional[Cluster], default: Optional[Dict[str, Any]] = None) -> GlobalConfig:
-    """Instantiates :class:`GlobalConfig` based on the input.
-
-    :param cluster: the currently known cluster state from DCS.
-    :param default: default configuration, which will be used if there is no valid *cluster.config*.
-
-    :returns: :class:`GlobalConfig` object.
-    """
-    # Try to protect from the case when DCS was wiped out
-    if cluster and cluster.config and cluster.config.modify_version:
-        config = cluster.config.data
-    else:
-        config = default or {}
-    return GlobalConfig(deepcopy(config))
-
-
 class Config(object):
     """Handle Patroni configuration.
 
@@ -290,10 +143,10 @@ class Config(object):
         self.__effective_configuration = self._build_effective_configuration({}, self._local_configuration)
         self._data_dir = self.__effective_configuration.get('postgresql', {}).get('data_dir', "")
         self._cache_file = os.path.join(self._data_dir, self.__CACHE_FILENAME)
-        if validator:  # patronictl uses validator=None and we don't want to load anything from local cache in this case
-            self._load_cache()
+        if validator:  # patronictl uses validator=None
+            self._load_cache()  # we don't want to load anything from local cache for ctl
+            self._validate_failover_tags()  # irrelevant for ctl
         self._cache_needs_saving = False
-        self._validate_failover_tags()
 
     @property
     def config_file(self) -> Optional[str]:
@@ -504,6 +357,7 @@ class Config(object):
                     new_configuration = self._build_effective_configuration(self._dynamic_configuration, configuration)
                     self._local_configuration = configuration
                     self.__effective_configuration = new_configuration
+                    self._validate_failover_tags()
                     return True
                 else:
                     logger.info('No local configuration items changed.')
@@ -591,14 +445,14 @@ class Config(object):
 
         for name, value in dynamic_configuration.items():
             if name == 'postgresql':
-                for name, value in (value or {}).items():
+                for name, value in (value or EMPTY_DICT).items():
                     if name == 'parameters':
                         config['postgresql'][name].update(self._process_postgresql_parameters(value))
                     elif name not in ('connect_address', 'proxy_address', 'listen',
                                       'config_dir', 'data_dir', 'pgpass', 'authentication'):
                         config['postgresql'][name] = deepcopy(value)
             elif name == 'standby_cluster':
-                for name, value in (value or {}).items():
+                for name, value in (value or EMPTY_DICT).items():
                     if name in self.__DEFAULT_CONFIG['standby_cluster']:
                         config['standby_cluster'][name] = deepcopy(value)
             elif name in config:  # only variables present in __DEFAULT_CONFIG allowed to be overridden from DCS
@@ -681,8 +535,8 @@ class Config(object):
         _set_section_values('ctl', ['insecure', 'cacert', 'certfile', 'keyfile', 'keyfile_password'])
         _set_section_values('postgresql', ['listen', 'connect_address', 'proxy_address',
                                            'config_dir', 'data_dir', 'pgpass', 'bin_dir'])
-        _set_section_values('log', ['level', 'traceback_level', 'format', 'dateformat', 'max_queue_size',
-                                    'dir', 'file_size', 'file_num', 'loggers'])
+        _set_section_values('log', ['type', 'level', 'traceback_level', 'format', 'dateformat', 'static_fields',
+                                    'max_queue_size', 'dir', 'file_size', 'file_num', 'loggers'])
         _set_section_values('raft', ['data_dir', 'self_addr', 'partner_addrs', 'password', 'bind_addr'])
 
         for binary in ('pg_ctl', 'initdb', 'pg_controldata', 'pg_basebackup', 'postgres', 'pg_isready', 'pg_rewind'):
@@ -729,6 +583,12 @@ class Config(object):
                 if value:
                     ret[first][second] = value
 
+        logformat = ret.get('log', {}).get('format')
+        if logformat and not re.search(r'%\(\w+\)', logformat):
+            logformat = _parse_list(logformat)
+            if logformat:
+                ret['log']['format'] = logformat
+
         def _parse_dict(value: str) -> Optional[Dict[str, Any]]:
             """Parse an YAML dictionary *value* as a :class:`dict`.
 
@@ -744,7 +604,12 @@ class Config(object):
                 logger.exception('Exception when parsing dict %s', value)
                 return None
 
-        for first, params in (('restapi', ('http_extra_headers', 'https_extra_headers')), ('log', ('loggers',))):
+        dict_configs = (
+            ('restapi', ('http_extra_headers', 'https_extra_headers')),
+            ('log', ('static_fields', 'loggers'))
+        )
+
+        for first, params in dict_configs:
             for second in params:
                 value = ret.get(first, {}).pop(second, None)
                 if value:
@@ -791,7 +656,7 @@ class Config(object):
                               'SERVICE_TAGS', 'NAMESPACE', 'CONTEXT', 'USE_ENDPOINTS', 'SCOPE_LABEL', 'ROLE_LABEL',
                               'POD_IP', 'PORTS', 'LABELS', 'BYPASS_API_SERVICE', 'RETRIABLE_HTTP_CODES', 'KEY_PASSWORD',
                               'USE_SSL', 'SET_ACLS', 'GROUP', 'DATABASE', 'LEADER_LABEL_VALUE', 'FOLLOWER_LABEL_VALUE',
-                              'STANDBY_LEADER_LABEL_VALUE', 'TMP_ROLE_LABEL') and name:
+                              'STANDBY_LEADER_LABEL_VALUE', 'TMP_ROLE_LABEL', 'AUTH_DATA') and name:
                     value = os.environ.pop(param)
                     if name == 'CITUS':
                         if suffix == 'GROUP':
@@ -802,7 +667,7 @@ class Config(object):
                         value = value and parse_int(value)
                     elif suffix in ('HOSTS', 'PORTS', 'CHECKS', 'SERVICE_TAGS', 'RETRIABLE_HTTP_CODES'):
                         value = value and _parse_list(value)
-                    elif suffix in ('LABELS', 'SET_ACLS'):
+                    elif suffix in ('LABELS', 'SET_ACLS', 'AUTH_DATA'):
                         value = _parse_dict(value)
                     elif suffix in ('USE_PROXIES', 'REGISTER_SERVICE', 'USE_ENDPOINTS', 'BYPASS_API_SERVICE', 'VERIFY'):
                         value = parse_bool(value)
@@ -949,18 +814,6 @@ class Config(object):
         """
         return deepcopy(self.__effective_configuration)
 
-    def get_global_config(self, cluster: Optional[Cluster]) -> GlobalConfig:
-        """Instantiate :class:`GlobalConfig` based on input.
-
-        Use the configuration from provided *cluster* (the most up-to-date) or from the
-        local cache if *cluster.config* is not initialized or doesn't have a valid config.
-
-        :param cluster: the currently known cluster state from DCS.
-
-        :returns: :class:`GlobalConfig` object.
-        """
-        return get_global_config(cluster, self._dynamic_configuration)
-
     def _validate_failover_tags(self) -> None:
         """Check ``nofailover``/``failover_priority`` config and warn user if it's contradictory.
 
@@ -974,10 +827,12 @@ class Config(object):
           bedrock source of truth)
         """
         tags = self.get('tags', {})
+        if 'nofailover' not in tags:
+            return
         nofailover_tag = tags.get('nofailover')
         failover_priority_tag = parse_int(tags.get('failover_priority'))
         if failover_priority_tag is not None \
-                and (nofailover_tag is True and failover_priority_tag > 0
-                     or nofailover_tag is False and failover_priority_tag <= 0):
+                and (bool(nofailover_tag) is True and failover_priority_tag > 0
+                     or bool(nofailover_tag) is False and failover_priority_tag <= 0):
             logger.warning('Conflicting configuration between nofailover: %s and failover_priority: %s. '
                            'Defaulting to nofailover: %s', nofailover_tag, failover_priority_tag, nofailover_tag)

patroni/config_generator.py

@@ -15,6 +15,7 @@ if TYPE_CHECKING:  # pragma: no cover
     from psycopg2 import cursor
 
 from . import psycopg
+from .collections import EMPTY_DICT
 from .config import Config
 from .exceptions import PatroniException
 from .log import PatroniLogger
@@ -99,6 +100,7 @@ class AbstractConfigGenerator(abc.ABC):
                 'listen': cls._IP + ':8008'
             },
             'log': {
+                'type': PatroniLogger.DEFAULT_TYPE,
                 'level': PatroniLogger.DEFAULT_LEVEL,
                 'traceback_level': PatroniLogger.DEFAULT_TRACEBACK_LEVEL,
                 'format': PatroniLogger.DEFAULT_FORMAT,
@@ -125,6 +127,7 @@ class AbstractConfigGenerator(abc.ABC):
                 'noloadbalance': False,
                 'clonefrom': True,
                 'nosync': False,
+                'nostream': False,
             }
         }
 
@@ -178,7 +181,7 @@ class AbstractConfigGenerator(abc.ABC):
 
         :yields: formatted lines or blocks that represent a text output of the YAML document.
         """
-        for name in ('scope', 'namespace', 'name', 'log', 'restapi', 'ctl' 'citus',
+        for name in ('scope', 'namespace', 'name', 'log', 'restapi', 'ctl', 'citus',
                      'consul', 'etcd', 'etcd3', 'exhibitor', 'kubernetes', 'raft', 'zookeeper'):
             yield from self._format_config_section(name)
 
@@ -242,7 +245,8 @@ class SampleConfigGenerator(AbstractConfigGenerator):
                   See :func:`~patroni.postgresql.misc.postgres_major_version_to_int` and
                   :func:`~patroni.utils.get_major_version`.
         """
-        postgres_bin = ((self.config.get('postgresql') or {}).get('bin_name') or {}).get('postgres', 'postgres')
+        postgres_bin = ((self.config.get('postgresql')
+                         or EMPTY_DICT).get('bin_name') or EMPTY_DICT).get('postgres', 'postgres')
         return postgres_major_version_to_int(get_major_version(self.config['postgresql'].get('bin_dir'), postgres_bin))
 
     def generate(self) -> None:
@@ -409,8 +413,10 @@ class RunningClusterConfigGenerator(AbstractConfigGenerator):
             val = self.parsed_dsn.get(conn_param, os.getenv(env_var))
             if val:
                 su_params[conn_param] = val
-        patroni_env_su_username = ((self.config.get('authentication') or {}).get('superuser') or {}).get('username')
-        patroni_env_su_pwd = ((self.config.get('authentication') or {}).get('superuser') or {}).get('password')
+        patroni_env_su_username = ((self.config.get('authentication')
+                                    or EMPTY_DICT).get('superuser') or EMPTY_DICT).get('username')
+        patroni_env_su_pwd = ((self.config.get('authentication')
+                               or EMPTY_DICT).get('superuser') or EMPTY_DICT).get('password')
         # because we use "username" in the config for some reason
         su_params['username'] = su_params.pop('user', patroni_env_su_username) or getuser()
         su_params['password'] = su_params.get('password', patroni_env_su_pwd) or \

patroni/ctl.py

@@ -41,15 +41,21 @@ if TYPE_CHECKING:  # pragma: no cover
     from psycopg import Cursor
     from psycopg2 import cursor
 
-try:
-    from ydiff import markup_to_pager, PatchStream  # pyright: ignore [reportMissingModuleSource]
+try:  # pragma: no cover
+    from ydiff import markup_to_pager  # pyright: ignore [reportMissingModuleSource]
+    try:
+        from ydiff import PatchStream  # pyright: ignore [reportMissingModuleSource]
+    except ImportError:
+        PatchStream = iter
 except ImportError:  # pragma: no cover
     from cdiff import markup_to_pager, PatchStream  # pyright: ignore [reportMissingModuleSource]
 
-from .config import Config, get_global_config
+from . import global_config
+from .config import Config
 from .dcs import get_dcs as _get_dcs, AbstractDCS, Cluster, Member
 from .exceptions import PatroniException
 from .postgresql.misc import postgres_version_to_int
+from .postgresql.mpp import get_mpp
 from .utils import cluster_as_json, patch_config, polling_loop
 from .request import PatroniRequest
 from .version import __version__
@@ -255,14 +261,23 @@ def load_config(path: str, dcs_url: Optional[str]) -> Dict[str, Any]:
     return config
 
 
-option_format = click.option('--format', '-f', 'fmt', help='Output format (pretty, tsv, json, yaml)', default='pretty')
+def _get_configuration() -> Dict[str, Any]:
+    """Get configuration object.
+
+    :returns: configuration object from the current context.
+    """
+    return click.get_current_context().obj['__config']
+
+
+option_format = click.option('--format', '-f', 'fmt', help='Output format', default='pretty',
+                             type=click.Choice(['pretty', 'tsv', 'json', 'yaml', 'yml']))
 option_watchrefresh = click.option('-w', '--watch', type=float, help='Auto update the screen every X seconds')
 option_watch = click.option('-W', is_flag=True, help='Auto update the screen every 2 seconds')
 option_force = click.option('--force', is_flag=True, help='Do not ask for confirmation at any point')
 arg_cluster_name = click.argument('cluster_name', required=False,
-                                  default=lambda: click.get_current_context().obj.get('scope'))
+                                  default=lambda: _get_configuration().get('scope'))
 option_default_citus_group = click.option('--group', required=False, type=int, help='Citus group',
-                                          default=lambda: click.get_current_context().obj.get('citus', {}).get('group'))
+                                          default=lambda: _get_configuration().get('citus', {}).get('group'))
 option_citus_group = click.option('--group', required=False, type=int, help='Citus group')
 role_choice = click.Choice(['leader', 'primary', 'standby-leader', 'replica', 'standby', 'any', 'master'])
 
@@ -300,15 +315,23 @@ def ctl(ctx: click.Context, config_file: str, dcs_url: Optional[str], insecure:
         level = os.environ.get(name, level)
     logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=level)
     logging.captureWarnings(True)  # Capture eventual SSL warning
-    ctx.obj = load_config(config_file, dcs_url)
+    config = load_config(config_file, dcs_url)
     # backward compatibility for configuration file where ctl section is not defined
-    ctx.obj.setdefault('ctl', {})['insecure'] = ctx.obj.get('ctl', {}).get('insecure') or insecure
+    config.setdefault('ctl', {})['insecure'] = config.get('ctl', {}).get('insecure') or insecure
+    ctx.obj = {'__config': config, '__mpp': get_mpp(config)}
 
 
-def get_dcs(config: Dict[str, Any], scope: str, group: Optional[int]) -> AbstractDCS:
+def is_citus_cluster() -> bool:
+    """Check if we are working with Citus cluster.
+
+    :returns: ``True`` if configuration has ``citus`` section, otherwise ``False``.
+    """
+    return click.get_current_context().obj['__mpp'].is_enabled()
+
+
+def get_dcs(scope: str, group: Optional[int]) -> AbstractDCS:
     """Get the DCS object.
 
-    :param config: Patroni configuration.
     :param scope: cluster name.
     :param group: if *group* is defined, use it to select which alternative Citus group this DCS refers to. If *group*
         is ``None`` and a Citus configuration exists, assume this is the coordinator. Coordinator has the group ``0``.
@@ -319,14 +342,16 @@ def get_dcs(config: Dict[str, Any], scope: str, group: Optional[int]) -> Abstrac
     :raises:
         :class:`PatroniCtlException`: if not suitable DCS configuration could be found.
     """
+    config = _get_configuration()
     config.update({'scope': scope, 'patronictl': True})
     if group is not None:
-        config['citus'] = {'group': group}
+        config['citus'] = {'group': group, 'database': 'postgres'}
     config.setdefault('name', scope)
     try:
         dcs = _get_dcs(config)
-        if config.get('citus') and group is None:
-            dcs.is_citus_coordinator = lambda: True
+        if is_citus_cluster() and group is None:
+            dcs.is_mpp_coordinator = lambda: True
+        click.get_current_context().obj['__mpp'] = dcs.mpp
         return dcs
     except PatroniException as e:
         raise PatroniCtlException(str(e))
@@ -346,7 +371,7 @@ def request_patroni(member: Member, method: str = 'GET',
     ctx = click.get_current_context()  # the current click context
     request_executor = ctx.obj.get('__request_patroni')
     if not request_executor:
-        request_executor = ctx.obj['__request_patroni'] = PatroniRequest(ctx.obj)
+        request_executor = ctx.obj['__request_patroni'] = PatroniRequest(_get_configuration())
     return request_executor(member, method, endpoint, data)
 
 
@@ -413,9 +438,9 @@ def print_output(columns: Optional[List[str]], rows: List[List[Any]], alignment:
 
 
 def watching(w: bool, watch: Optional[int], max_count: Optional[int] = None, clear: bool = True) -> Iterator[int]:
-    """Yield a value every ``x`` seconds.
+    """Yield a value every ``watch`` seconds.
 
-    Used to run a command with a watch-based aproach.
+    Used to run a command with a watch-based approach.
 
     :param w: if ``True`` and *watch* is ``None``, then *watch* assumes the value ``2``.
     :param watch: amount of seconds to wait before yielding another value.
@@ -451,11 +476,9 @@ def watching(w: bool, watch: Optional[int], max_count: Optional[int] = None, cle
         yield 0
 
 
-def get_all_members(obj: Dict[str, Any], cluster: Cluster,
-                    group: Optional[int], role: str = 'leader') -> Iterator[Member]:
+def get_all_members(cluster: Cluster, group: Optional[int], role: str = 'leader') -> Iterator[Member]:
     """Get all cluster members that have the given *role*.
 
-    :param obj: the Patroni configuration.
     :param cluster: the Patroni cluster.
     :param group: filter which Citus group we should get members from. If ``None`` get from all groups.
     :param role: role to filter members. Can be one among:
@@ -469,7 +492,7 @@ def get_all_members(obj: Dict[str, Any], cluster: Cluster,
     :yields: members that have the given *role*.
     """
     clusters = {0: cluster}
-    if obj.get('citus') and group is None:
+    if is_citus_cluster() and group is None:
         clusters.update(cluster.workers)
     if role in ('leader', 'master', 'primary', 'standby-leader'):
         # In the DCS the members' role can be one among: ``primary``, ``master``, ``replica`` or ``standby_leader``.
@@ -491,11 +514,10 @@ def get_all_members(obj: Dict[str, Any], cluster: Cluster,
                 yield m
 
 
-def get_any_member(obj: Dict[str, Any], cluster: Cluster, group: Optional[int],
+def get_any_member(cluster: Cluster, group: Optional[int],
                    role: Optional[str] = None, member: Optional[str] = None) -> Optional[Member]:
     """Get the first found cluster member that has the given *role*.
 
-    :param obj: the Patroni configuration.
     :param cluster: the Patroni cluster.
     :param group: filter which Citus group we should get members from. If ``None`` get from all groups.
     :param role: role to filter members. See :func:`get_all_members` for available options.
@@ -513,7 +535,7 @@ def get_any_member(obj: Dict[str, Any], cluster: Cluster, group: Optional[int],
     elif role is None:
         role = 'leader'
 
-    for m in get_all_members(obj, cluster, group, role):
+    for m in get_all_members(cluster, group, role):
         if member is None or m.name == member:
             return m
 
@@ -534,7 +556,7 @@ def get_all_members_leader_first(cluster: Cluster) -> Iterator[Member]:
             yield member
 
 
-def get_cursor(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], connect_parameters: Dict[str, Any],
+def get_cursor(cluster: Cluster, group: Optional[int], connect_parameters: Dict[str, Any],
                role: Optional[str] = None, member_name: Optional[str] = None) -> Union['cursor', 'Cursor[Any]', None]:
     """Get a cursor object to execute queries against a member that has the given *role* or *member_name*.
 
@@ -543,7 +565,6 @@ def get_cursor(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], conn
             * ``fallback_application_name``: as ``Patroni ctl``;
             * ``connect_timeout``: as ``5``.
 
-    :param obj: the Patroni configuration.
     :param cluster: the Patroni cluster.
     :param group: filter which Citus group we should get members to create a cursor against. If ``None`` consider
         members from all groups.
@@ -558,7 +579,7 @@ def get_cursor(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], conn
         * A :class:`psycopg2.extensions.cursor` if using :mod:`psycopg2`;
         * ``None`` if not able to get a cursor that attendees *role* and *member_name*.
     """
-    member = get_any_member(obj, cluster, group, role=role, member=member_name)
+    member = get_any_member(cluster, group, role=role, member=member_name)
     if member is None:
         return None
 
@@ -593,7 +614,7 @@ def get_cursor(obj: Dict[str, Any], cluster: Cluster, group: Optional[int], conn
     return None
 
 
-def get_members(obj: Dict[str, Any], cluster: Cluster, cluster_name: str, member_names: List[str], role: str,
+def get_members(cluster: Cluster, cluster_name: str, member_names: List[str], role: str,
                 force: bool, action: str, ask_confirmation: bool = True, group: Optional[int] = None) -> List[Member]:
     """Get the list of members based on the given filters.
 
@@ -617,7 +638,6 @@ def get_members(obj: Dict[str, Any], cluster: Cluster, cluster_name: str, member
         ``ask_confirmation=False``, and later call :func:`confirm_members_action` manually in the caller method. That
         way the workflow won't look broken to the user that is interacting with ``patronictl``.
 
-    :param obj: Patroni configuration.
     :param cluster: Patroni cluster.
     :param cluster_name: name of the Patroni cluster.
     :param member_names: used to filter which members should take the *action* based on their names. Each item is the
@@ -646,13 +666,13 @@ def get_members(obj: Dict[str, Any], cluster: Cluster, cluster_name: str, member
             * Cluster does not have members that match the given *member_names*; or
             * No member with given *role* is found among the specified *member_names*.
     """
-    members = list(get_all_members(obj, cluster, group, role))
+    members = list(get_all_members(cluster, group, role))
 
     candidates = {m.name for m in members}
     if not force or role:
         if not member_names and not candidates:
             raise PatroniCtlException('{0} cluster doesn\'t have any members'.format(cluster_name))
-        output_members(obj, cluster, cluster_name, group=group)
+        output_members(cluster, cluster_name, group=group)
 
     if member_names:
         member_names = list(set(member_names) & candidates)
@@ -712,9 +732,7 @@ def confirm_members_action(members: List[Member], force: bool, action: str,
 @click.option('--member', '-m', help='Generate a dsn for this member', type=str)
 @arg_cluster_name
 @option_citus_group
-@click.pass_obj
-def dsn(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
-        role: Optional[str], member: Optional[str]) -> None:
+def dsn(cluster_name: str, group: Optional[int], role: Optional[str], member: Optional[str]) -> None:
     """Process ``dsn`` command of ``patronictl`` utility.
 
     Get DSN to connect to *member*.
@@ -722,7 +740,6 @@ def dsn(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
     .. note::
         If no *role* nor *member* is given assume *role* as ``leader``.
 
-    :param obj: Patroni configuration.
     :param cluster_name: name of the Patroni cluster.
     :param group: filter which Citus group we should get members to get DSN from. Refer to the module note for more
         details.
@@ -735,8 +752,8 @@ def dsn(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
             * both *role* and *member* are provided; or
             * No member matches requested *member* or *role*.
     """
-    cluster = get_dcs(obj, cluster_name, group).get_cluster()
-    m = get_any_member(obj, cluster, group, role=role, member=member)
+    cluster = get_dcs(cluster_name, group).get_cluster()
+    m = get_any_member(cluster, group, role=role, member=member)
     if m is None:
         raise PatroniCtlException('Can not find a suitable member')
 
@@ -758,9 +775,7 @@ def dsn(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
 @click.option('--delimiter', help='The column delimiter', default='\t')
 @click.option('--command', '-c', help='The SQL commands to execute')
 @click.option('-d', '--dbname', help='database name to connect to', type=str)
-@click.pass_obj
 def query(
-    obj: Dict[str, Any],
     cluster_name: str,
     group: Optional[int],
     role: Optional[str],
@@ -779,7 +794,6 @@ def query(
 
     Perform a Postgres query in a Patroni node.
 
-    :param obj: Patroni configuration.
     :param cluster_name: name of the Patroni cluster.
     :param group: filter which Citus group we should get members from to perform the query. Refer to the module note for
         more details.
@@ -811,7 +825,7 @@ def query(
             raise PatroniCtlException('You need to specify either --command or --file')
         sql = command
 
-    connect_parameters = {}
+    connect_parameters: Dict[str, str] = {}
     if username:
         connect_parameters['username'] = username
     if password:
@@ -819,24 +833,22 @@ def query(
     if dbname:
         connect_parameters['dbname'] = dbname
 
-    dcs = get_dcs(obj, cluster_name, group)
+    dcs = get_dcs(cluster_name, group)
 
     cluster = cursor = None
     for _ in watching(w, watch, clear=False):
         if cluster is None:
             cluster = dcs.get_cluster()
-#            cursor = get_cursor(obj, cluster, group, connect_parameters, role=role, member=member)
 
-        output, header = query_member(obj, cluster, group, cursor, member, role, sql, connect_parameters)
+        output, header = query_member(cluster, group, cursor, member, role, sql, connect_parameters)
         print_output(header, output, fmt=fmt, delimiter=delimiter)
 
 
-def query_member(obj: Dict[str, Any], cluster: Cluster, group: Optional[int],
-                 cursor: Union['cursor', 'Cursor[Any]', None], member: Optional[str], role: Optional[str],
-                 command: str, connect_parameters: Dict[str, Any]) -> Tuple[List[List[Any]], Optional[List[Any]]]:
+def query_member(cluster: Cluster, group: Optional[int], cursor: Union['cursor', 'Cursor[Any]', None],
+                 member: Optional[str], role: Optional[str], command: str,
+                 connect_parameters: Dict[str, Any]) -> Tuple[List[List[Any]], Optional[List[Any]]]:
     """Execute SQL *command* against a member.
 
-    :param obj: Patroni configuration.
     :param cluster: the Patroni cluster.
     :param group: filter which Citus group we should get members from to perform the query. Refer to the module note for
         more details.
@@ -865,7 +877,7 @@ def query_member(obj: Dict[str, Any], cluster: Cluster, group: Optional[int],
     from . import psycopg
     try:
         if cursor is None:
-            cursor = get_cursor(obj, cluster, group, connect_parameters, role=role, member_name=member)
+            cursor = get_cursor(cluster, group, connect_parameters, role=role, member_name=member)
 
         if cursor is None:
             if member is not None:
@@ -892,13 +904,11 @@ def query_member(obj: Dict[str, Any], cluster: Cluster, group: Optional[int],
 @click.argument('cluster_name')
 @option_citus_group
 @option_format
-@click.pass_obj
-def remove(obj: Dict[str, Any], cluster_name: str, group: Optional[int], fmt: str) -> None:
+def remove(cluster_name: str, group: Optional[int], fmt: str) -> None:
     """Process ``remove`` command of ``patronictl`` utility.
 
     Remove cluster *cluster_name* from the DCS.
 
-    :param obj: Patroni configuration.
     :param cluster_name: name of the cluster which information will be wiped out of the DCS.
     :param group: which Citus group should have its information wiped out of the DCS. Refer to the module note for more
         details.
@@ -912,12 +922,12 @@ def remove(obj: Dict[str, Any], cluster_name: str, group: Optional[int], fmt: st
             * use did not type the correct leader name when requesting removal of a healthy cluster.
 
     """
-    dcs = get_dcs(obj, cluster_name, group)
+    dcs = get_dcs(cluster_name, group)
     cluster = dcs.get_cluster()
 
-    if obj.get('citus') and group is None:
+    if is_citus_cluster() and group is None:
         raise PatroniCtlException('For Citus clusters the --group must me specified')
-    output_members(obj, cluster, cluster_name, fmt=fmt)
+    output_members(cluster, cluster_name, fmt=fmt)
 
     confirm = click.prompt('Please confirm the cluster name to remove', type=str)
     if confirm != cluster_name:
@@ -1002,31 +1012,28 @@ def parse_scheduled(scheduled: Optional[str]) -> Optional[datetime.datetime]:
 @option_citus_group
 @click.option('--role', '-r', help='Reload only members with this role', type=role_choice, default='any')
 @option_force
-@click.pass_obj
-def reload(obj: Dict[str, Any], cluster_name: str, member_names: List[str],
-           group: Optional[int], force: bool, role: str) -> None:
+def reload(cluster_name: str, member_names: List[str], group: Optional[int], force: bool, role: str) -> None:
     """Process ``reload`` command of ``patronictl`` utility.
 
     Reload configuration of cluster members based on given filters.
 
-    :param obj: Patroni configuration.
     :param cluster_name: name of the Patroni cluster.
     :param member_names: name of the members which configuration should be reloaded.
     :param group: filter which Citus group we should reload members. Refer to the module note for more details.
     :param force: perform the reload without asking for confirmations.
     :param role: role to filter members. See :func:`get_all_members` for available options.
     """
-    dcs = get_dcs(obj, cluster_name, group)
+    dcs = get_dcs(cluster_name, group)
     cluster = dcs.get_cluster()
 
-    members = get_members(obj, cluster, cluster_name, member_names, role, force, 'reload', group=group)
+    members = get_members(cluster, cluster_name, member_names, role, force, 'reload', group=group)
 
     for member in members:
         r = request_patroni(member, 'post', 'reload')
         if r.status == 200:
             click.echo('No changes to apply on member {0}'.format(member.name))
         elif r.status == 202:
-            config = get_global_config(cluster)
+            config = global_config.from_cluster(cluster)
             click.echo('Reload request received for member {0} and will be processed within {1} seconds'.format(
                 member.name, config.get('loop_wait') or dcs.loop_wait)
             )
@@ -1049,15 +1056,13 @@ def reload(obj: Dict[str, Any], cluster_name: str, member_names: List[str],
 @click.option('--pending', help='Restart if pending', is_flag=True)
 @click.option('--timeout', help='Return error and fail over if necessary when restarting takes longer than this.')
 @option_force
-@click.pass_obj
-def restart(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member_names: List[str],
+def restart(cluster_name: str, group: Optional[int], member_names: List[str],
             force: bool, role: str, p_any: bool, scheduled: Optional[str], version: Optional[str],
             pending: bool, timeout: Optional[str]) -> None:
     """Process ``restart`` command of ``patronictl`` utility.
 
     Restart Postgres on cluster members based on given filters.
 
-    :param obj: Patroni configuration.
     :param cluster_name: name of the Patroni cluster.
     :param group: filter which Citus group we should restart members. Refer to the module note for more details.
     :param member_names: name of the members that should be restarted.
@@ -1075,9 +1080,9 @@ def restart(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member
             * *version* could not be parsed; or
             * a restart is attempted against a cluster that is in maintenance mode.
     """
-    cluster = get_dcs(obj, cluster_name, group).get_cluster()
+    cluster = get_dcs(cluster_name, group).get_cluster()
 
-    members = get_members(obj, cluster, cluster_name, member_names, role, force, 'restart', False, group=group)
+    members = get_members(cluster, cluster_name, member_names, role, force, 'restart', False, group=group)
     if scheduled is None and not force:
         next_hour = (datetime.datetime.now() + datetime.timedelta(hours=1)).strftime('%Y-%m-%dT%H:%M')
         scheduled = click.prompt('When should the restart take place (e.g. ' + next_hour + ') ',
@@ -1094,7 +1099,7 @@ def restart(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member
         version = click.prompt('Restart if the PostgreSQL version is less than provided (e.g. 9.5.2) ',
                                type=str, default='')
 
-    content = {}
+    content: Dict[str, Any] = {}
     if pending:
         content['restart_pending'] = True
 
@@ -1107,7 +1112,7 @@ def restart(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member
         content['postgres_version'] = version
 
     if scheduled_at:
-        if get_global_config(cluster).is_paused:
+        if global_config.from_cluster(cluster).is_paused:
             raise PatroniCtlException("Can't schedule restart in the paused state")
         content['schedule'] = scheduled_at.isoformat()
 
@@ -1139,9 +1144,7 @@ def restart(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member
 @click.argument('member_names', nargs=-1)
 @option_force
 @click.option('--wait', help='Wait until reinitialization completes', is_flag=True)
-@click.pass_obj
-def reinit(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
-           member_names: List[str], force: bool, wait: bool) -> None:
+def reinit(cluster_name: str, group: Optional[int], member_names: List[str], force: bool, wait: bool) -> None:
     """Process ``reinit`` command of ``patronictl`` utility.
 
     Reinitialize cluster members based on given filters.
@@ -1149,15 +1152,14 @@ def reinit(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
     .. note::
         Only reinitialize replica members, not a leader.
 
-    :param obj: Patroni configuration.
     :param cluster_name: name of the Patroni cluster.
     :param group: filter which Citus group we should reinit members. Refer to the module note for more details.
     :param member_names: name of the members that should be reinitialized.
     :param force: perform the restart without asking for confirmations.
     :param wait: wait for the operation to complete.
     """
-    cluster = get_dcs(obj, cluster_name, group).get_cluster()
-    members = get_members(obj, cluster, cluster_name, member_names, 'replica', force, 'reinitialize', group=group)
+    cluster = get_dcs(cluster_name, group).get_cluster()
+    members = get_members(cluster, cluster_name, member_names, 'replica', force, 'reinitialize', group=group)
 
     wait_on_members: List[Member] = []
     for member in members:
@@ -1188,8 +1190,8 @@ def reinit(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
                 wait_on_members.remove(member)
 
 
-def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: str,
-                               group: Optional[int], leader: Optional[str], candidate: Optional[str],
+def _do_failover_or_switchover(action: str, cluster_name: str, group: Optional[int],
+                               switchover_leader: Optional[str], candidate: Optional[str],
                                force: bool, scheduled: Optional[str] = None) -> None:
     """Perform a failover or a switchover operation in the cluster.
 
@@ -1199,12 +1201,11 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s
     .. note::
         If not able to perform the operation through the REST API, write directly to the DCS as a fall back.
 
-    :param obj: Patroni configuration.
     :param action: action to be taken -- ``failover`` or ``switchover``.
     :param cluster_name: name of the Patroni cluster.
     :param group: filter Citus group within we should perform a failover or switchover. If ``None``, user will be
         prompted for filling it -- unless *force* is ``True``, in which case an exception is raised.
-    :param leader: name of the current leader member.
+    :param switchover_leader: name of the leader member passed as switchover option.
     :param candidate: name of a standby member to be promoted. Nodes that are tagged with ``nofailover`` cannot be used.
     :param force: perform the failover or switchover without asking for confirmations.
     :param scheduled: timestamp when the switchover should be scheduled to occur. If ``now`` perform immediately.
@@ -1213,46 +1214,48 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s
         :class:`PatroniCtlException`: if:
             * Patroni is running on a Citus cluster, but no *group* was specified; or
             * a switchover was requested by the cluster has no leader; or
-            * *leader* does not match the current leader of the cluster; or
+            * *switchover_leader* does not match the current leader of the cluster; or
             * cluster has no candidates available for the operation; or
             * no *candidate* is given for a failover operation; or
-            * *leader* and *candidate* are the same; or
+            * current leader and *candidate* are the same; or
+            * *candidate* is tagged as nofailover; or
             * *candidate* is not a member of the cluster; or
             * trying to schedule a switchover in a cluster that is in maintenance mode; or
             * user aborts the operation.
     """
-    dcs = get_dcs(obj, cluster_name, group)
+    dcs = get_dcs(cluster_name, group)
     cluster = dcs.get_cluster()
     click.echo('Current cluster topology')
-    output_members(obj, cluster, cluster_name, group=group)
+    output_members(cluster, cluster_name, group=group)
 
-    if obj.get('citus') and group is None:
+    if is_citus_cluster() and group is None:
         if force:
             raise PatroniCtlException('For Citus clusters the --group must me specified')
         else:
             group = click.prompt('Citus group', type=int)
-            dcs = get_dcs(obj, cluster_name, group)
+            dcs = get_dcs(cluster_name, group)
             cluster = dcs.get_cluster()
 
-    global_config = get_global_config(cluster)
+    config = global_config.from_cluster(cluster)
 
+    cluster_leader = cluster.leader and cluster.leader.name
     # leader has to be be defined for switchover only
     if action == 'switchover':
-        if cluster.leader is None or not cluster.leader.name:
+        if not cluster_leader:
             raise PatroniCtlException('This cluster has no leader')
 
-        if leader is None:
+        if switchover_leader is None:
             if force:
-                leader = cluster.leader.name
+                switchover_leader = cluster_leader
             else:
-                prompt = 'Standby Leader' if global_config.is_standby_cluster else 'Primary'
-                leader = click.prompt(prompt, type=str, default=(cluster.leader and cluster.leader.name))
+                prompt = 'Standby Leader' if config.is_standby_cluster else 'Primary'
+                switchover_leader = click.prompt(prompt, type=str, default=cluster_leader)
 
-        if cluster.leader.name != leader:
-            raise PatroniCtlException(f'Member {leader} is not the leader of cluster {cluster_name}')
+        if cluster_leader != switchover_leader:
+            raise PatroniCtlException(f'Member {switchover_leader} is not the leader of cluster {cluster_name}')
 
     # excluding members with nofailover tag
-    candidate_names = [str(m.name) for m in cluster.members if m.name != leader and not m.nofailover]
+    candidate_names = [str(m.name) for m in cluster.members if m.name != cluster_leader and not m.nofailover]
     # We sort the names for consistent output to the client
     candidate_names.sort()
 
@@ -1265,19 +1268,19 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s
     if action == 'failover' and not candidate:
         raise PatroniCtlException('Failover could be performed only to a specific candidate')
 
-    if candidate == leader:
-        raise PatroniCtlException(action.title() + ' target and source are the same.')
-
     if candidate and candidate not in candidate_names:
+        if candidate == cluster_leader:
+            raise PatroniCtlException(
+                f'Member {candidate} is already the leader of cluster {cluster_name}')
         raise PatroniCtlException(
             f'Member {candidate} does not exist in cluster {cluster_name} or is tagged as nofailover')
 
     if all((not force,
             action == 'failover',
-            global_config.is_synchronous_mode,
+            config.is_synchronous_mode,
             not cluster.sync.is_empty,
             not cluster.sync.matches(candidate, True))):
-        if click.confirm(f'Are you sure you want to failover to the asynchronous node {candidate}'):
+        if not click.confirm(f'Are you sure you want to failover to the asynchronous node {candidate}?'):
             raise PatroniCtlException('Aborting ' + action)
 
     scheduled_at_str = None
@@ -1291,13 +1294,13 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s
 
         scheduled_at = parse_scheduled(scheduled)
         if scheduled_at:
-            if global_config.is_paused:
+            if config.is_paused:
                 raise PatroniCtlException("Can't schedule switchover in the paused state")
             scheduled_at_str = scheduled_at.isoformat()
 
     failover_value = {'candidate': candidate}
     if action == 'switchover':
-        failover_value['leader'] = leader
+        failover_value['leader'] = switchover_leader
     if scheduled_at_str:
         failover_value['scheduled_at'] = scheduled_at_str
 
@@ -1305,7 +1308,7 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s
 
     # By now we have established that the leader exists and the candidate exists
     if not force:
-        demote_msg = f', demoting current leader {cluster.leader.name}' if cluster.leader else ''
+        demote_msg = f', demoting current leader {cluster_leader}' if cluster_leader else ''
         if scheduled_at_str:
             # only switchover can be scheduled
             if not click.confirm(f'Are you sure you want to schedule switchover of cluster '
@@ -1339,9 +1342,9 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s
         logging.exception(r)
         logging.warning('Failing over to DCS')
         click.echo('{0} Could not {1} using Patroni api, falling back to DCS'.format(timestamp(), action))
-        dcs.manual_failover(leader, candidate, scheduled_at=scheduled_at)
+        dcs.manual_failover(switchover_leader, candidate, scheduled_at=scheduled_at)
 
-    output_members(obj, cluster, cluster_name, group=group)
+    output_members(cluster, cluster_name, group=group)
 
 
 @ctl.command('failover', help='Failover to a replica')
@@ -1350,8 +1353,7 @@ def _do_failover_or_switchover(obj: Dict[str, Any], action: str, cluster_name: s
 @click.option('--leader', '--primary', '--master', 'leader', help='The name of the current leader', default=None)
 @click.option('--candidate', help='The name of the candidate', default=None)
 @option_force
-@click.pass_obj
-def failover(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
+def failover(cluster_name: str, group: Optional[int],
              leader: Optional[str], candidate: Optional[str], force: bool) -> None:
     """Process ``failover`` command of ``patronictl`` utility.
 
@@ -1365,7 +1367,6 @@ def failover(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
     .. seealso::
         Refer to :func:`_do_failover_or_switchover` for details.
 
-    :param obj: Patroni configuration.
     :param cluster_name: name of the Patroni cluster.
     :param group: filter Citus group within we should perform a failover or switchover. If ``None``, user will be
         prompted for filling it -- unless *force* is ``True``, in which case an exception is raised by
@@ -1380,7 +1381,7 @@ def failover(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
         click.echo(click.style(
             'Supplying a leader name using this command is deprecated and will be removed in a future version of'
             ' Patroni, change your scripts to use `switchover` instead.\nExecuting switchover!', fg='red'))
-    _do_failover_or_switchover(obj, action, cluster_name, group, leader, candidate, force)
+    _do_failover_or_switchover(action, cluster_name, group, leader, candidate, force)
 
 
 @ctl.command('switchover', help='Switchover to a replica')
@@ -1391,9 +1392,8 @@ def failover(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
 @click.option('--scheduled', help='Timestamp of a scheduled switchover in unambiguous format (e.g. ISO 8601)',
               default=None)
 @option_force
-@click.pass_obj
-def switchover(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
-               leader: Optional[str], candidate: Optional[str], force: bool, scheduled: Optional[str]) -> None:
+def switchover(cluster_name: str, group: Optional[int], leader: Optional[str],
+               candidate: Optional[str], force: bool, scheduled: Optional[str]) -> None:
     """Process ``switchover`` command of ``patronictl`` utility.
 
     Perform a switchover operation in the cluster.
@@ -1401,7 +1401,6 @@ def switchover(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
     .. seealso::
         Refer to :func:`_do_failover_or_switchover` for details.
 
-    :param obj: Patroni configuration.
     :param cluster_name: name of the Patroni cluster.
     :param group: filter Citus group within we should perform a switchover. If ``None``, user will be prompted for
         filling it -- unless *force* is ``True``, in which case an exception is raised by
@@ -1411,11 +1410,11 @@ def switchover(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
     :param force: perform the switchover without asking for confirmations.
     :param scheduled: timestamp when the switchover should be scheduled to occur. If ``now`` perform immediately.
     """
-    _do_failover_or_switchover(obj, 'switchover', cluster_name, group, leader, candidate, force, scheduled)
+    _do_failover_or_switchover('switchover', cluster_name, group, leader, candidate, force, scheduled)
 
 
 def generate_topology(level: int, member: Dict[str, Any],
-                      topology: Dict[str, List[Dict[str, Any]]]) -> Iterator[Dict[str, Any]]:
+                      topology: Dict[Optional[str], List[Dict[str, Any]]]) -> Iterator[Dict[str, Any]]:
     """Recursively yield members with their names adjusted according to their *level* in the cluster topology.
 
     .. note::
@@ -1478,7 +1477,7 @@ def topology_sort(members: List[Dict[str, Any]]) -> Iterator[Dict[str, Any]]:
     :yields: *members* sorted by level in the topology, and with a new ``name`` value according to their level
         in the topology.
     """
-    topology: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
+    topology: Dict[Optional[str], List[Dict[str, Any]]] = defaultdict(list)
     leader = next((m for m in members if m['role'].endswith('leader')), {'name': None})
     replicas = set(member['name'] for member in members if not member['role'].endswith('leader'))
     for member in members:
@@ -1513,8 +1512,8 @@ def get_cluster_service_info(cluster: Dict[str, Any]) -> List[str]:
     return service_info
 
 
-def output_members(obj: Dict[str, Any], cluster: Cluster, name: str,
-                   extended: bool = False, fmt: str = 'pretty', group: Optional[int] = None) -> None:
+def output_members(cluster: Cluster, name: str, extended: bool = False,
+                   fmt: str = 'pretty', group: Optional[int] = None) -> None:
     """Print information about the Patroni cluster and its members.
 
     Information is printed to console through :func:`print_output`, and contains:
@@ -1539,7 +1538,6 @@ def output_members(obj: Dict[str, Any], cluster: Cluster, name: str,
     The 3 extended columns are always included if *extended*, even if the member has no value for a given column.
     If not *extended*, these columns may still be shown if any of the members has any information for them.
 
-    :param obj: Patroni configuration.
     :param cluster: Patroni cluster.
     :param name: name of the Patroni cluster.
     :param extended: if extended information (pending restarts, scheduled restarts, node tags) should be printed, if
@@ -1557,15 +1555,14 @@ def output_members(obj: Dict[str, Any], cluster: Cluster, name: str,
 
     clusters = {group or 0: cluster_as_json(cluster)}
 
-    is_citus_cluster = obj.get('citus')
-    if is_citus_cluster:
+    if is_citus_cluster():
         columns.insert(1, 'Group')
         if group is None:
             clusters.update({g: cluster_as_json(c) for g, c in cluster.workers.items()})
 
     all_members = [m for c in clusters.values() for m in c['members'] if 'host' in m]
 
-    for c in ('Pending restart', 'Scheduled restart', 'Tags'):
+    for c in ('Pending restart', 'Pending restart reason', 'Scheduled restart', 'Tags'):
         if extended or any(m.get(c.lower().replace(' ', '_')) for m in all_members):
             columns.append(c)
 
@@ -1579,11 +1576,19 @@ def output_members(obj: Dict[str, Any], cluster: Cluster, name: str,
             logging.debug(member)
 
             lag = member.get('lag', '')
+
+            def format_diff(param: str, values: Dict[str, str], hide_long: bool):
+                full_diff = param + ': ' + values['old_value'] + '->' + values['new_value']
+                return full_diff if not hide_long or len(full_diff) <= 50 else param + ': [hidden - too long]'
+            restart_reason = '\n'.join([format_diff(k, v, fmt in ('pretty', 'topology'))
+                                        for k, v in member.get('pending_restart_reason', {}).items()]) or ''
+
             member.update(cluster=name, member=member['name'], group=g,
                           host=member.get('host', ''), tl=member.get('timeline', ''),
                           role=member['role'].replace('_', ' ').title(),
                           lag_in_mb=round(lag / 1024 / 1024) if isinstance(lag, int) else lag,
-                          pending_restart='*' if member.get('pending_restart') else '')
+                          pending_restart='*' if member.get('pending_restart') else '',
+                          pending_restart_reason=restart_reason)
 
             if append_port and member['host'] and member.get('port'):
                 member['host'] = ':'.join([member['host'], str(member['port'])])
@@ -1596,10 +1601,12 @@ def output_members(obj: Dict[str, Any], cluster: Cluster, name: str,
 
             rows.append([member.get(n.lower().replace(' ', '_'), '') for n in columns])
 
-    title = 'Citus cluster' if is_citus_cluster else 'Cluster'
-    title_details = f' ({initialize})'
-    if is_citus_cluster:
+    if is_citus_cluster():
+        title = 'Citus cluster'
         title_details = '' if group is None else f' (group: {group}, {initialize})'
+    else:
+        title = 'Cluster'
+        title_details = f' ({initialize})'
 
     title = f' {title}: {name}{title_details} '
     print_output(columns, rows, {'Group': 'r', 'Lag in MB': 'r', 'TL': 'r'}, fmt, title)
@@ -1610,7 +1617,7 @@ def output_members(obj: Dict[str, Any], cluster: Cluster, name: str,
     for g, c in sorted(clusters.items()):
         service_info = get_cluster_service_info(c)
         if service_info:
-            if is_citus_cluster and group is None:
+            if is_citus_cluster() and group is None:
                 click.echo('Citus group: {0}'.format(g))
             click.echo(' ' + '\n '.join(service_info))
 
@@ -1623,16 +1630,14 @@ def output_members(obj: Dict[str, Any], cluster: Cluster, name: str,
 @option_format
 @option_watch
 @option_watchrefresh
-@click.pass_obj
-def members(obj: Dict[str, Any], cluster_names: List[str], group: Optional[int],
-            fmt: str, watch: Optional[int], w: bool, extended: bool, ts: bool) -> None:
+def members(cluster_names: List[str], group: Optional[int], fmt: str,
+            watch: Optional[int], w: bool, extended: bool, ts: bool) -> None:
     """Process ``list`` command of ``patronictl`` utility.
 
     Print information about the Patroni cluster through :func:`output_members`.
 
-    :param obj: Patroni configuration.
     :param cluster_names: name of clusters that should be printed. If ``None`` consider only the cluster present in
-        ``scope`` key of *obj*.
+        ``scope`` key of the configuration.
     :param group: filter which Citus group we should get members from. Refer to the module note for more details.
     :param fmt: the output table printing format. See :func:`print_output` for available options.
     :param watch: if given print output every *watch* seconds.
@@ -1641,9 +1646,10 @@ def members(obj: Dict[str, Any], cluster_names: List[str], group: Optional[int],
         more details.
     :param ts: if timestamp should be included in the output.
     """
+    config = _get_configuration()
     if not cluster_names:
-        if 'scope' in obj:
-            cluster_names = [obj['scope']]
+        if 'scope' in config:
+            cluster_names = [config['scope']]
         if not cluster_names:
             return logging.warning('Listing members: No cluster names were provided')
 
@@ -1652,10 +1658,10 @@ def members(obj: Dict[str, Any], cluster_names: List[str], group: Optional[int],
             click.echo(timestamp(0))
 
         for cluster_name in cluster_names:
-            dcs = get_dcs(obj, cluster_name, group)
+            dcs = get_dcs(cluster_name, group)
 
             cluster = dcs.get_cluster()
-            output_members(obj, cluster, cluster_name, extended, fmt, group)
+            output_members(cluster, cluster_name, extended, fmt, group)
 
 
 @ctl.command('topology', help='Prints ASCII topology for given cluster')
@@ -1697,14 +1703,12 @@ def timestamp(precision: int = 6) -> str:
 @click.argument('target', type=click.Choice(['restart', 'switchover']))
 @click.option('--role', '-r', help='Flush only members with this role', type=role_choice, default='any')
 @option_force
-@click.pass_obj
-def flush(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
+def flush(cluster_name: str, group: Optional[int],
           member_names: List[str], force: bool, role: str, target: str) -> None:
     """Process ``flush`` command of ``patronictl`` utility.
 
     Discard scheduled restart or switchover events.
 
-    :param obj: Patroni configuration.
     :param cluster_name: name of the Patroni cluster.
     :param group: filter which Citus group we should flush an event. Refer to the module note for more details.
     :param member_names: name of the members which events should be flushed.
@@ -1712,11 +1716,11 @@ def flush(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
     :param role: role to filter members. See :func:`get_all_members` for available options.
     :param target: the event that should be flushed -- ``restart`` or ``switchover``.
     """
-    dcs = get_dcs(obj, cluster_name, group)
+    dcs = get_dcs(cluster_name, group)
     cluster = dcs.get_cluster()
 
     if target == 'restart':
-        for member in get_members(obj, cluster, cluster_name, member_names, role, force, 'flush', group=group):
+        for member in get_members(cluster, cluster_name, member_names, role, force, 'flush', group=group):
             if member.data.get('scheduled_restart'):
                 r = request_patroni(member, 'delete', 'restart')
                 check_response(r, member.name, 'flush scheduled restart')
@@ -1752,7 +1756,7 @@ def wait_until_pause_is_applied(dcs: AbstractDCS, paused: bool, old_cluster: Clu
     :param old_cluster: original cluster information before pause or unpause has been requested. Used to report which
         nodes are still pending to have ``pause`` equal *paused* at a given point in time.
     """
-    config = get_global_config(old_cluster)
+    config = global_config.from_cluster(old_cluster)
 
     click.echo("'{0}' request sent, waiting until it is recognized by all nodes".format(paused and 'pause' or 'resume'))
     old = {m.name: m.version for m in old_cluster.members if m.api_url}
@@ -1774,10 +1778,9 @@ def wait_until_pause_is_applied(dcs: AbstractDCS, paused: bool, old_cluster: Clu
     return click.echo('Success: cluster management is {0}'.format(paused and 'paused' or 'resumed'))
 
 
-def toggle_pause(config: Dict[str, Any], cluster_name: str, group: Optional[int], paused: bool, wait: bool) -> None:
+def toggle_pause(cluster_name: str, group: Optional[int], paused: bool, wait: bool) -> None:
     """Toggle the ``pause`` state in the cluster members.
 
-    :param config: Patroni configuration.
     :param cluster_name: name of the Patroni cluster.
     :param group: filter which Citus group we should toggle the pause state of. Refer to the module note for more
         details.
@@ -1789,9 +1792,9 @@ def toggle_pause(config: Dict[str, Any], cluster_name: str, group: Optional[int]
             * ``pause`` state is already *paused*; or
             * cluster contains no accessible members.
     """
-    dcs = get_dcs(config, cluster_name, group)
+    dcs = get_dcs(cluster_name, group)
     cluster = dcs.get_cluster()
-    if get_global_config(cluster).is_paused == paused:
+    if global_config.from_cluster(cluster).is_paused == paused:
         raise PatroniCtlException('Cluster is {0} paused'.format(paused and 'already' or 'not'))
 
     for member in get_all_members_leader_first(cluster):
@@ -1818,37 +1821,33 @@ def toggle_pause(config: Dict[str, Any], cluster_name: str, group: Optional[int]
 @ctl.command('pause', help='Disable auto failover')
 @arg_cluster_name
 @option_default_citus_group
-@click.pass_obj
 @click.option('--wait', help='Wait until pause is applied on all nodes', is_flag=True)
-def pause(obj: Dict[str, Any], cluster_name: str, group: Optional[int], wait: bool) -> None:
+def pause(cluster_name: str, group: Optional[int], wait: bool) -> None:
     """Process ``pause`` command of ``patronictl`` utility.
 
     Put the cluster in maintenance mode.
 
-    :param obj: Patroni configuration.
     :param cluster_name: name of the Patroni cluster.
     :param group: filter which Citus group we should pause. Refer to the module note for more details.
     :param wait: ``True`` if it should block until the operation is finished or ``false`` for returning immediately.
     """
-    return toggle_pause(obj, cluster_name, group, True, wait)
+    return toggle_pause(cluster_name, group, True, wait)
 
 
 @ctl.command('resume', help='Resume auto failover')
 @arg_cluster_name
 @option_default_citus_group
 @click.option('--wait', help='Wait until pause is cleared on all nodes', is_flag=True)
-@click.pass_obj
-def resume(obj: Dict[str, Any], cluster_name: str, group: Optional[int], wait: bool) -> None:
+def resume(cluster_name: str, group: Optional[int], wait: bool) -> None:
     """Process ``unpause`` command of ``patronictl`` utility.
 
     Put the cluster out of maintenance mode.
 
-    :param obj: Patroni configuration.
     :param cluster_name: name of the Patroni cluster.
     :param group: filter which Citus group we should unpause. Refer to the module note for more details.
     :param wait: ``True`` if it should block until the operation is finished or ``false`` for returning immediately.
     """
-    return toggle_pause(obj, cluster_name, group, False, wait)
+    return toggle_pause(cluster_name, group, False, wait)
 
 
 @contextmanager
@@ -2080,15 +2079,12 @@ def invoke_editor(before_editing: str, cluster_name: str) -> Tuple[str, Dict[str
 @click.option('--replace', 'replace_filename', help='Apply configuration from file, replacing existing configuration.'
               ' Use - for stdin.')
 @option_force
-@click.pass_obj
-def edit_config(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
-                force: bool, quiet: bool, kvpairs: List[str], pgkvpairs: List[str],
-                apply_filename: Optional[str], replace_filename: Optional[str]) -> None:
+def edit_config(cluster_name: str, group: Optional[int], force: bool, quiet: bool, kvpairs: List[str],
+                pgkvpairs: List[str], apply_filename: Optional[str], replace_filename: Optional[str]) -> None:
     """Process ``edit-config`` command of ``patronictl`` utility.
 
     Update or replace Patroni configuration in the DCS.
 
-    :param obj: Patroni configuration.
     :param cluster_name: name of the Patroni cluster.
     :param group: filter which Citus group configuration we should edit. Refer to the module note for more details.
     :param force: if ``True`` apply config changes without asking for confirmations.
@@ -2105,7 +2101,7 @@ def edit_config(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
             * Configuration is absent from DCS; or
             * Detected a concurrent modification of the configuration in the DCS.
     """
-    dcs = get_dcs(obj, cluster_name, group)
+    dcs = get_dcs(cluster_name, group)
     cluster = dcs.get_cluster()
 
     if not cluster.config:
@@ -2143,7 +2139,7 @@ def edit_config(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
         return
 
     if force or click.confirm('Apply these changes?'):
-        if not dcs.set_config_value(json.dumps(changed_data), cluster.config.version):
+        if not dcs.set_config_value(json.dumps(changed_data, separators=(',', ':')), cluster.config.version):
             raise PatroniCtlException("Config modification aborted due to concurrent changes")
         click.echo("Configuration changed")
 
@@ -2151,17 +2147,15 @@ def edit_config(obj: Dict[str, Any], cluster_name: str, group: Optional[int],
 @ctl.command('show-config', help="Show cluster configuration")
 @arg_cluster_name
 @option_default_citus_group
-@click.pass_obj
-def show_config(obj: Dict[str, Any], cluster_name: str, group: Optional[int]) -> None:
+def show_config(cluster_name: str, group: Optional[int]) -> None:
     """Process ``show-config`` command of ``patronictl`` utility.
 
     Show Patroni configuration stored in the DCS.
 
-    :param obj: Patroni configuration.
     :param cluster_name: name of the Patroni cluster.
     :param group: filter which Citus group configuration we should show. Refer to the module note for more details.
     """
-    cluster = get_dcs(obj, cluster_name, group).get_cluster()
+    cluster = get_dcs(cluster_name, group).get_cluster()
     if cluster.config:
         click.echo(format_config_for_editing(cluster.config.data))
 
@@ -2170,8 +2164,7 @@ def show_config(obj: Dict[str, Any], cluster_name: str, group: Optional[int]) ->
 @click.argument('cluster_name', required=False)
 @click.argument('member_names', nargs=-1)
 @option_citus_group
-@click.pass_obj
-def version(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member_names: List[str]) -> None:
+def version(cluster_name: str, group: Optional[int], member_names: List[str]) -> None:
     """Process ``version`` command of ``patronictl`` utility.
 
     Show version of:
@@ -2179,7 +2172,6 @@ def version(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member
         * ``patroni`` on all members of the cluster;
         * ``PostgreSQL`` on all members of the cluster.
 
-    :param obj: Patroni configuration.
     :param cluster_name: name of the Patroni cluster.
     :param group: filter which Citus group we should get members from. Refer to the module note for more details.
     :param member_names: filter which members we should get version information from.
@@ -2190,8 +2182,8 @@ def version(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member
         return
 
     click.echo("")
-    cluster = get_dcs(obj, cluster_name, group).get_cluster()
-    for m in get_all_members(obj, cluster, group, 'any'):
+    cluster = get_dcs(cluster_name, group).get_cluster()
+    for m in get_all_members(cluster, group, 'any'):
         if m.api_url:
             if not member_names or m.name in member_names:
                 try:
@@ -2209,8 +2201,7 @@ def version(obj: Dict[str, Any], cluster_name: str, group: Optional[int], member
 @arg_cluster_name
 @option_default_citus_group
 @option_format
-@click.pass_obj
-def history(obj: Dict[str, Any], cluster_name: str, group: Optional[int], fmt: str) -> None:
+def history(cluster_name: str, group: Optional[int], fmt: str) -> None:
     """Process ``history`` command of ``patronictl`` utility.
 
     Show the history of failover/switchover events in the cluster.
@@ -2222,12 +2213,11 @@ def history(obj: Dict[str, Any], cluster_name: str, group: Optional[int], fmt: s
         * ``Timestamp``: timestamp when the event occurred;
         * ``New Leader``: the Postgres node that was promoted during the event.
 
-    :param obj: Patroni configuration.
     :param cluster_name: name of the Patroni cluster.
     :param group: filter which Citus group we should get events from. Refer to the module note for more details.
     :param fmt: the output table printing format. See :func:`print_output` for available options.
     """
-    cluster = get_dcs(obj, cluster_name, group).get_cluster()
+    cluster = get_dcs(cluster_name, group).get_cluster()
     cluster_history = cluster.history.lines if cluster.history else []
     history: List[List[Any]] = list(map(list, cluster_history))
     table_header_row = ['TL', 'LSN', 'Reason', 'Timestamp', 'New Leader']

patroni/dcs/__init__.py

@@ -1,26 +1,22 @@
 """Abstract classes for Distributed Configuration Store."""
 import abc
 import datetime
-import importlib
-import inspect
 import json
 import logging
-import os
-import pkgutil
 import re
-import sys
 import time
 from collections import defaultdict
 from copy import deepcopy
 from random import randint
 from threading import Event, Lock
-from types import ModuleType
-from typing import Any, Callable, Collection, Dict, List, NamedTuple, Optional, Set, Tuple, Union, TYPE_CHECKING, \
-    Type, Iterator
+from typing import Any, Callable, Collection, Dict, Iterator, List, \
+    NamedTuple, Optional, Tuple, Type, TYPE_CHECKING, Union
 from urllib.parse import urlparse, urlunparse, parse_qsl
 
 import dateutil.parser
 
+from .. import global_config
+from ..dynamic_loader import iter_classes, iter_modules
 from ..exceptions import PatroniFatalException
 from ..utils import deep_compare, uri
 from ..tags import Tags
@@ -28,10 +24,10 @@ from ..utils import parse_int
 
 if TYPE_CHECKING:  # pragma: no cover
     from ..config import Config
+    from ..postgresql import Postgresql
+    from ..postgresql.mpp import AbstractMPP
 
 SLOT_ADVANCE_AVAILABLE_VERSION = 110000
-CITUS_COORDINATOR_GROUP_ID = 0
-citus_group_re = re.compile('^(0|[1-9][0-9]*)$')
 slot_name_re = re.compile('^[a-z0-9_]{1,63}$')
 logger = logging.getLogger(__name__)
 
@@ -87,28 +83,11 @@ def parse_connection_string(value: str) -> Tuple[str, Union[str, None]]:
 def dcs_modules() -> List[str]:
     """Get names of DCS modules, depending on execution environment.
 
-    .. note::
-        If being packaged with PyInstaller, modules aren't discoverable dynamically by scanning source directory because
-        :class:`importlib.machinery.FrozenImporter` doesn't implement :func:`iter_modules`. But it is still possible to
-        find all potential DCS modules by iterating through ``toc``, which contains list of all "frozen" resources.
-
     :returns: list of known module names with absolute python module path namespace, e.g. ``patroni.dcs.etcd``.
     """
-    dcs_dirname = os.path.dirname(__file__)
-    module_prefix = __package__ + '.'
-
-    if getattr(sys, 'frozen', False):
-        toc: Set[str] = set()
-        # dcs_dirname may contain a dot, which causes pkgutil.iter_importers()
-        # to misinterpret the path as a package name. This can be avoided
-        # altogether by not passing a path at all, because PyInstaller's
-        # FrozenImporter is a singleton and registered as top-level finder.
-        for importer in pkgutil.iter_importers():
-            if hasattr(importer, 'toc'):
-                toc |= getattr(importer, 'toc')
-        return [module for module in toc if module.startswith(module_prefix) and module.count('.') == 2]
-
-    return [module_prefix + name for _, name, is_pkg in pkgutil.iter_modules([dcs_dirname]) if not is_pkg]
+    if TYPE_CHECKING:  # pragma: no cover
+        assert isinstance(__package__, str)
+    return iter_modules(__package__)
 
 
 def iter_dcs_classes(
@@ -122,44 +101,18 @@ def iter_dcs_classes(
     :param config: configuration information with possible DCS names as keys. If given, only attempt to import DCS
                    modules defined in the configuration. Else, if ``None``, attempt to import any supported DCS module.
 
-    :yields: a tuple containing the module ``name`` and the imported DCS class object.
+    :returns: an iterator of tuples, each containing the module ``name`` and the imported DCS class object.
     """
-    for mod_name in dcs_modules():
-        name = mod_name.rpartition('.')[2]
-        if config is None or name in config:
-
-            try:
-                module = importlib.import_module(mod_name)
-                dcs_module = find_dcs_class_in_module(module)
-                if dcs_module:
-                    yield name, dcs_module
-
-            except ImportError:
-                logger.log(logging.DEBUG if config is not None else logging.INFO,
-                           'Failed to import %s', mod_name)
-
-
-def find_dcs_class_in_module(module: ModuleType) -> Optional[Type['AbstractDCS']]:
-    """Try to find the implementation of :class:`AbstractDCS` interface in *module* matching the *module* name.
-
-    :param module: Imported DCS module.
-
-    :returns: class with a name matching the name of *module* that implements :class:`AbstractDCS` or ``None`` if not
-              found.
-    """
-    module_name = module.__name__.rpartition('.')[2]
-    return next(
-        (obj for obj_name, obj in module.__dict__.items()
-         if (obj_name.lower() == module_name
-             and inspect.isclass(obj) and issubclass(obj, AbstractDCS))),
-        None)
+    if TYPE_CHECKING:  # pragma: no cover
+        assert isinstance(__package__, str)
+    return iter_classes(__package__, AbstractDCS, config)
 
 
 def get_dcs(config: Union['Config', Dict[str, Any]]) -> 'AbstractDCS':
     """Attempt to load a Distributed Configuration Store from known available implementations.
 
     .. note::
-        Using the list of available DCS classes returned by :func:`iter_dcs_classes` attempt to dynamically
+        Using the list of available DCS classes returned by :func:`iter_classes` attempt to dynamically
         instantiate the class that implements a DCS using the abstract class :class:`AbstractDCS`.
 
         Basic top-level configuration parameters retrieved from *config* are propagated to the DCS specific config
@@ -180,14 +133,13 @@ def get_dcs(config: Union['Config', Dict[str, Any]]) -> 'AbstractDCS':
             p: config[p] for p in ('namespace', 'name', 'scope', 'loop_wait',
                                    'patronictl', 'ttl', 'retry_timeout')
             if p in config})
-        # From citus section we only need "group" parameter, but will propagate everything just in case.
-        if isinstance(config.get('citus'), dict):
-            config[name].update(config['citus'])
-        return dcs_class(config[name])
 
-    raise PatroniFatalException(
-        f"Can not find suitable configuration of distributed configuration store\n"
-        f"Available implementations: {', '.join(sorted([n for n, _ in iter_dcs_classes()]))}")
+        from patroni.postgresql.mpp import get_mpp
+        return dcs_class(config[name], get_mpp(config))
+
+    available_implementations = ', '.join(sorted([n for n, _ in iter_dcs_classes()]))
+    raise PatroniFatalException("Can not find suitable configuration of distributed configuration store\n"
+                                f"Available implementations: {available_implementations}")
 
 
 _Version = Union[int, str]
@@ -590,24 +542,6 @@ class ClusterConfig(NamedTuple):
             modify_version = 0
         return ClusterConfig(version, data, version if modify_version is None else modify_version)
 
-    @property
-    def permanent_slots(self) -> Dict[str, Any]:
-        """Dictionary of permanent slots information looked up from :attr:`~ClusterConfig.data`."""
-        return (self.data.get('permanent_replication_slots')
-                or self.data.get('permanent_slots')
-                or self.data.get('slots')
-                or {})
-
-    @property
-    def ignore_slots_matchers(self) -> List[Dict[str, Any]]:
-        """The value for ``ignore_slots`` from :attr:`~ClusterConfig.data` if defined or an empty list."""
-        return self.data.get('ignore_slots') or []
-
-    @property
-    def max_timelines_history(self) -> int:
-        """The value for ``max_timelines_history`` from :attr:`~ClusterConfig.data` if defined or ``0``."""
-        return self.data.get('max_timelines_history', 0)
-
 
 class SyncState(NamedTuple):
     """Immutable object (namedtuple) which represents last observed synchronous replication state.
@@ -626,7 +560,7 @@ class SyncState(NamedTuple):
         """Factory method to parse *value* as synchronisation state information.
 
         :param version: optional *version* number for the object.
-        :param value: (optionally JSON serialised) sychronisation state information
+        :param value: (optionally JSON serialised) synchronisation state information
 
         :returns: constructed :class:`SyncState` object.
 
@@ -852,7 +786,7 @@ class Cluster(NamedTuple('Cluster',
                           ('history', Optional[TimelineHistory]),
                           ('failsafe', Optional[Dict[str, str]]),
                           ('workers', Dict[int, 'Cluster'])])):
-    """Immutable object (namedtuple) which represents PostgreSQL or Citus cluster.
+    """Immutable object (namedtuple) which represents PostgreSQL or MPP cluster.
 
     .. note::
         We are using an old-style attribute declaration here because otherwise it is not possible to override `__new__`
@@ -869,8 +803,8 @@ class Cluster(NamedTuple('Cluster',
     :ivar sync: reference to :class:`SyncState` object, last observed synchronous replication state.
     :ivar history: reference to `TimelineHistory` object.
     :ivar failsafe: failsafe topology. Node is allowed to become the leader only if its name is found in this list.
-    :ivar workers: dictionary of workers of the Citus cluster, optional. Each key is an :class:`int` representing
-                   the group, and the corresponding value is a :class:`Cluster` instance.
+    :ivar workers: dictionary of workers of the MPP cluster, optional. Each key representing the group and the
+                   corresponding value is a :class:`Cluster` instance.
     """
 
     def __new__(cls, *args: Any, **kwargs: Any):
@@ -996,7 +930,7 @@ class Cluster(NamedTuple('Cluster',
     @property
     def __permanent_slots(self) -> Dict[str, Union[Dict[str, Any], Any]]:
         """Dictionary of permanent replication slots with their known LSN."""
-        ret: Dict[str, Union[Dict[str, Any], Any]] = deepcopy(self.config.permanent_slots if self.config else {})
+        ret: Dict[str, Union[Dict[str, Any], Any]] = global_config.permanent_slots
 
         members: Dict[str, int] = {slot_name_from_member_name(m.name): m.lsn or 0 for m in self.members}
         slots: Dict[str, int] = {k: parse_int(v) or 0 for k, v in (self.slots or {}).items()}
@@ -1025,36 +959,29 @@ class Cluster(NamedTuple('Cluster',
         """Dictionary of permanent ``logical`` replication slots."""
         return {name: value for name, value in self.__permanent_slots.items() if self.is_logical_slot(value)}
 
-    @property
-    def use_slots(self) -> bool:
-        """``True`` if cluster is configured to use replication slots."""
-        return bool(self.config and (self.config.data.get('postgresql') or {}).get('use_slots', True))
-
-    def get_replication_slots(self, my_name: str, role: str, nofailover: bool, major_version: int, *,
-                              is_standby_cluster: bool = False, show_error: bool = False) -> Dict[str, Dict[str, Any]]:
+    def get_replication_slots(self, postgresql: 'Postgresql', member: Tags, *,
+                              role: Optional[str] = None, show_error: bool = False) -> Dict[str, Dict[str, Any]]:
         """Lookup configured slot names in the DCS, report issues found and merge with permanent slots.
 
         Will log an error if:
 
             * Any logical slots are disabled, due to version compatibility, and *show_error* is ``True``.
 
-        :param my_name: name of this node.
-        :param role: role of this node.
-        :param nofailover: ``True`` if this node is tagged to not be a failover candidate.
-        :param major_version: postgresql major version.
-        :param is_standby_cluster: ``True`` if it is known that this is a standby cluster. We pass the value from
-                                   the outside because we want to protect from the ``/config`` key removal.
+        :param postgresql: reference to :class:`Postgresql` object.
+        :param member: reference to an object implementing :class:`Tags` interface.
+        :param role: role of the node, if not set will be taken from *postgresql*.
         :param show_error: if ``True`` report error if any disabled logical slots or conflicting slot names are found.
 
         :returns: final dictionary of slot names, after merging with permanent slots and performing sanity checks.
         """
-        slots: Dict[str, Dict[str, str]] = self._get_members_slots(my_name, role)
-        permanent_slots: Dict[str, Any] = self._get_permanent_slots(is_standby_cluster=is_standby_cluster,
-                                                                    role=role, nofailover=nofailover,
-                                                                    major_version=major_version)
+        name = member.name if isinstance(member, Member) else postgresql.name
+        role = role or postgresql.role
+
+        slots: Dict[str, Dict[str, str]] = self._get_members_slots(name, role)
+        permanent_slots: Dict[str, Any] = self._get_permanent_slots(postgresql, member, role)
 
         disabled_permanent_logical_slots: List[str] = self._merge_permanent_slots(
-            slots, permanent_slots, my_name, major_version)
+            slots, permanent_slots, name, postgresql.major_version)
 
         if disabled_permanent_logical_slots and show_error:
             logger.error("Permanent logical replication slots supported by Patroni only starting from PostgreSQL 11. "
@@ -1062,7 +989,7 @@ class Cluster(NamedTuple('Cluster',
 
         return slots
 
-    def _merge_permanent_slots(self, slots: Dict[str, Dict[str, str]], permanent_slots: Dict[str, Any], my_name: str,
+    def _merge_permanent_slots(self, slots: Dict[str, Dict[str, str]], permanent_slots: Dict[str, Any], name: str,
                                major_version: int) -> List[str]:
         """Merge replication *slots* for members with *permanent_slots*.
 
@@ -1072,7 +999,7 @@ class Cluster(NamedTuple('Cluster',
         Type is assumed to be ``physical`` if there are no attributes stored as the slot value.
 
         :param slots: Slot names with existing attributes if known.
-        :param my_name: name of this node.
+        :param name: name of this node.
         :param permanent_slots: dictionary containing slot name key and slot information values.
         :param major_version: postgresql major version.
 
@@ -1080,9 +1007,9 @@ class Cluster(NamedTuple('Cluster',
         """
         disabled_permanent_logical_slots: List[str] = []
 
-        for name, value in permanent_slots.items():
-            if not slot_name_re.match(name):
-                logger.error("Invalid permanent replication slot name '%s'", name)
+        for slot_name, value in permanent_slots.items():
+            if not slot_name_re.match(slot_name):
+                logger.error("Invalid permanent replication slot name '%s'", slot_name)
                 logger.error("Slot name may only contain lower case letters, numbers, and the underscore chars")
                 continue
 
@@ -1093,55 +1020,54 @@ class Cluster(NamedTuple('Cluster',
 
                 if value['type'] == 'physical':
                     # Don't try to create permanent physical replication slot for yourself
-                    if name != slot_name_from_member_name(my_name):
-                        slots[name] = value
+                    if slot_name != slot_name_from_member_name(name):
+                        slots[slot_name] = value
                     continue
 
                 if self.is_logical_slot(value):
                     if major_version < SLOT_ADVANCE_AVAILABLE_VERSION:
-                        disabled_permanent_logical_slots.append(name)
-                    elif name in slots:
+                        disabled_permanent_logical_slots.append(slot_name)
+                    elif slot_name in slots:
                         logger.error("Permanent logical replication slot {'%s': %s} is conflicting with"
-                                     " physical replication slot for cluster member", name, value)
+                                     " physical replication slot for cluster member", slot_name, value)
                     else:
-                        slots[name] = value
+                        slots[slot_name] = value
                     continue
 
-            logger.error("Bad value for slot '%s' in permanent_slots: %s", name, permanent_slots[name])
+            logger.error("Bad value for slot '%s' in permanent_slots: %s", slot_name, permanent_slots[slot_name])
         return disabled_permanent_logical_slots
 
-    def _get_permanent_slots(self, *, is_standby_cluster: bool, role: str,
-                             nofailover: bool, major_version: int) -> Dict[str, Any]:
+    def _get_permanent_slots(self, postgresql: 'Postgresql', tags: Tags, role: str) -> Dict[str, Any]:
         """Get configured permanent replication slots.
 
         .. note::
             Permanent replication slots are only considered if ``use_slots`` configuration is enabled.
             A node that is not supposed to become a leader (*nofailover*) will not have permanent replication slots.
+            Also node with disabled streaming (*nostream*) and its cascading followers must not have permanent
+            logical slots due to lack of feedback from node to primary, which makes them unsafe to use.
 
             In a standby cluster we only support physical replication slots.
 
             The returned dictionary for a non-standby cluster always contains permanent logical replication slots in
             order to show a warning if they are not supported by PostgreSQL before v11.
 
-        :param is_standby_cluster: ``True`` if it is known that this is a standby cluster. We pass the value from
-                                   the outside because we want to protect from the ``/config`` key removal.
-        :param role: role of this node -- ``primary``, ``standby_leader`` or ``replica``.
-        :param nofailover: ``True`` if this node is tagged to not be a failover candidate.
-        :param major_version: postgresql major version.
+        :param postgresql: reference to :class:`Postgresql` object.
+        :param tags: reference to an object implementing :class:`Tags` interface.
+        :param role: role of the node -- ``primary``, ``standby_leader`` or ``replica``.
 
         :returns: dictionary of permanent slot names mapped to attributes.
         """
-        if not self.use_slots or nofailover:
+        if not global_config.use_slots or tags.nofailover:
             return {}
 
-        if is_standby_cluster:
+        if global_config.is_standby_cluster or self.get_slot_name_on_primary(postgresql.name, tags) is None:
             return self.__permanent_physical_slots \
-                if major_version >= SLOT_ADVANCE_AVAILABLE_VERSION or role == 'standby_leader' else {}
+                if postgresql.major_version >= SLOT_ADVANCE_AVAILABLE_VERSION or role == 'standby_leader' else {}
 
-        return self.__permanent_slots if major_version >= SLOT_ADVANCE_AVAILABLE_VERSION\
+        return self.__permanent_slots if postgresql.major_version >= SLOT_ADVANCE_AVAILABLE_VERSION\
             or role in ('master', 'primary') else self.__permanent_logical_slots
 
-    def _get_members_slots(self, my_name: str, role: str) -> Dict[str, Dict[str, str]]:
+    def _get_members_slots(self, name: str, role: str) -> Dict[str, Dict[str, str]]:
         """Get physical replication slots configuration for members that sourcing from this node.
 
         If the ``replicatefrom`` tag is set on the member - we should not create the replication slot for it on
@@ -1149,29 +1075,34 @@ class Cluster(NamedTuple('Cluster',
         the ``replicatefrom`` destination member is currently not a member of the cluster (fallback to the
         primary), or if ``replicatefrom`` destination member happens to be the current primary.
 
+        If the ``nostream`` tag is set on the member - we should not create the replication slot for it on
+        the current primary or any other member even if ``replicatefrom`` is set, because ``nostream`` disables
+        WAL streaming.
+
         Will log an error if:
 
             * Conflicting slot names between members are found
 
-        :param my_name: name of this node.
+        :param name: name of this node.
         :param role: role of this node, if this is a ``primary`` or ``standby_leader`` return list of members
                      replicating from this node. If not then return a list of members replicating as cascaded
                      replicas from this node.
 
         :returns: dictionary of physical replication slots that should exist on a given node.
         """
-        if not self.use_slots:
+        if not global_config.use_slots:
             return {}
 
-        # we always want to exclude the member with our name from the list
-        members = filter(lambda m: m.name != my_name, self.members)
+        # we always want to exclude the member with our name from the list,
+        # also exlude members with disabled WAL streaming
+        members = filter(lambda m: m.name != name and not m.nostream, self.members)
 
         if role in ('master', 'primary', 'standby_leader'):
             members = [m for m in members if m.replicatefrom is None
-                       or m.replicatefrom == my_name or not self.has_member(m.replicatefrom)]
+                       or m.replicatefrom == name or not self.has_member(m.replicatefrom)]
         else:
             # only manage slots for replicas that replicate from this one, except for the leader among them
-            members = [m for m in members if m.replicatefrom == my_name and m.name != self.leader_name]
+            members = [m for m in members if m.replicatefrom == name and m.name != self.leader_name]
 
         slots = {slot_name_from_member_name(m.name): {'type': 'physical'} for m in members}
         if len(slots) < len(members):
@@ -1184,84 +1115,76 @@ class Cluster(NamedTuple('Cluster',
                                    for k, v in slot_conflicts.items() if len(v) > 1))
         return slots
 
-    def has_permanent_slots(self, my_name: str, *, is_standby_cluster: bool = False, nofailover: bool = False,
-                            major_version: int = SLOT_ADVANCE_AVAILABLE_VERSION) -> bool:
-        """Check if the given member node has permanent replication slots configured.
+    def has_permanent_slots(self, postgresql: 'Postgresql', member: Tags) -> bool:
+        """Check if our node has permanent replication slots configured.
 
-        :param my_name: name of the member node to check.
-        :param is_standby_cluster: ``True`` if it is known that this is a standby cluster. We pass the value from
-                                   the outside because we want to protect from the ``/config`` key removal.
-        :param nofailover: ``True`` if this node is tagged to not be a failover candidate.
-        :param major_version: postgresql major version.
+        :param postgresql: reference to :class:`Postgresql` object.
+        :param member: reference to an object implementing :class:`Tags` interface for
+                       the node that we are checking permanent logical replication slots for.
 
         :returns: ``True`` if there are permanent replication slots configured, otherwise ``False``.
         """
         role = 'replica'
-        members_slots: Dict[str, Dict[str, str]] = self._get_members_slots(my_name, role)
-        permanent_slots: Dict[str, Any] = self._get_permanent_slots(is_standby_cluster=is_standby_cluster,
-                                                                    role=role, nofailover=nofailover,
-                                                                    major_version=major_version)
+        members_slots: Dict[str, Dict[str, str]] = self._get_members_slots(postgresql.name, role)
+        permanent_slots: Dict[str, Any] = self._get_permanent_slots(postgresql, member, role)
         slots = deepcopy(members_slots)
-        self._merge_permanent_slots(slots, permanent_slots, my_name, major_version)
+        self._merge_permanent_slots(slots, permanent_slots, postgresql.name, postgresql.major_version)
         return len(slots) > len(members_slots) or any(self.is_physical_slot(v) for v in permanent_slots.values())
 
-    def filter_permanent_slots(self, slots: Dict[str, int], is_standby_cluster: bool,
-                               major_version: int) -> Dict[str, int]:
+    def filter_permanent_slots(self, postgresql: 'Postgresql', slots: Dict[str, int]) -> Dict[str, int]:
         """Filter out all non-permanent slots from provided *slots* dict.
 
-        :param slots: slot names with LSN values
-        :param is_standby_cluster: ``True`` if it is known that this is a standby cluster. We pass the value from
-                                   the outside because we want to protect from the ``/config`` key removal.
-        :param major_version: postgresql major version.
+        :param postgresql: reference to :class:`Postgresql` object.
+        :param slots: slot names with LSN values.
 
         :returns: a :class:`dict` object that contains only slots that are known to be permanent.
         """
-        if major_version < SLOT_ADVANCE_AVAILABLE_VERSION:
+        if postgresql.major_version < SLOT_ADVANCE_AVAILABLE_VERSION:
             return {}  # for legacy PostgreSQL we don't support permanent slots on standby nodes
 
-        permanent_slots: Dict[str, Any] = self._get_permanent_slots(is_standby_cluster=is_standby_cluster,
-                                                                    role='replica',
-                                                                    nofailover=False,
-                                                                    major_version=major_version)
+        permanent_slots: Dict[str, Any] = self._get_permanent_slots(postgresql, RemoteMember('', {}), 'replica')
         members_slots = {slot_name_from_member_name(m.name) for m in self.members}
 
         return {name: value for name, value in slots.items() if name in permanent_slots
                 and (self.is_physical_slot(permanent_slots[name])
                      or self.is_logical_slot(permanent_slots[name]) and name not in members_slots)}
 
-    def _has_permanent_logical_slots(self, my_name: str, nofailover: bool) -> bool:
+    def _has_permanent_logical_slots(self, postgresql: 'Postgresql', member: Tags) -> bool:
         """Check if the given member node has permanent ``logical`` replication slots configured.
 
-        :param my_name: name of the member node to check.
-        :param nofailover: ``True`` if this node is tagged to not be a failover candidate.
+        :param postgresql: reference to a :class:`Postgresql` object.
+        :param member: reference to an object implementing :class:`Tags` interface for
+                       the node that we are checking permanent logical replication slots for.
 
         :returns: ``True`` if any detected replications slots are ``logical``, otherwise ``False``.
         """
-        slots = self.get_replication_slots(my_name, 'replica', nofailover, SLOT_ADVANCE_AVAILABLE_VERSION).values()
+        slots = self.get_replication_slots(postgresql, member, role='replica').values()
         return any(v for v in slots if v.get("type") == "logical")
 
-    def should_enforce_hot_standby_feedback(self, my_name: str, nofailover: bool) -> bool:
+    def should_enforce_hot_standby_feedback(self, postgresql: 'Postgresql', member: Tags) -> bool:
         """Determine whether ``hot_standby_feedback`` should be enabled for the given member.
 
         The ``hot_standby_feedback`` must be enabled if the current replica has ``logical`` slots,
         or it is working as a cascading replica for the other node that has ``logical`` slots.
 
-        :param my_name: name of the member node to check.
-        :param nofailover: ``True`` if this node is tagged to not be a failover candidate.
+        :param postgresql: reference to a :class:`Postgresql` object.
+        :param member: reference to an object implementing :class:`Tags` interface for
+                       the node that we are checking permanent logical replication slots for.
 
         :returns: ``True`` if this node or any member replicating from this node has
                   permanent logical slots, otherwise ``False``.
         """
-        if self._has_permanent_logical_slots(my_name, nofailover):
+        if self._has_permanent_logical_slots(postgresql, member):
             return True
 
-        if self.use_slots:
-            members = [m for m in self.members if m.replicatefrom == my_name and m.name != self.leader_name]
-            return any(self.should_enforce_hot_standby_feedback(m.name, m.nofailover) for m in members)
+        if global_config.use_slots:
+            name = member.name if isinstance(member, Member) else postgresql.name
+            members = [m for m in self.members if m.replicatefrom == name and m.name != self.leader_name]
+            return any(self.should_enforce_hot_standby_feedback(postgresql, m) for m in members)
         return False
 
-    def get_my_slot_name_on_primary(self, my_name: str, replicatefrom: Optional[str]) -> str:
-        """Canonical slot name for physical replication.
+    def get_slot_name_on_primary(self, name: str, tags: Tags) -> Optional[str]:
+        """Get the name of physical replication slot for this node on the primary.
 
         .. note::
             P <-- I <-- L
@@ -1269,14 +1192,16 @@ class Cluster(NamedTuple('Cluster',
             In case of cascading replication we have to check not our physical slot, but slot of the replica that
             connects us to the primary.
 
-        :param my_name: the member node name that is replicating.
-        :param replicatefrom: the Intermediate member name that is configured to replicate for cascading replication.
+        :param name: name of the member node to check.
+        :param tags: reference to an object implementing :class:`Tags` interface.
 
-        :returns: The slot name that is in use for physical replication on this no`de.
+        :returns: the slot name on the primary that is in use for physical replication on this node.
         """
-        m = self.get_member(replicatefrom, False) if replicatefrom else None
-        return self.get_my_slot_name_on_primary(m.name, m.replicatefrom) \
-            if isinstance(m, Member) else slot_name_from_member_name(my_name)
+        if tags.nostream:
+            return None
+        replicatefrom = self.get_member(tags.replicatefrom, False) if tags.replicatefrom else None
+        return self.get_slot_name_on_primary(replicatefrom.name, replicatefrom) \
+            if isinstance(replicatefrom, Member) else slot_name_from_member_name(name)
 
     @property
     def timeline(self) -> int:
@@ -1351,11 +1276,11 @@ class AbstractDCS(abc.ABC):
     Functional methods that are critical in their timing, required to complete within ``retry_timeout`` period in order
     to prevent the DCS considered inaccessible, each perform construction of complex data objects:
 
-        * :meth:`~AbstractDCS._cluster_loader`:
+        * :meth:`~AbstractDCS._postgresql_cluster_loader`:
             method which processes the structure of data stored in the DCS used to build the :class:`Cluster` object
             with all relevant associated data.
-        * :meth:`~AbstractDCS._citus_cluster_loader`:
-            Similar to above but specifically representing Citus group and workers information.
+        * :meth:`~AbstractDCS._mpp_cluster_loader`:
+            Similar to above but specifically representing MPP group and workers information.
         * :meth:`~AbstractDCS._load_cluster`:
             main method for calling specific ``loader`` method to build the :class:`Cluster` object representing the
             state and topology of the cluster.
@@ -1424,15 +1349,15 @@ class AbstractDCS(abc.ABC):
     _SYNC = 'sync'
     _FAILSAFE = 'failsafe'
 
-    def __init__(self, config: Dict[str, Any]) -> None:
-        """Prepare DCS paths, Citus group ID, initial values for state information and processing dependencies.
+    def __init__(self, config: Dict[str, Any], mpp: 'AbstractMPP') -> None:
+        """Prepare DCS paths, MPP object, initial values for state information and processing dependencies.
 
         :ivar config: :class:`dict`, reference to config section of selected DCS.
                       i.e.: ``zookeeper`` for zookeeper, ``etcd`` for etcd, etc...
         """
+        self._mpp = mpp
         self._name = config['name']
         self._base_path = re.sub('/+', '/', '/'.join(['', config.get('namespace', 'service'), config['scope']]))
-        self._citus_group = str(config['group']) if isinstance(config.get('group'), int) else None
         self._set_loop_wait(config.get('loop_wait', 10))
 
         self._ctl = bool(config.get('patronictl', False))
@@ -1445,6 +1370,11 @@ class AbstractDCS(abc.ABC):
         self._last_failsafe: Optional[Dict[str, str]] = {}
         self.event = Event()
 
+    @property
+    def mpp(self) -> 'AbstractMPP':
+        """Get the effective underlying MPP, if any has been configured."""
+        return self._mpp
+
     def client_path(self, path: str) -> str:
         """Construct the absolute key name from appropriate parts for the DCS type.
 
@@ -1453,8 +1383,8 @@ class AbstractDCS(abc.ABC):
         :returns: absolute key name for the current Patroni cluster.
         """
         components = [self._base_path]
-        if self._citus_group:
-            components.append(self._citus_group)
+        if self._mpp.is_enabled():
+            components.append(str(self._mpp.group))
         components.append(path.lstrip('/'))
         return '/'.join(components)
 
@@ -1555,22 +1485,21 @@ class AbstractDCS(abc.ABC):
         return self._last_seen
 
     @abc.abstractmethod
-    def _cluster_loader(self, path: Any) -> Cluster:
-        """Load and build the :class:`Cluster` object from DCS, which represents a single Patroni or Citus cluster.
+    def _postgresql_cluster_loader(self, path: Any) -> Cluster:
+        """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster.
 
-        :param path: the path in DCS where to load Cluster(s) from.
+        :param path: the path in DCS where to load :class:`Cluster` from.
 
         :returns: :class:`Cluster` instance.
         """
 
     @abc.abstractmethod
-    def _citus_cluster_loader(self, path: Any) -> Union[Cluster, Dict[int, Cluster]]:
-        """Load and build all Patroni clusters from a single Citus cluster.
+    def _mpp_cluster_loader(self, path: Any) -> Dict[int, Cluster]:
+        """Load and build all PostgreSQL clusters from a single MPP cluster.
 
         :param path: the path in DCS where to load Cluster(s) from.
 
-        :returns: all Citus groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values or a
-                  :class:`Cluster` object representing the coordinator with filled `Cluster.workers` attribute.
+        :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values.
         """
 
     @abc.abstractmethod
@@ -1585,13 +1514,14 @@ class AbstractDCS(abc.ABC):
             the :meth:`~AbstractDCS.get_cluster` method.
 
         :param path: the path in DCS where to load Cluster(s) from.
-        :param loader: one of :meth:`~AbstractDCS._cluster_loader` or :meth:`~AbstractDCS._citus_cluster_loader`.
+        :param loader: one of :meth:`~AbstractDCS._postgresql_cluster_loader` or
+                       :meth:`~AbstractDCS._mpp_cluster_loader`.
 
         :raise: :exc:`~DCSError` in case of communication problems with DCS. If the current node was running as a
                 primary and exception raised, instance would be demoted.
         """
 
-    def __get_patroni_cluster(self, path: Optional[str] = None) -> Cluster:
+    def __get_postgresql_cluster(self, path: Optional[str] = None) -> Cluster:
         """Low level method to load a :class:`Cluster` object from DCS.
 
         :param path: optional client path in DCS backend to load from.
@@ -1600,42 +1530,43 @@ class AbstractDCS(abc.ABC):
         """
         if path is None:
             path = self.client_path('')
-        cluster = self._load_cluster(path, self._cluster_loader)
+        cluster = self._load_cluster(path, self._postgresql_cluster_loader)
         if TYPE_CHECKING:  # pragma: no cover
             assert isinstance(cluster, Cluster)
         return cluster
 
-    def is_citus_coordinator(self) -> bool:
-        """:class:`Cluster` instance has a Citus Coordinator group ID.
+    def is_mpp_coordinator(self) -> bool:
+        """:class:`Cluster` instance has a Coordinator group ID.
 
-        :returns: ``True`` if the given node is running as Citus Coordinator (``group=0``).
+        :returns: ``True`` if the given node is running as the MPP Coordinator.
         """
-        return self._citus_group == str(CITUS_COORDINATOR_GROUP_ID)
+        return self._mpp.is_coordinator()
 
-    def get_citus_coordinator(self) -> Optional[Cluster]:
-        """Load the Patroni cluster for the Citus Coordinator.
+    def get_mpp_coordinator(self) -> Optional[Cluster]:
+        """Load the PostgreSQL cluster for the MPP Coordinator.
 
-          .. note::
-              This method is only executed on the worker nodes (``group!=0``) to find the coordinator.
+        .. note::
+            This method is only executed on the worker nodes to find the coordinator.
 
-        :returns: Select :class:`Cluster` instance associated with the Citus Coordinator group ID.
+        :returns: Select :class:`Cluster` instance associated with the MPP Coordinator group ID.
         """
         try:
-            return self.__get_patroni_cluster(f'{self._base_path}/{CITUS_COORDINATOR_GROUP_ID}/')
+            return self.__get_postgresql_cluster(f'{self._base_path}/{self._mpp.coordinator_group_id}/')
         except Exception as e:
-            logger.error('Failed to load Citus coordinator cluster from %s: %r', self.__class__.__name__, e)
+            logger.error('Failed to load %s coordinator cluster from %s: %r',
+                         self._mpp.type, self.__class__.__name__, e)
         return None
 
-    def _get_citus_cluster(self) -> Cluster:
-        """Load Citus cluster from DCS.
+    def _get_mpp_cluster(self) -> Cluster:
+        """Load MPP cluster from DCS.
 
-        :returns: A Citus :class:`Cluster` instance for the coordinator with workers clusters in the `Cluster.workers`
+        :returns: A MPP :class:`Cluster` instance for the coordinator with workers clusters in the `Cluster.workers`
                   dict.
         """
-        groups = self._load_cluster(self._base_path + '/', self._citus_cluster_loader)
+        groups = self._load_cluster(self._base_path + '/', self._mpp_cluster_loader)
         if TYPE_CHECKING:  # pragma: no cover
             assert isinstance(groups, dict)
-        cluster = groups.pop(CITUS_COORDINATOR_GROUP_ID, Cluster.empty())
+        cluster = groups.pop(self._mpp.coordinator_group_id, Cluster.empty())
         cluster.workers.update(groups)
         return cluster
 
@@ -1646,12 +1577,12 @@ class AbstractDCS(abc.ABC):
             Stores copy of time, status and failsafe values for comparison in DCS update decisions.
             Caching is required to avoid overhead placed upon the REST API.
 
-            Returns either a Citus or Patroni implementation of :class:`Cluster` depending on availability.
+            Returns either a PostgreSQL or MPP implementation of :class:`Cluster` depending on availability.
 
         :returns:
         """
         try:
-            cluster = self._get_citus_cluster() if self.is_citus_coordinator() else self.__get_patroni_cluster()
+            cluster = self._get_mpp_cluster() if self.is_mpp_coordinator() else self.__get_postgresql_cluster()
         except Exception:
             self.reset_cluster()
             raise

patroni/dcs/consul.py

@@ -16,8 +16,9 @@ from urllib.parse import urlencode, urlparse, quote
 from typing import Any, Callable, Dict, List, Mapping, NamedTuple, Optional, Union, Tuple, TYPE_CHECKING
 
 from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, Status, SyncState, \
-    TimelineHistory, ReturnFalseException, catch_return_false_exception, citus_group_re
+    TimelineHistory, ReturnFalseException, catch_return_false_exception
 from ..exceptions import DCSError
+from ..postgresql.mpp import AbstractMPP
 from ..utils import deep_compare, parse_bool, Retry, RetryFailedError, split_host_port, uri, USER_AGENT
 if TYPE_CHECKING:  # pragma: no cover
     from ..config import Config
@@ -232,8 +233,8 @@ def service_name_from_scope_name(scope_name: str) -> str:
 
 class Consul(AbstractDCS):
 
-    def __init__(self, config: Dict[str, Any]) -> None:
-        super(Consul, self).__init__(config)
+    def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None:
+        super(Consul, self).__init__(config, mpp)
         self._base_path = self._base_path[1:]
         self._scope = config['scope']
         self._session = None
@@ -419,23 +420,36 @@ class Consul(AbstractDCS):
     def _consistency(self) -> str:
         return 'consistent' if self._ctl else self._client.consistency
 
-    def _cluster_loader(self, path: str) -> Cluster:
+    def _postgresql_cluster_loader(self, path: str) -> Cluster:
+        """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster.
+
+        :param path: the path in DCS where to load :class:`Cluster` from.
+
+        :returns: :class:`Cluster` instance.
+        """
         _, results = self.retry(self._client.kv.get, path, recurse=True, consistency=self._consistency)
         if results is None:
-            raise NotFound
-        nodes = {}
+            return Cluster.empty()
+        nodes: Dict[str, Dict[str, Any]] = {}
         for node in results:
             node['Value'] = (node['Value'] or b'').decode('utf-8')
             nodes[node['Key'][len(path):]] = node
 
         return self._cluster_from_nodes(nodes)
 
-    def _citus_cluster_loader(self, path: str) -> Dict[int, Cluster]:
+    def _mpp_cluster_loader(self, path: str) -> Dict[int, Cluster]:
+        """Load and build all PostgreSQL clusters from a single MPP cluster.
+
+        :param path: the path in DCS where to load Cluster(s) from.
+
+        :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values.
+        """
+        results: Optional[List[Dict[str, Any]]]
         _, results = self.retry(self._client.kv.get, path, recurse=True, consistency=self._consistency)
-        clusters: Dict[int, Dict[str, Cluster]] = defaultdict(dict)
+        clusters: Dict[int, Dict[str, Dict[str, Any]]] = defaultdict(dict)
         for node in results or []:
             key = node['Key'][len(path):].split('/', 1)
-            if len(key) == 2 and citus_group_re.match(key[0]):
+            if len(key) == 2 and self._mpp.group_re.match(key[0]):
                 node['Value'] = (node['Value'] or b'').decode('utf-8')
                 clusters[int(key[0])][key[1]] = node
         return {group: self._cluster_from_nodes(nodes) for group, nodes in clusters.items()}
@@ -445,8 +459,6 @@ class Consul(AbstractDCS):
     ) -> Union[Cluster, Dict[int, Cluster]]:
         try:
             return loader(path)
-        except NotFound:
-            return Cluster.empty()
         except Exception:
             logger.exception('get_cluster')
             raise ConsulError('Consul is not responding properly')
@@ -566,14 +578,17 @@ class Consul(AbstractDCS):
         try:
             return retry(self._client.kv.put, self.leader_path, self._name, acquire=self._session)
         except InvalidSession:
-            logger.error('Our session disappeared from Consul. Will try to get a new one and retry attempt')
             self._session = None
-            retry.ensure_deadline(0)
+
+            if not retry.ensure_deadline(0):
+                logger.error('Our session disappeared from Consul. Deadline exceeded, giving up')
+                return False
+
+            logger.error('Our session disappeared from Consul. Will try to get a new one and retry attempt')
 
             retry(self._do_refresh_session)
 
             retry.ensure_deadline(1, ConsulError('_do_attempt_to_acquire_leader timeout'))
-
             return retry(self._client.kv.put, self.leader_path, self._name, acquire=self._session)
 
     @catch_return_false_exception
@@ -668,7 +683,7 @@ class Consul(AbstractDCS):
         if ret:  # We have no other choise, only read after write :(
             if not retry.ensure_deadline(0.5):
                 return False
-            _, ret = self.retry(self._client.kv.get, self.sync_path)
+            _, ret = self.retry(self._client.kv.get, self.sync_path, consistency='consistent')
             if ret and (ret.get('Value') or b'').decode('utf-8') == value:
                 return ret['ModifyIndex']
         return False

patroni/dcs/etcd.py

@@ -22,8 +22,9 @@ from urllib3 import Timeout
 from urllib3.exceptions import HTTPError, ReadTimeoutError, ProtocolError
 
 from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, Status, SyncState, \
-    TimelineHistory, ReturnFalseException, catch_return_false_exception, citus_group_re
+    TimelineHistory, ReturnFalseException, catch_return_false_exception
 from ..exceptions import DCSError
+from ..postgresql.mpp import AbstractMPP
 from ..request import get as requests_get
 from ..utils import Retry, RetryFailedError, split_host_port, uri, USER_AGENT
 if TYPE_CHECKING:  # pragma: no cover
@@ -470,9 +471,9 @@ class EtcdClient(AbstractEtcdClientWithFailover):
 
 class AbstractEtcd(AbstractDCS):
 
-    def __init__(self, config: Dict[str, Any], client_cls: Type[AbstractEtcdClientWithFailover],
+    def __init__(self, config: Dict[str, Any], mpp: AbstractMPP, client_cls: Type[AbstractEtcdClientWithFailover],
                  retry_errors_cls: Union[Type[Exception], Tuple[Type[Exception], ...]]) -> None:
-        super(AbstractEtcd, self).__init__(config)
+        super(AbstractEtcd, self).__init__(config, mpp)
         self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
                             retry_exceptions=retry_errors_cls)
         self._ttl = int(config.get('ttl') or 30)
@@ -645,8 +646,8 @@ def catch_etcd_errors(func: Callable[..., Any]) -> Any:
 
 class Etcd(AbstractEtcd):
 
-    def __init__(self, config: Dict[str, Any]) -> None:
-        super(Etcd, self).__init__(config, EtcdClient, (etcd.EtcdLeaderElectionInProgress, EtcdRaftInternal))
+    def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None:
+        super(Etcd, self).__init__(config, mpp, EtcdClient, (etcd.EtcdLeaderElectionInProgress, EtcdRaftInternal))
         self.__do_not_watch = False
 
     @property
@@ -709,17 +710,36 @@ class Etcd(AbstractEtcd):
 
         return Cluster(initialize, config, leader, status, members, failover, sync, history, failsafe)
 
-    def _cluster_loader(self, path: str) -> Cluster:
-        result = self.retry(self._client.read, path, recursive=True, quorum=self._ctl)
+    def _postgresql_cluster_loader(self, path: str) -> Cluster:
+        """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster.
+
+        :param path: the path in DCS where to load :class:`Cluster` from.
+
+        :returns: :class:`Cluster` instance.
+        """
+        try:
+            result = self.retry(self._client.read, path, recursive=True, quorum=self._ctl)
+        except etcd.EtcdKeyNotFound:
+            return Cluster.empty()
         nodes = {node.key[len(result.key):].lstrip('/'): node for node in result.leaves}
         return self._cluster_from_nodes(result.etcd_index, nodes)
 
-    def _citus_cluster_loader(self, path: str) -> Dict[int, Cluster]:
+    def _mpp_cluster_loader(self, path: str) -> Dict[int, Cluster]:
+        """Load and build all PostgreSQL clusters from a single MPP cluster.
+
+        :param path: the path in DCS where to load Cluster(s) from.
+
+        :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values.
+        """
+        try:
+            result = self.retry(self._client.read, path, recursive=True, quorum=self._ctl)
+        except etcd.EtcdKeyNotFound:
+            return {}
+
         clusters: Dict[int, Dict[str, etcd.EtcdResult]] = defaultdict(dict)
-        result = self.retry(self._client.read, path, recursive=True, quorum=self._ctl)
         for node in result.leaves:
             key = node.key[len(result.key):].lstrip('/').split('/', 1)
-            if len(key) == 2 and citus_group_re.match(key[0]):
+            if len(key) == 2 and self._mpp.group_re.match(key[0]):
                 clusters[int(key[0])][key[1]] = node
         return {group: self._cluster_from_nodes(result.etcd_index, nodes) for group, nodes in clusters.items()}
 
@@ -729,8 +749,6 @@ class Etcd(AbstractEtcd):
         cluster = None
         try:
             cluster = loader(path)
-        except etcd.EtcdKeyNotFound:
-            cluster = Cluster.empty()
         except Exception as e:
             self._handle_exception(e, 'get_cluster', raise_ex=EtcdError('Etcd is not responding properly'))
         self._has_failed = False

patroni/dcs/etcd3.py

@@ -16,9 +16,10 @@ from threading import Condition, Lock, Thread
 from typing import Any, Callable, Collection, Dict, Iterator, List, Optional, Tuple, Type, TYPE_CHECKING, Union
 
 from . import ClusterConfig, Cluster, Failover, Leader, Member, Status, SyncState, \
-    TimelineHistory, catch_return_false_exception, citus_group_re
+    TimelineHistory, catch_return_false_exception
 from .etcd import AbstractEtcdClientWithFailover, AbstractEtcd, catch_etcd_errors, DnsCachingResolver, Retry
 from ..exceptions import DCSError, PatroniException
+from ..postgresql.mpp import AbstractMPP
 from ..utils import deep_compare, enable_keepalive, iter_response_objects, RetryFailedError, USER_AGENT
 
 logger = logging.getLogger(__name__)
@@ -197,12 +198,6 @@ def build_range_request(key: str, range_end: Union[bytes, str, None] = None) ->
     return fields
 
 
-class ReAuthenticateMode(IntEnum):
-    NOT_REQUIRED = 0
-    REQUIRED = 1
-    WITHOUT_WATCHER_RESTART = 2
-
-
 def _handle_auth_errors(func: Callable[..., Any]) -> Any:
     def wrapper(self: 'Etcd3Client', *args: Any, **kwargs: Any) -> Any:
         return self.handle_auth_errors(func, *args, **kwargs)
@@ -214,7 +209,7 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
     ERROR_CLS = Etcd3Error
 
     def __init__(self, config: Dict[str, Any], dns_resolver: DnsCachingResolver, cache_ttl: int = 300) -> None:
-        self._reauthenticate_reason = ReAuthenticateMode.NOT_REQUIRED
+        self._reauthenticate = False
         self._token = None
         self._cluster_version: Tuple[int, ...] = tuple()
         super(Etcd3Client, self).__init__({**config, 'version_prefix': '/v3beta'}, dns_resolver, cache_ttl)
@@ -293,7 +288,7 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
         fields['retry'] = retry
         return self.api_execute(self.version_prefix + method, self._MPOST, fields)
 
-    def authenticate(self, *, restart_watcher: bool = True, retry: Optional[Retry] = None) -> bool:
+    def authenticate(self, *, retry: Optional[Retry] = None) -> bool:
         if self._use_proxies and not self._cluster_version:
             kwargs = self._prepare_common_parameters(1)
             self._ensure_version_prefix(self._base_uri, **kwargs)
@@ -315,20 +310,18 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
 
     def handle_auth_errors(self: 'Etcd3Client', func: Callable[..., Any], *args: Any,
                            retry: Optional[Retry] = None, **kwargs: Any) -> Any:
+        reauthenticated = False
         exc = None
         while True:
-            if self._reauthenticate_reason:
+            if self._reauthenticate:
                 if self.username and self.password:
-                    self.authenticate(
-                        restart_watcher=self._reauthenticate_reason != ReAuthenticateMode.WITHOUT_WATCHER_RESTART,
-                        retry=retry)
-                    self._reauthenticate_reason = ReAuthenticateMode.NOT_REQUIRED
-                    if retry:
-                        retry.ensure_deadline(0)
+                    self.authenticate(retry=retry)
+                    self._reauthenticate = False
                 else:
                     msg = 'Username or password not set, authentication is not possible'
                     logger.fatal(msg)
                     raise exc or Etcd3Exception(msg)
+                reauthenticated = True
 
             try:
                 return func(self, *args, retry=retry, **kwargs)
@@ -346,11 +339,12 @@ class Etcd3Client(AbstractEtcdClientWithFailover):
             except AuthOldRevision as e:
                 logger.error('Auth token is for old revision of auth store')
                 exc = e
-            self._reauthenticate_reason = ReAuthenticateMode.WITHOUT_WATCHER_RESTART \
-                if isinstance(exc, AuthOldRevision) else ReAuthenticateMode.REQUIRED
-            if not retry:
+            self._reauthenticate = True
+            if retry:
+                logger.error('retry = %s', retry)
+                retry.ensure_deadline(0.5, exc)
+            elif reauthenticated:
                 raise exc
-            retry.ensure_deadline(0.5, exc)
 
     @_handle_auth_errors
     def range(self, key: str, range_end: Union[bytes, str, None] = None, serializable: bool = True,
@@ -602,12 +596,6 @@ class PatroniEtcd3Client(Etcd3Client):
         super(PatroniEtcd3Client, self).set_base_uri(value)
         self._restart_watcher()
 
-    def authenticate(self, *, restart_watcher: bool = True, retry: Optional[Retry] = None) -> bool:
-        ret = super(PatroniEtcd3Client, self).authenticate(restart_watcher=restart_watcher, retry=retry)
-        if ret and restart_watcher:
-            self._restart_watcher()
-        return ret
-
     def _wait_cache(self, timeout: float) -> None:
         stop_time = time.time() + timeout
         while self._kv_cache and not self._kv_cache.is_ready():
@@ -671,8 +659,9 @@ class PatroniEtcd3Client(Etcd3Client):
 
 class Etcd3(AbstractEtcd):
 
-    def __init__(self, config: Dict[str, Any]) -> None:
-        super(Etcd3, self).__init__(config, PatroniEtcd3Client, (DeadlineExceeded, Unavailable, FailedPrecondition))
+    def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None:
+        super(Etcd3, self).__init__(config, mpp, PatroniEtcd3Client,
+                                    (DeadlineExceeded, Unavailable, FailedPrecondition))
         self.__do_not_watch = False
         self._lease = None
         self._last_lease_refresh = 0
@@ -731,7 +720,11 @@ class Etcd3(AbstractEtcd):
 
     @property
     def cluster_prefix(self) -> str:
-        return self._base_path + '/' if self.is_citus_coordinator() else self.client_path('')
+        """Construct the cluster prefix for the cluster.
+
+        :returns: path in the DCS under which we store information about this Patroni cluster.
+        """
+        return self._base_path + '/' if self.is_mpp_coordinator() else self.client_path('')
 
     @staticmethod
     def member(node: Dict[str, str]) -> Member:
@@ -785,18 +778,30 @@ class Etcd3(AbstractEtcd):
 
         return Cluster(initialize, config, leader, status, members, failover, sync, history, failsafe)
 
-    def _cluster_loader(self, path: str) -> Cluster:
+    def _postgresql_cluster_loader(self, path: str) -> Cluster:
+        """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster.
+
+        :param path: the path in DCS where to load :class:`Cluster` from.
+
+        :returns: :class:`Cluster` instance.
+        """
         nodes = {node['key'][len(path):]: node
                  for node in self._client.get_cluster(path)
                  if node['key'].startswith(path)}
         return self._cluster_from_nodes(nodes)
 
-    def _citus_cluster_loader(self, path: str) -> Dict[int, Cluster]:
+    def _mpp_cluster_loader(self, path: str) -> Dict[int, Cluster]:
+        """Load and build all PostgreSQL clusters from a single MPP cluster.
+
+        :param path: the path in DCS where to load Cluster(s) from.
+
+        :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values.
+        """
         clusters: Dict[int, Dict[str, Dict[str, Any]]] = defaultdict(dict)
         path = self._base_path + '/'
         for node in self._client.get_cluster(path):
             key = node['key'][len(path):].split('/', 1)
-            if len(key) == 2 and citus_group_re.match(key[0]):
+            if len(key) == 2 and self._mpp.group_re.match(key[0]):
                 clusters[int(key[0])][key[1]] = node
         return {group: self._cluster_from_nodes(nodes) for group, nodes in clusters.items()}
 
@@ -848,14 +853,16 @@ class Etcd3(AbstractEtcd):
         try:
             return _retry(self._client.put, self.leader_path, self._name, self._lease, create_revision='0')
         except LeaseNotFound:
-            logger.error('Our lease disappeared from Etcd. Will try to get a new one and retry attempt')
             self._lease = None
-            retry.ensure_deadline(0)
+            if not retry.ensure_deadline(0):
+                logger.error('Our lease disappeared from Etcd. Deadline exceeded, giving up')
+                return False
+
+            logger.error('Our lease disappeared from Etcd. Will try to get a new one and retry attempt')
 
             _retry(self._do_refresh_lease)
 
             retry.ensure_deadline(1, Etcd3Error('_do_attempt_to_acquire_leader timeout'))
-
             return _retry(self._client.put, self.leader_path, self._name, self._lease, create_revision='0')
 
     @catch_return_false_exception

patroni/dcs/exhibitor.py

@@ -7,6 +7,7 @@ from typing import Any, Callable, Dict, List, Union
 
 from . import Cluster
 from .zookeeper import ZooKeeper
+from ..postgresql.mpp import AbstractMPP
 from ..request import get as requests_get
 from ..utils import uri
 
@@ -66,10 +67,10 @@ class ExhibitorEnsembleProvider(object):
 
 class Exhibitor(ZooKeeper):
 
-    def __init__(self, config: Dict[str, Any]) -> None:
+    def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None:
         interval = config.get('poll_interval', 300)
         self._ensemble_provider = ExhibitorEnsembleProvider(config['hosts'], config['port'], poll_interval=interval)
-        super(Exhibitor, self).__init__({**config, 'hosts': self._ensemble_provider.zookeeper_hosts})
+        super(Exhibitor, self).__init__({**config, 'hosts': self._ensemble_provider.zookeeper_hosts}, mpp)
 
     def _load_cluster(
             self, path: str, loader: Callable[[str], Union[Cluster, Dict[int, Cluster]]]

patroni/dcs/kubernetes.py

@@ -19,9 +19,10 @@ from urllib3.exceptions import HTTPError
 from threading import Condition, Lock, Thread
 from typing import Any, Callable, Collection, Dict, List, Optional, Tuple, Type, Union, TYPE_CHECKING
 
-from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, Status, SyncState, \
-    TimelineHistory, CITUS_COORDINATOR_GROUP_ID, citus_group_re
+from . import AbstractDCS, Cluster, ClusterConfig, Failover, Leader, Member, Status, SyncState, TimelineHistory
+from ..collections import EMPTY_DICT
 from ..exceptions import DCSError
+from ..postgresql.mpp import AbstractMPP
 from ..utils import deep_compare, iter_response_objects, keepalive_socket_options, \
     Retry, RetryFailedError, tzutc, uri, USER_AGENT
 if TYPE_CHECKING:  # pragma: no cover
@@ -470,7 +471,7 @@ class K8sClient(object):
                 if len(args) == 3:  # name, namespace, body
                     body = args[2]
                 elif action == 'create':  # namespace, body
-                    body = args[1]
+                    body = args[1]  # pyright: ignore [reportGeneralTypeIssues]
                 elif action == 'delete':  # name, namespace
                     body = kwargs.pop('body', None)
                 else:
@@ -509,7 +510,7 @@ class KubernetesRetriableException(k8s_client.rest.ApiException):
     @property
     def sleeptime(self) -> Optional[int]:
         try:
-            return int((self.headers or {}).get('retry-after', ''))
+            return int((self.headers or EMPTY_DICT).get('retry-after', ''))
         except Exception:
             return None
 
@@ -654,7 +655,7 @@ class ObjectCache(Thread):
             obj = K8sObject(obj)
             success, old_value = self.set(name, obj)
             if success:
-                new_value = (obj.metadata.annotations or {}).get(self._annotations_map.get(name))
+                new_value = (obj.metadata.annotations or EMPTY_DICT).get(self._annotations_map.get(name, ''))
         elif ev_type == 'DELETED':
             success, old_value = self.delete(name, obj['metadata']['resourceVersion'])
         else:
@@ -662,7 +663,7 @@ class ObjectCache(Thread):
 
         if success and obj.get('kind') != 'Pod':
             if old_value:
-                old_value = (old_value.metadata.annotations or {}).get(self._annotations_map.get(name))
+                old_value = (old_value.metadata.annotations or EMPTY_DICT).get(self._annotations_map.get(name, ''))
 
             value_changed = old_value != new_value and \
                 (name != self._dcs.config_path or old_value is not None and new_value is not None)
@@ -746,9 +747,7 @@ class ObjectCache(Thread):
 
 class Kubernetes(AbstractDCS):
 
-    _CITUS_LABEL = 'citus-group'
-
-    def __init__(self, config: Dict[str, Any]) -> None:
+    def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None:
         self._labels = deepcopy(config['labels'])
         self._labels[config.get('scope_label', 'cluster-name')] = config['scope']
         self._label_selector = ','.join('{0}={1}'.format(k, v) for k, v in self._labels.items())
@@ -759,9 +758,9 @@ class Kubernetes(AbstractDCS):
         self._standby_leader_label_value = config.get('standby_leader_label_value', 'master')
         self._tmp_role_label = config.get('tmp_role_label')
         self._ca_certs = os.environ.get('PATRONI_KUBERNETES_CACERT', config.get('cacert')) or SERVICE_CERT_FILENAME
-        super(Kubernetes, self).__init__({**config, 'namespace': ''})
-        if self._citus_group:
-            self._labels[self._CITUS_LABEL] = self._citus_group
+        super(Kubernetes, self).__init__({**config, 'namespace': ''}, mpp)
+        if self._mpp.is_enabled():
+            self._labels[self._mpp.k8s_group_label] = str(self._mpp.group)
 
         self._retry = Retry(deadline=config['retry_timeout'], max_delay=1, max_tries=-1,
                             retry_exceptions=KubernetesRetriableException)
@@ -846,7 +845,7 @@ class Kubernetes(AbstractDCS):
 
     @staticmethod
     def member(pod: K8sObject) -> Member:
-        annotations = pod.metadata.annotations or {}
+        annotations = pod.metadata.annotations or EMPTY_DICT
         member = Member.from_node(pod.metadata.resource_version, pod.metadata.name, None, annotations.get('status', ''))
         member.data['pod_labels'] = pod.metadata.labels
         return member
@@ -927,7 +926,7 @@ class Kubernetes(AbstractDCS):
         failover = nodes.get(path + self._FAILOVER)
         metadata = failover and failover.metadata
         failover = metadata and Failover.from_node(metadata.resource_version,
-                                                   (metadata.annotations or {}).copy())
+                                                   (metadata.annotations or EMPTY_DICT).copy())
 
         # get synchronization state
         sync = nodes.get(path + self._SYNC)
@@ -936,20 +935,32 @@ class Kubernetes(AbstractDCS):
 
         return Cluster(initialize, config, leader, status, members, failover, sync, history, failsafe)
 
-    def _cluster_loader(self, path: Dict[str, Any]) -> Cluster:
+    def _postgresql_cluster_loader(self, path: Dict[str, Any]) -> Cluster:
+        """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster.
+
+        :param path: the path in DCS where to load :class:`Cluster` from.
+
+        :returns: :class:`Cluster` instance.
+        """
         return self._cluster_from_nodes(path['group'], path['nodes'], path['pods'].values())
 
-    def _citus_cluster_loader(self, path: Dict[str, Any]) -> Dict[int, Cluster]:
+    def _mpp_cluster_loader(self, path: Dict[str, Any]) -> Dict[int, Cluster]:
+        """Load and build all PostgreSQL clusters from a single MPP cluster.
+
+        :param path: the path in DCS where to load Cluster(s) from.
+
+        :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values.
+        """
         clusters: Dict[str, Dict[str, Dict[str, K8sObject]]] = defaultdict(lambda: defaultdict(dict))
 
         for name, pod in path['pods'].items():
-            group = pod.metadata.labels.get(self._CITUS_LABEL)
-            if group and citus_group_re.match(group):
+            group = pod.metadata.labels.get(self._mpp.k8s_group_label)
+            if group and self._mpp.group_re.match(group):
                 clusters[group]['pods'][name] = pod
 
         for name, kind in path['nodes'].items():
-            group = kind.metadata.labels.get(self._CITUS_LABEL)
-            if group and citus_group_re.match(group):
+            group = kind.metadata.labels.get(self._mpp.k8s_group_label)
+            if group and self._mpp.group_re.match(group):
                 clusters[group]['nodes'][name] = kind
         return {int(group): self._cluster_from_nodes(group, value['nodes'], value['pods'].values())
                 for group, value in clusters.items()}
@@ -965,9 +976,9 @@ class Kubernetes(AbstractDCS):
             with self._condition:
                 self._wait_caches(stop_time)
                 pods = {name: pod for name, pod in self._pods.copy().items()
-                        if not group or pod.metadata.labels.get(self._CITUS_LABEL) == group}
+                        if not group or pod.metadata.labels.get(self._mpp.k8s_group_label) == group}
                 nodes = {name: kind for name, kind in self._kinds.copy().items()
-                         if not group or kind.metadata.labels.get(self._CITUS_LABEL) == group}
+                         if not group or kind.metadata.labels.get(self._mpp.k8s_group_label) == group}
             return loader({'group': group, 'pods': pods, 'nodes': nodes})
         except Exception:
             logger.exception('get_cluster')
@@ -976,17 +987,24 @@ class Kubernetes(AbstractDCS):
     def _load_cluster(
             self, path: str, loader: Callable[[Any], Union[Cluster, Dict[int, Cluster]]]
     ) -> Union[Cluster, Dict[int, Cluster]]:
-        group = self._citus_group if path == self.client_path('') else None
+        group = str(self._mpp.group) if self._mpp.is_enabled() and path == self.client_path('') else None
         return self.__load_cluster(group, loader)
 
-    def get_citus_coordinator(self) -> Optional[Cluster]:
+    def get_mpp_coordinator(self) -> Optional[Cluster]:
+        """Load the PostgreSQL cluster for the MPP Coordinator.
+
+        .. note::
+            This method is only executed on the worker nodes to find the coordinator.
+
+        :returns: Select :class:`Cluster` instance associated with the MPP Coordinator group ID.
+        """
         try:
-            ret = self.__load_cluster(str(CITUS_COORDINATOR_GROUP_ID), self._cluster_loader)
+            ret = self.__load_cluster(str(self._mpp.coordinator_group_id), self._postgresql_cluster_loader)
             if TYPE_CHECKING:  # pragma: no cover
                 assert isinstance(ret, Cluster)
             return ret
         except Exception as e:
-            logger.error('Failed to load Citus coordinator cluster from Kubernetes: %r', e)
+            logger.error('Failed to load %s coordinator cluster from Kubernetes: %r', self._mpp.type, e)
 
     @staticmethod
     def compare_ports(p1: K8sObject, p2: K8sObject) -> bool:
@@ -1030,8 +1048,9 @@ class Kubernetes(AbstractDCS):
 
     def __target_ref(self, leader_ip: str, latest_subsets: List[K8sObject], pod: K8sObject) -> K8sObject:
         # we want to re-use existing target_ref if possible
+        empty_addresses: List[K8sObject] = []
         for subset in latest_subsets:
-            for address in subset.addresses or []:
+            for address in subset.addresses or empty_addresses:
                 if address.ip == leader_ip and address.target_ref and address.target_ref.name == self._name:
                     return address.target_ref
         return k8s_client.V1ObjectReference(kind='Pod', uid=pod.metadata.uid, namespace=self._namespace,
@@ -1039,7 +1058,8 @@ class Kubernetes(AbstractDCS):
 
     def _map_subsets(self, endpoints: Dict[str, Any], ips: List[str]) -> None:
         leader = self._kinds.get(self.leader_path)
-        latest_subsets = leader and leader.subsets or []
+        empty_addresses: List[K8sObject] = []
+        latest_subsets = leader and leader.subsets or empty_addresses
         if not ips:
             # We want to have subsets empty
             if latest_subsets:
@@ -1195,7 +1215,7 @@ class Kubernetes(AbstractDCS):
         if not retry.ensure_deadline(0.5):
             return False
 
-        kind_annotations = kind and kind.metadata.annotations or {}
+        kind_annotations = kind and kind.metadata.annotations or EMPTY_DICT
         kind_resource_version = kind and kind.metadata.resource_version
 
         # There is different leader or resource_version in cache didn't change
@@ -1208,7 +1228,7 @@ class Kubernetes(AbstractDCS):
     def update_leader(self, leader: Leader, last_lsn: Optional[int],
                       slots: Optional[Dict[str, int]] = None, failsafe: Optional[Dict[str, str]] = None) -> bool:
         kind = self._kinds.get(self.leader_path)
-        kind_annotations = kind and kind.metadata.annotations or {}
+        kind_annotations = kind and kind.metadata.annotations or EMPTY_DICT
 
         if kind and kind_annotations.get(self._LEADER) != self._name:
             return False
@@ -1329,7 +1349,7 @@ class Kubernetes(AbstractDCS):
     def delete_leader(self, leader: Optional[Leader], last_lsn: Optional[int] = None) -> bool:
         ret = False
         kind = self._kinds.get(self.leader_path)
-        if kind and (kind.metadata.annotations or {}).get(self._LEADER) == self._name:
+        if kind and (kind.metadata.annotations or EMPTY_DICT).get(self._LEADER) == self._name:
             annotations: Dict[str, Optional[str]] = {self._LEADER: None}
             if last_lsn:
                 annotations[self._OPTIME] = str(last_lsn)

patroni/dcs/raft.py

@@ -12,9 +12,9 @@ from pysyncobj.transport import TCPTransport, CONNECTION_STATE
 from pysyncobj.utility import TcpUtility
 from typing import Any, Callable, Collection, Dict, List, Optional, Set, Union, TYPE_CHECKING
 
-from . import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, Status, SyncState, \
-    TimelineHistory, citus_group_re
+from . import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, Status, SyncState, TimelineHistory
 from ..exceptions import DCSError
+from ..postgresql.mpp import AbstractMPP
 from ..utils import validate_directory
 if TYPE_CHECKING:  # pragma: no cover
     from ..config import Config
@@ -285,8 +285,8 @@ class KVStoreTTL(DynMemberSyncObj):
 
 class Raft(AbstractDCS):
 
-    def __init__(self, config: Dict[str, Any]) -> None:
-        super(Raft, self).__init__(config)
+    def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None:
+        super(Raft, self).__init__(config, mpp)
         self._ttl = int(config.get('ttl') or 30)
 
         ready_event = threading.Event()
@@ -375,19 +375,31 @@ class Raft(AbstractDCS):
 
         return Cluster(initialize, config, leader, status, members, failover, sync, history, failsafe)
 
-    def _cluster_loader(self, path: str) -> Cluster:
+    def _postgresql_cluster_loader(self, path: str) -> Cluster:
+        """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster.
+
+        :param path: the path in DCS where to load :class:`Cluster` from.
+
+        :returns: :class:`Cluster` instance.
+        """
         response = self._sync_obj.get(path, recursive=True)
         if not response:
             return Cluster.empty()
         nodes = {key[len(path):]: value for key, value in response.items()}
         return self._cluster_from_nodes(nodes)
 
-    def _citus_cluster_loader(self, path: str) -> Dict[int, Cluster]:
+    def _mpp_cluster_loader(self, path: str) -> Dict[int, Cluster]:
+        """Load and build all PostgreSQL clusters from a single MPP cluster.
+
+        :param path: the path in DCS where to load Cluster(s) from.
+
+        :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values.
+        """
         clusters: Dict[int, Dict[str, Any]] = defaultdict(dict)
         response = self._sync_obj.get(path, recursive=True)
         for key, value in (response or {}).items():
             key = key[len(path):].split('/', 1)
-            if len(key) == 2 and citus_group_re.match(key[0]):
+            if len(key) == 2 and self._mpp.group_re.match(key[0]):
                 clusters[int(key[0])][key[1]] = value
         return {group: self._cluster_from_nodes(nodes) for group, nodes in clusters.items()}
 

patroni/dcs/zookeeper.py

@@ -12,9 +12,9 @@ from kazoo.retry import RetryFailedError
 from kazoo.security import ACL, make_acl
 from typing import Any, Callable, Dict, List, Optional, Union, Tuple, TYPE_CHECKING
 
-from . import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, Status, SyncState, \
-    TimelineHistory, citus_group_re
+from . import AbstractDCS, ClusterConfig, Cluster, Failover, Leader, Member, Status, SyncState, TimelineHistory
 from ..exceptions import DCSError
+from ..postgresql.mpp import AbstractMPP
 from ..utils import deep_compare
 if TYPE_CHECKING:  # pragma: no cover
     from ..config import Config
@@ -87,8 +87,8 @@ class PatroniKazooClient(KazooClient):
 
 class ZooKeeper(AbstractDCS):
 
-    def __init__(self, config: Dict[str, Any]) -> None:
-        super(ZooKeeper, self).__init__(config)
+    def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None:
+        super(ZooKeeper, self).__init__(config, mpp)
 
         hosts: Union[str, List[str]] = config.get('hosts', [])
         if isinstance(hosts, list):
@@ -115,7 +115,8 @@ class ZooKeeper(AbstractDCS):
         self._client = PatroniKazooClient(hosts, handler=PatroniSequentialThreadingHandler(config['retry_timeout']),
                                           timeout=config['ttl'], connection_retry=KazooRetry(max_delay=1, max_tries=-1,
                                           sleep_func=time.sleep), command_retry=KazooRetry(max_delay=1, max_tries=-1,
-                                          deadline=config['retry_timeout'], sleep_func=time.sleep), **kwargs)
+                                          deadline=config['retry_timeout'], sleep_func=time.sleep),
+                                          auth_data=list(config.get('auth_data', {}).items()), **kwargs)
 
         self.__last_member_data: Optional[Dict[str, Any]] = None
 
@@ -213,7 +214,13 @@ class ZooKeeper(AbstractDCS):
                 members.append(self.member(member, *data))
         return members
 
-    def _cluster_loader(self, path: str) -> Cluster:
+    def _postgresql_cluster_loader(self, path: str) -> Cluster:
+        """Load and build the :class:`Cluster` object from DCS, which represents a single PostgreSQL cluster.
+
+        :param path: the path in DCS where to load :class:`Cluster` from.
+
+        :returns: :class:`Cluster` instance.
+        """
         nodes = set(self.get_children(path))
 
         # get initialize flag
@@ -257,11 +264,17 @@ class ZooKeeper(AbstractDCS):
 
         return Cluster(initialize, config, leader, status, members, failover, sync, history, failsafe)
 
-    def _citus_cluster_loader(self, path: str) -> Dict[int, Cluster]:
+    def _mpp_cluster_loader(self, path: str) -> Dict[int, Cluster]:
+        """Load and build all PostgreSQL clusters from a single MPP cluster.
+
+        :param path: the path in DCS where to load Cluster(s) from.
+
+        :returns: all MPP groups as :class:`dict`, with group IDs as keys and :class:`Cluster` objects as values.
+        """
         ret: Dict[int, Cluster] = {}
         for node in self.get_children(path):
-            if citus_group_re.match(node):
-                ret[int(node)] = self._cluster_loader(path + node + '/')
+            if self._mpp.group_re.match(node):
+                ret[int(node)] = self._postgresql_cluster_loader(path + node + '/')
         return ret
 
     def _load_cluster(

patroni/dynamic_loader.py

@@ -0,0 +1,96 @@
+"""Helper functions to search for implementations of specific abstract interface in a package."""
+import importlib
+import inspect
+import logging
+import os
+import pkgutil
+import sys
+from types import ModuleType
+
+from typing import Any, Dict, Iterator, List, Optional, Set, Tuple, TYPE_CHECKING, Type, TypeVar, Union
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .config import Config
+
+logger = logging.getLogger(__name__)
+
+
+def iter_modules(package: str) -> List[str]:
+    """Get names of modules from *package*, depending on execution environment.
+
+    .. note::
+        If being packaged with PyInstaller, modules aren't discoverable dynamically by scanning source directory because
+        :class:`importlib.machinery.FrozenImporter` doesn't implement :func:`iter_modules`. But it is still possible to
+        find all potential modules by iterating through ``toc``, which contains list of all "frozen" resources.
+
+    :param package: a package name to search modules in, e.g. ``patroni.dcs``.
+
+    :returns: list of known module names with absolute python module path namespace, e.g. ``patroni.dcs.etcd``.
+    """
+    module_prefix = package + '.'
+
+    if getattr(sys, 'frozen', False):
+        toc: Set[str] = set()
+        # dirname may contain a few dots, which causes pkgutil.iter_importers()
+        # to misinterpret the path as a package name. This can be avoided
+        # altogether by not passing a path at all, because PyInstaller's
+        # FrozenImporter is a singleton and registered as top-level finder.
+        for importer in pkgutil.iter_importers():
+            if hasattr(importer, 'toc'):
+                toc |= getattr(importer, 'toc')
+        dots = module_prefix.count('.')  # search for modules only on the same level
+        return [module for module in toc if module.startswith(module_prefix) and module.count('.') == dots]
+
+    # here we are making an assumption that the package which is calling this function is already imported
+    pkg_file = sys.modules[package].__file__
+    if TYPE_CHECKING:  # pragma: no cover
+        assert isinstance(pkg_file, str)
+    return [name for _, name, is_pkg in pkgutil.iter_modules([os.path.dirname(pkg_file)], module_prefix) if not is_pkg]
+
+
+ClassType = TypeVar("ClassType")
+
+
+def find_class_in_module(module: ModuleType, cls_type: Type[ClassType]) -> Optional[Type[ClassType]]:
+    """Try to find the implementation of *cls_type* class interface in *module* matching the *module* name.
+
+    :param module: imported module.
+    :param cls_type: a class type we are looking for.
+
+    :returns: class with a name matching the name of *module* that implements *cls_type* or ``None`` if not found.
+    """
+    module_name = module.__name__.rpartition('.')[2]
+    return next(
+        (obj for obj_name, obj in module.__dict__.items()
+         if (obj_name.lower() == module_name
+             and inspect.isclass(obj) and issubclass(obj, cls_type))),
+        None)
+
+
+def iter_classes(
+        package: str, cls_type: Type[ClassType],
+        config: Optional[Union['Config', Dict[str, Any]]] = None
+) -> Iterator[Tuple[str, Type[ClassType]]]:
+    """Attempt to import modules and find implementations of *cls_type* that are present in the given configuration.
+
+    .. note::
+            If a module successfully imports we can assume that all its requirements are installed.
+
+    :param package: a package name to search modules in, e.g. ``patroni.dcs``.
+    :param cls_type: a class type we are looking for.
+    :param config: configuration information with possible module names as keys. If given, only attempt to import
+                   modules defined in the configuration. Else, if ``None``, attempt to import any supported module.
+
+    :yields: a tuple containing the module ``name`` and the imported class object.
+    """
+    for mod_name in iter_modules(package):
+        name = mod_name.rpartition('.')[2]
+        if config is None or name in config:
+            try:
+                module = importlib.import_module(mod_name)
+                module_cls = find_class_in_module(module, cls_type)
+                if module_cls:
+                    yield name, module_cls
+            except ImportError:
+                logger.log(logging.DEBUG if config is not None else logging.INFO,
+                           'Failed to import %s', mod_name)

patroni/global_config.py

@@ -0,0 +1,229 @@
+"""Implements *global_config* facilities.
+
+The :class:`GlobalConfig` object is instantiated on import and replaces
+``patroni.global_config`` module in :data:`sys.modules`, what allows to use
+its properties and methods like they were module variables and functions.
+"""
+import sys
+import types
+
+from copy import deepcopy
+from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING
+
+from .collections import EMPTY_DICT
+from .utils import parse_bool, parse_int
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .dcs import Cluster
+
+
+def __getattr__(mod: types.ModuleType, name: str) -> Any:
+    """This function exists just to make pyright happy.
+
+    Without it pyright complains about access to unknown members of global_config module.
+    """
+    return getattr(sys.modules[__name__], name)  # pragma: no cover
+
+
+class GlobalConfig(types.ModuleType):
+    """A class that wraps global configuration and provides convenient methods to access/check values."""
+
+    __file__ = __file__  # just to make unittest and pytest happy
+
+    def __init__(self) -> None:
+        """Initialize :class:`GlobalConfig` object."""
+        super().__init__(__name__)
+        self.__config = {}
+
+    @staticmethod
+    def _cluster_has_valid_config(cluster: Optional['Cluster']) -> bool:
+        """Check if provided *cluster* object has a valid global configuration.
+
+        :param cluster: the currently known cluster state from DCS.
+
+        :returns: ``True`` if provided *cluster* object has a valid global configuration, otherwise ``False``.
+        """
+        return bool(cluster and cluster.config and cluster.config.modify_version)
+
+    def update(self, cluster: Optional['Cluster'], default: Optional[Dict[str, Any]] = None) -> None:
+        """Update with the new global configuration from the :class:`Cluster` object view.
+
+        .. note::
+            Update happens in-place and is executed only from the main heartbeat thread.
+
+        :param cluster: the currently known cluster state from DCS.
+        :param default: default configuration, which will be used if there is no valid *cluster.config*.
+        """
+        # Try to protect from the case when DCS was wiped out
+        if self._cluster_has_valid_config(cluster):
+            self.__config = cluster.config.data  # pyright: ignore [reportOptionalMemberAccess]
+        elif default:
+            self.__config = default
+
+    def from_cluster(self, cluster: Optional['Cluster']) -> 'GlobalConfig':
+        """Return :class:`GlobalConfig` instance from the provided :class:`Cluster` object view.
+
+        .. note::
+            If the provided *cluster* object doesn't have a valid global configuration we return
+            the last known valid state of the :class:`GlobalConfig` object.
+
+            This method is used when we need to have the most up-to-date values in the global configuration,
+            but we don't want to update the global object.
+
+        :param cluster: the currently known cluster state from DCS.
+
+        :returns: :class:`GlobalConfig` object.
+        """
+        if not self._cluster_has_valid_config(cluster):
+            return self
+
+        ret = GlobalConfig()
+        ret.update(cluster)
+        return ret
+
+    def get(self, name: str) -> Any:
+        """Gets global configuration value by *name*.
+
+        :param name: parameter name.
+
+        :returns: configuration value or ``None`` if it is missing.
+        """
+        return self.__config.get(name)
+
+    def check_mode(self, mode: str) -> bool:
+        """Checks whether the certain parameter is enabled.
+
+        :param mode: parameter name, e.g. ``synchronous_mode``, ``failsafe_mode``, ``pause``, ``check_timeline``, and
+            so on.
+
+        :returns: ``True`` if parameter *mode* is enabled in the global configuration.
+        """
+        return bool(parse_bool(self.__config.get(mode)))
+
+    @property
+    def is_paused(self) -> bool:
+        """``True`` if cluster is in maintenance mode."""
+        return self.check_mode('pause')
+
+    @property
+    def is_synchronous_mode(self) -> bool:
+        """``True`` if synchronous replication is requested and it is not a standby cluster config."""
+        return self.check_mode('synchronous_mode') and not self.is_standby_cluster
+
+    @property
+    def is_synchronous_mode_strict(self) -> bool:
+        """``True`` if at least one synchronous node is required."""
+        return self.check_mode('synchronous_mode_strict')
+
+    def get_standby_cluster_config(self) -> Union[Dict[str, Any], Any]:
+        """Get ``standby_cluster`` configuration.
+
+        :returns: a copy of ``standby_cluster`` configuration.
+        """
+        return deepcopy(self.get('standby_cluster'))
+
+    @property
+    def is_standby_cluster(self) -> bool:
+        """``True`` if global configuration has a valid ``standby_cluster`` section."""
+        config = self.get_standby_cluster_config()
+        return isinstance(config, dict) and\
+            bool(config.get('host') or config.get('port') or config.get('restore_command'))
+
+    def get_int(self, name: str, default: int = 0) -> int:
+        """Gets current value of *name* from the global configuration and try to return it as :class:`int`.
+
+        :param name: name of the parameter.
+        :param default: default value if *name* is not in the configuration or invalid.
+
+        :returns: currently configured value of *name* from the global configuration or *default* if it is not set or
+            invalid.
+        """
+        ret = parse_int(self.get(name))
+        return default if ret is None else ret
+
+    @property
+    def min_synchronous_nodes(self) -> int:
+        """The minimum number of synchronous nodes based on whether ``synchronous_mode_strict`` is enabled or not."""
+        return 1 if self.is_synchronous_mode_strict else 0
+
+    @property
+    def synchronous_node_count(self) -> int:
+        """Currently configured value of ``synchronous_node_count`` from the global configuration.
+
+        Assume ``1`` if it is not set or invalid.
+        """
+        return max(self.get_int('synchronous_node_count', 1), self.min_synchronous_nodes)
+
+    @property
+    def maximum_lag_on_failover(self) -> int:
+        """Currently configured value of ``maximum_lag_on_failover`` from the global configuration.
+
+        Assume ``1048576`` if it is not set or invalid.
+        """
+        return self.get_int('maximum_lag_on_failover', 1048576)
+
+    @property
+    def maximum_lag_on_syncnode(self) -> int:
+        """Currently configured value of ``maximum_lag_on_syncnode`` from the global configuration.
+
+        Assume ``-1`` if it is not set or invalid.
+        """
+        return self.get_int('maximum_lag_on_syncnode', -1)
+
+    @property
+    def primary_start_timeout(self) -> int:
+        """Currently configured value of ``primary_start_timeout`` from the global configuration.
+
+        Assume ``300`` if it is not set or invalid.
+
+        .. note::
+            ``master_start_timeout`` is still supported to keep backward compatibility.
+        """
+        default = 300
+        return self.get_int('primary_start_timeout', default)\
+            if 'primary_start_timeout' in self.__config else self.get_int('master_start_timeout', default)
+
+    @property
+    def primary_stop_timeout(self) -> int:
+        """Currently configured value of ``primary_stop_timeout`` from the global configuration.
+
+        Assume ``0`` if it is not set or invalid.
+
+        .. note::
+            ``master_stop_timeout`` is still supported to keep backward compatibility.
+        """
+        default = 0
+        return self.get_int('primary_stop_timeout', default)\
+            if 'primary_stop_timeout' in self.__config else self.get_int('master_stop_timeout', default)
+
+    @property
+    def ignore_slots_matchers(self) -> List[Dict[str, Any]]:
+        """Currently configured value of ``ignore_slots`` from the global configuration.
+
+        Assume an empty :class:`list` if not set.
+        """
+        return self.get('ignore_slots') or []
+
+    @property
+    def max_timelines_history(self) -> int:
+        """Currently configured value of ``max_timelines_history`` from the global configuration.
+
+        Assume ``0`` if not set or invalid.
+        """
+        return self.get_int('max_timelines_history', 0)
+
+    @property
+    def use_slots(self) -> bool:
+        """``True`` if cluster is configured to use replication slots."""
+        return bool(parse_bool((self.get('postgresql') or EMPTY_DICT).get('use_slots', True)))
+
+    @property
+    def permanent_slots(self) -> Dict[str, Any]:
+        """Dictionary of permanent slots information from the global configuration."""
+        return deepcopy(self.get('permanent_replication_slots')
+                        or self.get('permanent_slots')
+                        or self.get('slots')
+                        or EMPTY_DICT.copy())
+
+
+sys.modules[__name__] = GlobalConfig()

patroni/ha.py

@@ -10,7 +10,7 @@ from multiprocessing.pool import ThreadPool
 from threading import RLock
 from typing import Any, Callable, Collection, Dict, List, NamedTuple, Optional, Union, Tuple, TYPE_CHECKING
 
-from . import psycopg
+from . import global_config, psycopg
 from .__main__ import Patroni
 from .async_executor import AsyncExecutor, CriticalTask
 from .collections import CaseInsensitiveSet
@@ -156,7 +156,6 @@ class Ha(object):
         self._rewind = Rewind(self.state_handler)
         self.dcs = patroni.dcs
         self.cluster = Cluster.empty()
-        self.global_config = self.patroni.config.get_global_config(None)
         self.old_cluster = Cluster.empty()
         self._leader_expiry = 0
         self._leader_expiry_lock = RLock()
@@ -176,7 +175,7 @@ class Ha(object):
         # Count of concurrent sync disabling requests. Value above zero means that we don't want to be synchronous
         # standby. Changes protected by _member_state_lock.
         self._disable_sync = 0
-        # Remember the last known member role and state written to the DCS in order to notify Citus coordinator
+        # Remember the last known member role and state written to the DCS in order to notify MPP coordinator
         self._last_state = None
 
         # We need following property to avoid shutdown of postgres when join of Patroni to the postgres
@@ -186,22 +185,25 @@ class Ha(object):
         # used only in backoff after failing a pre_promote script
         self._released_leader_key_timestamp = 0
 
+        # Initialize global config
+        global_config.update(None, self.patroni.config.dynamic_configuration)
+
     def primary_stop_timeout(self) -> Union[int, None]:
         """:returns: "primary_stop_timeout" from the global configuration or `None` when not in synchronous mode."""
-        ret = self.global_config.primary_stop_timeout
+        ret = global_config.primary_stop_timeout
         return ret if ret > 0 and self.is_synchronous_mode() else None
 
     def is_paused(self) -> bool:
         """:returns: `True` if in maintenance mode."""
-        return self.global_config.is_paused
+        return global_config.is_paused
 
     def check_timeline(self) -> bool:
         """:returns: `True` if should check whether the timeline is latest during the leader race."""
-        return self.global_config.check_mode('check_timeline')
+        return global_config.check_mode('check_timeline')
 
     def is_standby_cluster(self) -> bool:
         """:returns: `True` if global configuration has a valid "standby_cluster" section."""
-        return self.global_config.is_standby_cluster
+        return global_config.is_standby_cluster
 
     def is_leader(self) -> bool:
         """:returns: `True` if the current node is the leader, based on expiration set when it last held the key."""
@@ -295,9 +297,8 @@ class Ha(object):
             try:
                 last_lsn = self.state_handler.last_operation()
                 slots = self.cluster.filter_permanent_slots(
-                    {**self.state_handler.slots(), slot_name_from_member_name(self.state_handler.name): last_lsn},
-                    self.is_standby_cluster(),
-                    self.state_handler.major_version)
+                    self.state_handler,
+                    {**self.state_handler.slots(), slot_name_from_member_name(self.state_handler.name): last_lsn})
             except Exception:
                 logger.exception('Exception when called state_handler.last_operation()')
         if TYPE_CHECKING:  # pragma: no cover
@@ -328,20 +329,26 @@ class Ha(object):
             tags['nosync'] = True
         return tags
 
-    def notify_citus_coordinator(self, event: str) -> None:
-        if self.state_handler.citus_handler.is_worker():
-            coordinator = self.dcs.get_citus_coordinator()
+    def notify_mpp_coordinator(self, event: str) -> None:
+        """Send an event to the MPP coordinator.
+
+        :param event: the type of event for coordinator to parse.
+        """
+        mpp_handler = self.state_handler.mpp_handler
+        if mpp_handler.is_worker():
+            coordinator = self.dcs.get_mpp_coordinator()
             if coordinator and coordinator.leader and coordinator.leader.conn_url:
                 try:
                     data = {'type': event,
-                            'group': self.state_handler.citus_handler.group(),
+                            'group': mpp_handler.group,
                             'leader': self.state_handler.name,
                             'timeout': self.dcs.ttl,
                             'cooldown': self.patroni.config['retry_timeout']}
                     timeout = self.dcs.ttl if event == 'before_demote' else 2
-                    self.patroni.request(coordinator.leader.member, 'post', 'citus', data, timeout=timeout, retries=0)
+                    endpoint = 'citus' if mpp_handler.type == 'Citus' else 'mpp'
+                    self.patroni.request(coordinator.leader.member, 'post', endpoint, data, timeout=timeout, retries=0)
                 except Exception as e:
-                    logger.warning('Request to Citus coordinator leader %s %s failed: %r',
+                    logger.warning('Request to %s coordinator leader %s %s failed: %r', mpp_handler.type,
                                    coordinator.leader.name, coordinator.leader.member.api_url, e)
 
     def touch_member(self) -> bool:
@@ -363,8 +370,9 @@ class Ha(object):
             tags = self.get_effective_tags()
             if tags:
                 data['tags'] = tags
-            if self.state_handler.pending_restart:
+            if self.state_handler.pending_restart_reason:
                 data['pending_restart'] = True
+                data['pending_restart_reason'] = dict(self.state_handler.pending_restart_reason)
             if self._async_executor.scheduled_action in (None, 'promote') \
                     and data['state'] in ['running', 'restarting', 'starting']:
                 try:
@@ -404,7 +412,7 @@ class Ha(object):
             if ret:
                 new_state = (data['state'], {'master': 'primary'}.get(data['role'], data['role']))
                 if self._last_state != new_state and new_state == ('running', 'primary'):
-                    self.notify_citus_coordinator('after_promote')
+                    self.notify_mpp_coordinator('after_promote')
                 self._last_state = new_state
             return ret
 
@@ -450,7 +458,7 @@ class Ha(object):
             return ret or 'trying to bootstrap {0}'.format(msg)
 
         # no leader, but configuration may allowed replica creation using backup tools
-        create_replica_methods = self.global_config.get_standby_cluster_config().get('create_replica_methods', []) \
+        create_replica_methods = global_config.get_standby_cluster_config().get('create_replica_methods', []) \
             if self.is_standby_cluster() else None
         can_bootstrap = self.state_handler.can_create_replica_without_replication_connection(create_replica_methods)
         concurrent_bootstrap = self.cluster.initialize == ""
@@ -525,7 +533,7 @@ class Ha(object):
         :returns: action message, describing what was performed.
         """
         if self.has_lock() and self.update_lock():
-            timeout = self.global_config.primary_start_timeout
+            timeout = global_config.primary_start_timeout
             if timeout == 0:
                 # We are requested to prefer failing over to restarting primary. But see first if there
                 # is anyone to fail over to.
@@ -602,9 +610,12 @@ class Ha(object):
 
         :returns: the node which we should be replicating from.
         """
+        # nostream is set, the node must not use WAL streaming
+        if self.patroni.nostream:
+            return None
         # The standby leader or when there is no standby leader we want to follow
         # the remote member, except when there is no standby leader in pause.
-        if self.is_standby_cluster() \
+        elif self.is_standby_cluster() \
                 and (cluster.leader and cluster.leader.name and cluster.leader.name == self.state_handler.name
                      or cluster.is_unlocked() and not self.is_paused()):
             node_to_follow = self.get_remote_member()
@@ -622,7 +633,7 @@ class Ha(object):
             for param in params:  # It is highly unlikely to happen, but we want to protect from the case
                 node_to_follow.data.pop(param, None)  # when above-mentioned params came from outside.
             if self.is_standby_cluster():
-                standby_config = self.global_config.get_standby_cluster_config()
+                standby_config = global_config.get_standby_cluster_config()
                 node_to_follow.data.update({p: standby_config[p] for p in params if standby_config.get(p)})
 
         return node_to_follow
@@ -684,11 +695,11 @@ class Ha(object):
 
     def is_synchronous_mode(self) -> bool:
         """:returns: `True` if synchronous replication is requested."""
-        return self.global_config.is_synchronous_mode
+        return global_config.is_synchronous_mode
 
     def is_failsafe_mode(self) -> bool:
         """:returns: `True` if failsafe_mode is enabled in global configuration."""
-        return self.global_config.check_mode('failsafe_mode')
+        return global_config.check_mode('failsafe_mode')
 
     def process_sync_replication(self) -> None:
         """Process synchronous standby beahvior.
@@ -732,7 +743,7 @@ class Ha(object):
                         return logger.info('Synchronous replication key updated by someone else.')
 
                 # When strict mode and no suitable replication connections put "*" to synchronous_standby_names
-                if self.global_config.is_synchronous_mode_strict and not picked:
+                if global_config.is_synchronous_mode_strict and not picked:
                     picked = CaseInsensitiveSet('*')
                     logger.warning("No standbys available!")
 
@@ -805,7 +816,7 @@ class Ha(object):
             cluster_history_dict: Dict[int, List[Any]] = {line[0]: list(line) for line in cluster_history}
             history: List[List[Any]] = list(map(list, self.state_handler.get_history(primary_timeline)))
             if self.cluster.config:
-                history = history[-self.cluster.config.max_timelines_history:]
+                history = history[-global_config.max_timelines_history:]
             for line in history:
                 # enrich current history with promotion timestamps stored in DCS
                 cluster_history_line = cluster_history_dict.get(line[0], [])
@@ -849,7 +860,7 @@ class Ha(object):
             self.state_handler.set_role('master')
             self.process_sync_replication()
             self.update_cluster_history()
-            self.state_handler.citus_handler.sync_pg_dist_node(self.cluster)
+            self.state_handler.mpp_handler.sync_meta_data(self.cluster)
             return message
         elif self.state_handler.role in ('master', 'promoted', 'primary'):
             self.process_sync_replication()
@@ -863,13 +874,13 @@ class Ha(object):
                     # promotion until next cycle. TODO: trigger immediate retry of run_cycle
                     return 'Postponing promotion because synchronous replication state was updated by somebody else'
                 self.state_handler.sync_handler.set_synchronous_standby_names(
-                    CaseInsensitiveSet('*') if self.global_config.is_synchronous_mode_strict else CaseInsensitiveSet())
+                    CaseInsensitiveSet('*') if global_config.is_synchronous_mode_strict else CaseInsensitiveSet())
             if self.state_handler.role not in ('master', 'promoted', 'primary'):
                 # reset failsafe state when promote
                 self._failsafe.set_is_active(0)
 
                 def before_promote():
-                    self.notify_citus_coordinator('before_promote')
+                    self.notify_mpp_coordinator('before_promote')
 
                 with self._async_response:
                     self._async_response.reset()
@@ -974,7 +985,7 @@ class Ha(object):
         :returns True when node is lagging
         """
         lag = (self.cluster.last_lsn or 0) - wal_position
-        return lag > self.global_config.maximum_lag_on_failover
+        return lag > global_config.maximum_lag_on_failover
 
     def _is_healthiest_node(self, members: Collection[Member], check_replication_lag: bool = True) -> bool:
         """This method tries to determine whether I am healthy enough to became a new leader candidate or not."""
@@ -1227,22 +1238,23 @@ class Ha(object):
 
         status = {'released': False}
 
-        def on_shutdown(checkpoint_location: int) -> None:
+        def on_shutdown(checkpoint_location: int, prev_location: int) -> None:
             # Postmaster is still running, but pg_control already reports clean "shut down".
             # It could happen if Postgres is still archiving the backlog of WAL files.
             # If we know that there are replicas that received the shutdown checkpoint
             # location, we can remove the leader key and allow them to start leader race.
+            time.sleep(1)  # give replicas some more time to catch up
             if self.is_failover_possible(cluster_lsn=checkpoint_location):
                 self.state_handler.set_role('demoted')
                 with self._async_executor:
-                    self.release_leader_key_voluntarily(checkpoint_location)
+                    self.release_leader_key_voluntarily(prev_location)
                     status['released'] = True
 
         def before_shutdown() -> None:
-            if self.state_handler.citus_handler.is_coordinator():
-                self.state_handler.citus_handler.on_demote()
+            if self.state_handler.mpp_handler.is_coordinator():
+                self.state_handler.mpp_handler.on_demote()
             else:
-                self.notify_citus_coordinator('before_demote')
+                self.notify_mpp_coordinator('before_demote')
 
         self.state_handler.stop(str(mode_control['stop']), checkpoint=bool(mode_control['checkpoint']),
                                 on_safepoint=self.watchdog.disable if self.watchdog.is_running else None,
@@ -1486,7 +1498,7 @@ class Ha(object):
         if postgres_version and postgres_version_to_int(postgres_version) <= int(self.state_handler.server_version):
             reason_to_cancel = "postgres version mismatch"
 
-        if pending_restart and not self.state_handler.pending_restart:
+        if pending_restart and not self.state_handler.pending_restart_reason:
             reason_to_cancel = "pending restart flag is not set"
 
         if not reason_to_cancel:
@@ -1540,14 +1552,14 @@ class Ha(object):
 
         # Now that restart is scheduled we can set timeout for startup, it will get reset
         # once async executor runs and main loop notices PostgreSQL as up.
-        timeout = restart_data.get('timeout', self.global_config.primary_start_timeout)
+        timeout = restart_data.get('timeout', global_config.primary_start_timeout)
         self.set_start_timeout(timeout)
 
         def before_shutdown() -> None:
-            self.notify_citus_coordinator('before_demote')
+            self.notify_mpp_coordinator('before_demote')
 
         def after_start() -> None:
-            self.notify_citus_coordinator('after_promote')
+            self.notify_mpp_coordinator('after_promote')
 
         # For non async cases we want to wait for restart to complete or timeout before returning.
         do_restart = functools.partial(self.state_handler.restart, timeout, self._async_executor.critical_task,
@@ -1604,7 +1616,7 @@ class Ha(object):
         """Figure out what to do with the task AsyncExecutor is performing."""
         if self.has_lock() and self.update_lock():
             if self._async_executor.scheduled_action == 'doing crash recovery in a single user mode':
-                time_left = self.global_config.primary_start_timeout - (time.time() - self._crash_recovery_started)
+                time_left = global_config.primary_start_timeout - (time.time() - self._crash_recovery_started)
                 if time_left <= 0 and self.is_failover_possible():
                     logger.info("Demoting self because crash recovery is taking too long")
                     self.state_handler.cancellable.cancel(True)
@@ -1689,7 +1701,7 @@ class Ha(object):
         self.set_is_leader(True)
         if self.is_synchronous_mode():
             self.state_handler.sync_handler.set_synchronous_standby_names(
-                CaseInsensitiveSet('*') if self.global_config.is_synchronous_mode_strict else CaseInsensitiveSet())
+                CaseInsensitiveSet('*') if global_config.is_synchronous_mode_strict else CaseInsensitiveSet())
         self.state_handler.call_nowait(CallbackAction.ON_START)
         self.load_cluster_from_dcs()
 
@@ -1712,7 +1724,7 @@ class Ha(object):
                 self.demote('immediate-nolock')
                 return 'stopped PostgreSQL while starting up because leader key was lost'
 
-            timeout = self._start_timeout or self.global_config.primary_start_timeout
+            timeout = self._start_timeout or global_config.primary_start_timeout
             time_left = timeout - self.state_handler.time_in_state()
 
             if time_left <= 0:
@@ -1745,8 +1757,8 @@ class Ha(object):
         try:
             try:
                 self.load_cluster_from_dcs()
-                self.global_config = self.patroni.config.get_global_config(self.cluster)
-                self.state_handler.reset_cluster_info_state(self.cluster, self.patroni.nofailover, self.global_config)
+                global_config.update(self.cluster)
+                self.state_handler.reset_cluster_info_state(self.cluster, self.patroni)
             except Exception:
                 self.state_handler.reset_cluster_info_state(None)
                 raise
@@ -1766,10 +1778,10 @@ class Ha(object):
                 self.touch_member()
 
             # cluster has leader key but not initialize key
-            if not (self.cluster.is_unlocked() or self.sysid_valid(self.cluster.initialize)) and self.has_lock():
+            if self.has_lock(False) and not self.sysid_valid(self.cluster.initialize):
                 self.dcs.initialize(create_new=(self.cluster.initialize is None), sysid=self.state_handler.sysid)
 
-            if not (self.cluster.is_unlocked() or self.cluster.config and self.cluster.config.data) and self.has_lock():
+            if self.has_lock(False) and not (self.cluster.config and self.cluster.config.data):
                 self.dcs.set_config_value(json.dumps(self.patroni.config.dynamic_configuration, separators=(',', ':')))
                 self.cluster = self.dcs.get_cluster()
 
@@ -1850,10 +1862,9 @@ class Ha(object):
                             logger.fatal('system ID mismatch, node %s belongs to a different cluster: %s != %s',
                                          self.state_handler.name, self.cluster.initialize, data_sysid)
                             sys.exit(1)
-                elif self.cluster.is_unlocked() and not self.is_paused():
+                elif self.cluster.is_unlocked() and not self.is_paused() and not self.state_handler.cb_called:
                     # "bootstrap", but data directory is not empty
-                    if not self.state_handler.cb_called and self.state_handler.is_running() \
-                            and not self.state_handler.is_primary():
+                    if self.state_handler.is_running() and not self.state_handler.is_primary():
                         self._join_aborted = True
                         logger.error('No initialize key in DCS and PostgreSQL is running as replica, aborting start')
                         logger.error('Please first start Patroni on the node running as primary')
@@ -1903,7 +1914,7 @@ class Ha(object):
                 if not is_promoting and create_slots and self.cluster.leader:
                     err = self._async_executor.try_run_async('copy_logical_slots',
                                                              self.state_handler.slots_handler.copy_logical_slots,
-                                                             args=(self.cluster, create_slots))
+                                                             args=(self.cluster, self.patroni, create_slots))
                     if not err:
                         ret = 'Copying logical slots {0} from the primary'.format(create_slots)
             return ret
@@ -1959,10 +1970,7 @@ class Ha(object):
         cluster = self._failsafe.update_cluster(self.cluster)\
             if self.is_failsafe_mode() and not self.is_leader() else self.cluster
         if cluster:
-            slots = self.state_handler.slots_handler.sync_replication_slots(cluster,
-                                                                            self.patroni.nofailover,
-                                                                            self.patroni.replicatefrom,
-                                                                            self.is_paused())
+            slots = self.state_handler.slots_handler.sync_replication_slots(cluster, self.patroni)
         # Don't copy replication slots if failsafe_mode is active
         return [] if self.failsafe_is_active() else slots
 
@@ -1990,21 +1998,21 @@ class Ha(object):
 
             status = {'deleted': False}
 
-            def _on_shutdown(checkpoint_location: int) -> None:
+            def _on_shutdown(checkpoint_location: int, prev_location: int) -> None:
                 if self.is_leader():
                     # Postmaster is still running, but pg_control already reports clean "shut down".
                     # It could happen if Postgres is still archiving the backlog of WAL files.
                     # If we know that there are replicas that received the shutdown checkpoint
                     # location, we can remove the leader key and allow them to start leader race.
-
+                    time.sleep(1)  # give replicas some more time to catch up
                     if self.is_failover_possible(cluster_lsn=checkpoint_location):
-                        self.dcs.delete_leader(self.cluster.leader, checkpoint_location)
+                        self.dcs.delete_leader(self.cluster.leader, prev_location)
                         status['deleted'] = True
                     else:
-                        self.dcs.write_leader_optime(checkpoint_location)
+                        self.dcs.write_leader_optime(prev_location)
 
             def _before_shutdown() -> None:
-                self.notify_citus_coordinator('before_demote')
+                self.notify_mpp_coordinator('before_demote')
 
             on_shutdown = _on_shutdown if self.is_leader() else None
             before_shutdown = _before_shutdown if self.is_leader() else None
@@ -2046,7 +2054,7 @@ class Ha(object):
         config or cluster.config.data.
         """
         data: Dict[str, Any] = {}
-        cluster_params = self.global_config.get_standby_cluster_config()
+        cluster_params = global_config.get_standby_cluster_config()
 
         if cluster_params:
             data.update({k: v for k, v in cluster_params.items() if k in RemoteMember.ALLOWED_KEYS})

patroni/log.py

@@ -9,12 +9,15 @@ import sys
 
 from copy import deepcopy
 from logging.handlers import RotatingFileHandler
-from patroni.utils import deep_compare
 from queue import Queue, Full
 from threading import Lock, Thread
 
 from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING
 
+from .utils import deep_compare
+
+type_logformat = Union[List[Union[str, Dict[str, Any], Any]], str, Any]
+
 _LOGGER = logging.getLogger(__name__)
 
 
@@ -157,6 +160,7 @@ class PatroniLogger(Thread):
     .. seealso::
         :class:`QueueHandler`: object used for enqueueing messages in-memory.
 
+    :cvar DEFAULT_TYPE: default type of log format (``plain``).
     :cvar DEFAULT_LEVEL: default logging level (``INFO``).
     :cvar DEFAULT_TRACEBACK_LEVEL: default traceback logging level (``ERROR``).
     :cvar DEFAULT_FORMAT: default format of log messages (``%(asctime)s %(levelname)s: %(message)s``).
@@ -169,6 +173,7 @@ class PatroniLogger(Thread):
     :ivar log_handler_lock: lock used to modify ``log_handler``.
     """
 
+    DEFAULT_TYPE = 'plain'
     DEFAULT_LEVEL = 'INFO'
     DEFAULT_TRACEBACK_LEVEL = 'ERROR'
     DEFAULT_FORMAT = '%(asctime)s %(levelname)s: %(message)s'
@@ -202,28 +207,186 @@ class PatroniLogger(Thread):
         self._proxy_handler = ProxyHandler(self)
         self._root_logger.addHandler(self._proxy_handler)
 
-    def update_loggers(self) -> None:
-        """Configure loggers' log level as defined in ``log.loggers`` section of Patroni configuration.
+    def update_loggers(self, config: Dict[str, Any]) -> None:
+        """Configure custom loggers' log levels.
 
         .. note::
             It creates logger objects that are not defined yet in the log manager.
+
+        :param config: :class:`dict` object with custom loggers configuration, is set either from:
+
+                       * ``log.loggers`` section of Patroni configuration; or
+
+                       * from the method that is trying to make sure that the node name
+                         isn't duplicated (to silence annoying ``urllib3`` WARNING's).
+
+        :Example:
+
+            .. code-block:: python
+
+                update_loggers({'urllib3.connectionpool': 'WARNING'})
         """
-        loggers = deepcopy((self._config or {}).get('loggers') or {})
+        loggers = deepcopy(config)
         for name, logger in self._root_logger.manager.loggerDict.items():
             # ``Placeholder`` is a node in the log manager for which no logger has been defined. We are interested only
             # in the ones that were defined
             if not isinstance(logger, logging.PlaceHolder):
-                # if this logger is present in ``log.loggers`` Patroni configuration, use the configured level,
-                # otherwise use ``logging.NOTSET``, which means it will inherit the level from any parent node up to
-                # the root for which log level is defined.
+                # if this logger is present in *config*, use the configured level, otherwise
+                # use ``logging.NOTSET``, which means it will inherit the level
+                # from any parent node up to the root for which log level is defined.
                 level = loggers.pop(name, logging.NOTSET)
                 logger.setLevel(level)
 
-        # define loggers that do not exist yet and set level as configured in ``log.loggers`` section of configuration.
+        # define loggers that do not exist yet and set level as configured in the *config*
         for name, level in loggers.items():
             logger = self._root_logger.manager.getLogger(name)
             logger.setLevel(level)
 
+    def _is_config_changed(self, config: Dict[str, Any]) -> bool:
+        """Checks if the given config is different from the current one.
+
+        :param config: ``log`` section from Patroni configuration.
+
+        :returns: ``True`` if the config is changed, ``False`` otherwise.
+        """
+        old_config = self._config or {}
+
+        oldlogtype = old_config.get('type', PatroniLogger.DEFAULT_TYPE)
+        logtype = config.get('type', PatroniLogger.DEFAULT_TYPE)
+
+        oldlogformat: type_logformat = old_config.get('format', PatroniLogger.DEFAULT_FORMAT)
+        logformat: type_logformat = config.get('format', PatroniLogger.DEFAULT_FORMAT)
+
+        olddateformat = old_config.get('dateformat') or None
+        dateformat = config.get('dateformat') or None  # Convert empty string to `None`
+
+        old_static_fields = old_config.get('static_fields', {})
+        static_fields = config.get('static_fields', {})
+
+        old_log_config = {
+            'type': oldlogtype,
+            'format': oldlogformat,
+            'dateformat': olddateformat,
+            'static_fields': old_static_fields
+        }
+
+        log_config = {
+            'type': logtype,
+            'format': logformat,
+            'dateformat': dateformat,
+            'static_fields': static_fields
+        }
+
+        return not deep_compare(old_log_config, log_config)
+
+    def _get_plain_formatter(self, logformat: type_logformat, dateformat: Optional[str]) -> logging.Formatter:
+        """Returns a logging formatter with the specified format and date format.
+
+        .. note::
+            If the log format isn't a string, prints a warning message and uses the default log format instead.
+
+        :param logformat: The format of the log messages.
+        :param dateformat: The format of the timestamp in the log messages.
+
+        :returns: A logging formatter object that can be used to format log records.
+        """
+
+        if not isinstance(logformat, str):
+            _LOGGER.warning('Expected log format to be a string when log type is plain, but got "%s"', type(logformat))
+            logformat = PatroniLogger.DEFAULT_FORMAT
+
+        return logging.Formatter(logformat, dateformat)
+
+    def _get_json_formatter(self, logformat: type_logformat, dateformat: Optional[str],
+                            static_fields: Dict[str, Any]) -> logging.Formatter:
+        """Returns a logging formatter that outputs JSON formatted messages.
+
+        .. note::
+            If :mod:`pythonjsonlogger` library is not installed, prints an error message and returns
+            a plain log formatter instead.
+
+        :param logformat: Specifies the log fields and their key names in the JSON log message.
+        :param dateformat: The format of the timestamp in the log messages.
+        :param static_fields: A dictionary of static fields that are added to every log message.
+
+        :returns: A logging formatter object that can be used to format log records as JSON strings.
+        """
+
+        if isinstance(logformat, str):
+            jsonformat = logformat
+            rename_fields = {}
+        elif isinstance(logformat, list):
+            log_fields: List[str] = []
+            rename_fields: Dict[str, str] = {}
+
+            for field in logformat:
+                if isinstance(field, str):
+                    log_fields.append(field)
+                elif isinstance(field, dict):
+                    for original_field, renamed_field in field.items():
+                        if isinstance(renamed_field, str):
+                            log_fields.append(original_field)
+                            rename_fields[original_field] = renamed_field
+                        else:
+                            _LOGGER.warning(
+                                'Expected renamed log field to be a string, but got "%s"',
+                                type(renamed_field)
+                            )
+
+                else:
+                    _LOGGER.warning(
+                        'Expected each item of log format to be a string or dictionary, but got "%s"',
+                        type(field)
+                    )
+
+            if len(log_fields) > 0:
+                jsonformat = ' '.join([f'%({field})s' for field in log_fields])
+            else:
+                jsonformat = PatroniLogger.DEFAULT_FORMAT
+        else:
+            jsonformat = PatroniLogger.DEFAULT_FORMAT
+            rename_fields = {}
+            _LOGGER.warning('Expected log format to be a string or a list, but got "%s"', type(logformat))
+
+        try:
+            from pythonjsonlogger import jsonlogger
+
+            return jsonlogger.JsonFormatter(
+                jsonformat,
+                dateformat,
+                rename_fields=rename_fields,
+                static_fields=static_fields
+            )
+        except ImportError as e:
+            _LOGGER.error('Failed to import "python-json-logger" library: %r. Falling back to the plain logger', e)
+        except Exception as e:
+            _LOGGER.error('Failed to initialize JsonFormatter: %r. Falling back to the plain logger', e)
+
+        return self._get_plain_formatter(jsonformat, dateformat)
+
+    def _get_formatter(self, config: Dict[str, Any]) -> logging.Formatter:
+        """Returns a logging formatter based on the type of logger in the given configuration.
+
+        :param config: ``log`` section from Patroni configuration.
+
+        :returns: A :class:`logging.Formatter` object that can be used to format log records.
+        """
+        logtype = config.get('type', PatroniLogger.DEFAULT_TYPE)
+        logformat: type_logformat = config.get('format', PatroniLogger.DEFAULT_FORMAT)
+        dateformat = config.get('dateformat') or None  # Convert empty string to `None`
+        static_fields = config.get('static_fields', {})
+
+        if dateformat is not None and not isinstance(dateformat, str):
+            _LOGGER.warning('Expected log dateformat to be a string, but got "%s"', type(dateformat))
+            dateformat = None
+
+        if logtype == 'json':
+            formatter = self._get_json_formatter(logformat, dateformat, static_fields)
+        else:
+            formatter = self._get_plain_formatter(logformat, dateformat)
+
+        return formatter
+
     def reload_config(self, config: Dict[str, Any]) -> None:
         """Apply log related configuration.
 
@@ -244,37 +407,34 @@ class PatroniLogger(Thread):
                 # show stack traces as ``ERROR`` log messages
                 logging.Logger.exception = error_exception
 
-            new_handler = None
+            handler = self.log_handler
+
             if 'dir' in config:
-                if not isinstance(self.log_handler, RotatingFileHandler):
-                    new_handler = RotatingFileHandler(os.path.join(config['dir'], __name__))
-                handler = new_handler or self.log_handler
-                if TYPE_CHECKING:  # pragma: no cover
-                    assert isinstance(handler, RotatingFileHandler)
-                handler.maxBytes = int(config.get('file_size', 25000000))  # pyright: ignore [reportGeneralTypeIssues]
+                if not isinstance(handler, RotatingFileHandler):
+                    handler = RotatingFileHandler(os.path.join(config['dir'], __name__))
+
+                max_file_size = int(config.get('file_size', 25000000))
+                handler.maxBytes = max_file_size  # pyright: ignore [reportAttributeAccessIssue]
                 handler.backupCount = int(config.get('file_num', 4))
-            else:
-                if self.log_handler is None or isinstance(self.log_handler, RotatingFileHandler):
-                    new_handler = logging.StreamHandler()
-                handler = new_handler or self.log_handler
+            # we can't use `if not isinstance(handler, logging.StreamHandler)` below,
+            # because RotatingFileHandler is a child of StreamHandler!!!
+            elif handler is None or isinstance(handler, RotatingFileHandler):
+                handler = logging.StreamHandler()
 
-            oldlogformat = (self._config or {}).get('format', PatroniLogger.DEFAULT_FORMAT)
-            logformat = config.get('format', PatroniLogger.DEFAULT_FORMAT)
+            is_new_handler = handler != self.log_handler
 
-            olddateformat = (self._config or {}).get('dateformat') or None
-            dateformat = config.get('dateformat') or None  # Convert empty string to `None`
+            if (self._is_config_changed(config) or is_new_handler) and handler:
+                formatter = self._get_formatter(config)
+                handler.setFormatter(formatter)
 
-            if (oldlogformat != logformat or olddateformat != dateformat or new_handler) and handler:
-                handler.setFormatter(logging.Formatter(logformat, dateformat))
-
-            if new_handler:
+            if is_new_handler:
                 with self.log_handler_lock:
                     if self.log_handler:
                         self._old_handlers.append(self.log_handler)
-                    self.log_handler = new_handler
+                    self.log_handler = handler
 
             self._config = config.copy()
-            self.update_loggers()
+            self.update_loggers(config.get('loggers') or {})
 
     def _close_old_handlers(self) -> None:
         """Close old log handlers.

patroni/postgresql/__init__.py

@@ -19,22 +19,22 @@ from .callback_executor import CallbackAction, CallbackExecutor
 from .cancellable import CancellableSubprocess
 from .config import ConfigHandler, mtime
 from .connection import ConnectionPool, get_connection_cursor
-from .citus import CitusHandler
 from .misc import parse_history, parse_lsn, postgres_major_version_to_int
+from .mpp import AbstractMPP
 from .postmaster import PostmasterProcess
 from .slots import SlotsHandler
 from .sync import SyncHandler
-from .. import psycopg
+from .. import global_config, psycopg
 from ..async_executor import CriticalTask
-from ..collections import CaseInsensitiveSet
+from ..collections import CaseInsensitiveSet, CaseInsensitiveDict, EMPTY_DICT
 from ..dcs import Cluster, Leader, Member, SLOT_ADVANCE_AVAILABLE_VERSION
 from ..exceptions import PostgresConnectionException
 from ..utils import Retry, RetryFailedError, polling_loop, data_directory_is_empty, parse_int
+from ..tags import Tags
 
 if TYPE_CHECKING:  # pragma: no cover
     from psycopg import Connection as Connection3, Cursor
     from psycopg2 import connection as connection3, cursor
-    from ..config import GlobalConfig
 
 logger = logging.getLogger(__name__)
 
@@ -63,7 +63,7 @@ class Postgresql(object):
               "pg_catalog.pg_{0}_{1}_diff(COALESCE(pg_catalog.pg_last_{0}_receive_{1}(), '0/0'), '0/0')::bigint, "
               "pg_catalog.pg_is_in_recovery() AND pg_catalog.pg_is_{0}_replay_paused()")
 
-    def __init__(self, config: Dict[str, Any]) -> None:
+    def __init__(self, config: Dict[str, Any], mpp: AbstractMPP) -> None:
         self.name: str = config['name']
         self.scope: str = config['scope']
         self._data_dir: str = config['data_dir']
@@ -73,15 +73,14 @@ class Postgresql(object):
         self.connection_string: str
         self.proxy_url: Optional[str]
         self._major_version = self.get_major_version()
-        self._global_config = None
 
         self._state_lock = Lock()
         self.set_state('stopped')
 
-        self._pending_restart = False
+        self._pending_restart_reason = CaseInsensitiveDict()
         self.connection_pool = ConnectionPool()
         self._connection = self.connection_pool.get('heartbeat')
-        self.citus_handler = CitusHandler(self, config.get('citus'))
+        self.mpp_handler = mpp.get_handler_impl(self)
         self.config = ConfigHandler(self, config)
         self.config.check_directories()
 
@@ -119,6 +118,8 @@ class Postgresql(object):
         # Last known running process
         self._postmaster_proc = None
 
+        self._available_gucs = None
+
         if self.is_running():
             # If we found postmaster process we need to figure out whether postgres is accepting connections
             self.set_state('starting')
@@ -217,7 +218,7 @@ class Postgresql(object):
                          "FROM pg_catalog.pg_stat_get_wal_senders() w,"
                          " pg_catalog.pg_stat_get_activity(w.pid)"
                          " WHERE w.state = 'streaming') r)").format(self.wal_name, self.lsn_name)
-                        if (not self.global_config or self.global_config.is_synchronous_mode)
+                        if global_config.is_synchronous_mode
                         and self.role in ('master', 'primary', 'promoted') else "'on', '', NULL")
 
         if self._major_version >= 90600:
@@ -241,7 +242,9 @@ class Postgresql(object):
     @property
     def available_gucs(self) -> CaseInsensitiveSet:
         """GUCs available in this Postgres server."""
-        return self._get_gucs()
+        if not self._available_gucs:
+            self._available_gucs = self._get_gucs()
+        return self._available_gucs
 
     def _version_file_exists(self) -> bool:
         return not self.data_directory_empty() and os.path.isfile(self._version_file)
@@ -269,7 +272,7 @@ class Postgresql(object):
 
         :returns: path to Postgres binary named *cmd*.
         """
-        return os.path.join(self._bin_dir, (self.config.get('bin_name', {}) or {}).get(cmd, cmd))
+        return os.path.join(self._bin_dir, (self.config.get('bin_name', {}) or EMPTY_DICT).get(cmd, cmd))
 
     def pg_ctl(self, cmd: str, *args: str, **kwargs: Any) -> bool:
         """Builds and executes pg_ctl command
@@ -318,11 +321,22 @@ class Postgresql(object):
         self._is_leader_retry.deadline = self.retry.deadline = config['retry_timeout'] / 2.0
 
     @property
-    def pending_restart(self) -> bool:
-        return self._pending_restart
+    def pending_restart_reason(self) -> CaseInsensitiveDict:
+        """Get :attr:`_pending_restart_reason` value.
 
-    def set_pending_restart(self, value: bool) -> None:
-        self._pending_restart = value
+        :attr:`_pending_restart_reason` is a :class:`CaseInsensitiveDict` object of the PG parameters that are
+        causing pending restart state. Every key is a parameter name, value - a dictionary containing the old
+        and the new value (see :func:`~patroni.postgresql.config.get_param_diff`).
+        """
+        return self._pending_restart_reason
+
+    def set_pending_restart_reason(self, diff_dict: CaseInsensitiveDict) -> None:
+        """Set new or update current :attr:`_pending_restart_reason`.
+
+        :param diff_dict: :class:``CaseInsensitiveDict`` object with the parameters that are causing pending restart
+            state with the diff of their values. Used to reset/update the :attr:`_pending_restart_reason`.
+        """
+        self._pending_restart_reason = diff_dict
 
     @property
     def sysid(self) -> str:
@@ -400,7 +414,7 @@ class Postgresql(object):
         return data_directory_is_empty(self._data_dir)
 
     def replica_method_options(self, method: str) -> Dict[str, Any]:
-        return deepcopy(self.config.get(method, {}) or {})
+        return deepcopy(self.config.get(method, {}) or EMPTY_DICT.copy())
 
     def replica_method_can_work_without_replication_connection(self, method: str) -> bool:
         return method != 'basebackup' and bool(self.replica_method_options(method).get('no_master')
@@ -426,46 +440,30 @@ class Postgresql(object):
                 self.config.write_postgresql_conf()
                 self.reload()
 
-    @property
-    def global_config(self) -> Optional['GlobalConfig']:
-        return self._global_config
-
-    def reset_cluster_info_state(self, cluster: Union[Cluster, None], nofailover: bool = False,
-                                 global_config: Optional['GlobalConfig'] = None) -> None:
+    def reset_cluster_info_state(self, cluster: Optional[Cluster], tags: Optional[Tags] = None) -> None:
         """Reset monitoring query cache.
 
-        It happens in the beginning of heart-beat loop and on change of `synchronous_standby_names`.
+        .. note::
+            It happens in the beginning of heart-beat loop and on change of `synchronous_standby_names`.
 
         :param cluster: currently known cluster state from DCS
-        :param nofailover: whether this node could become a new primary.
-                           Important when there are logical permanent replication slots because "nofailover"
-                           node could do cascading replication and should enable `hot_standby_feedback`
-        :param global_config: last known :class:`GlobalConfig` object
+        :param tags: reference to an object implementing :class:`Tags` interface.
         """
         self._cluster_info_state = {}
 
-        if global_config:
-            self._global_config = global_config
-
-        if not self._global_config:
+        if not tags:
             return
 
-        if self._global_config.is_standby_cluster:
+        if global_config.is_standby_cluster:
             # Standby cluster can't have logical replication slots, and we don't need to enforce hot_standby_feedback
             self.set_enforce_hot_standby_feedback(False)
 
         if cluster and cluster.config and cluster.config.modify_version:
             # We want to enable hot_standby_feedback if the replica is supposed
             # to have a logical slot or in case if it is the cascading replica.
-            self.set_enforce_hot_standby_feedback(not self._global_config.is_standby_cluster and self.can_advance_slots
-                                                  and cluster.should_enforce_hot_standby_feedback(self.name,
-                                                                                                  nofailover))
-
-            self._has_permanent_slots = cluster.has_permanent_slots(
-                my_name=self.name,
-                is_standby_cluster=self._global_config.is_standby_cluster,
-                nofailover=nofailover,
-                major_version=self.major_version)
+            self.set_enforce_hot_standby_feedback(not global_config.is_standby_cluster and self.can_advance_slots
+                                                  and cluster.should_enforce_hot_standby_feedback(self, tags))
+            self._has_permanent_slots = cluster.has_permanent_slots(self, tags)
 
     def _cluster_info_state_get(self, name: str) -> Optional[Any]:
         if not self._cluster_info_state:
@@ -591,14 +589,17 @@ class Postgresql(object):
                 return match.group(1), match.group(2), match.group(3), match.group(4)
         return None, None, None, None
 
-    def latest_checkpoint_location(self) -> Optional[int]:
-        """Returns checkpoint location for the cleanly shut down primary.
-           But, if we know that the checkpoint was written to the new WAL
-           due to the archive_mode=on, we will return the LSN of prev wal record (SWITCH)."""
+    def _checkpoint_locations_from_controldata(self, data: Dict[str, str]) -> Optional[Tuple[int, int]]:
+        """Get shutdown checkpoint location.
 
-        data = self.controldata()
+        :param data: :class:`dict` object with values returned by `pg_controldata` tool.
+
+        :returns: a tuple of checkpoint LSN for the cleanly shut down primary, and LSN of prev wal record (SWITCH)
+                  if we know that the checkpoint was written to the new WAL file due to the archive_mode=on.
+        """
         timeline = data.get("Latest checkpoint's TimeLineID")
         lsn = checkpoint_lsn = data.get('Latest checkpoint location')
+        prev_lsn = None
         if data.get('Database cluster state') == 'shut down' and lsn and timeline and checkpoint_lsn:
             try:
                 checkpoint_lsn = parse_lsn(checkpoint_lsn)
@@ -609,13 +610,26 @@ class Postgresql(object):
                     _, lsn, _, desc = self.parse_wal_record(timeline, prev)
                     prev = parse_lsn(prev)
                     # If the cluster is shutdown with archive_mode=on, WAL is switched before writing the checkpoint.
-                    # In this case we want to take the LSN of previous record (switch) as the last known WAL location.
+                    # In this case we want to take the LSN of previous record (SWITCH) as the last known WAL location.
                     if lsn and parse_lsn(lsn) == prev and str(desc).strip() in ('xlog switch', 'SWITCH'):
-                        return prev
+                        prev_lsn = prev
             except Exception as e:
                 logger.error('Exception when parsing WAL pg_%sdump output: %r', self.wal_name, e)
             if isinstance(checkpoint_lsn, int):
-                return checkpoint_lsn
+                return checkpoint_lsn, (prev_lsn or checkpoint_lsn)
+
+    def latest_checkpoint_location(self) -> Optional[int]:
+        """Get shutdown checkpoint location.
+
+        .. note::
+            In case if checkpoint was written to the new WAL file due to the archive_mode=on
+            we return LSN of the previous wal record (SWITCH).
+
+        :returns: checkpoint LSN for the cleanly shut down primary.
+        """
+        checkpoint_locations = self._checkpoint_locations_from_controldata(self.controldata())
+        if checkpoint_locations:
+            return checkpoint_locations[1]
 
     def is_running(self) -> Optional[PostmasterProcess]:
         """Returns PostmasterProcess if one is running on the data directory or None. If most recently seen process
@@ -724,7 +738,7 @@ class Postgresql(object):
         self.set_role(role or self.get_postgres_role_from_data_directory())
 
         self.set_state('starting')
-        self._pending_restart = False
+        self.set_pending_restart_reason(CaseInsensitiveDict())
 
         try:
             if not self.ensure_major_version_is_known():
@@ -801,7 +815,7 @@ class Postgresql(object):
             return 'not accessible or not healty'
 
     def stop(self, mode: str = 'fast', block_callbacks: bool = False, checkpoint: Optional[bool] = None,
-             on_safepoint: Optional[Callable[..., Any]] = None, on_shutdown: Optional[Callable[[int], Any]] = None,
+             on_safepoint: Optional[Callable[..., Any]] = None, on_shutdown: Optional[Callable[[int, int], Any]] = None,
              before_shutdown: Optional[Callable[..., Any]] = None, stop_timeout: Optional[int] = None) -> bool:
         """Stop PostgreSQL
 
@@ -831,7 +845,7 @@ class Postgresql(object):
         return success
 
     def _do_stop(self, mode: str, block_callbacks: bool, checkpoint: bool,
-                 on_safepoint: Optional[Callable[..., Any]], on_shutdown: Optional[Callable[..., Any]],
+                 on_safepoint: Optional[Callable[..., Any]], on_shutdown: Optional[Callable[[int, int], Any]],
                  before_shutdown: Optional[Callable[..., Any]], stop_timeout: Optional[int]) -> Tuple[bool, bool]:
         postmaster = self.is_running()
         if not postmaster:
@@ -871,7 +885,9 @@ class Postgresql(object):
             while postmaster.is_running():
                 data = self.controldata()
                 if data.get('Database cluster state', '') == 'shut down':
-                    on_shutdown(self.latest_checkpoint_location())
+                    checkpoint_locations = self._checkpoint_locations_from_controldata(data)
+                    if checkpoint_locations:
+                        on_shutdown(*checkpoint_locations)
                     break
                 elif data.get('Database cluster state', '').startswith('shut down'):  # shut down in recovery
                     break
@@ -1192,7 +1208,7 @@ class Postgresql(object):
             before_promote()
 
         self.slots_handler.on_promote()
-        self.citus_handler.schedule_cache_rebuild()
+        self.mpp_handler.schedule_cache_rebuild()
 
         ret = self.pg_ctl('promote', '-W')
         if ret:
@@ -1339,7 +1355,7 @@ class Postgresql(object):
         """
         self.ensure_major_version_is_known()
         self.slots_handler.schedule()
-        self.citus_handler.schedule_cache_rebuild()
+        self.mpp_handler.schedule_cache_rebuild()
         self._sysid = ''
 
     def _get_gucs(self) -> CaseInsensitiveSet:

patroni/postgresql/available_parameters/__init__.py

@@ -0,0 +1,59 @@
+import logging
+import sys
+
+from typing import Iterator
+
+logger = logging.getLogger(__name__)
+
+if sys.version_info < (3, 9):  # pragma: no cover
+    from pathlib import Path
+
+    PathLikeObj = Path
+    conf_dir = Path(__file__).parent
+else:
+    from importlib.resources import files
+
+    if sys.version_info < (3, 11):  # pragma: no cover
+        from importlib.abc import Traversable
+    else:  # pragma: no cover
+        from importlib.resources.abc import Traversable
+
+    PathLikeObj = Traversable
+    conf_dir = files(__name__)
+
+
+def get_validator_files() -> Iterator[PathLikeObj]:
+    """Recursively find YAML files from the current package directory.
+
+    :returns: an iterator of :class:`PathLikeObj` objects representing validator files.
+    """
+    return _traversable_walk(conf_dir.iterdir())
+
+
+def _traversable_walk(tvbs: Iterator[PathLikeObj]) -> Iterator[PathLikeObj]:
+    """Recursively walk through Path/Traversable objects, yielding all YAML files in deterministic order.
+
+    :param tvbs: An iterator over :class:`PathLikeObj` objects, where each object is a file or directory
+                 that potentially contains YAML files.
+
+    :yields: :class:`PathLikeObj` objects representing YAML files found during the traversal.
+    """
+    for tvb in _filter_and_sort_files(tvbs):
+        if tvb.is_file():
+            yield tvb
+        elif tvb.is_dir():
+            yield from _traversable_walk(tvb.iterdir())
+
+
+def _filter_and_sort_files(files: Iterator[PathLikeObj]) -> Iterator[PathLikeObj]:
+    """Sort files by name, and filter out non-YAML files and Python files.
+
+    :param files: A list of files and/or directories to be filtered and sorted.
+
+    :yields: filtered and sorted objects.
+    """
+    for file in sorted(files, key=lambda x: x.name):
+        if file.name.lower().endswith((".yml", ".yaml")) or file.is_dir():
+            yield file
+        elif not file.name.lower().endswith((".py", ".pyc")):
+            logger.info("Ignored a non-YAML file found under `%s` directory: `%s`.", __name__.split('.')[-1], file)

patroni/postgresql/bootstrap.py

@@ -7,6 +7,7 @@ import time
 from typing import Any, Callable, Dict, List, Optional, Union, Tuple, TYPE_CHECKING
 
 from ..async_executor import CriticalTask
+from ..collections import EMPTY_DICT
 from ..dcs import Leader, Member, RemoteMember
 from ..psycopg import quote_ident, quote_literal
 from ..utils import deep_compare, unquote
@@ -100,10 +101,11 @@ class Bootstrap(object):
                     user_options.append('--{0}'.format(opt))
                 elif isinstance(opt, dict):
                     keys = list(opt.keys())
-                    if len(keys) != 1 or not isinstance(opt[keys[0]], str) or not option_is_allowed(keys[0]):
+                    if len(keys) == 1 and isinstance(opt[keys[0]], str) and option_is_allowed(keys[0]):
+                        user_options.append('--{0}={1}'.format(keys[0], unquote(opt[keys[0]])))
+                    else:
                         error_handler('Error when parsing {0} key-value option {1}: only one key-value is allowed'
                                       ' and value should be a string'.format(tool, opt[keys[0]]))
-                    user_options.append('--{0}={1}'.format(keys[0], unquote(opt[keys[0]])))
                 else:
                     error_handler('Error when parsing {0} option {1}: value should be string value'
                                   ' or a single key-value pair'.format(tool, opt))
@@ -145,7 +147,7 @@ class Bootstrap(object):
 
         # make sure there is no trigger file or postgres will be automatically promoted
         trigger_file = self._postgresql.config.triggerfile_good_name
-        trigger_file = (self._postgresql.config.get('recovery_conf') or {}).get(trigger_file) or 'promote'
+        trigger_file = (self._postgresql.config.get('recovery_conf') or EMPTY_DICT).get(trigger_file) or 'promote'
         trigger_file = os.path.abspath(os.path.join(self._postgresql.data_dir, trigger_file))
         if os.path.exists(trigger_file):
             os.unlink(trigger_file)
@@ -188,10 +190,9 @@ class Bootstrap(object):
         params = [] if config.get('no_params') else ['--scope=' + self._postgresql.scope,
                                                      '--datadir=' + self._postgresql.data_dir]
         # Add custom parameters specified by the user
-        reserved_args = {'no_params', 'keep_existing_recovery_conf', 'recovery_conf', 'scope', 'datadir'}
-        for arg, val in config.items():
-            if arg not in reserved_args:
-                params.append(f"--{arg}={val}")
+        reserved_args = {'command', 'no_params', 'keep_existing_recovery_conf', 'recovery_conf', 'scope', 'datadir'}
+        params += [f"--{arg}={val}" for arg, val in config.items() if arg not in reserved_args]
+
         try:
             logger.info('Running custom bootstrap script: %s', config['command'])
             if self._postgresql.cancellable.call(shlex.split(config['command']) + params) != 0:
@@ -441,7 +442,7 @@ END;$$""".format(f, quote_ident(rewind['username'], postgresql.connection()))
                 if config.get('users'):
                     logger.warning('User creation via "bootstrap.users" will be removed in v4.0.0')
 
-                for name, value in (config.get('users') or {}).items():
+                for name, value in (config.get('users') or EMPTY_DICT).items():
                     if all(name != a.get('username') for a in (superuser, replication, rewind)):
                         self.create_or_update_role(name, value.get('password'), value.get('options', []))
 
@@ -464,15 +465,15 @@ END;$$""".format(f, quote_ident(rewind['username'], postgresql.connection()))
                         postgresql.restart()
                     else:
                         postgresql.config.replace_pg_hba()
-                        if postgresql.pending_restart:
+                        if postgresql.pending_restart_reason:
                             postgresql.restart()
                         else:
                             postgresql.reload()
                             time.sleep(1)  # give a time to postgres to "reload" configuration files
                             postgresql.connection().close()  # close connection to reconnect with a new password
                 else:  # initdb
-                    # We may want create database and extension for citus
-                    self._postgresql.citus_handler.bootstrap()
+                    # We may want create database and extension for some MPP clusters
+                    self._postgresql.mpp_handler.bootstrap()
         except Exception:
             logger.exception('post_bootstrap')
             task.complete(False)

patroni/postgresql/cancellable.py

@@ -100,7 +100,8 @@ class CancellableSubprocess(CancellableExecutor):
 
             if started and self._process is not None:
                 if isinstance(communicate, dict):
-                    communicate['stdout'], communicate['stderr'] = self._process.communicate(input_data)
+                    communicate['stdout'], communicate['stderr'] = \
+                        self._process.communicate(input_data)  # pyright: ignore [reportGeneralTypeIssues]
                 return self._process.wait()
         finally:
             with self._lock:

patroni/postgresql/citus.py

@@ -1,414 +0,0 @@
-import logging
-import re
-import time
-
-from threading import Condition, Event, Thread
-from urllib.parse import urlparse
-from typing import Any, Dict, List, Optional, Union, Tuple, TYPE_CHECKING
-
-from ..dcs import CITUS_COORDINATOR_GROUP_ID, Cluster
-from ..psycopg import connect, quote_ident
-
-if TYPE_CHECKING:  # pragma: no cover
-    from . import Postgresql
-
-CITUS_SLOT_NAME_RE = re.compile(r'^citus_shard_(move|split)_slot(_[1-9][0-9]*){2,3}$')
-logger = logging.getLogger(__name__)
-
-
-class PgDistNode(object):
-    """Represents a single row in the `pg_dist_node` table"""
-
-    def __init__(self, group: int, host: str, port: int, event: str, nodeid: Optional[int] = None,
-                 timeout: Optional[float] = None, cooldown: Optional[float] = None) -> None:
-        self.group = group
-        # A weird way of pausing client connections by adding the `-demoted` suffix to the hostname
-        self.host = host + ('-demoted' if event == 'before_demote' else '')
-        self.port = port
-        # Event that is trying to change or changed the given row.
-        # Possible values: before_demote, before_promote, after_promote.
-        self.event = event
-        self.nodeid = nodeid
-
-        # If transaction was started, we need to COMMIT/ROLLBACK before the deadline
-        self.timeout = timeout
-        self.cooldown = cooldown or 10000  # 10s by default
-        self.deadline: float = 0
-
-        # All changes in the pg_dist_node are serialized on the Patroni
-        # side by performing them from a thread. The thread, that is
-        # requested a change, sometimes needs to wait for a result.
-        # For example, we want to pause client connections before demoting
-        # the worker, and once it is done notify the calling thread.
-        self._event = Event()
-
-    def wait(self) -> None:
-        self._event.wait()
-
-    def wakeup(self) -> None:
-        self._event.set()
-
-    def __eq__(self, other: Any) -> bool:
-        return isinstance(other, PgDistNode) and self.event == other.event\
-            and self.host == other.host and self.port == other.port
-
-    def __ne__(self, other: Any) -> bool:
-        return not self == other
-
-    def __str__(self) -> str:
-        return ('PgDistNode(nodeid={0},group={1},host={2},port={3},event={4})'
-                .format(self.nodeid, self.group, self.host, self.port, self.event))
-
-    def __repr__(self) -> str:
-        return str(self)
-
-
-class CitusHandler(Thread):
-
-    def __init__(self, postgresql: 'Postgresql', config: Optional[Dict[str, Union[str, int]]]) -> None:
-        super(CitusHandler, self).__init__()
-        self.daemon = True
-        self._postgresql = postgresql
-        self._config = config
-        if config:
-            self._connection = postgresql.connection_pool.get(
-                'citus', {'dbname': config['database'],
-                          'options': '-c statement_timeout=0 -c idle_in_transaction_session_timeout=0'})
-        self._pg_dist_node: Dict[int, PgDistNode] = {}  # Cache of pg_dist_node: {groupid: PgDistNode()}
-        self._tasks: List[PgDistNode] = []  # Requests to change pg_dist_node, every task is a `PgDistNode`
-        self._in_flight: Optional[PgDistNode] = None  # Reference to the `PgDistNode` being changed in a transaction
-        self._schedule_load_pg_dist_node = True  # Flag that "pg_dist_node" should be queried from the database
-        self._condition = Condition()  # protects _pg_dist_node, _tasks, _in_flight, and _schedule_load_pg_dist_node
-        self.schedule_cache_rebuild()
-
-    def is_enabled(self) -> bool:
-        return isinstance(self._config, dict)
-
-    def group(self) -> Optional[int]:
-        return int(self._config['group']) if isinstance(self._config, dict) else None
-
-    def is_coordinator(self) -> bool:
-        return self.is_enabled() and self.group() == CITUS_COORDINATOR_GROUP_ID
-
-    def is_worker(self) -> bool:
-        return self.is_enabled() and not self.is_coordinator()
-
-    def schedule_cache_rebuild(self) -> None:
-        with self._condition:
-            self._schedule_load_pg_dist_node = True
-
-    def on_demote(self) -> None:
-        with self._condition:
-            self._pg_dist_node.clear()
-            self._tasks[:] = []
-            self._in_flight = None
-
-    def query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]:
-        try:
-            logger.debug('query(%s, %s)', sql, params)
-            return self._connection.query(sql, *params)
-        except Exception as e:
-            logger.error('Exception when executing query "%s", (%s): %r', sql, params, e)
-            self._connection.close()
-            with self._condition:
-                self._in_flight = None
-            self.schedule_cache_rebuild()
-            raise e
-
-    def load_pg_dist_node(self) -> bool:
-        """Read from the `pg_dist_node` table and put it into the local cache"""
-
-        with self._condition:
-            if not self._schedule_load_pg_dist_node:
-                return True
-            self._schedule_load_pg_dist_node = False
-
-        try:
-            rows = self.query("SELECT nodeid, groupid, nodename, nodeport, noderole"
-                              " FROM pg_catalog.pg_dist_node WHERE noderole = 'primary'")
-        except Exception:
-            return False
-
-        with self._condition:
-            self._pg_dist_node = {r[1]: PgDistNode(r[1], r[2], r[3], 'after_promote', r[0]) for r in rows}
-        return True
-
-    def sync_pg_dist_node(self, cluster: Cluster) -> None:
-        """Maintain the `pg_dist_node` from the coordinator leader every heartbeat loop.
-
-        We can't always rely on REST API calls from worker nodes in order
-        to maintain `pg_dist_node`, therefore at least once per heartbeat
-        loop we make sure that workes registered in `self._pg_dist_node`
-        cache are matching the cluster view from DCS by creating tasks
-        the same way as it is done from the REST API."""
-
-        if not self.is_coordinator():
-            return
-
-        with self._condition:
-            if not self.is_alive():
-                self.start()
-
-        self.add_task('after_promote', CITUS_COORDINATOR_GROUP_ID, self._postgresql.connection_string)
-
-        for group, worker in cluster.workers.items():
-            leader = worker.leader
-            if leader and leader.conn_url\
-                    and leader.data.get('role') in ('master', 'primary') and leader.data.get('state') == 'running':
-                self.add_task('after_promote', group, leader.conn_url)
-
-    def find_task_by_group(self, group: int) -> Optional[int]:
-        for i, task in enumerate(self._tasks):
-            if task.group == group:
-                return i
-
-    def pick_task(self) -> Tuple[Optional[int], Optional[PgDistNode]]:
-        """Returns the tuple(i, task), where `i` - is the task index in the self._tasks list
-
-        Tasks are picked by following priorities:
-
-        1. If there is already a transaction in progress, pick a task
-           that that will change already affected worker primary.
-        2. If the coordinator address should be changed - pick a task
-           with group=0 (coordinators are always in group 0).
-        3. Pick a task that is the oldest (first from the self._tasks)
-        """
-
-        with self._condition:
-            if self._in_flight:
-                i = self.find_task_by_group(self._in_flight.group)
-            else:
-                while True:
-                    i = self.find_task_by_group(CITUS_COORDINATOR_GROUP_ID)  # set_coordinator
-                    if i is None and self._tasks:
-                        i = 0
-                    if i is None:
-                        break
-                    task = self._tasks[i]
-                    if task == self._pg_dist_node.get(task.group):
-                        self._tasks.pop(i)  # nothing to do because cached version of pg_dist_node already matches
-                    else:
-                        break
-            task = self._tasks[i] if i is not None else None
-
-            # When tasks are added it could happen that self._pg_dist_node
-            # wasn't ready (self._schedule_load_pg_dist_node is False)
-            # and hence the nodeid wasn't filled.
-            if task and task.group in self._pg_dist_node:
-                task.nodeid = self._pg_dist_node[task.group].nodeid
-            return i, task
-
-    def update_node(self, task: PgDistNode) -> None:
-        if task.nodeid is not None:
-            self.query('SELECT pg_catalog.citus_update_node(%s, %s, %s, true, %s)',
-                       task.nodeid, task.host, task.port, task.cooldown)
-        elif task.event != 'before_demote':
-            task.nodeid = self.query("SELECT pg_catalog.citus_add_node(%s, %s, %s, 'primary', 'default')",
-                                     task.host, task.port, task.group)[0][0]
-
-    def process_task(self, task: PgDistNode) -> bool:
-        """Updates a single row in `pg_dist_node` table, optionally in a transaction.
-
-        The transaction is started if we do a demote of the worker node or before promoting the other worker if
-        there is no transaction in progress. And, the transaction is committed when the switchover/failover completed.
-
-        .. note:
-            The maximum lifetime of the transaction in progress is controlled outside of this method.
-
-        .. note:
-            Read access to `self._in_flight` isn't protected because we know it can't be changed outside of our thread.
-
-        :param task: reference to a :class:`PgDistNode` object that represents a row to be updated/created.
-        :returns: `True` if the row was succesfully created/updated or transaction in progress
-            was committed as an indicator that the `self._pg_dist_node` cache should be updated,
-            or, if the new transaction was opened, this method returns `False`.
-        """
-
-        if task.event == 'after_promote':
-            # The after_promote may happen without previous before_demote and/or
-            # before_promore.  In this case we just call self.update_node() method.
-            # If there is a transaction in progress, it could be that it already did
-            # required changes and we can simply COMMIT.
-            if not self._in_flight or self._in_flight.host != task.host or self._in_flight.port != task.port:
-                self.update_node(task)
-            if self._in_flight:
-                self.query('COMMIT')
-            return True
-        else:  # before_demote, before_promote
-            if task.timeout:
-                task.deadline = time.time() + task.timeout
-            if not self._in_flight:
-                self.query('BEGIN')
-            self.update_node(task)
-        return False
-
-    def process_tasks(self) -> None:
-        while True:
-            # Read access to `_in_flight` isn't protected because we know it can't be changed outside of our thread.
-            if not self._in_flight and not self.load_pg_dist_node():
-                break
-
-            i, task = self.pick_task()
-            if not task or i is None:
-                break
-            try:
-                update_cache = self.process_task(task)
-            except Exception as e:
-                logger.error('Exception when working with pg_dist_node: %r', e)
-                update_cache = None
-            with self._condition:
-                if self._tasks:
-                    if update_cache:
-                        self._pg_dist_node[task.group] = task
-
-                    if update_cache is False:  # an indicator that process_tasks has started a transaction
-                        self._in_flight = task
-                    else:
-                        self._in_flight = None
-
-                    if id(self._tasks[i]) == id(task):
-                        self._tasks.pop(i)
-            task.wakeup()
-
-    def run(self) -> None:
-        while True:
-            try:
-                with self._condition:
-                    if self._schedule_load_pg_dist_node:
-                        timeout = -1
-                    elif self._in_flight:
-                        timeout = self._in_flight.deadline - time.time() if self._tasks else None
-                    else:
-                        timeout = -1 if self._tasks else None
-
-                    if timeout is None or timeout > 0:
-                        self._condition.wait(timeout)
-                    elif self._in_flight:
-                        logger.warning('Rolling back transaction. Last known status: %s', self._in_flight)
-                        self.query('ROLLBACK')
-                        self._in_flight = None
-                self.process_tasks()
-            except Exception:
-                logger.exception('run')
-
-    def _add_task(self, task: PgDistNode) -> bool:
-        with self._condition:
-            i = self.find_task_by_group(task.group)
-
-            # The `PgDistNode.timeout` == None is an indicator that it was scheduled from the sync_pg_dist_node().
-            if task.timeout is None:
-                # We don't want to override the already existing task created from REST API.
-                if i is not None and self._tasks[i].timeout is not None:
-                    return False
-
-                # There is a little race condition with tasks created from REST API - the call made "before" the member
-                # key is updated in DCS. Therefore it is possible that :func:`sync_pg_dist_node` will try to create a
-                # task based on the outdated values of "state"/"role". To solve it we introduce an artificial timeout.
-                # Only when the timeout is reached new tasks could be scheduled from sync_pg_dist_node()
-                if self._in_flight and self._in_flight.group == task.group and self._in_flight.timeout is not None\
-                        and self._in_flight.deadline > time.time():
-                    return False
-
-            # Override already existing task for the same worker group
-            if i is not None:
-                if task != self._tasks[i]:
-                    logger.debug('Overriding existing task: %s != %s', self._tasks[i], task)
-                    self._tasks[i] = task
-                    self._condition.notify()
-                    return True
-            # Add the task to the list if Worker node state is different from the cached `pg_dist_node`
-            elif self._schedule_load_pg_dist_node or task != self._pg_dist_node.get(task.group)\
-                    or self._in_flight and task.group == self._in_flight.group:
-                logger.debug('Adding the new task: %s', task)
-                self._tasks.append(task)
-                self._condition.notify()
-                return True
-        return False
-
-    def add_task(self, event: str, group: int, conn_url: str,
-                 timeout: Optional[float] = None, cooldown: Optional[float] = None) -> Optional[PgDistNode]:
-        try:
-            r = urlparse(conn_url)
-        except Exception as e:
-            return logger.error('Failed to parse connection url %s: %r', conn_url, e)
-        host = r.hostname
-        if host:
-            port = r.port or 5432
-            task = PgDistNode(group, host, port, event, timeout=timeout, cooldown=cooldown)
-            return task if self._add_task(task) else None
-
-    def handle_event(self, cluster: Cluster, event: Dict[str, Any]) -> None:
-        if not self.is_alive():
-            return
-
-        worker = cluster.workers.get(event['group'])
-        if not (worker and worker.leader and worker.leader.name == event['leader'] and worker.leader.conn_url):
-            return
-
-        task = self.add_task(event['type'], event['group'],
-                             worker.leader.conn_url,
-                             event['timeout'], event['cooldown'] * 1000)
-        if task and event['type'] == 'before_demote':
-            task.wait()
-
-    def bootstrap(self) -> None:
-        if not isinstance(self._config, dict):  # self.is_enabled()
-            return
-
-        conn_kwargs = {**self._postgresql.connection_pool.conn_kwargs,
-                       'options': '-c synchronous_commit=local -c statement_timeout=0'}
-        if self._config['database'] != self._postgresql.database:
-            conn = connect(**conn_kwargs)
-            try:
-                with conn.cursor() as cur:
-                    cur.execute('CREATE DATABASE {0}'.format(
-                        quote_ident(self._config['database'], conn)).encode('utf-8'))
-            finally:
-                conn.close()
-
-        conn_kwargs['dbname'] = self._config['database']
-        conn = connect(**conn_kwargs)
-        try:
-            with conn.cursor() as cur:
-                cur.execute('CREATE EXTENSION citus')
-
-                superuser = self._postgresql.config.superuser
-                params = {k: superuser[k] for k in ('password', 'sslcert', 'sslkey') if k in superuser}
-                if params:
-                    cur.execute("INSERT INTO pg_catalog.pg_dist_authinfo VALUES"
-                                "(0, pg_catalog.current_user(), %s)",
-                                (self._postgresql.config.format_dsn(params),))
-
-                if self.is_coordinator():
-                    r = urlparse(self._postgresql.connection_string)
-                    cur.execute("SELECT pg_catalog.citus_set_coordinator_host(%s, %s, 'primary', 'default')",
-                                (r.hostname, r.port or 5432))
-        finally:
-            conn.close()
-
-    def adjust_postgres_gucs(self, parameters: Dict[str, Any]) -> None:
-        if not self.is_enabled():
-            return
-
-        # citus extension must be on the first place in shared_preload_libraries
-        shared_preload_libraries = list(filter(
-            lambda el: el and el != 'citus',
-            [p.strip() for p in parameters.get('shared_preload_libraries', '').split(',')]))
-        parameters['shared_preload_libraries'] = ','.join(['citus'] + shared_preload_libraries)
-
-        # if not explicitly set Citus overrides max_prepared_transactions to max_connections*2
-        if parameters['max_prepared_transactions'] == 0:
-            parameters['max_prepared_transactions'] = parameters['max_connections'] * 2
-
-        # Resharding in Citus implemented using logical replication
-        parameters['wal_level'] = 'logical'
-
-        # Sometimes Citus needs to connect to the local postgres. We will do it the same way as Patroni does.
-        parameters['citus.local_hostname'] = self._postgresql.connection_pool.conn_kwargs.get('host', 'localhost')
-
-    def ignore_replication_slot(self, slot: Dict[str, str]) -> bool:
-        if isinstance(self._config, dict) and self._postgresql.is_primary() and\
-                slot['type'] == 'logical' and slot['database'] == self._config['database']:
-            m = CITUS_SLOT_NAME_RE.match(slot['name'])
-            return bool(m and {'move': 'pgoutput', 'split': 'citus'}.get(m.group(1)) == slot['plugin'])
-        return False

patroni/postgresql/config.py

@@ -9,14 +9,16 @@ import time
 from contextlib import contextmanager
 from urllib.parse import urlparse, parse_qsl, unquote
 from types import TracebackType
-from typing import Any, Collection, Dict, Iterator, List, Optional, Union, Tuple, Type, TYPE_CHECKING
+from typing import Any, Callable, Collection, Dict, Iterator, List, Optional, Union, Tuple, Type, TYPE_CHECKING
 
 from .validator import recovery_parameters, transform_postgresql_parameter_value, transform_recovery_parameter_value
-from ..collections import CaseInsensitiveDict, CaseInsensitiveSet
+from .. import global_config
+from ..collections import CaseInsensitiveDict, CaseInsensitiveSet, EMPTY_DICT
 from ..dcs import Leader, Member, RemoteMember, slot_name_from_member_name
 from ..exceptions import PatroniFatalException, PostgresConnectionException
 from ..file_perm import pg_perm
-from ..utils import compare_values, parse_bool, parse_int, split_host_port, uri, validate_directory, is_subpath
+from ..utils import (compare_values, maybe_convert_from_base_unit, parse_bool, parse_int,
+                     split_host_port, uri, validate_directory, is_subpath)
 from ..validator import IntValidator, EnumValidator
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -269,6 +271,29 @@ def _bool_is_true_validator(value: Any) -> bool:
     return parse_bool(value) is True
 
 
+def get_param_diff(old_value: Any, new_value: Any,
+                   vartype: Optional[str] = None, unit: Optional[str] = None) -> Dict[str, str]:
+    """Get a dictionary representing a single PG parameter's value diff.
+
+    :param old_value: current :class:`str` parameter value.
+    :param new_value: :class:`str` value of the paramater after a restart.
+    :param vartype: the target type to parse old/new_value. See ``vartype`` argument of
+        :func:`~patroni.utils.maybe_convert_from_base_unit`.
+    :param unit: unit of *old/new_value*. See ``base_unit`` argument of
+        :func:`~patroni.utils.maybe_convert_from_base_unit`.
+
+    :returns: a :class:`dict` object that contains two keys: ``old_value`` and ``new_value``
+        with their values casted to :class:`str` and converted from base units (if possible).
+    """
+    str_value: Callable[[Any], str] = lambda x: '' if x is None else str(x)
+    return {
+        'old_value': (maybe_convert_from_base_unit(str_value(old_value), vartype, unit)
+                      if vartype else str_value(old_value)),
+        'new_value': (maybe_convert_from_base_unit(str_value(new_value), vartype, unit)
+                      if vartype else str_value(new_value))
+    }
+
+
 class ConfigHandler(object):
 
     # List of parameters which must be always passed to postmaster as command line options
@@ -336,12 +361,24 @@ class ConfigHandler(object):
 
     def load_current_server_parameters(self) -> None:
         """Read GUC's values from ``pg_settings`` when Patroni is joining the the postgres that is already running."""
-        exclude = [name.lower() for name, value in self.CMDLINE_OPTIONS.items() if value[1] == _false_validator] \
-            + [name.lower() for name in self._RECOVERY_PARAMETERS]
-        self._server_parameters = CaseInsensitiveDict({r[0]: r[1] for r in self._postgresql.query(
+        exclude = [name.lower() for name, value in self.CMDLINE_OPTIONS.items() if value[1] == _false_validator]
+        keep_values = {k: self._server_parameters[k] for k in exclude}
+        server_parameters = CaseInsensitiveDict({r[0]: r[1] for r in self._postgresql.query(
             "SELECT name, pg_catalog.current_setting(name) FROM pg_catalog.pg_settings"
             " WHERE (source IN ('command line', 'environment variable') OR sourcefile = %s)"
             " AND pg_catalog.lower(name) != ALL(%s)", self._postgresql_conf, exclude)})
+        recovery_params = CaseInsensitiveDict({k: server_parameters.pop(k) for k in self._RECOVERY_PARAMETERS
+                                               if k in server_parameters})
+        # We also want to load current settings of recovery parameters, including primary_conninfo
+        # and primary_slot_name, otherwise patronictl restart will update postgresql.conf
+        # and remove them, what in the worst case will cause another restart.
+        # We are doing it only for PostgresSQL v12 onwards, because older version still have recovery.conf
+        if not self._postgresql.is_primary() and self._postgresql.major_version >= 120000:
+            # primary_conninfo is expected to be a dict, therefore we need to parse it
+            recovery_params['primary_conninfo'] = parse_dsn(recovery_params.pop('primary_conninfo', '')) or {}
+            self._recovery_params = recovery_params
+
+        self._server_parameters = CaseInsensitiveDict({**server_parameters, **keep_values})
 
     def setup_server_parameters(self) -> None:
         self._server_parameters = self.get_server_parameters(self._config)
@@ -582,7 +619,8 @@ class ConfigHandler(object):
             fd.write_param(name, value)
 
     def build_recovery_params(self, member: Union[Leader, Member, None]) -> CaseInsensitiveDict:
-        recovery_params = CaseInsensitiveDict({p: v for p, v in (self.get('recovery_conf') or {}).items()
+        default: Dict[str, Any] = {}
+        recovery_params = CaseInsensitiveDict({p: v for p, v in (self.get('recovery_conf') or default).items()
                                                if not p.lower().startswith('recovery_target')
                                                and p.lower() not in ('primary_conninfo', 'primary_slot_name')})
         recovery_params.update({'standby_mode': 'on', 'recovery_target_timeline': 'latest'})
@@ -595,15 +633,14 @@ class ConfigHandler(object):
         is_remote_member = isinstance(member, RemoteMember)
         primary_conninfo = self.primary_conninfo_params(member)
         if primary_conninfo:
-            use_slots = self.get('use_slots', True) and self._postgresql.major_version >= 90400
+            use_slots = global_config.use_slots and self._postgresql.major_version >= 90400
             if use_slots and not (is_remote_member and member.no_replication_slot):
                 primary_slot_name = member.primary_slot_name if is_remote_member else self._postgresql.name
                 recovery_params['primary_slot_name'] = slot_name_from_member_name(primary_slot_name)
                 # We are a standby leader and are using a replication slot. Make sure we connect to
                 # the leader of the main cluster (in case more than one host is specified in the
                 # connstr) by adding 'target_session_attrs=read-write' to primary_conninfo.
-                if is_remote_member and 'target_sesions_attrs' not in primary_conninfo and\
-                        self._postgresql.major_version >= 100000:
+                if is_remote_member and ',' in primary_conninfo['host'] and self._postgresql.major_version >= 100000:
                     primary_conninfo['target_session_attrs'] = 'read-write'
             recovery_params['primary_conninfo'] = primary_conninfo
 
@@ -809,7 +846,7 @@ class ConfigHandler(object):
             required['restart' if mtype else 'reload'] += 1
 
         wanted_recovery_params = self.build_recovery_params(member)
-        for param, value in (self._current_recovery_params or {}).items():
+        for param, value in (self._current_recovery_params or EMPTY_DICT).items():
             # Skip certain parameters defined in the included postgres config files
             # if we know that they are not specified in the patroni configuration.
             if len(value) > 2 and value[2] not in (self._postgresql_conf, self._auto_conf) and \
@@ -930,10 +967,10 @@ class ConfigHandler(object):
         parameters = config['parameters'].copy()
         listen_addresses, port = split_host_port(config['listen'], 5432)
         parameters.update(cluster_name=self._postgresql.scope, listen_addresses=listen_addresses, port=str(port))
-        if not self._postgresql.global_config or self._postgresql.global_config.is_synchronous_mode:
+        if global_config.is_synchronous_mode:
             synchronous_standby_names = self._server_parameters.get('synchronous_standby_names')
             if synchronous_standby_names is None:
-                if self._postgresql.global_config and self._postgresql.global_config.is_synchronous_mode_strict\
+                if global_config.is_synchronous_mode_strict\
                         and self._postgresql.role in ('master', 'primary', 'promoted'):
                     parameters['synchronous_standby_names'] = '*'
                 else:
@@ -955,7 +992,7 @@ class ConfigHandler(object):
             wal_keep_size = parse_int(parameters.pop('wal_keep_size', self.CMDLINE_OPTIONS['wal_keep_size'][0]), 'MB')
             parameters.setdefault('wal_keep_segments', int(((wal_keep_size or 0) + 8) / 16))
 
-        self._postgresql.citus_handler.adjust_postgres_gucs(parameters)
+        self._postgresql.mpp_handler.adjust_postgres_gucs(parameters)
 
         ret = CaseInsensitiveDict({k: v for k, v in parameters.items() if not self._postgresql.major_version
                                    or self._postgresql.major_version >= self.CMDLINE_OPTIONS.get(k, (0, 1, 90100))[2]})
@@ -1064,13 +1101,15 @@ class ConfigHandler(object):
     def reload_config(self, config: Dict[str, Any], sighup: bool = False) -> None:
         self._superuser = config['authentication'].get('superuser', {})
         server_parameters = self.get_server_parameters(config)
+        params_skip_changes = CaseInsensitiveSet((*self._RECOVERY_PARAMETERS, 'hot_standby', 'wal_log_hints'))
 
-        conf_changed = hba_changed = ident_changed = local_connection_address_changed = pending_restart = False
+        conf_changed = hba_changed = ident_changed = local_connection_address_changed = False
+        param_diff = CaseInsensitiveDict()
         if self._postgresql.state == 'running':
             changes = CaseInsensitiveDict({p: v for p, v in server_parameters.items()
-                                           if p.lower() not in self._RECOVERY_PARAMETERS})
+                                           if p not in params_skip_changes})
             changes.update({p: None for p in self._server_parameters.keys()
-                            if not (p in changes or p.lower() in self._RECOVERY_PARAMETERS)})
+                            if not (p in changes or p in params_skip_changes)})
             if changes:
                 undef = []
                 if 'wal_buffers' in changes:  # we need to calculate the default value of wal_buffers
@@ -1089,20 +1128,28 @@ class ConfigHandler(object):
                         if new_value is None or not compare_values(r[3], r[2], r[1], new_value):
                             conf_changed = True
                             if r[4] == 'postmaster':
-                                pending_restart = True
-                                logger.info('Changed %s from %s to %s (restart might be required)',
-                                            r[0], r[1], new_value)
+                                param_diff[r[0]] = get_param_diff(r[1], new_value, r[3], r[2])
+                                logger.info("Changed %s from '%s' to '%s' (restart might be required)",
+                                            r[0], param_diff[r[0]]['old_value'], new_value)
                                 if config.get('use_unix_socket') and r[0] == 'unix_socket_directories'\
                                         or r[0] in ('listen_addresses', 'port'):
                                     local_connection_address_changed = True
                             else:
-                                logger.info('Changed %s from %s to %s', r[0], r[1], new_value)
+                                logger.info("Changed %s from '%s' to '%s'",
+                                            r[0], maybe_convert_from_base_unit(r[1], r[3], r[2]), new_value)
+                        elif r[0] in self._server_parameters \
+                                and not compare_values(r[3], r[2], r[1], self._server_parameters[r[0]]):
+                            # Check if any parameter was set back to the current pg_settings value
+                            # We can use pg_settings value here, as it is proved to be equal to new_value
+                            logger.info("Changed %s from '%s' to '%s'", r[0], self._server_parameters[r[0]], new_value)
+                            conf_changed = True
                 for param, value in changes.items():
                     if '.' in param:
-                        # Check that user-defined-paramters have changed (parameters with period in name)
+                        # Check that user-defined-parameters have changed (parameters with period in name)
                         if value is None or param not in self._server_parameters \
                                 or str(value) != str(self._server_parameters[param]):
-                            logger.info('Changed %s from %s to %s', param, self._server_parameters.get(param), value)
+                            logger.info("Changed %s from '%s' to '%s'",
+                                        param, self._server_parameters.get(param), value)
                             conf_changed = True
                     elif param in server_parameters:
                         logger.warning('Removing invalid parameter `%s` from postgresql.parameters', param)
@@ -1117,7 +1164,6 @@ class ConfigHandler(object):
                 ident_changed = self._config.get('pg_ident', []) != config['pg_ident']
 
         self._config = config
-        self._postgresql.set_pending_restart(pending_restart)
         self._server_parameters = server_parameters
         self._adjust_recovery_parameters()
         self._krbsrvname = config.get('krbsrvname')
@@ -1147,16 +1193,28 @@ class ConfigHandler(object):
             if self._postgresql.major_version >= 90500:
                 time.sleep(1)
                 try:
-                    pending_restart = self._postgresql.query(
-                        'SELECT COUNT(*) FROM pg_catalog.pg_settings'
-                        ' WHERE pg_catalog.lower(name) != ALL(%s) AND pending_restart',
-                        [n.lower() for n in self._RECOVERY_PARAMETERS])[0][0] > 0
-                    self._postgresql.set_pending_restart(pending_restart)
+                    settings_diff: CaseInsensitiveDict = CaseInsensitiveDict()
+                    for param, value, unit, vartype in self._postgresql.query(
+                            'SELECT name, pg_catalog.current_setting(name), unit, vartype FROM pg_catalog.pg_settings'
+                            ' WHERE pg_catalog.lower(name) != ALL(%s) AND pending_restart',
+                            [n.lower() for n in params_skip_changes]):
+                        new_value = self._postgresql.get_guc_value(param)
+                        new_value = '?' if new_value is None else new_value
+                        settings_diff[param] = get_param_diff(value, new_value, vartype, unit)
+                    external_change = {param: value for param, value in settings_diff.items()
+                                       if param not in param_diff or value != param_diff[param]}
+                    if external_change:
+                        logger.info("PostgreSQL configuration parameters requiring restart"
+                                    " (%s) seem to be changed bypassing Patroni config."
+                                    " Setting 'Pending restart' flag", ', '.join(external_change))
+                    param_diff = settings_diff
                 except Exception as e:
                     logger.warning('Exception %r when running query', e)
         else:
             logger.info('No PostgreSQL configuration items changed, nothing to reload.')
 
+        self._postgresql.set_pending_restart_reason(param_diff)
+
     def set_synchronous_standby_names(self, value: Optional[str]) -> Optional[bool]:
         """Updates synchronous_standby_names and reloads if necessary.
         :returns: True if value was updated."""
@@ -1199,6 +1257,7 @@ class ConfigHandler(object):
         data = self._postgresql.controldata()
         effective_configuration = self._server_parameters.copy()
 
+        param_diff = CaseInsensitiveDict()
         for name, cname in options_mapping.items():
             value = parse_int(effective_configuration[name])
             if cname not in data:
@@ -1208,7 +1267,10 @@ class ConfigHandler(object):
             cvalue = parse_int(data[cname])
             if cvalue is not None and value is not None and cvalue > value:
                 effective_configuration[name] = cvalue
-                self._postgresql.set_pending_restart(True)
+                logger.info("%s value in pg_controldata: %d, in the global configuration: %d."
+                            " pg_controldata value will be used. Setting 'Pending restart' flag", name, cvalue, value)
+                param_diff[name] = get_param_diff(cvalue, value)
+        self._postgresql.set_pending_restart_reason(param_diff)
 
         # If we are using custom bootstrap with PITR it could fail when values like max_connections
         # are increased, therefore we disable hot_standby if recovery_target_action == 'promote'.
@@ -1225,7 +1287,6 @@ class ConfigHandler(object):
 
             if disable_hot_standby:
                 effective_configuration['hot_standby'] = 'off'
-                self._postgresql.set_pending_restart(True)
 
         return effective_configuration
 
@@ -1264,4 +1325,4 @@ class ConfigHandler(object):
         return self._config.get(key, default)
 
     def restore_command(self) -> Optional[str]:
-        return (self.get('recovery_conf') or {}).get('restore_command')
+        return (self.get('recovery_conf') or EMPTY_DICT).get('restore_command')

patroni/postgresql/connection.py

@@ -147,7 +147,8 @@ class ConnectionPool:
     def close(self) -> None:
         """Close all named connections from Patroni to PostgreSQL registered in the pool."""
         with self._lock:
-            if any(conn.close(True) for conn in self._connections.values()):
+            closed_connections = [conn.close(True) for conn in self._connections.values()]
+            if any(closed_connections):
                 logger.info("closed patroni connections to postgres")
 
 

patroni/postgresql/mpp/__init__.py

@@ -0,0 +1,317 @@
+"""Abstract classes for MPP handler.
+
+MPP stands for Massively Parallel Processing, and Citus belongs to this architecture. Currently, Citus is the only
+supported MPP cluster. However, we may consider adapting other databases such as TimescaleDB, GPDB, etc. into Patroni.
+"""
+import abc
+
+from typing import Any, Dict, Iterator, Optional, Union, Tuple, Type, TYPE_CHECKING
+
+from ...dcs import Cluster
+from ...dynamic_loader import iter_classes
+from ...exceptions import PatroniException
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .. import Postgresql
+    from ...config import Config
+
+
+class AbstractMPP(abc.ABC):
+    """An abstract class which should be passed to :class:`AbstractDCS`.
+
+    .. note::
+        We create :class:`AbstractMPP` and :class:`AbstractMPPHandler` to solve the chicken-egg initialization problem.
+        When initializing DCS, we dynamically create an object implementing :class:`AbstractMPP`, later this object is
+        used to instantiate an object implementing :class:`AbstractMPPHandler`.
+    """
+
+    group_re: Any  # re.Pattern[str]
+
+    def __init__(self, config: Dict[str, Union[str, int]]) -> None:
+        """Init method for :class:`AbstractMPP`.
+
+        :param config: configuration of MPP section.
+        """
+        self._config = config
+
+    def is_enabled(self) -> bool:
+        """Check if MPP is enabled for a given MPP.
+
+        .. note::
+            We just check that the :attr:`_config` object isn't empty and expect
+            it to be empty only in case of :class:`Null`.
+
+        :returns: ``True`` if MPP is enabled, otherwise ``False``.
+        """
+        return bool(self._config)
+
+    @staticmethod
+    @abc.abstractmethod
+    def validate_config(config: Any) -> bool:
+        """Check whether provided config is good for a given MPP.
+
+        :param config: configuration of MPP section.
+
+        :returns: ``True`` is config passes validation, otherwise ``False``.
+        """
+
+    @property
+    @abc.abstractmethod
+    def group(self) -> Any:
+        """The group for a given MPP implementation."""
+
+    @property
+    @abc.abstractmethod
+    def coordinator_group_id(self) -> Any:
+        """The group id of the coordinator PostgreSQL cluster."""
+
+    @property
+    def type(self) -> str:
+        """The type of the MPP cluster.
+
+        :returns: A string representation of the type of a given MPP implementation.
+        """
+        for base in self.__class__.__bases__:
+            if not base.__name__.startswith('Abstract'):
+                return base.__name__
+        return self.__class__.__name__
+
+    @property
+    def k8s_group_label(self):
+        """Group label used for kubernetes DCS of the MPP cluster.
+
+        :returns: A string representation of the k8s group label of a given MPP implementation.
+        """
+        return self.type.lower() + '-group'
+
+    def is_coordinator(self) -> bool:
+        """Check whether this node is running in the coordinator PostgreSQL cluster.
+
+        :returns: ``True`` if MPP is enabled and the group id of this node
+                  matches with the :attr:`coordinator_group_id`, otherwise ``False``.
+        """
+        return self.is_enabled() and self.group == self.coordinator_group_id
+
+    def is_worker(self) -> bool:
+        """Check whether this node is running as a MPP worker PostgreSQL cluster.
+
+        :returns: ``True`` if MPP is enabled and this node is known to be not running
+                  as the coordinator PostgreSQL cluster, otherwise ``False``.
+        """
+        return self.is_enabled() and not self.is_coordinator()
+
+    def _get_handler_cls(self) -> Iterator[Type['AbstractMPPHandler']]:
+        """Find Handler classes inherited from a class type of this object.
+
+        :yields: handler classes for this object.
+        """
+        for cls in self.__class__.__subclasses__():
+            if issubclass(cls, AbstractMPPHandler) and cls.__name__.startswith(self.__class__.__name__):
+                yield cls
+
+    def get_handler_impl(self, postgresql: 'Postgresql') -> 'AbstractMPPHandler':
+        """Find and instantiate Handler implementation of this object.
+
+        :param postgresql: a reference to :class:`Postgresql` object.
+
+        :raises:
+            :exc:`PatroniException`: if the Handler class haven't been found.
+
+        :returns: an instantiated class that implements Handler for this object.
+        """
+        for cls in self._get_handler_cls():
+            return cls(postgresql, self._config)
+        raise PatroniException(f'Failed to initialize {self.__class__.__name__}Handler object')
+
+
+class AbstractMPPHandler(AbstractMPP):
+    """An abstract class which defines interfaces that should be implemented by real handlers."""
+
+    def __init__(self, postgresql: 'Postgresql', config: Dict[str, Union[str, int]]) -> None:
+        """Init method for :class:`AbstractMPPHandler`.
+
+        :param postgresql: a reference to :class:`Postgresql` object.
+        :param config: configuration of MPP section.
+        """
+        super().__init__(config)
+        self._postgresql = postgresql
+
+    @abc.abstractmethod
+    def handle_event(self, cluster: Cluster, event: Dict[str, Any]) -> None:
+        """Handle an event sent from a worker node.
+
+        :param cluster: the currently known cluster state from DCS.
+        :param event: the event to be handled.
+        """
+
+    @abc.abstractmethod
+    def sync_meta_data(self, cluster: Cluster) -> None:
+        """Sync meta data on the coordinator.
+
+        :param cluster: the currently known cluster state from DCS.
+        """
+
+    @abc.abstractmethod
+    def on_demote(self) -> None:
+        """On demote handler.
+
+        Is called when the primary was demoted.
+        """
+
+    @abc.abstractmethod
+    def schedule_cache_rebuild(self) -> None:
+        """Cache rebuild handler.
+
+        Is called to notify handler that it has to refresh its metadata cache from the database.
+        """
+
+    @abc.abstractmethod
+    def bootstrap(self) -> None:
+        """Bootstrap handler.
+
+        Is called when the new cluster is initialized (through ``initdb`` or a custom bootstrap method).
+        """
+
+    @abc.abstractmethod
+    def adjust_postgres_gucs(self, parameters: Dict[str, Any]) -> None:
+        """Adjust GUCs in the current PostgreSQL configuration.
+
+        :param parameters: dictionary of GUCs, with key as GUC name and the corresponding value as current GUC value.
+        """
+
+    @abc.abstractmethod
+    def ignore_replication_slot(self, slot: Dict[str, str]) -> bool:
+        """Check whether provided replication *slot* existing in the database should not be removed.
+
+        .. note::
+            MPP database may create replication slots for its own use, for example to migrate data between workers
+            using logical replication, and we don't want to suddenly drop them.
+
+        :param slot: dictionary containing the replication slot settings, like ``name``, ``database``, ``type``, and
+                     ``plugin``.
+
+        :returns: ``True`` if the replication slots should not be removed, otherwise ``False``.
+        """
+
+
+class Null(AbstractMPP):
+    """Dummy implementation of :class:`AbstractMPP`."""
+
+    def __init__(self) -> None:
+        """Init method for :class:`Null`."""
+        super().__init__({})
+
+    @staticmethod
+    def validate_config(config: Any) -> bool:
+        """Check whether provided config is good for :class:`Null`.
+
+        :returns: always ``True``.
+        """
+        return True
+
+    @property
+    def group(self) -> None:
+        """The group for :class:`Null`.
+
+        :returns: always ``None``.
+        """
+        return None
+
+    @property
+    def coordinator_group_id(self) -> None:
+        """The group id of the coordinator PostgreSQL cluster.
+
+        :returns: always ``None``.
+        """
+        return None
+
+
+class NullHandler(Null, AbstractMPPHandler):
+    """Dummy implementation of :class:`AbstractMPPHandler`."""
+
+    def __init__(self, postgresql: 'Postgresql', config: Dict[str, Union[str, int]]) -> None:
+        """Init method for :class:`NullHandler`.
+
+        :param postgresql: a reference to :class:`Postgresql` object.
+        :param config: configuration of MPP section.
+        """
+        AbstractMPPHandler.__init__(self, postgresql, config)
+
+    def handle_event(self, cluster: Cluster, event: Dict[str, Any]) -> None:
+        """Handle an event sent from a worker node.
+
+        :param cluster: the currently known cluster state from DCS.
+        :param event: the event to be handled.
+        """
+
+    def sync_meta_data(self, cluster: Cluster) -> None:
+        """Sync meta data on the coordinator.
+
+        :param cluster: the currently known cluster state from DCS.
+        """
+
+    def on_demote(self) -> None:
+        """On demote handler.
+
+        Is called when the primary was demoted.
+        """
+
+    def schedule_cache_rebuild(self) -> None:
+        """Cache rebuild handler.
+
+        Is called to notify handler that it has to refresh its metadata cache from the database.
+        """
+
+    def bootstrap(self) -> None:
+        """Bootstrap handler.
+
+        Is called when the new cluster is initialized (through ``initdb`` or a custom bootstrap method).
+        """
+
+    def adjust_postgres_gucs(self, parameters: Dict[str, Any]) -> None:
+        """Adjust GUCs in the current PostgreSQL configuration.
+
+        :param parameters: dictionary of GUCs, with key as GUC name and corresponding value as current GUC value.
+        """
+
+    def ignore_replication_slot(self, slot: Dict[str, str]) -> bool:
+        """Check whether provided replication *slot* existing in the database should not be removed.
+
+        .. note::
+            MPP database may create replication slots for its own use, for example to migrate data between workers
+            using logical replication, and we don't want to suddenly drop them.
+
+        :param slot: dictionary containing the replication slot settings, like ``name``, ``database``, ``type``, and
+                     ``plugin``.
+
+        :returns: always ``False``.
+        """
+        return False
+
+
+def iter_mpp_classes(
+        config: Optional[Union['Config', Dict[str, Any]]] = None
+) -> Iterator[Tuple[str, Type[AbstractMPP]]]:
+    """Attempt to import MPP modules that are present in the given configuration.
+
+    :param config: configuration information with possible MPP names as keys. If given, only attempt to import MPP
+                   modules defined in the configuration. Else, if ``None``, attempt to import any supported MPP module.
+
+    :yields: tuples, each containing the module ``name`` and the imported MPP class object.
+    """
+    if TYPE_CHECKING:  # pragma: no cover
+        assert isinstance(__package__, str)
+    yield from iter_classes(__package__, AbstractMPP, config)
+
+
+def get_mpp(config: Union['Config', Dict[str, Any]]) -> AbstractMPP:
+    """Attempt to load and instantiate a MPP module from known available implementations.
+
+    :param config: object or dictionary with Patroni configuration.
+
+    :returns: The successfully loaded MPP or fallback to :class:`Null`.
+    """
+    for name, mpp_class in iter_mpp_classes(config):
+        if mpp_class.validate_config(config[name]):
+            return mpp_class(config[name])
+    return Null()

patroni/postgresql/mpp/citus.py

@@ -0,0 +1,773 @@
+import logging
+import re
+import time
+
+from threading import Condition, Event, Thread
+from urllib.parse import urlparse
+from typing import Any, Collection, Dict, Iterator, List, Optional, Union, Set, Tuple, TYPE_CHECKING
+
+from . import AbstractMPP, AbstractMPPHandler
+from ...dcs import Cluster
+from ...psycopg import connect, quote_ident, ProgrammingError
+from ...utils import parse_int
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .. import Postgresql
+
+CITUS_COORDINATOR_GROUP_ID = 0
+CITUS_SLOT_NAME_RE = re.compile(r'^citus_shard_(move|split)_slot(_[1-9][0-9]*){2,3}$')
+logger = logging.getLogger(__name__)
+
+
+class PgDistNode:
+    """Represents a single row in "pg_dist_node" table.
+
+    .. note::
+
+        Unlike "noderole" possible values of ``role`` are ``primary``, ``secondary``, and ``demoted``.
+        The last one is used to pause client connections on the coordinator to the worker by
+        appending ``-demoted`` suffix to the "nodename". The actual "noderole" in DB remains ``primary``.
+
+    :ivar host: "nodename" value
+    :ivar port: "nodeport" value
+    :ivar role: "noderole" value
+    :ivar nodeid: "nodeid" value
+    """
+
+    def __init__(self, host: str, port: int, role: str, nodeid: Optional[int] = None) -> None:
+        """Create a :class:`PgDistNode` object based on given arguments.
+
+        :param host: "nodename" of the Citus coordinator or worker.
+        :param port: "nodeport" of the Citus coordinator or worker.
+        :param role: "noderole" value.
+        :param nodeid: id of the row in the "pg_dist_node".
+        """
+        self.host = host
+        self.port = port
+        self.role = role
+        self.nodeid = nodeid
+
+    def __hash__(self) -> int:
+        """Defines a hash function to put :class:`PgDistNode` objects to :class:`PgDistGroup` set-like object.
+
+        .. note::
+            We use (:attr:`host`, :attr:`port`) tuple here because it is one of the UNIQUE constraints on the
+            "pg_dist_node" table. The :attr:`role` value is irrelevant here because nodes may change their roles.
+        """
+        return hash((self.host, self.port))
+
+    def __eq__(self, other: Any) -> bool:
+        """Defines a comparison function.
+
+        :returns: ``True`` if :attr:`host` and :attr:`port` between two instances are the same.
+        """
+        return isinstance(other, PgDistNode) and self.host == other.host and self.port == other.port
+
+    def __str__(self) -> str:
+        return ('PgDistNode(nodeid={0},host={1},port={2},role={3})'
+                .format(self.nodeid, self.host, self.port, self.role))
+
+    def __repr__(self) -> str:
+        return str(self)
+
+    def is_primary(self) -> bool:
+        """Checks whether this object represents "primary" in a corresponding group.
+
+        :returns: ``True`` if this object represents the ``primary``.
+        """
+        return self.role in ('primary', 'demoted')
+
+    def as_tuple(self, include_nodeid: bool = False) -> Tuple[str, int, str, Optional[int]]:
+        """Helper method to compare two :class:`PgDistGroup` objects.
+
+        .. note::
+
+            *include_nodeid* is set to ``True`` only in unit-tests.
+
+        :param include_nodeid: whether :attr:`nodeid` should be taken into account when comparison is performed.
+
+        :returns: :class:`tuple` object with :attr:`host`, :attr:`port`, :attr:`role`, and optionally :attr:`nodeid`.
+        """
+        return self.host, self.port, self.role, (self.nodeid if include_nodeid else None)
+
+
+class PgDistGroup(Set[PgDistNode]):
+    """A :class:`set`-like object that represents a Citus group in "pg_dist_node" table.
+
+    This class implements a set of methods to compare topology and if it is necessary
+    to transition from the old to the new topology in a "safe" manner:
+
+        * register new primary/secondaries
+        * replace gone secondaries with added secondaries
+        * failover and switchover
+
+    Typically there will be at least one :class:`PgDistNode` object registered (``primary``).
+    In addition to that there could be one or more ``secondary`` nodes.
+
+    :ivar failover: whether the ``primary`` row should be updated as a result of :func:`transition` method call.
+    :ivar groupid: the "groupid" from "pg_dist_node".
+    """
+
+    def __init__(self, groupid: int, nodes: Optional[Collection[PgDistNode]] = None) -> None:
+        """Creates a :class:`PgDistGroup` object based on given arguments.
+
+        :param groupid: the groupid from "pg_dist_node".
+        :param nodes: a collection of :class:`PgDistNode` objects that belog to a *groupid*.
+        """
+        self.failover = False
+        self.groupid = groupid
+
+        if nodes:
+            self.update(nodes)
+
+    def equals(self, other: 'PgDistGroup', check_nodeid: bool = False) -> bool:
+        """Compares two :class:`PgDistGroup` objects.
+
+        :param other: what we want to compare with.
+        :param check_nodeid: whether :attr:`PgDistNode.nodeid` should be compared in addition to
+                             :attr:`PgDistNode.host`, :attr:`PgDistNode.port`, and :attr:`PgDistNode.role`.
+
+        :returns: ``True`` if two :class:`PgDistGroup` objects are fully identical.
+        """
+        return self.groupid == other.groupid\
+            and set(v.as_tuple(check_nodeid) for v in self) == set(v.as_tuple(check_nodeid) for v in other)
+
+    def primary(self) -> Optional[PgDistNode]:
+        """Finds and returns :class:`PgDistNode` object that represents the "primary".
+
+        :returns: :class:`PgDistNode` object which represents the "primary" or ``None`` if not found.
+        """
+        return next(iter(v for v in self if v.is_primary()), None)
+
+    def get(self, value: PgDistNode) -> Optional[PgDistNode]:
+        """Performs a lookup of the actual value in a set.
+
+        .. note::
+            It is necessary because :func:`__hash__` and :func:`__eq__` methods in :class:`PgDistNode` are
+            redefined and effectively they check only :attr:`PgDistNode.host` and :attr:`PgDistNode.port` attributes.
+
+        :param value: the key we search for.
+        :returns: the actual :class:`PgDistNode` value from this :class:`PgDistGroup` object or ``None`` if not found.
+        """
+        return next(iter(v for v in self if v == value), None)
+
+    def transition(self, old: 'PgDistGroup') -> Iterator[PgDistNode]:
+        """Compares this topology with the old one and yields transitions that transform the old to the new one.
+
+        .. note::
+            The actual yielded object is :class:`PgDistNode` that will be passed to
+            the :meth:`CitusHandler.update_node` to execute all transitions in a transaction.
+
+            In addition to the yielding transactions this method fills up :attr:`PgDistNode.nodeid`
+            attribute for nodes that are presented in the old and in the new topology.
+
+            There are a few simple rules/constraints that are imposed by Citus and must be followed:
+            - adding/removing nodes is only possible when metadata is synced to all registered "priorities".
+
+            - the "primary" row in "pg_dist_node" always keeps the nodeid (unless it is
+              removed, but it is not supported by Patroni).
+
+            - "nodename", "nodeport" must be unique across all rows in the "pg_dist_node". This means that
+              every time we want to change the nodeid of an existing node (i.e. to change it from secondary
+              to primary), we should first write some other "nodename"/"nodeport" to the row it's currently in.
+
+            - updating "broken" nodes always works and metadata is synced asynchnonously after the commit.
+
+        Following these rules below is an example of the switchover between node1 (primary, nodeid=4)
+        and node2 (secondary, nodeid=5).
+
+        .. code-block:: SQL
+
+            BEGIN;
+                SELECT citus_update_node(4, 'node1-demoted', 5432);
+                SELECT citus_update_node(5, 'node1', 5432);
+                SELECT citus_update_node(4, 'node2', 5432);
+            COMMIT;
+
+        :param old: the last known topology registered in "pg_dist_node" for a given :attr:`groupid`.
+
+        :yields: :class:`PgDistNode` objects that must be updated/added/removed in "pg_dist_node".
+        """
+        self.failover = old.failover
+
+        new_primary = self.primary()
+        assert new_primary is not None
+        old_primary = old.primary()
+
+        gone_nodes = old - self - {old_primary}
+        added_nodes = self - old - {new_primary}
+
+        if not old_primary:
+            # We did not have any nodes in the group yet and we're adding one now
+            yield new_primary
+        elif old_primary == new_primary:
+            new_primary.nodeid = old_primary.nodeid
+            # Controlled switchover with pausing client connections.
+            # Achieved by updating the primary row and putting hostname = '${host}-demoted' in a transaction.
+            if old_primary.role != new_primary.role:
+                self.failover = True
+                yield new_primary
+        elif old_primary != new_primary:
+            self.failover = True
+
+            new_primary_old_node = old.get(new_primary)
+            old_primary_new_node = self.get(old_primary)
+
+            # The new primary was registered as a secondary before failover
+            if new_primary_old_node:
+                new_node = None
+                # Old primary is gone and some new secondaries were added.
+                # We can use the row of promoted secondary to add the new secondary.
+                if not old_primary_new_node and added_nodes:
+                    new_node = added_nodes.pop()
+                    new_node.nodeid = new_primary_old_node.nodeid
+                    yield new_node
+
+                    # notify _maybe_register_old_primary_as_secondary that the old primary should not be re-registered
+                    old_primary.role = 'secondary'
+                # In opposite case we need to change the primary record to '${host}-demoted:${port}'
+                # before we can put its host:port to the row of promoted secondary.
+                elif old_primary.role == 'primary':
+                    old_primary.role = 'demoted'
+                    yield old_primary
+
+                # The old primary is gone and the promoted secondary row wasn't yet used.
+                if not old_primary_new_node and not new_node:
+                    # We have to "add" the gone primary to the row of promoted secondary because
+                    # nodes could not be removed while the metadata isn't synced.
+                    old_primary_new_node = PgDistNode(old_primary.host, old_primary.port, new_primary_old_node.role)
+                    self.add(old_primary_new_node)
+
+                # put the old primary instead of promoted secondary
+                if old_primary_new_node:
+                    old_primary_new_node.nodeid = new_primary_old_node.nodeid
+                    yield old_primary_new_node
+
+            # update the primary record with the new information
+            new_primary.nodeid = old_primary.nodeid
+            yield new_primary
+
+            # The new primary was never registered as a standby and there are secondaries that have gone away. Since
+            # nodes can't be removed while metadata isn't synced we have to temporarily "add" the old primary back.
+            if not new_primary_old_node and gone_nodes:
+                # We were in the middle of controlled switchover while the primary disappeared.
+                # If there are any gone nodes that can't be reused for new secondaries we will
+                # use one of them to temporarily "add" the old primary back as a secondary.
+                if not old_primary_new_node and old_primary.role == 'demoted' and len(gone_nodes) > len(added_nodes):
+                    old_primary_new_node = PgDistNode(old_primary.host, old_primary.port, 'secondary')
+                    self.add(old_primary_new_node)
+
+                # Use one of the gone secondaries to put host:port of the old primary there.
+                if old_primary_new_node:
+                    old_primary_new_node.nodeid = gone_nodes.pop().nodeid
+                    yield old_primary_new_node
+
+        # Fill nodeid for standbys in the new topology from the old ones
+        old_replicas = {v: v for v in old if not v.is_primary()}
+        for n in self:
+            if not n.is_primary() and not n.nodeid and n in old_replicas:
+                n.nodeid = old_replicas[n].nodeid
+
+        # Reuse nodeid's of gone standbys to "add" new standbys
+        while gone_nodes and added_nodes:
+            a = added_nodes.pop()
+            a.nodeid = gone_nodes.pop().nodeid
+            yield a
+
+        # Adding or removing nodes operations are executed on primaries in all Citus groups in 2PC.
+        # If we know that the primary was updated (self.failover is True) that automatically means that
+        # adding/removing nodes calls will fail and the whole transaction will be aborted. Therefore
+        # we discard operations that add/remove secondaries if we know that the primary was just updated.
+        # The inconsistency will be automatically resolved on the next Patroni heartbeat loop.
+
+        # Remove remaining nodes that are gone, but only in case if metadata is in sync (self.failover is False).
+        for g in gone_nodes:
+            if not self.failover:
+                # Remove the node if we expect metadata to be in sync
+                yield PgDistNode(g.host, g.port, '')
+            else:
+                # Otherwise add these nodes to the new topology
+                self.add(g)
+
+        # Add new nodes to the metadata, but only in case if metadata is in sync (self.failover is False).
+        for a in added_nodes:
+            if not self.failover:
+                # Add the node if we expect metadata to be in sync
+                yield a
+            else:
+                # Otherwise remove them from the new topology
+                self.discard(a)
+
+
+class PgDistTask(PgDistGroup):
+    """A "task" that represents the current or desired state of "pg_dist_node" for a provided *groupid*.
+
+    :ivar group: the "groupid" in "pg_dist_node".
+    :ivar event: an "event" that resulted in creating this task.
+                 possible values: "before_demote", "before_promote", "after_promote".
+    :ivar timeout: a transaction timeout if the task resulted in starting a transaction.
+    :ivar cooldown: the cooldown value for ``citus_update_node()`` UDF call.
+    :ivar deadline: the time in unix seconds when the transaction is allowed to be rolled back.
+    """
+
+    def __init__(self, groupid: int, nodes: Optional[Collection[PgDistNode]], event: str,
+                 timeout: Optional[float] = None, cooldown: Optional[float] = None) -> None:
+        """Create a :class:`PgDistTask` object based on given arguments.
+
+        :param groupid: the groupid from "pg_dist_node".
+        :param nodes: a collection of :class:`PgDistNode` objects that belog to a *groupid*.
+        :param event: an "event" that resulted in creating this task.
+        :param timeout: a transaction timeout if the task resulted in starting a transaction.
+        :param cooldown: the cooldown value for ``citus_update_node()`` UDF call.
+        """
+        super(PgDistTask, self).__init__(groupid, nodes)
+
+        # Event that is trying to change or changed the given row.
+        # Possible values: before_demote, before_promote, after_promote.
+        self.event = event
+
+        # If transaction was started, we need to COMMIT/ROLLBACK before the deadline
+        self.timeout = timeout
+        self.cooldown = cooldown or 10000  # 10s by default
+        self.deadline: float = 0
+
+        # All changes in the pg_dist_node are serialized on the Patroni
+        # side by performing them from a thread. The thread, that is
+        # requested a change, sometimes needs to wait for a result.
+        # For example, we want to pause client connections before demoting
+        # the worker, and once it is done notify the calling thread.
+        self._event = Event()
+
+    def wait(self) -> None:
+        """Wait until this task is processed by a dedicated thread."""
+        self._event.wait()
+
+    def wakeup(self) -> None:
+        """Notify a thread that created a task that it was processed."""
+        self._event.set()
+
+    def __eq__(self, other: Any) -> bool:
+        return isinstance(other, PgDistTask) and self.event == other.event\
+            and super(PgDistTask, self).equals(other)
+
+    def __ne__(self, other: Any) -> bool:
+        return not self == other
+
+
+class Citus(AbstractMPP):
+
+    group_re = re.compile('^(0|[1-9][0-9]*)$')
+
+    @staticmethod
+    def validate_config(config: Union[Any, Dict[str, Union[str, int]]]) -> bool:
+        """Check whether provided config is good for a given MPP.
+
+        :param config: configuration of ``citus`` MPP section.
+
+        :returns: ``True`` is config passes validation, otherwise ``False``.
+        """
+        return isinstance(config, dict) \
+            and isinstance(config.get('database'), str) \
+            and parse_int(config.get('group')) is not None
+
+    @property
+    def group(self) -> int:
+        """The group of this Citus node."""
+        return int(self._config['group'])
+
+    @property
+    def coordinator_group_id(self) -> int:
+        """The group id of the Citus coordinator PostgreSQL cluster."""
+        return CITUS_COORDINATOR_GROUP_ID
+
+
+class CitusHandler(Citus, AbstractMPPHandler, Thread):
+    """Define the interfaces for handling an underlying Citus cluster."""
+
+    def __init__(self, postgresql: 'Postgresql', config: Dict[str, Union[str, int]]) -> None:
+        """"Initialize a new instance of :class:`CitusHandler`.
+
+        :param postgresql: the Postgres node.
+        :param config: the ``citus`` MPP config section.
+        """
+        Thread.__init__(self)
+        AbstractMPPHandler.__init__(self, postgresql, config)
+        self.daemon = True
+        if config:
+            self._connection = postgresql.connection_pool.get(
+                'citus', {'dbname': config['database'],
+                          'options': '-c statement_timeout=0 -c idle_in_transaction_session_timeout=0'})
+        self._pg_dist_group: Dict[int, PgDistTask] = {}  # Cache of pg_dist_node: {groupid: PgDistTask()}
+        self._tasks: List[PgDistTask] = []  # Requests to change pg_dist_group, every task is a `PgDistTask`
+        self._in_flight: Optional[PgDistTask] = None  # Reference to the `PgDistTask` being changed in a transaction
+        self._schedule_load_pg_dist_group = True  # Flag that "pg_dist_group" should be queried from the database
+        self._condition = Condition()  # protects _pg_dist_group, _tasks, _in_flight, and _schedule_load_pg_dist_group
+        self.schedule_cache_rebuild()
+
+    def schedule_cache_rebuild(self) -> None:
+        """Cache rebuild handler.
+
+        Is called to notify handler that it has to refresh its metadata cache from the database.
+        """
+        with self._condition:
+            self._schedule_load_pg_dist_group = True
+
+    def on_demote(self) -> None:
+        with self._condition:
+            self._pg_dist_group.clear()
+            empty_tasks: List[PgDistTask] = []
+            self._tasks[:] = empty_tasks
+            self._in_flight = None
+
+    def query(self, sql: str, *params: Any) -> List[Tuple[Any, ...]]:
+        try:
+            logger.debug('query(%s, %s)', sql, params)
+            return self._connection.query(sql, *params)
+        except Exception as e:
+            logger.error('Exception when executing query "%s", (%s): %r', sql, params, e)
+            self._connection.close()
+            with self._condition:
+                self._in_flight = None
+            self.schedule_cache_rebuild()
+            raise e
+
+    def load_pg_dist_group(self) -> bool:
+        """Read from the `pg_dist_node` table and put it into the local cache"""
+
+        with self._condition:
+            if not self._schedule_load_pg_dist_group:
+                return True
+            self._schedule_load_pg_dist_group = False
+
+        try:
+            rows = self.query('SELECT groupid, nodename, nodeport, noderole, nodeid FROM pg_catalog.pg_dist_node')
+        except Exception:
+            return False
+
+        pg_dist_group: Dict[int, PgDistTask] = {}
+
+        for row in rows:
+            if row[0] not in pg_dist_group:
+                pg_dist_group[row[0]] = PgDistTask(row[0], nodes=set(), event='after_promote')
+            pg_dist_group[row[0]].add(PgDistNode(*row[1:]))
+
+        with self._condition:
+            self._pg_dist_group = pg_dist_group
+        return True
+
+    def sync_meta_data(self, cluster: Cluster) -> None:
+        """Maintain the ``pg_dist_node`` from the coordinator leader every heartbeat loop.
+
+        We can't always rely on REST API calls from worker nodes in order
+        to maintain `pg_dist_node`, therefore at least once per heartbeat
+        loop we make sure that workes registered in `self._pg_dist_group`
+        cache are matching the cluster view from DCS by creating tasks
+        the same way as it is done from the REST API."""
+
+        if not self.is_coordinator():
+            return
+
+        with self._condition:
+            if not self.is_alive():
+                self.start()
+
+        self.add_task('after_promote', CITUS_COORDINATOR_GROUP_ID, cluster,
+                      self._postgresql.name, self._postgresql.connection_string)
+
+        for groupid, worker in cluster.workers.items():
+            leader = worker.leader
+            if leader and leader.conn_url\
+                    and leader.data.get('role') in ('master', 'primary') and leader.data.get('state') == 'running':
+                self.add_task('after_promote', groupid, worker, leader.name, leader.conn_url)
+
+    def find_task_by_groupid(self, groupid: int) -> Optional[int]:
+        for i, task in enumerate(self._tasks):
+            if task.groupid == groupid:
+                return i
+
+    def pick_task(self) -> Tuple[Optional[int], Optional[PgDistTask]]:
+        """Returns the tuple(i, task), where `i` - is the task index in the self._tasks list
+
+        Tasks are picked by following priorities:
+
+        1. If there is already a transaction in progress, pick a task
+           that that will change already affected worker primary.
+        2. If the coordinator address should be changed - pick a task
+           with groupid=0 (coordinators are always in groupid 0).
+        3. Pick a task that is the oldest (first from the self._tasks)
+        """
+
+        with self._condition:
+            if self._in_flight:
+                i = self.find_task_by_groupid(self._in_flight.groupid)
+            else:
+                while True:
+                    i = self.find_task_by_groupid(CITUS_COORDINATOR_GROUP_ID)  # set_coordinator
+                    if i is None and self._tasks:
+                        i = 0
+                    if i is None:
+                        break
+                    task = self._tasks[i]
+                    if task == self._pg_dist_group.get(task.groupid):
+                        self._tasks.pop(i)  # nothing to do because cached version of pg_dist_group already matches
+                    else:
+                        break
+            task = self._tasks[i] if i is not None else None
+
+            return i, task
+
+    def update_node(self, groupid: int, node: PgDistNode, cooldown: float = 10000) -> None:
+        if node.role not in ('primary', 'secondary', 'demoted'):
+            self.query('SELECT pg_catalog.citus_remove_node(%s, %s)', node.host, node.port)
+        elif node.nodeid is not None:
+            host = node.host + ('-demoted' if node.role == 'demoted' else '')
+            self.query('SELECT pg_catalog.citus_update_node(%s, %s, %s, true, %s)',
+                       node.nodeid, host, node.port, cooldown)
+        elif node.role != 'demoted':
+            node.nodeid = self.query("SELECT pg_catalog.citus_add_node(%s, %s, %s, %s, 'default')",
+                                     node.host, node.port, groupid, node.role)[0][0]
+
+    def update_group(self, task: PgDistTask, transaction: bool) -> None:
+        current_state = self._in_flight\
+            or self._pg_dist_group.get(task.groupid)\
+            or PgDistTask(task.groupid, set(), 'after_promote')
+        transitions = list(task.transition(current_state))
+        if transitions:
+            if not transaction and len(transitions) > 1:
+                self.query('BEGIN')
+            for node in transitions:
+                self.update_node(task.groupid, node, task.cooldown)
+            if not transaction and len(transitions) > 1:
+                task.failover = False
+                self.query('COMMIT')
+
+    def process_task(self, task: PgDistTask) -> bool:
+        """Updates a single row in `pg_dist_group` table, optionally in a transaction.
+
+        The transaction is started if we do a demote of the worker node or before promoting the other worker if
+        there is no transaction in progress. And, the transaction is committed when the switchover/failover completed.
+
+        .. note:
+            The maximum lifetime of the transaction in progress is controlled outside of this method.
+
+        .. note:
+            Read access to `self._in_flight` isn't protected because we know it can't be changed outside of our thread.
+
+        :param task: reference to a :class:`PgDistTask` object that represents a row to be updated/created.
+        :returns: ``True`` if the row was succesfully created/updated or transaction in progress
+            was committed as an indicator that the `self._pg_dist_group` cache should be updated,
+            or, if the new transaction was opened, this method returns `False`.
+        """
+
+        if task.event == 'after_promote':
+            self.update_group(task, self._in_flight is not None)
+            if self._in_flight:
+                self.query('COMMIT')
+            task.failover = False
+            return True
+        else:  # before_demote, before_promote
+            if task.timeout:
+                task.deadline = time.time() + task.timeout
+            if not self._in_flight:
+                self.query('BEGIN')
+            self.update_group(task, True)
+        return False
+
+    def process_tasks(self) -> None:
+        while True:
+            # Read access to `_in_flight` isn't protected because we know it can't be changed outside of our thread.
+            if not self._in_flight and not self.load_pg_dist_group():
+                break
+
+            i, task = self.pick_task()
+            if not task or i is None:
+                break
+            try:
+                update_cache = self.process_task(task)
+            except Exception as e:
+                logger.error('Exception when working with pg_dist_node: %r', e)
+                update_cache = None
+            with self._condition:
+                if self._tasks:
+                    if update_cache:
+                        self._pg_dist_group[task.groupid] = task
+
+                    if update_cache is False:  # an indicator that process_tasks has started a transaction
+                        self._in_flight = task
+                    else:
+                        self._in_flight = None
+
+                    if id(self._tasks[i]) == id(task):
+                        self._tasks.pop(i)
+            task.wakeup()
+
+    def run(self) -> None:
+        while True:
+            try:
+                with self._condition:
+                    if self._schedule_load_pg_dist_group:
+                        timeout = -1
+                    elif self._in_flight:
+                        timeout = self._in_flight.deadline - time.time() if self._tasks else None
+                    else:
+                        timeout = -1 if self._tasks else None
+
+                    if timeout is None or timeout > 0:
+                        self._condition.wait(timeout)
+                    elif self._in_flight:
+                        logger.warning('Rolling back transaction. Last known status: %s', self._in_flight)
+                        self.query('ROLLBACK')
+                        self._in_flight = None
+                self.process_tasks()
+            except Exception:
+                logger.exception('run')
+
+    def _add_task(self, task: PgDistTask) -> bool:
+        with self._condition:
+            i = self.find_task_by_groupid(task.groupid)
+
+            # The `PgDistNode.timeout` == None is an indicator that it was scheduled from the sync_meta_data().
+            if task.timeout is None:
+                # We don't want to override the already existing task created from REST API.
+                if i is not None and self._tasks[i].timeout is not None:
+                    return False
+
+                # There is a little race condition with tasks created from REST API - the call made "before" the member
+                # key is updated in DCS. Therefore it is possible that :func:`sync_meta_data` will try to create a task
+                # based on the outdated values of "state"/"role". To solve it we introduce an artificial timeout.
+                # Only when the timeout is reached new tasks could be scheduled from sync_meta_data()
+                if self._in_flight and self._in_flight.groupid == task.groupid and self._in_flight.timeout is not None\
+                        and self._in_flight.deadline > time.time():
+                    return False
+
+            # Override already existing task for the same worker groupid
+            if i is not None:
+                if task != self._tasks[i]:
+                    logger.debug('Overriding existing task: %s != %s', self._tasks[i], task)
+                    self._tasks[i] = task
+                    self._condition.notify()
+                    return True
+            # Add the task to the list if Worker node state is different from the cached `pg_dist_group`
+            elif self._schedule_load_pg_dist_group or task != self._pg_dist_group.get(task.groupid)\
+                    or self._in_flight and task.groupid == self._in_flight.groupid:
+                logger.debug('Adding the new task: %s', task)
+                self._tasks.append(task)
+                self._condition.notify()
+                return True
+        return False
+
+    @staticmethod
+    def _pg_dist_node(role: str, conn_url: str) -> Optional[PgDistNode]:
+        try:
+            r = urlparse(conn_url)
+            if r.hostname:
+                return PgDistNode(r.hostname, r.port or 5432, role)
+        except Exception as e:
+            logger.error('Failed to parse connection url %s: %r', conn_url, e)
+
+    def add_task(self, event: str, groupid: int, cluster: Cluster, leader_name: str, leader_url: str,
+                 timeout: Optional[float] = None, cooldown: Optional[float] = None) -> Optional[PgDistTask]:
+        primary = self._pg_dist_node('demoted' if event == 'before_demote' else 'primary', leader_url)
+        if not primary:
+            return
+
+        task = PgDistTask(groupid, {primary}, event=event, timeout=timeout, cooldown=cooldown)
+        for member in cluster.members:
+            secondary = self._pg_dist_node('secondary', member.conn_url)\
+                if member.name != leader_name and member.is_running and member.conn_url else None
+            if secondary:
+                task.add(secondary)
+        return task if self._add_task(task) else None
+
+    def handle_event(self, cluster: Cluster, event: Dict[str, Any]) -> None:
+        if not self.is_alive():
+            return
+
+        worker = cluster.workers.get(event['group'])
+        if not (worker and worker.leader and worker.leader.name == event['leader'] and worker.leader.conn_url):
+            return logger.info('Discarding event %s', event)
+
+        task = self.add_task(event['type'], event['group'], worker,
+                             worker.leader.name, worker.leader.conn_url,
+                             event['timeout'], event['cooldown'] * 1000)
+        if task and event['type'] == 'before_demote':
+            task.wait()
+
+    def bootstrap(self) -> None:
+        """Bootstrap handler.
+
+        Is called when the new cluster is initialized (through ``initdb`` or a custom bootstrap method).
+        """
+        conn_kwargs = {**self._postgresql.connection_pool.conn_kwargs,
+                       'options': '-c synchronous_commit=local -c statement_timeout=0'}
+        if self._config['database'] != self._postgresql.database:
+            conn = connect(**conn_kwargs)
+            try:
+                with conn.cursor() as cur:
+                    cur.execute('CREATE DATABASE {0}'.format(
+                        quote_ident(self._config['database'], conn)).encode('utf-8'))
+            except ProgrammingError as exc:
+                if exc.diag.sqlstate == '42P04':  # DuplicateDatabase
+                    logger.debug('Exception when creating database: %r', exc)
+                else:
+                    raise exc
+            finally:
+                conn.close()
+
+        conn_kwargs['dbname'] = self._config['database']
+        conn = connect(**conn_kwargs)
+        try:
+            with conn.cursor() as cur:
+                cur.execute('CREATE EXTENSION IF NOT EXISTS citus')
+
+                superuser = self._postgresql.config.superuser
+                params = {k: superuser[k] for k in ('password', 'sslcert', 'sslkey') if k in superuser}
+                if params:
+                    cur.execute("INSERT INTO pg_catalog.pg_dist_authinfo VALUES"
+                                "(0, pg_catalog.current_user(), %s)",
+                                (self._postgresql.config.format_dsn(params),))
+
+                if self.is_coordinator():
+                    r = urlparse(self._postgresql.connection_string)
+                    cur.execute("SELECT pg_catalog.citus_set_coordinator_host(%s, %s, 'primary', 'default')",
+                                (r.hostname, r.port or 5432))
+        finally:
+            conn.close()
+
+    def adjust_postgres_gucs(self, parameters: Dict[str, Any]) -> None:
+        """Adjust GUCs in the current PostgreSQL configuration.
+
+        :param parameters: dictionary of GUCs, with key as GUC name and the corresponding value as current GUC value.
+        """
+        # citus extension must be on the first place in shared_preload_libraries
+        shared_preload_libraries = list(filter(
+            lambda el: el and el != 'citus',
+            [p.strip() for p in parameters.get('shared_preload_libraries', '').split(',')]))
+        parameters['shared_preload_libraries'] = ','.join(['citus'] + shared_preload_libraries)
+
+        # if not explicitly set Citus overrides max_prepared_transactions to max_connections*2
+        if parameters['max_prepared_transactions'] == 0:
+            parameters['max_prepared_transactions'] = parameters['max_connections'] * 2
+
+        # Resharding in Citus implemented using logical replication
+        parameters['wal_level'] = 'logical'
+
+        # Sometimes Citus needs to connect to the local postgres. We will do it the same way as Patroni does.
+        parameters['citus.local_hostname'] = self._postgresql.connection_pool.conn_kwargs.get('host', 'localhost')
+
+    def ignore_replication_slot(self, slot: Dict[str, str]) -> bool:
+        """Check whether provided replication *slot* existing in the database should not be removed.
+
+        .. note::
+            MPP database may create replication slots for its own use, for example to migrate data between workers
+            using logical replication, and we don't want to suddenly drop them.
+
+        :param slot: dictionary containing the replication slot settings, like ``name``, ``database``, ``type``, and
+                     ``plugin``.
+
+        :returns: ``True`` if the replication slots should not be removed, otherwise ``False``.
+        """
+        if self._postgresql.is_primary() and slot['type'] == 'logical' and slot['database'] == self._config['database']:
+            m = CITUS_SLOT_NAME_RE.match(slot['name'])
+            return bool(m and {'move': 'pgoutput', 'split': 'citus'}.get(m.group(1)) == slot['plugin'])
+        return False

patroni/postgresql/postmaster.py

@@ -176,7 +176,7 @@ class PostmasterProcess(psutil.Process):
             return not self.is_running()
 
     def wait_for_user_backends_to_close(self, stop_timeout: Optional[float]) -> None:
-        # These regexps are cross checked against versions PostgreSQL 9.1 .. 15
+        # These regexps are cross checked against versions PostgreSQL 9.1 .. 16
         aux_proc_re = re.compile("(?:postgres:)( .*:)? (?:(?:archiver|startup|autovacuum launcher|autovacuum worker|"
                                  "checkpointer|logger|stats collector|wal receiver|wal writer|writer)(?: process  )?|"
                                  "walreceiver|wal sender process|walsender|walwriter|background writer|"

patroni/postgresql/rewind.py

@@ -13,6 +13,7 @@ from . import Postgresql
 from .connection import get_connection_cursor
 from .misc import format_lsn, fsync_dir, parse_history, parse_lsn
 from ..async_executor import CriticalTask
+from ..collections import EMPTY_DICT
 from ..dcs import Leader, RemoteMember
 
 logger = logging.getLogger(__name__)
@@ -101,12 +102,26 @@ class Rewind(object):
             return 'not accessible or not healty'
 
     def _get_checkpoint_end(self, timeline: int, lsn: int) -> int:
-        """The checkpoint record size in WAL depends on postgres major version and platform (memory alignment).
-        Hence, the only reliable way to figure out where it ends, read the record from file with the help of pg_waldump
-        and parse the output. We are trying to read two records, and expect that it will fail to read the second one:
-        `pg_waldump: fatal: error in WAL record at 0/182E220: invalid record length at 0/182E298: wanted 24, got 0`
-        The error message contains information about LSN of the next record, which is exactly where checkpoint ends."""
+        """Get the end of checkpoint record from WAL.
 
+        .. note::
+            The checkpoint record size in WAL depends on postgres major version and platform (memory alignment).
+            Hence, the only reliable way to figure out where it ends, is to read the record from file with the
+            help of ``pg_waldump`` and parse the output.
+
+            We are trying to read two records, and expect that it will fail to read the second record with message:
+
+                fatal: error in WAL record at 0/182E220: invalid record length at 0/182E298: wanted 24, got 0; or
+
+                fatal: error in WAL record at 0/182E220: invalid record length at 0/182E298: expected at least 24, got 0
+
+            The error message contains information about LSN of the next record, which is exactly where checkpoint ends.
+
+        :param timeline: the checkpoint *timeline* from ``pg_controldata``.
+        :param lsn: the checkpoint *location* as :class:`int` from ``pg_controldata``.
+
+        :returns: the end of checkpoint record as :class:`int` or ``0`` if failed to parse ``pg_waldump`` output.
+        """
         lsn8 = format_lsn(lsn, True)
         lsn_str = format_lsn(lsn)
         out, err = self._postgresql.waldump(timeline, lsn_str, 2)
@@ -117,12 +132,17 @@ class Rewind(object):
 
             if len(out) == 1 and len(err) == 1 and ', lsn: {0}, prev '.format(lsn8) in out[0] and pattern in err[0]:
                 i = err[0].find(pattern) + len(pattern)
-                j = err[0].find(": wanted ", i)
-                if j > -1:
-                    try:
-                        return parse_lsn(err[0][i:j])
-                    except Exception as e:
-                        logger.error('Failed to parse lsn %s: %r', err[0][i:j], e)
+                # Message format depends on the major version:
+                # * expected at least -- starting from v16
+                # * wanted -- before v16
+                # We will simply check all possible combinations.
+                for pattern in (': expected at least ', ': wanted '):
+                    j = err[0].find(pattern, i)
+                    if j > -1:
+                        try:
+                            return parse_lsn(err[0][i:j])
+                        except Exception as e:
+                            logger.error('Failed to parse lsn %s: %r', err[0][i:j], e)
             logger.error('Failed to parse pg_%sdump output', self._postgresql.wal_name)
             logger.error(' stdout=%s', '\n'.join(out))
             logger.error(' stderr=%s', '\n'.join(err))
@@ -190,9 +210,10 @@ class Rewind(object):
         ret = member.conn_kwargs(auth)
         if not ret.get('dbname'):
             ret['dbname'] = self._postgresql.database
-        # Add target_session_attrs in case more than one hostname is specified
-        # (libpq client-side failover) making sure we hit the primary
-        if 'target_session_attrs' not in ret and self._postgresql.major_version >= 100000:
+        # Add target_session_attrs to make sure we hit the primary.
+        # It is not strictly necessary for starting from PostgreSQL v14, which made it possible
+        # to rewind from standby, but doing it from the real primary is always safer.
+        if self._postgresql.major_version >= 100000:
             ret['target_session_attrs'] = 'read-write'
         return ret
 
@@ -398,7 +419,7 @@ class Rewind(object):
         dsn = self._postgresql.config.format_dsn(r, True)
         logger.info('running pg_rewind from %s', dsn)
 
-        restore_command = (self._postgresql.config.get('recovery_conf') or {}).get('restore_command') \
+        restore_command = (self._postgresql.config.get('recovery_conf') or EMPTY_DICT).get('restore_command') \
             if self._postgresql.major_version < 120000 else self._postgresql.get_guc_value('restore_command')
 
         # Until v15 pg_rewind expected postgresql.conf to be inside $PGDATA, which is not the case on e.g. Debian

patroni/postgresql/slots.py

@@ -13,9 +13,11 @@ from typing import Any, Dict, Iterator, List, Optional, Union, Tuple, TYPE_CHECK
 
 from .connection import get_connection_cursor
 from .misc import format_lsn, fsync_dir
+from .. import global_config
 from ..dcs import Cluster, Leader
 from ..file_perm import pg_perm
 from ..psycopg import OperationalError
+from ..tags import Tags
 
 if TYPE_CHECKING:  # pragma: no cover
     from psycopg import Cursor
@@ -289,18 +291,18 @@ class SlotsHandler:
         :param name: name of the slot to ignore
 
         :returns: ``True`` if slot *name* matches any slot specified in ``ignore_slots`` configuration,
-                 otherwise will pass through and return result of :meth:`CitusHandler.ignore_replication_slot`.
+                 otherwise will pass through and return result of :meth:`AbstractMPPHandler.ignore_replication_slot`.
         """
         slot = self._replication_slots[name]
         if cluster.config:
-            for matcher in cluster.config.ignore_slots_matchers:
+            for matcher in global_config.ignore_slots_matchers:
                 if (
                         (matcher.get("name") is None or matcher["name"] == name)
                         and all(not matcher.get(a) or matcher[a] == slot.get(a)
                                 for a in ('database', 'plugin', 'type'))
                 ):
                     return True
-        return self._postgresql.citus_handler.ignore_replication_slot(slot)
+        return self._postgresql.mpp_handler.ignore_replication_slot(slot)
 
     def drop_replication_slot(self, name: str) -> Tuple[bool, bool]:
         """Drop a named slot from Postgres.
@@ -319,7 +321,7 @@ class SlotsHandler:
                             ' FULL OUTER JOIN dropped ON true'), name)
         return (rows[0][0], rows[0][1]) if rows else (False, False)
 
-    def _drop_incorrect_slots(self, cluster: Cluster, slots: Dict[str, Any], paused: bool) -> None:
+    def _drop_incorrect_slots(self, cluster: Cluster, slots: Dict[str, Any]) -> None:
         """Compare required slots and configured as permanent slots with those found, dropping extraneous ones.
 
         .. note::
@@ -330,11 +332,10 @@ class SlotsHandler:
 
         :param cluster: cluster state information object.
         :param slots: dictionary of desired slot names as keys with slot attributes as a dictionary value, if known.
-        :param paused: ``True`` if the patroni cluster is currently in a paused state.
         """
         # drop old replication slots which are not presented in desired slots.
         for name in set(self._replication_slots) - set(slots):
-            if not paused and not self.ignore_replication_slot(cluster, name):
+            if not global_config.is_paused and not self.ignore_replication_slot(cluster, name):
                 active, dropped = self.drop_replication_slot(name)
                 if dropped:
                     logger.info("Dropped unknown replication slot '%s'", name)
@@ -492,8 +493,7 @@ class SlotsHandler:
             self._schedule_load_slots = True
         return create_slots + copy_slots
 
-    def sync_replication_slots(self, cluster: Cluster, nofailover: bool,
-                               replicatefrom: Optional[str] = None, paused: bool = False) -> List[str]:
+    def sync_replication_slots(self, cluster: Cluster, tags: Tags) -> List[str]:
         """During the HA loop read, check and alter replication slots found in the cluster.
 
         Read physical and logical slots from ``pg_replication_slots``, then compare to those configured in the DCS.
@@ -503,22 +503,18 @@ class SlotsHandler:
         them on replica nodes by copying slot files from the primary.
 
         :param cluster: object containing stateful information for the cluster.
-        :param nofailover: ``True`` if this node has been tagged to not be a failover candidate.
-        :param replicatefrom: the tag containing the node to replicate from.
-        :param paused: ``True`` if the cluster is in maintenance mode.
+        :param tags: reference to an object implementing :class:`Tags` interface.
 
         :returns: list of logical replication slots names that should be copied from the primary.
         """
         ret = []
-        if self._postgresql.major_version >= 90400 and self._postgresql.global_config and cluster.config:
+        if self._postgresql.major_version >= 90400 and cluster.config:
             try:
                 self.load_replication_slots()
 
-                slots = cluster.get_replication_slots(
-                    self._postgresql.name, self._postgresql.role, nofailover, self._postgresql.major_version,
-                    is_standby_cluster=self._postgresql.global_config.is_standby_cluster, show_error=True)
+                slots = cluster.get_replication_slots(self._postgresql, tags, show_error=True)
 
-                self._drop_incorrect_slots(cluster, slots, paused)
+                self._drop_incorrect_slots(cluster, slots)
 
                 self._ensure_physical_slots(slots)
 
@@ -526,7 +522,7 @@ class SlotsHandler:
                     self._logical_slots_processing_queue.clear()
                     self._ensure_logical_slots_primary(slots)
                 else:
-                    self.check_logical_slots_readiness(cluster, replicatefrom)
+                    self.check_logical_slots_readiness(cluster, tags)
                     ret = self._ensure_logical_slots_replica(slots)
 
                 self._replication_slots = slots
@@ -552,7 +548,7 @@ class SlotsHandler:
         with get_connection_cursor(connect_timeout=3, options="-c statement_timeout=2000", **conn_kwargs) as cur:
             yield cur
 
-    def check_logical_slots_readiness(self, cluster: Cluster, replicatefrom: Optional[str]) -> bool:
+    def check_logical_slots_readiness(self, cluster: Cluster, tags: Tags) -> bool:
         """Determine whether all known logical slots are synchronised from the leader.
 
         1) Retrieve the current ``catalog_xmin`` value for the physical slot from the cluster leader, and
@@ -561,13 +557,13 @@ class SlotsHandler:
         3) store logical slot ``catalog_xmin`` when the physical slot ``catalog_xmin`` becomes valid.
 
         :param cluster: object containing stateful information for the cluster.
-        :param replicatefrom: name of the member that should be used to replicate from.
+        :param tags: reference to an object implementing :class:`Tags` interface.
 
         :returns: ``False`` if any issue while checking logical slots readiness, ``True`` otherwise.
         """
         catalog_xmin = None
         if self._logical_slots_processing_queue and cluster.leader:
-            slot_name = cluster.get_my_slot_name_on_primary(self._postgresql.name, replicatefrom)
+            slot_name = cluster.get_slot_name_on_primary(self._postgresql.name, tags)
             try:
                 with self._get_leader_connection_cursor(cluster.leader) as cur:
                     cur.execute("SELECT slot_name, catalog_xmin FROM pg_catalog.pg_get_replication_slots()"
@@ -645,16 +641,17 @@ class SlotsHandler:
                 if standby_logical_slot:
                     logger.info('Logical slot %s is safe to be used after a failover', name)
 
-    def copy_logical_slots(self, cluster: Cluster, create_slots: List[str]) -> None:
+    def copy_logical_slots(self, cluster: Cluster, tags: Tags, create_slots: List[str]) -> None:
         """Create logical replication slots on standby nodes.
 
         :param cluster: object containing stateful information for the cluster.
+        :param tags: reference to an object implementing :class:`Tags` interface.
         :param create_slots: list of slot names to copy from the primary.
         """
         leader = cluster.leader
         if not leader:
             return
-        slots = cluster.get_replication_slots(self._postgresql.name, 'replica', False, self._postgresql.major_version)
+        slots = cluster.get_replication_slots(self._postgresql, tags, role='replica')
         copy_slots: Dict[str, Dict[str, Any]] = {}
         with self._get_leader_connection_cursor(leader) as cur:
             try:

patroni/postgresql/sync.py

@@ -5,6 +5,7 @@ import time
 from copy import deepcopy
 from typing import Collection, List, NamedTuple, Tuple, TYPE_CHECKING
 
+from .. import global_config
 from ..collections import CaseInsensitiveDict, CaseInsensitiveSet
 from ..dcs import Cluster
 from ..psycopg import quote_ident as _quote_ident
@@ -303,11 +304,8 @@ END;$$""")
         replica_list = _ReplicaList(self._postgresql, cluster)
         self._process_replica_readiness(cluster, replica_list)
 
-        if TYPE_CHECKING:  # pragma: no cover
-            assert self._postgresql.global_config is not None
-        sync_node_count = self._postgresql.global_config.synchronous_node_count\
-            if self._postgresql.supports_multiple_sync else 1
-        sync_node_maxlag = self._postgresql.global_config.maximum_lag_on_syncnode
+        sync_node_count = global_config.synchronous_node_count if self._postgresql.supports_multiple_sync else 1
+        sync_node_maxlag = global_config.maximum_lag_on_syncnode
 
         candidates = CaseInsensitiveSet()
         sync_nodes = CaseInsensitiveSet()

patroni/postgresql/validator.py

@@ -1,11 +1,11 @@
 import abc
 from copy import deepcopy
 import logging
-import os
 import yaml
 
 from typing import Any, Dict, Iterator, List, MutableMapping, Optional, Tuple, Type, Union
 
+from .available_parameters import get_validator_files, PathLikeObj
 from ..collections import CaseInsensitiveDict, CaseInsensitiveSet
 from ..exceptions import PatroniException
 from ..utils import parse_bool, parse_int, parse_real
@@ -258,10 +258,10 @@ class InvalidGucValidatorsFile(PatroniException):
     """Raised when reading or parsing of a YAML file faces an issue."""
 
 
-def _read_postgres_gucs_validators_file(file: str) -> Dict[str, Any]:
+def _read_postgres_gucs_validators_file(file: PathLikeObj) -> Dict[str, Any]:
     """Read an YAML file and return the corresponding Python object.
 
-    :param file: path to the file to be read. It is expected to be encoded with ``UTF-8``, and to be a YAML document.
+    :param file: path-like object to read from. It is expected to be encoded with ``UTF-8``, and to be a YAML document.
 
     :returns: the YAML content parsed into a Python object. If any issue is faced while reading/parsing the file, then
         return ``None``.
@@ -270,7 +270,7 @@ def _read_postgres_gucs_validators_file(file: str) -> Dict[str, Any]:
         :class:`InvalidGucValidatorsFile`: if faces an issue while reading or parsing *file*.
     """
     try:
-        with open(file, encoding='UTF-8') as stream:
+        with file.open(encoding='UTF-8') as stream:
             return yaml.safe_load(stream)
     except Exception as exc:
         raise InvalidGucValidatorsFile(
@@ -385,21 +385,7 @@ def _load_postgres_gucs_validators() -> None:
                     version_till: null
 
     """
-    conf_dir = os.path.join(
-        os.path.dirname(os.path.abspath(__file__)),
-        'available_parameters',
-    )
-    yaml_files: List[str] = []
-
-    for root, _, files in os.walk(conf_dir):
-        for file in sorted(files):
-            full_path = os.path.join(root, file)
-            if file.lower().endswith(('.yml', '.yaml')):
-                yaml_files.append(full_path)
-            else:
-                logger.info('Ignored a non-YAML file found under `available_parameters` directory: `%s`.', full_path)
-
-    for file in yaml_files:
+    for file in get_validator_files():
         try:
             config: Dict[str, Any] = _read_postgres_gucs_validators_file(file)
         except InvalidGucValidatorsFile as exc:

patroni/psycopg.py

@@ -42,7 +42,8 @@ try:
             value.prepare(conn)
         return value.getquoted().decode('utf-8')
 except ImportError:
-    from psycopg import connect as __connect, sql, Error, DatabaseError, OperationalError, ProgrammingError
+    from psycopg import connect as __connect  # pyright: ignore [reportUnknownVariableType]
+    from psycopg import sql, Error, DatabaseError, OperationalError, ProgrammingError
 
     def _connect(dsn: Optional[str] = None, **kwargs: Any) -> 'Connection[Any]':
         """Call :func:`psycopg.connect` with *dsn* and ``**kwargs``.
@@ -56,7 +57,7 @@ except ImportError:
 
         :returns: a connection to the database.
         """
-        ret = __connect(dsn or "", **kwargs)
+        ret: 'Connection[Any]' = __connect(dsn or "", **kwargs)
         setattr(ret, 'server_version', ret.pgconn.server_version)  # compatibility with psycopg2
         return ret
 

patroni/scripts/barman/__init__.py

@@ -0,0 +1 @@
+"""Create :mod:`patroni.scripts.barman`."""

patroni/scripts/barman/cli.py

@@ -0,0 +1,240 @@
+#!/usr/bin/env python
+
+"""Perform operations on Barman through ``pg-backup-api``.
+
+The actual operations are implemented by separate modules. This module only
+builds the CLI that makes an interface with the actual commands.
+
+.. note::
+    See :class:ExitCode` for possible exit codes of this main script.
+"""
+
+from argparse import ArgumentParser
+from enum import IntEnum
+import logging
+import sys
+
+from .config_switch import run_barman_config_switch
+from .recover import run_barman_recover
+from .utils import ApiNotOk, PgBackupApi, set_up_logging
+
+
+class ExitCode(IntEnum):
+    """Possible exit codes of this script.
+
+    :cvar NO_COMMAND: if no sub-command of ``patroni_barman`` application has
+        been selected by the user.
+    :cvar API_NOT_OK: ``pg-backup-api`` status is not ``OK``.
+    """
+
+    NO_COMMAND = -1
+    API_NOT_OK = -2
+
+
+def main() -> None:
+    """Entry point of ``patroni_barman`` application.
+
+    Implements the parser for the application and for its sub-commands.
+
+    The script exit code may be one of:
+
+    * :attr:`ExitCode.NO_COMMAND`: if no sub-command was specified in the
+        ``patroni_barman`` call;
+    * :attr:`ExitCode.API_NOT_OK`: if ``pg-backup-api`` is not correctly up and
+        running;
+    * Value returned by :func:`~patroni.scripts.barman.config_switch.run_barman_config_switch`,
+        if running ``patroni_barman config-switch``;
+    * Value returned by :func:`~patroni.scripts.barman.recover.run_barman_recover`,
+        if running ``patroni_barman recover``.
+
+    The called sub-command is expected to exit execution once finished using
+    its own set of exit codes.
+    """
+    parser = ArgumentParser(
+        description=(
+            "Wrapper application for pg-backup-api. Communicate with the API "
+            "running at the given URL to perform remote Barman operations."
+        ),
+    )
+    parser.add_argument(
+        "--api-url",
+        type=str,
+        required=True,
+        help="URL to reach the pg-backup-api, e.g. 'http://localhost:7480'",
+        dest="api_url",
+    )
+    parser.add_argument(
+        "--cert-file",
+        type=str,
+        required=False,
+        help="Certificate to authenticate against the API, if required.",
+        dest="cert_file",
+    )
+    parser.add_argument(
+        "--key-file",
+        type=str,
+        required=False,
+        help="Certificate key to authenticate against the API, if required.",
+        dest="key_file",
+    )
+    parser.add_argument(
+        "--retry-wait",
+        type=int,
+        required=False,
+        default=2,
+        help="How long in seconds to wait before retrying a failed "
+             "pg-backup-api request (default: '%(default)s')",
+        dest="retry_wait",
+    )
+    parser.add_argument(
+        "--max-retries",
+        type=int,
+        required=False,
+        default=5,
+        help="Maximum number of retries when receiving malformed responses "
+             "from the pg-backup-api (default: '%(default)s')",
+        dest="max_retries",
+    )
+    parser.add_argument(
+        "--log-file",
+        type=str,
+        required=False,
+        help="File where to log messages produced by this application, if any.",
+        dest="log_file",
+    )
+
+    subparsers = parser.add_subparsers(title="Sub-commands")
+
+    recover_parser = subparsers.add_parser(
+        "recover",
+        help="Remote 'barman recover'",
+        description="Restore a Barman backup of a given Barman server"
+    )
+    recover_parser.add_argument(
+        "--barman-server",
+        type=str,
+        required=True,
+        help="Name of the Barman server from which to restore the backup.",
+        dest="barman_server",
+    )
+    recover_parser.add_argument(
+        "--backup-id",
+        type=str,
+        required=False,
+        default="latest",
+        help="ID of the Barman backup to be restored. You can use any value "
+             "supported by 'barman recover' command "
+             "(default: '%(default)s')",
+        dest="backup_id",
+    )
+    recover_parser.add_argument(
+        "--ssh-command",
+        type=str,
+        required=True,
+        help="Value to be passed as '--remote-ssh-command' to 'barman recover'.",
+        dest="ssh_command",
+    )
+    recover_parser.add_argument(
+        "--data-directory",
+        "--datadir",
+        type=str,
+        required=True,
+        help="Destination path where to restore the barman backup in the "
+             "local host.",
+        dest="data_directory",
+    )
+    recover_parser.add_argument(
+        "--loop-wait",
+        type=int,
+        required=False,
+        default=10,
+        help="How long to wait before checking again the status of the "
+             "recovery process, in seconds. Use higher values if your "
+             "recovery is expected to take long (default: '%(default)s')",
+        dest="loop_wait",
+    )
+    recover_parser.set_defaults(func=run_barman_recover)
+
+    config_switch_parser = subparsers.add_parser(
+        "config-switch",
+        help="Remote 'barman config-switch'",
+        description="Switch the configuration of a given Barman server. "
+                    "Intended to be used as a 'on_role_change' callback."
+    )
+    config_switch_parser.add_argument(
+        "action",
+        type=str,
+        choices=["on_role_change"],
+        help="Name of the callback (automatically filled by Patroni)",
+    )
+    config_switch_parser.add_argument(
+        "role",
+        type=str,
+        choices=["master", "primary", "promoted", "standby_leader", "replica",
+                 "demoted"],
+        help="Name of the new role of this node (automatically filled by "
+             "Patroni)",
+    )
+    config_switch_parser.add_argument(
+        "cluster",
+        type=str,
+        help="Name of the Patroni cluster involved in the callback "
+             "(automatically filled by Patroni)",
+    )
+    config_switch_parser.add_argument(
+        "--barman-server",
+        type=str,
+        required=True,
+        help="Name of the Barman server which config is to be switched.",
+        dest="barman_server",
+    )
+    group = config_switch_parser.add_mutually_exclusive_group(required=True)
+    group.add_argument(
+        "--barman-model",
+        type=str,
+        help="Name of the Barman config model to be applied to the server.",
+        dest="barman_model",
+    )
+    group.add_argument(
+        "--reset",
+        action="store_true",
+        help="Unapply the currently active model for the server, if any.",
+        dest="reset",
+    )
+    config_switch_parser.add_argument(
+        "--switch-when",
+        type=str,
+        required=True,
+        default="promoted",
+        choices=["promoted", "demoted", "always"],
+        help="Controls under which circumstances the 'on_role_change' callback "
+             "should actually switch config in Barman. 'promoted' means the "
+             "'role' is either 'master', 'primary' or 'promoted'. 'demoted' "
+             "means the 'role' is either 'replica' or 'demoted' "
+             "(default: '%(default)s')",
+        dest="switch_when",
+    )
+    config_switch_parser.set_defaults(func=run_barman_config_switch)
+
+    args, _ = parser.parse_known_args()
+
+    set_up_logging(args.log_file)
+
+    if not hasattr(args, "func"):
+        parser.print_help()
+        sys.exit(ExitCode.NO_COMMAND)
+
+    api = None
+
+    try:
+        api = PgBackupApi(args.api_url, args.cert_file, args.key_file,
+                          args.retry_wait, args.max_retries)
+    except ApiNotOk as exc:
+        logging.error("pg-backup-api is not working: %r", exc)
+        sys.exit(ExitCode.API_NOT_OK)
+
+    sys.exit(args.func(api, args))
+
+
+if __name__ == "__main__":
+    main()

patroni/scripts/barman/config_switch.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+
+"""Implements ``patroni_barman config-switch`` sub-command.
+
+Apply a Barman configuration model through ``pg-backup-api``.
+
+This sub-command is specially useful as a ``on_role_change`` callback to change
+Barman configuration in response to failovers and switchovers. Check the output
+of ``--help`` to understand the parameters supported by the sub-command.
+
+It requires that you have previously configured a Barman server and Barman
+config models, and that you have ``pg-backup-api`` configured and running in
+the same host as Barman.
+
+Refer to :class:`ExitCode` for possible exit codes of this sub-command.
+"""
+from argparse import Namespace
+from enum import IntEnum
+import logging
+import time
+from typing import Optional, TYPE_CHECKING
+
+from .utils import OperationStatus, RetriesExceeded
+
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .utils import PgBackupApi
+
+
+class ExitCode(IntEnum):
+    """Possible exit codes of this script.
+
+    :cvar CONFIG_SWITCH_DONE: config switch was successfully performed.
+    :cvar CONFIG_SWITCH_SKIPPED: if the execution was skipped because of not
+        matching user expectations.
+    :cvar CONFIG_SWITCH_FAILED: config switch faced an issue.
+    :cvar HTTP_ERROR: an error has occurred while communicating with
+        ``pg-backup-api``
+    :cvar INVALID_ARGS: an invalid set of arguments has been given to the
+        operation.
+    """
+
+    CONFIG_SWITCH_DONE = 0
+    CONFIG_SWITCH_SKIPPED = 1
+    CONFIG_SWITCH_FAILED = 2
+    HTTP_ERROR = 3
+    INVALID_ARGS = 4
+
+
+def _should_skip_switch(args: Namespace) -> bool:
+    """Check if we should skip the config switch operation.
+
+    :param args: arguments received from the command-line of
+        ``patroni_barman config-switch`` command.
+
+    :returns: if the operation should be skipped.
+    """
+    if args.switch_when == "promoted":
+        return args.role not in {"master", "primary", "promoted"}
+    if args.switch_when == "demoted":
+        return args.role not in {"replica", "demoted"}
+    return False
+
+
+def _switch_config(api: "PgBackupApi", barman_server: str,
+                   barman_model: Optional[str], reset: Optional[bool]) -> int:
+    """Switch configuration of Barman server through ``pg-backup-api``.
+
+    .. note::
+        If requests to ``pg-backup-api`` fail recurrently or we face HTTP
+        errors, then exit with :attr:`ExitCode.HTTP_ERROR`.
+
+    :param api: a :class:`PgBackupApi` instance to handle communication with
+        the API.
+    :param barman_server: name of the Barman server which config is to be
+        switched.
+    :param barman_model: name of the Barman model to be applied to the server,
+        if any.
+    :param reset: ``True`` if you would like to unapply the currently active
+        model for the server, if any.
+
+    :returns: the return code to be used when exiting the ``patroni_barman``
+        application. Refer to :class:`ExitCode`.
+    """
+    operation_id = None
+
+    try:
+        operation_id = api.create_config_switch_operation(
+            barman_server,
+            barman_model,
+            reset,
+        )
+    except RetriesExceeded as exc:
+        logging.error("An issue was faced while trying to create a config "
+                      "switch operation: %r", exc)
+        return ExitCode.HTTP_ERROR
+
+    logging.info("Created the config switch operation with ID %s",
+                 operation_id)
+
+    status = None
+
+    while True:
+        try:
+            status = api.get_operation_status(barman_server, operation_id)
+        except RetriesExceeded:
+            logging.error("Maximum number of retries exceeded, exiting.")
+            return ExitCode.HTTP_ERROR
+
+        if status != OperationStatus.IN_PROGRESS:
+            break
+
+        logging.info("Config switch operation %s is still in progress",
+                     operation_id)
+        time.sleep(5)
+
+    if status == OperationStatus.DONE:
+        logging.info("Config switch operation finished successfully.")
+        return ExitCode.CONFIG_SWITCH_DONE
+    else:
+        logging.error("Config switch operation failed.")
+        return ExitCode.CONFIG_SWITCH_FAILED
+
+
+def run_barman_config_switch(api: "PgBackupApi", args: Namespace) -> int:
+    """Run a remote ``barman config-switch`` through the ``pg-backup-api``.
+
+    :param api: a :class:`PgBackupApi` instance to handle communication with
+        the API.
+    :param args: arguments received from the command-line of
+        ``patroni_barman config-switch`` command.
+
+    :returns: the return code to be used when exiting the ``patroni_barman``
+        application. Refer to :class:`ExitCode`.
+    """
+    if _should_skip_switch(args):
+        logging.info("Config switch operation was skipped (role=%s, "
+                     "switch_when=%s).", args.role, args.switch_when)
+        return ExitCode.CONFIG_SWITCH_SKIPPED
+
+    if not bool(args.barman_model) ^ bool(args.reset):
+        logging.error("One, and only one among 'barman_model' ('%s') and "
+                      "'reset' ('%s') should be given", args.barman_model, args.reset)
+        return ExitCode.INVALID_ARGS
+
+    return _switch_config(api, args.barman_server, args.barman_model, args.reset)

patroni/scripts/barman/recover.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+
+"""Implements ``patroni_barman recover`` sub-command.
+
+Restore a Barman backup to the local node through ``pg-backup-api``.
+
+This sub-command can be used both as a custom bootstrap method, and as a custom
+create replica method. Check the output of ``--help`` to understand the
+parameters supported by the sub-command. ``--datadir`` is a special parameter
+and it is automatically filled by Patroni in both cases.
+
+It requires that you have previously configured a Barman server, and that you
+have ``pg-backup-api`` configured and running in the same host as Barman.
+
+Refer to :class:`ExitCode` for possible exit codes of this sub-command.
+"""
+from argparse import Namespace
+from enum import IntEnum
+import logging
+import time
+from typing import TYPE_CHECKING
+
+from .utils import OperationStatus, RetriesExceeded
+
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .utils import PgBackupApi
+
+
+class ExitCode(IntEnum):
+    """Possible exit codes of this script.
+
+    :cvar RECOVERY_DONE: backup was successfully restored.
+    :cvar RECOVERY_FAILED: recovery of the backup faced an issue.
+    :cvar HTTP_ERROR: an error has occurred while communicating with
+        ``pg-backup-api``
+    """
+
+    RECOVERY_DONE = 0
+    RECOVERY_FAILED = 1
+    HTTP_ERROR = 2
+
+
+def _restore_backup(api: "PgBackupApi", barman_server: str, backup_id: str,
+                    ssh_command: str, data_directory: str,
+                    loop_wait: int) -> int:
+    """Restore the configured Barman backup through ``pg-backup-api``.
+
+    .. note::
+        If requests to ``pg-backup-api`` fail recurrently or we face HTTP
+        errors, then exit with :attr:`ExitCode.HTTP_ERROR`.
+
+    :param api: a :class:`PgBackupApi` instance to handle communication with
+        the API.
+    :param barman_server: name of the Barman server which backup is to be
+        restored.
+    :param backup_id: ID of the backup from the Barman server.
+    :param ssh_command: SSH command to connect from the Barman host to the
+        target host.
+    :param data_directory: path to the Postgres data directory where to restore
+        the backup in.
+    :param loop_wait: how long in seconds to wait before checking again the
+        status of the recovery process. Higher values are useful for backups
+        that are expected to take longer to restore.
+
+    :returns: the return code to be used when exiting the ``patroni_barman``
+        application. Refer to :class:`ExitCode`.
+    """
+    operation_id = None
+
+    try:
+        operation_id = api.create_recovery_operation(
+            barman_server,
+            backup_id,
+            ssh_command,
+            data_directory,
+        )
+    except RetriesExceeded as exc:
+        logging.error("An issue was faced while trying to create a recovery "
+                      "operation: %r", exc)
+        return ExitCode.HTTP_ERROR
+
+    logging.info("Created the recovery operation with ID %s", operation_id)
+
+    status = None
+
+    while True:
+        try:
+            status = api.get_operation_status(barman_server, operation_id)
+        except RetriesExceeded:
+            logging.error("Maximum number of retries exceeded, exiting.")
+            return ExitCode.HTTP_ERROR
+
+        if status != OperationStatus.IN_PROGRESS:
+            break
+
+        logging.info("Recovery operation %s is still in progress",
+                     operation_id)
+        time.sleep(loop_wait)
+
+    if status == OperationStatus.DONE:
+        logging.info("Recovery operation finished successfully.")
+        return ExitCode.RECOVERY_DONE
+    else:
+        logging.error("Recovery operation failed.")
+        return ExitCode.RECOVERY_FAILED
+
+
+def run_barman_recover(api: "PgBackupApi", args: Namespace) -> int:
+    """Run a remote ``barman recover`` through the ``pg-backup-api``.
+
+    :param api: a :class:`PgBackupApi` instance to handle communication with
+        the API.
+    :param args: arguments received from the command-line of
+        ``patroni_barman recover`` command.
+
+    :returns: the return code to be used when exiting the ``patroni_barman``
+        application. Refer to :class:`ExitCode`.
+    """
+    return _restore_backup(api, args.barman_server, args.backup_id,
+                           args.ssh_command, args.data_directory,
+                           args.loop_wait)

patroni/scripts/barman/utils.py

@@ -0,0 +1,308 @@
+#!/usr/bin/env python
+
+"""Utilitary stuff to be used by Barman related scripts."""
+
+from enum import IntEnum
+import json
+import logging
+from typing import Any, Callable, Dict, Optional, Tuple, Type, Union
+import time
+from urllib.parse import urljoin
+
+from urllib3 import PoolManager
+from urllib3.exceptions import MaxRetryError
+from urllib3.response import HTTPResponse
+
+
+class RetriesExceeded(Exception):
+    """Maximum number of retries exceeded."""
+
+
+def retry(exceptions: Union[Type[Exception], Tuple[Type[Exception], ...]]) \
+        -> Any:
+    """Retry an operation n times if expected *exceptions* are faced.
+
+    .. note::
+        Should be used as a decorator of a class' method as it expects the
+        first argument to be a class instance.
+
+        The class which method is going to be decorated should contain a couple
+        attributes:
+
+        * ``max_retries``: maximum retry attempts before failing;
+        * ``retry_wait``: how long in seconds to wait before retrying.
+
+    :param exceptions: exceptions that could trigger a retry attempt.
+
+    :raises:
+        :exc:`RetriesExceeded`: if the maximum number of attempts has been
+            exhausted.
+    """
+    def decorator(func: Callable[..., Any]) -> Any:
+        def inner_func(instance: object, *args: Any, **kwargs: Any) -> Any:
+            times: int = getattr(instance, "max_retries")
+            retry_wait: int = getattr(instance, "retry_wait")
+            method_name = f"{instance.__class__.__name__}.{func.__name__}"
+
+            attempt = 1
+
+            while attempt <= times:
+                try:
+                    return func(instance, *args, **kwargs)
+                except exceptions as exc:
+                    logging.warning("Attempt %d of %d on method %s failed "
+                                    "with %r.",
+                                    attempt, times, method_name, exc)
+                    attempt += 1
+
+                time.sleep(retry_wait)
+
+            raise RetriesExceeded("Maximum number of retries exceeded for "
+                                  f"method {method_name}.")
+        return inner_func
+    return decorator
+
+
+def set_up_logging(log_file: Optional[str] = None) -> None:
+    """Set up logging to file, if *log_file* is given, otherwise to console.
+
+    :param log_file: file where to log messages, if any.
+    """
+    logging.basicConfig(filename=log_file, level=logging.INFO,
+                        format="%(asctime)s %(levelname)s: %(message)s")
+
+
+class OperationStatus(IntEnum):
+    """Possible status of ``pg-backup-api`` operations.
+
+    :cvar IN_PROGRESS: the operation is still ongoing.
+    :cvar FAILED: the operation failed.
+    :cvar DONE: the operation finished successfully.
+    """
+
+    IN_PROGRESS = 0
+    FAILED = 1
+    DONE = 2
+
+
+class ApiNotOk(Exception):
+    """The ``pg-backup-api`` is not currently up and running."""
+
+
+class PgBackupApi:
+    """Facilities for communicating with the ``pg-backup-api``.
+
+    :ivar api_url: base URL to reach the ``pg-backup-api``.
+    :ivar cert_file: certificate to authenticate against the ``pg-backup-api``,
+        if required.
+    :ivar key_file: certificate key to authenticate against the
+        ``pg-backup-api``, if required.
+    :ivar retry_wait: how long in seconds to wait before retrying a failed
+        request to the ``pg-backup-api``.
+    :ivar max_retries: maximum number of retries when ``pg-backup-api`` returns
+        malformed responses.
+    :ivar http: a HTTP pool manager for performing web requests.
+    """
+
+    def __init__(self, api_url: str, cert_file: Optional[str],
+                 key_file: Optional[str], retry_wait: int,
+                 max_retries: int) -> None:
+        """Create a new instance of :class:`BarmanRecover`.
+
+        Make sure the ``pg-backup-api`` is reachable and running fine.
+
+        .. note::
+            When using any method which send requests to the API, be aware that
+            they might raise :exc:`RetriesExceeded` upon HTTP request errors.
+
+            Similarly, when instantiating this class you may face an
+            :exc:`ApiNotOk`, if the API is down or returns a bogus status.
+
+        :param api_url: base URL to reach the ``pg-backup-api``.
+        :param cert_file: certificate to authenticate against the
+            ``pg-backup-api``, if required.
+        :param key_file: certificate key to authenticate against the
+            ``pg-backup-api``, if required.
+        :param retry_wait: how long in seconds to wait before retrying a failed
+            request to the ``pg-backup-api``.
+        :param max_retries: maximum number of retries when ``pg-backup-api``
+            returns malformed responses.
+        """
+        self.api_url = api_url
+        self.cert_file = cert_file
+        self.key_file = key_file
+        self.retry_wait = retry_wait
+        self.max_retries = max_retries
+        self._http = PoolManager(cert_file=cert_file, key_file=key_file)
+        self._ensure_api_ok()
+
+    def _build_full_url(self, url_path: str) -> str:
+        """Build the full URL by concatenating *url_path* with the base URL.
+
+        :param url_path: path to be accessed in the ``pg-backup-api``.
+
+        :returns: the full URL after concatenating.
+        """
+        return urljoin(self.api_url, url_path)
+
+    @staticmethod
+    def _deserialize_response(response: HTTPResponse) -> Any:
+        """Retrieve body from *response* as a deserialized JSON object.
+
+        :param response: response from which JSON body will be deserialized.
+
+        :returns: the deserialized JSON body.
+        """
+        return json.loads(response.data.decode("utf-8"))
+
+    @staticmethod
+    def _serialize_request(body: Any) -> Any:
+        """Serialize a request body.
+
+        :param body: content of the request body to be serialized.
+
+        :returns: the serialized request body.
+        """
+        return json.dumps(body).encode("utf-8")
+
+    def _get_request(self, url_path: str) -> Any:
+        """Perform a ``GET`` request to *url_path*.
+
+        :param url_path: URL to perform the ``GET`` request against.
+
+        :returns: the deserialized response body.
+
+        :raises:
+            :exc:`RetriesExceeded`: raised from the corresponding :mod:`urllib3`
+                exception.
+        """
+        url = self._build_full_url(url_path)
+        response = None
+
+        try:
+            response = self._http.request("GET", url)
+        except MaxRetryError as exc:
+            msg = f"Failed to perform a GET request to {url}"
+            raise RetriesExceeded(msg) from exc
+
+        return self._deserialize_response(response)
+
+    def _post_request(self, url_path: str, body: Any) -> Any:
+        """Perform a ``POST`` request to *url_path* serializing *body* as JSON.
+
+        :param url_path: URL to perform the ``POST`` request against.
+        :param body: the body to be serialized as JSON and sent in the request.
+
+        :returns: the deserialized response body.
+
+        :raises:
+            :exc:`RetriesExceeded`: raised from the corresponding :mod:`urllib3`
+                exception.
+        """
+        body = self._serialize_request(body)
+
+        url = self._build_full_url(url_path)
+        response = None
+
+        try:
+            response = self._http.request("POST",
+                                          url,
+                                          body=body,
+                                          headers={
+                                              "Content-Type": "application/json"
+                                          })
+        except MaxRetryError as exc:
+            msg = f"Failed to perform a POST request to {url} with {body}"
+            raise RetriesExceeded(msg) from exc
+
+        return self._deserialize_response(response)
+
+    def _ensure_api_ok(self) -> None:
+        """Ensure ``pg-backup-api`` is reachable and ``OK``.
+
+        :raises:
+            :exc:`ApiNotOk`: if ``pg-backup-api`` status is not ``OK``.
+        """
+        response = self._get_request("status")
+
+        if response != "OK":
+            msg = (
+                "pg-backup-api is currently not up and running at "
+                f"{self.api_url}: {response}"
+            )
+
+            raise ApiNotOk(msg)
+
+    @retry(KeyError)
+    def get_operation_status(self, barman_server: str,
+                             operation_id: str) -> OperationStatus:
+        """Get status of the operation which ID is *operation_id*.
+
+        :param barman_server: name of the Barman server related with the
+            operation.
+        :param operation_id: ID of the operation to be checked.
+
+        :returns: the status of the operation.
+        """
+        response = self._get_request(
+            f"servers/{barman_server}/operations/{operation_id}",
+        )
+
+        status = response["status"]
+        return OperationStatus[status]
+
+    @retry(KeyError)
+    def create_recovery_operation(self, barman_server: str, backup_id: str,
+                                  ssh_command: str, data_directory: str) -> str:
+        """Create a recovery operation on the ``pg-backup-api``.
+
+        :param barman_server: name of the Barman server which backup is to be
+            restored.
+        :param backup_id: ID of the backup from the Barman server.
+        :param ssh_command: SSH command to connect from the Barman host to the
+            target host.
+        :param data_directory: path to the Postgres data directory where to
+            restore the backup at.
+
+        :returns: the ID of the recovery operation that has been created.
+        """
+        response = self._post_request(
+            f"servers/{barman_server}/operations",
+            {
+                "type": "recovery",
+                "backup_id": backup_id,
+                "remote_ssh_command": ssh_command,
+                "destination_directory": data_directory,
+            },
+        )
+
+        return response["operation_id"]
+
+    @retry(KeyError)
+    def create_config_switch_operation(self, barman_server: str,
+                                       barman_model: Optional[str],
+                                       reset: Optional[bool]) -> str:
+        """Create a config switch operation on the ``pg-backup-api``.
+
+        :param barman_server: name of the Barman server which config is to be
+            switched.
+        :param barman_model: name of the Barman model to be applied to the
+            server, if any.
+        :param reset: ``True`` if you would like to unapply the currently active
+            model for the server, if any.
+
+        :returns: the ID of the config switch operation that has been created.
+        """
+        body: Dict[str, Any] = {"type": "config_switch"}
+
+        if barman_model:
+            body["model_name"] = barman_model
+        elif reset:
+            body["reset"] = reset
+
+        response = self._post_request(
+            f"servers/{barman_server}/operations",
+            body,
+        )
+
+        return response["operation_id"]

patroni/tags.py

@@ -3,13 +3,16 @@ import abc
 
 from typing import Any, Dict, Optional
 
-from patroni.utils import parse_int
+from patroni.utils import parse_int, parse_bool
 
 
 class Tags(abc.ABC):
     """An abstract class that encapsulates all the ``tags`` logic.
 
     Child classes that want to use provided facilities must implement ``tags`` abstract property.
+
+    .. note::
+        Due to backward-compatibility reasons, old tags may have a less strict type conversion than new ones.
     """
 
     @staticmethod
@@ -20,16 +23,20 @@ class Tags(abc.ABC):
 
         .. note::
             A custom tag is any tag added to the configuration ``tags`` section that is not one of ``clonefrom``,
-            ``nofailover``, ``noloadbalance`` or ``nosync``.
+            ``nofailover``, ``noloadbalance``,``nosync`` or ``nostream``.
 
-            For the Patroni predefined tags, the returning object will only contain them if they are enabled as they
-            all are boolean values that default to disabled.
+            For most of the Patroni predefined tags, the returning object will only contain them if they are enabled as
+            they all are boolean values that default to disabled.
+            However ``nofailover`` tag is always returned if ``failover_priority`` tag is defined. In this case, we need
+            both values to see if they are contradictory and the ``nofailover`` value should be used.
 
         :returns: a dictionary of tags set for this node. The key is the tag name, and the value is the corresponding
             tag value.
         """
         return {tag: value for tag, value in tags.items()
-                if tag not in ('clonefrom', 'nofailover', 'noloadbalance', 'nosync') or value}
+                if any((tag not in ('clonefrom', 'nofailover', 'noloadbalance', 'nosync', 'nostream'),
+                        value,
+                        tag == 'nofailover' and 'failover_priority' in tags))}
 
     @property
     @abc.abstractmethod
@@ -85,3 +92,8 @@ class Tags(abc.ABC):
     def replicatefrom(self) -> Optional[str]:
         """Value of ``replicatefrom`` tag, if any."""
         return self.tags.get('replicatefrom')
+
+    @property
+    def nostream(self) -> bool:
+        """``True`` if ``nostream`` is ``True``, else ``False``."""
+        return parse_bool(self.tags.get('nostream')) or False

patroni/utils.py

@@ -10,6 +10,7 @@
 :var WHITESPACE_RE: regular expression to match whitespace characters
 """
 import errno
+import itertools
 import logging
 import os
 import platform
@@ -24,6 +25,7 @@ from shlex import split
 
 from typing import Any, Callable, Dict, Iterator, List, Optional, Union, Tuple, Type, TYPE_CHECKING
 
+from collections import OrderedDict
 from dateutil import tz
 from json import JSONDecoder
 from urllib3.response import HTTPResponse
@@ -33,7 +35,6 @@ from .version import __version__
 
 if TYPE_CHECKING:  # pragma: no cover
     from .dcs import Cluster
-    from .config import GlobalConfig
 
 tzutc = tz.tzutc()
 
@@ -47,6 +48,37 @@ DBL_RE = re.compile(r'^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?')
 WHITESPACE_RE = re.compile(r'[ \t\n\r]*', re.VERBOSE | re.MULTILINE | re.DOTALL)
 
 
+def get_conversion_table(base_unit: str) -> Dict[str, Dict[str, Union[int, float]]]:
+    """Get conversion table for the specified base unit.
+
+    If no conversion table exists for the passed unit, return an empty :class:`OrderedDict`.
+
+    :param base_unit: unit to choose the conversion table for.
+
+    :returns: :class:`OrderedDict` object.
+    """
+    memory_unit_conversion_table: Dict[str, Dict[str, Union[int, float]]] = OrderedDict([
+        ('TB', {'B': 1024**4, 'kB': 1024**3, 'MB': 1024**2}),
+        ('GB', {'B': 1024**3, 'kB': 1024**2, 'MB': 1024}),
+        ('MB', {'B': 1024**2, 'kB': 1024, 'MB': 1}),
+        ('kB', {'B': 1024, 'kB': 1, 'MB': 1024**-1}),
+        ('B', {'B': 1, 'kB': 1024**-1, 'MB': 1024**-2})
+    ])
+    time_unit_conversion_table: Dict[str, Dict[str, Union[int, float]]] = OrderedDict([
+        ('d', {'ms': 1000 * 60**2 * 24, 's': 60**2 * 24, 'min': 60 * 24}),
+        ('h', {'ms': 1000 * 60**2, 's': 60**2, 'min': 60}),
+        ('min', {'ms': 1000 * 60, 's': 60, 'min': 1}),
+        ('s', {'ms': 1000, 's': 1, 'min': 60**-1}),
+        ('ms', {'ms': 1, 's': 1000**-1, 'min': 1 / (1000 * 60)}),
+        ('us', {'ms': 1000**-1, 's': 1000**-2, 'min': 1 / (1000**2 * 60)})
+    ])
+    if base_unit in ('B', 'kB', 'MB'):
+        return memory_unit_conversion_table
+    elif base_unit in ('ms', 's', 'min'):
+        return time_unit_conversion_table
+    return OrderedDict()
+
+
 def deep_compare(obj1: Dict[Any, Union[Any, Dict[Any, Any]]], obj2: Dict[Any, Union[Any, Dict[Any, Any]]]) -> bool:
     """Recursively compare two dictionaries to check if they are equal in terms of keys and values.
 
@@ -273,35 +305,154 @@ def convert_to_base_unit(value: Union[int, float], unit: str, base_unit: Optiona
         >>> convert_to_base_unit(1, 'GB', '512 MB') is None
         True
     """
-    convert: Dict[str, Dict[str, Union[int, float]]] = {
-        'B': {'B': 1, 'kB': 1024, 'MB': 1024 * 1024, 'GB': 1024 * 1024 * 1024, 'TB': 1024 * 1024 * 1024 * 1024},
-        'kB': {'B': 1.0 / 1024, 'kB': 1, 'MB': 1024, 'GB': 1024 * 1024, 'TB': 1024 * 1024 * 1024},
-        'MB': {'B': 1.0 / (1024 * 1024), 'kB': 1.0 / 1024, 'MB': 1, 'GB': 1024, 'TB': 1024 * 1024},
-        'ms': {'us': 1.0 / 1000, 'ms': 1, 's': 1000, 'min': 1000 * 60, 'h': 1000 * 60 * 60, 'd': 1000 * 60 * 60 * 24},
-        's': {'us': 1.0 / (1000 * 1000), 'ms': 1.0 / 1000, 's': 1, 'min': 60, 'h': 60 * 60, 'd': 60 * 60 * 24},
-        'min': {'us': 1.0 / (1000 * 1000 * 60), 'ms': 1.0 / (1000 * 60), 's': 1.0 / 60, 'min': 1, 'h': 60, 'd': 60 * 24}
-    }
-
-    round_order = {
-        'TB': 'GB', 'GB': 'MB', 'MB': 'kB', 'kB': 'B',
-        'd': 'h', 'h': 'min', 'min': 's', 's': 'ms', 'ms': 'us'
-    }
-
-    if base_unit and base_unit not in convert:
-        base_value, base_unit = strtol(base_unit, False)
-    else:
-        base_value = 1
-
-    if base_value is not None and base_unit in convert and unit in convert[base_unit]:
-        value *= convert[base_unit][unit] / float(base_value)
+    base_value, base_unit = strtol(base_unit, False)
+    if TYPE_CHECKING:  # pragma: no cover
+        assert isinstance(base_value, int)
 
+    convert_tbl = get_conversion_table(base_unit)
+    # {'TB': 'GB', 'GB': 'MB', ...}
+    round_order = dict(zip(convert_tbl, itertools.islice(convert_tbl, 1, None)))
+    if unit in convert_tbl and base_unit in convert_tbl[unit]:
+        value *= convert_tbl[unit][base_unit] / float(base_value)
         if unit in round_order:
-            multiplier = convert[base_unit][round_order[unit]]
+            multiplier = convert_tbl[round_order[unit]][base_unit]
             value = round(value / float(multiplier)) * multiplier
-
         return value
 
 
+def convert_int_from_base_unit(base_value: int, base_unit: Optional[str]) -> Optional[str]:
+    """Convert an integer value in some base unit to a human-friendly unit.
+
+    The output unit is chosen so that it's the greatest unit that can represent
+    the value without loss.
+
+    :param base_value: value to be converted from a base unit
+    :param base_unit: unit of *value*. Should be one of the base units (case sensitive):
+
+            * For space: ``B``, ``kB``, ``MB``;
+            * For time: ``ms``, ``s``, ``min``.
+
+    :returns: :class:`str` value representing *base_value* converted from *base_unit* to the greatest
+        possible human-friendly unit, or ``None`` if conversion failed.
+
+    :Example:
+
+        >>> convert_int_from_base_unit(1024, 'kB')
+        '1MB'
+
+        >>> convert_int_from_base_unit(1025, 'kB')
+        '1025kB'
+
+        >>> convert_int_from_base_unit(4, '256MB')
+        '1GB'
+
+        >>> convert_int_from_base_unit(4, '256 MB') is None
+        True
+
+        >>> convert_int_from_base_unit(1024, 'KB') is None
+        True
+    """
+    base_value_mult, base_unit = strtol(base_unit, False)
+    if TYPE_CHECKING:  # pragma: no cover
+        assert isinstance(base_value_mult, int)
+    base_value *= base_value_mult
+
+    convert_tbl = get_conversion_table(base_unit)
+    for unit in convert_tbl:
+        multiplier = convert_tbl[unit][base_unit]
+        if multiplier <= 1.0 or base_value % multiplier == 0:
+            return str(round(base_value / multiplier)) + unit
+
+
+def convert_real_from_base_unit(base_value: float, base_unit: Optional[str]) -> Optional[str]:
+    """Convert an floating-point value in some base unit to a human-friendly unit.
+
+    Same as :func:`convert_int_from_base_unit`, except we have to do the math a bit differently,
+    and there's a possibility that we don't find any exact divisor.
+
+    :param base_value: value to be converted from a base unit
+    :param base_unit: unit of *value*. Should be one of the base units (case sensitive):
+
+            * For space: ``B``, ``kB``, ``MB``;
+            * For time: ``ms``, ``s``, ``min``.
+
+    :returns: :class:`str` value representing *base_value* converted from *base_unit* to the greatest
+        possible human-friendly unit, or ``None`` if conversion failed.
+
+    :Example:
+
+        >>> convert_real_from_base_unit(5, 'ms')
+        '5ms'
+
+        >>> convert_real_from_base_unit(2.5, 'ms')
+        '2500us'
+
+        >>> convert_real_from_base_unit(4.0, '256MB')
+        '1GB'
+
+        >>> convert_real_from_base_unit(4.0, '256 MB') is None
+        True
+    """
+    base_value_mult, base_unit = strtol(base_unit, False)
+    if TYPE_CHECKING:  # pragma: no cover
+        assert isinstance(base_value_mult, int)
+    base_value *= base_value_mult
+
+    result = None
+    convert_tbl = get_conversion_table(base_unit)
+    for unit in convert_tbl:
+        value = base_value / convert_tbl[unit][base_unit]
+        result = f'{value:g}{unit}'
+        if value > 0 and abs((round(value) / value) - 1.0) <= 1e-8:
+            break
+    return result
+
+
+def maybe_convert_from_base_unit(base_value: str, vartype: str, base_unit: Optional[str]) -> str:
+    """Try to convert integer or real value in a base unit to a human-readable unit.
+
+    Value is passed as a string. If parsing or subsequent conversion fails, the original
+    value is returned.
+
+    :param base_value: value to be converted from a base unit.
+    :param vartype: the target type to parse *base_value* before converting (``integer``
+        or ``real`` is expected, any other type results in return value being equal to the
+        *base_value* string).
+    :param base_unit: unit of *value*. Should be one of the base units (case sensitive):
+
+            * For space: ``B``, ``kB``, ``MB``;
+            * For time: ``ms``, ``s``, ``min``.
+
+    :returns: :class:`str` value representing *base_value* converted from *base_unit* to the greatest
+        possible human-friendly unit, or *base_value* string if conversion failed.
+
+    :Example:
+
+        >>> maybe_convert_from_base_unit('5', 'integer', 'ms')
+        '5ms'
+
+        >>> maybe_convert_from_base_unit('4.2', 'real', 'ms')
+        '4200us'
+
+        >>> maybe_convert_from_base_unit('on', 'bool', None)
+        'on'
+
+        >>> maybe_convert_from_base_unit('', 'integer', '256MB')
+        ''
+    """
+    converters: Dict[str, Tuple[Callable[[str, Optional[str]], Union[int, float, str, None]],
+                                Callable[[Any, Optional[str]], Optional[str]]]] = {
+        'integer': (parse_int, convert_int_from_base_unit),
+        'real': (parse_real, convert_real_from_base_unit),
+        'default': (lambda v, _: v, lambda v, _: v)
+    }
+    parser, converter = converters.get(vartype, converters['default'])
+    parsed_value = parser(base_value, None)
+    if parsed_value:
+        return converter(parsed_value, base_unit) or base_value
+    return base_value
+
+
 def parse_int(value: Any, base_unit: Optional[str] = None) -> Optional[int]:
     """Parse *value* as an :class:`int`.
 
@@ -401,22 +552,23 @@ def parse_real(value: Any, base_unit: Optional[str] = None) -> Optional[float]:
         return convert_to_base_unit(val, unit, base_unit)
 
 
-def compare_values(vartype: str, unit: Optional[str], old_value: Any, new_value: Any) -> bool:
-    """Check if *old_value* and *new_value* are equivalent after parsing them as *vartype*.
+def compare_values(vartype: str, unit: Optional[str], settings_value: Any, config_value: Any) -> bool:
+    """Check if the value from ``pg_settings`` and from Patroni config are equivalent after parsing them as *vartype*.
 
-    :param vartpe: the target type to parse *old_value* and *new_value* before comparing them. Accepts any among of the
-        following (case sensitive):
+    :param vartype: the target type to parse *settings_value* and *config_value* before comparing them.
+        Accepts any among of the following (case sensitive):
 
         * ``bool``: parse values using :func:`parse_bool`; or
         * ``integer``: parse values using :func:`parse_int`; or
         * ``real``: parse values using :func:`parse_real`; or
         * ``enum``: parse values as lowercase strings; or
         * ``string``: parse values as strings. This one is used by default if no valid value is passed as *vartype*.
-    :param unit: base unit to be used as argument when calling :func:`parse_int` or :func:`parse_real` for *new_value*.
-    :param old_value: value to be compared with *new_value*.
-    :param new_value: value to be compared with *old_value*.
+    :param unit: base unit to be used as argument when calling :func:`parse_int` or :func:`parse_real`
+        for *config_value*.
+    :param settings_value: value to be compared with *config_value*.
+    :param config_value: value to be compared with *settings_value*.
 
-    :returns: ``True`` if *old_value* is equivalent to *new_value* when both are parsed as *vartype*.
+    :returns: ``True`` if *settings_value* is equivalent to *config_value* when both are parsed as *vartype*.
 
     :Example:
 
@@ -456,8 +608,8 @@ def compare_values(vartype: str, unit: Optional[str], old_value: Any, new_value:
     }
 
     converter = converters.get(vartype) or converters['string']
-    old_converted = converter(old_value, None)
-    new_converted = converter(new_value, unit)
+    old_converted = converter(settings_value, None)
+    new_converted = converter(config_value, unit)
 
     return old_converted is not None and new_converted is not None and old_converted == new_converted
 
@@ -564,7 +716,7 @@ class Retry(object):
         return self._cur_stoptime or 0
 
     def ensure_deadline(self, timeout: float, raise_ex: Optional[Exception] = None) -> bool:
-        """Calculates, sets, and checks the remaining deadline time.
+        """Calculates and checks the remaining deadline time.
 
         :param timeout: if the *deadline* is smaller than the provided *timeout* value raise *raise_ex* exception.
         :param raise_ex: the exception object that will be raised if the *deadline* is smaller than provided *timeout*.
@@ -575,8 +727,7 @@ class Retry(object):
         :raises:
             :class:`Exception`: *raise_ex* if calculated deadline is smaller than provided *timeout*.
         """
-        self.deadline = self.stoptime - time.time()
-        if self.deadline < timeout:
+        if self.stoptime - time.time() < timeout:
             if raise_ex:
                 raise raise_ex
             return False
@@ -759,12 +910,10 @@ def iter_response_objects(response: HTTPResponse) -> Iterator[Dict[str, Any]]:
         prev = chunk[idx:]
 
 
-def cluster_as_json(cluster: 'Cluster', global_config: Optional['GlobalConfig'] = None) -> Dict[str, Any]:
+def cluster_as_json(cluster: 'Cluster') -> Dict[str, Any]:
     """Get a JSON representation of *cluster*.
 
     :param cluster: the :class:`~patroni.dcs.Cluster` object to be parsed as JSON.
-    :param global_config: optional :class:`~patroni.config.GlobalConfig` object to check the cluster state.
-                          if not provided will be instantiated from the `Cluster.config`.
 
     :returns: JSON representation of *cluster*.
 
@@ -793,16 +942,16 @@ def cluster_as_json(cluster: 'Cluster', global_config: Optional['GlobalConfig']
             * ``from``: name of the member to be demoted;
             * ``to``: name of the member to be promoted.
     """
-    if not global_config:
-        from patroni.config import get_global_config
-        global_config = get_global_config(cluster)
+    from . import global_config
+
+    config = global_config.from_cluster(cluster)
     leader_name = cluster.leader.name if cluster.leader else None
     cluster_lsn = cluster.last_lsn or 0
 
     ret: Dict[str, Any] = {'members': []}
     for m in cluster.members:
         if m.name == leader_name:
-            role = 'standby_leader' if global_config.is_standby_cluster else 'leader'
+            role = 'standby_leader' if config.is_standby_cluster else 'leader'
         elif cluster.sync.matches(m.name):
             role = 'sync_standby'
         else:
@@ -815,7 +964,7 @@ def cluster_as_json(cluster: 'Cluster', global_config: Optional['GlobalConfig']
             member['host'] = conn_kwargs['host']
             if conn_kwargs.get('port'):
                 member['port'] = int(conn_kwargs['port'])
-        optional_attributes = ('timeline', 'pending_restart', 'scheduled_restart', 'tags')
+        optional_attributes = ('timeline', 'pending_restart', 'pending_restart_reason', 'scheduled_restart', 'tags')
         member.update({n: m.data[n] for n in optional_attributes if n in m.data})
 
         if m.name != leader_name:
@@ -832,7 +981,7 @@ def cluster_as_json(cluster: 'Cluster', global_config: Optional['GlobalConfig']
     # sort members by name for consistency
     cmp: Callable[[Dict[str, Any]], bool] = lambda m: m['name']
     ret['members'].sort(key=cmp)
-    if global_config.is_paused:
+    if config.is_paused:
         ret['pause'] = True
     if cluster.failover and cluster.failover.scheduled_at:
         ret['scheduled_switchover'] = {'at': cluster.failover.scheduled_at.isoformat()}

patroni/validator.py

@@ -9,13 +9,55 @@ import os
 import shutil
 import socket
 
-from typing import Any, Dict, Union, Iterator, List, Optional as OptionalType, Tuple
-
-from .collections import CaseInsensitiveSet
+from typing import Any, Dict, Union, Iterator, List, Optional as OptionalType, Tuple, TYPE_CHECKING
 
+from .collections import CaseInsensitiveSet, EMPTY_DICT
 from .dcs import dcs_modules
 from .exceptions import ConfigParseError
 from .utils import parse_int, split_host_port, data_directory_is_empty, get_major_version
+from .log import type_logformat
+
+
+def validate_log_field(field: Union[str, Dict[str, Any], Any]) -> bool:
+    """Checks if log field is valid.
+
+    :param field: A log field to be validated.
+
+    :returns: ``True`` if the field is either a string or a dictionary with exactly one key
+              that has string value, ``False`` otherwise.
+    """
+    if isinstance(field, str):
+        return True
+    elif isinstance(field, dict):
+        return len(field) == 1 and isinstance(next(iter(field.values())), str)
+    return False
+
+
+def validate_log_format(logformat: type_logformat) -> bool:
+    """Checks if log format is valid.
+
+    :param logformat: A log format to be validated.
+
+    :returns: ``True`` if the log format is either a string or a list of valid log fields.
+
+    :raises:
+        :exc:`~patroni.exceptions.ConfigParseError`:
+            * If the logformat is not a string or a list; or
+            * If the logformat is an empty list; or
+            * If the log format is a list and it with values that don't pass validation using
+              :func:`validate_log_field`.
+    """
+    if isinstance(logformat, str):
+        return True
+    elif isinstance(logformat, list):
+        if len(logformat) == 0:
+            raise ConfigParseError('should contain at least one item')
+        if not all(map(validate_log_field, logformat)):
+            raise ConfigParseError('each item should be a string or a dictionary with string values')
+
+        return True
+    else:
+        raise ConfigParseError('Should be a string or a list')
 
 
 def data_directory_empty(data_dir: str) -> bool:
@@ -200,7 +242,9 @@ def get_bin_name(bin_name: str) -> str:
 
     :returns: value of ``postgresql.bin_name[*bin_name*]``, if present, otherwise *bin_name*.
     """
-    return (schema.data.get('postgresql', {}).get('bin_name', {}) or {}).get(bin_name, bin_name)
+    if TYPE_CHECKING:  # pragma: no cover
+        assert isinstance(schema.data, dict)
+    return (schema.data.get('postgresql', {}).get('bin_name', {}) or EMPTY_DICT).get(bin_name, bin_name)
 
 
 def validate_data_dir(data_dir: str) -> bool:
@@ -239,6 +283,8 @@ def validate_data_dir(data_dir: str) -> bool:
             if not os.path.isdir(os.path.join(data_dir, waldir)):
                 raise ConfigParseError("data dir for the cluster is not empty, but doesn't contain"
                                        " \"{}\" directory".format(waldir))
+            if TYPE_CHECKING:  # pragma: no cover
+                assert isinstance(schema.data, dict)
             bin_dir = schema.data.get("postgresql", {}).get("bin_dir", None)
             major_version = get_major_version(bin_dir, get_bin_name('postgres'))
             if pgversion != major_version:
@@ -274,6 +320,8 @@ def validate_binary_name(bin_name: str) -> bool:
     """
     if not bin_name:
         raise ConfigParseError("is an empty string")
+    if TYPE_CHECKING:  # pragma: no cover
+        assert isinstance(schema.data, dict)
     bin_dir = schema.data.get('postgresql', {}).get('bin_dir', None)
     if not shutil.which(bin_name, path=bin_dir):
         raise ConfigParseError(f"does not contain '{bin_name}' in '{bin_dir or '$PATH'}'")
@@ -523,7 +571,7 @@ class Schema(object):
         * :class:`dict`: dictionary representing the YAML configuration tree.
     """
 
-    def __init__(self, validator: Any) -> None:
+    def __init__(self, validator: Union[Dict[Any, Any], List[Any], Any]) -> None:
         """Create a :class:`Schema` object.
 
         .. note::
@@ -614,7 +662,7 @@ class Schema(object):
                 errors.append(str(i))
         return errors
 
-    def validate(self, data: Any) -> Iterator[Result]:
+    def validate(self, data: Union[Dict[Any, Any], Any]) -> Iterator[Result]:
         """Perform all validations from the schema against the given configuration.
 
         It first checks that *data* argument type is compliant with the type of ``validator`` attribute.
@@ -638,11 +686,8 @@ class Schema(object):
         # iterable objects in the structure, until we eventually reach a leaf node to validate its value.
         if isinstance(self.validator, str):
             yield Result(isinstance(self.data, str), "is not a string", level=1, data=self.data)
-        elif issubclass(type(self.validator), type):
-            validator = self.validator
-            if self.validator == str:
-                validator = str
-            yield Result(isinstance(self.data, validator),
+        elif isinstance(self.validator, type):
+            yield Result(isinstance(self.data, self.validator),
                          "is not {}".format(_get_type_name(self.validator)), level=1, data=self.data)
         elif callable(self.validator):
             if hasattr(self.validator, "expected_type"):
@@ -689,7 +734,7 @@ class Schema(object):
                     for v in Schema(self.validator[0]).validate(value):
                         yield Result(v.status, v.error,
                                      path=(str(key) + ("." + v.path if v.path else "")), level=v.level, data=value)
-        elif isinstance(self.validator, Directory):
+        elif isinstance(self.validator, Directory) and isinstance(self.data, str):
             yield from self.validator.validate(self.data)
         elif isinstance(self.validator, Or):
             yield from self.iter_or()
@@ -701,6 +746,9 @@ class Schema(object):
         """
         # One key in `validator` attribute (`key` variable) can be mapped to one or more keys in `data` attribute (`d`
         # variable), depending on the `key` type.
+        if TYPE_CHECKING:  # pragma: no cover
+            assert isinstance(self.validator, dict)
+            assert isinstance(self.data, dict)
         for key in self.validator.keys():
             if isinstance(key, AtMostOne) and len(list(self._data_key(key))) > 1:
                 yield Result(False, f"Multiple of {key.args} provided")
@@ -730,6 +778,8 @@ class Schema(object):
 
         :yields: objects with the error message related to the failure, if any check fails.
         """
+        if TYPE_CHECKING:  # pragma: no cover
+            assert isinstance(self.validator, Or)
         results: List[Result] = []
         for a in self.validator.args:
             r: List[Result] = []
@@ -766,7 +816,7 @@ class Schema(object):
             yield key.name
         # If the key was defined as an `Or` object in `validator` attribute, then each of its values are the keys to
         # access the `data` dictionary.
-        elif isinstance(key, Or):
+        elif isinstance(key, Or) and isinstance(self.data, dict):
             # At least one of the `Or` entries should be available in the `data` dictionary. If we find at least one of
             # them in `data`, then we return all found entries so the caller method can validate them all.
             if any([item in self.data for item in key.args]):
@@ -780,7 +830,7 @@ class Schema(object):
                     yield item
         # If the key was defined as a `AtMostOne` object in `validator` attribute, then each of its values
         # are the keys to access the `data` dictionary.
-        elif isinstance(key, AtMostOne):
+        elif isinstance(key, AtMostOne) and isinstance(self.data, dict):
             # Yield back all of the entries from the `data` dictionary, each will be validated and then counted
             # to inform us if we've provided too many
             for item in key.args:
@@ -929,6 +979,20 @@ validate_etcd = {
 schema = Schema({
     "name": str,
     "scope": str,
+    Optional("log"): {
+        Optional("type"): EnumValidator(('plain', 'json'), case_sensitive=True, raise_assert=True),
+        Optional("level"): EnumValidator(('DEBUG', 'INFO', 'WARN', 'WARNING', 'ERROR', 'FATAL', 'CRITICAL'),
+                                         case_sensitive=True, raise_assert=True),
+        Optional("traceback_level"): EnumValidator(('DEBUG', 'ERROR'), raise_assert=True),
+        Optional("format"): validate_log_format,
+        Optional("dateformat"): str,
+        Optional("static_fields"): dict,
+        Optional("max_queue_size"): int,
+        Optional("dir"): str,
+        Optional("file_num"): int,
+        Optional("file_size"): int,
+        Optional("loggers"): dict
+    },
     Optional("ctl"): {
         Optional("insecure"): bool,
         Optional("cacert"): str,
@@ -1042,7 +1106,8 @@ schema = Schema({
             Optional("key"): str,
             Optional("key_password"): str,
             Optional("verify"): bool,
-            Optional("set_acls"): dict
+            Optional("set_acls"): dict,
+            Optional("auth_data"): dict,
         },
         "kubernetes": {
             "labels": {},
@@ -1106,6 +1171,7 @@ schema = Schema({
         Optional("clonefrom"): bool,
         Optional("noloadbalance"): bool,
         Optional("replicatefrom"): str,
-        Optional("nosync"): bool
+        Optional("nosync"): bool,
+        Optional("nostream"): bool
     }
 })

patroni/version.py

@@ -2,4 +2,4 @@
 
 :var __version__: the current Patroni version.
 """
-__version__ = '3.2.0'
+__version__ = '3.2.2'

postgres0.yml

@@ -132,7 +132,8 @@ postgresql:
 #  safety_margin: 5
 
 tags:
-    nofailover: false
+    # failover_priority: 1
     noloadbalance: false
     clonefrom: false
     nosync: false
+    nostream: false

postgres1.yml

@@ -124,6 +124,6 @@ postgresql:
   #pre_promote: /path/to/pre_promote.sh
 
 tags:
-    nofailover: false
+    # failover_priority: 1
     noloadbalance: false
     clonefrom: false

postgres2.yml

@@ -114,7 +114,7 @@ postgresql:
 #    krb_server_keyfile: /var/spool/keytabs/postgres
     unix_socket_directories: '..'  # parent directory of data_dir
 tags:
-    nofailover: false
+    # failover_priority: 1
     noloadbalance: false
     clonefrom: false
 #    replicatefrom: postgresql1

requirements.txt

@@ -11,3 +11,4 @@ pysyncobj>=0.3.8
 cryptography>=1.4
 psutil>=2.0.0
 ydiff>=1.2.0
+python-json-logger>=2.0.2

setup.py

@@ -25,7 +25,7 @@ KEYWORDS = 'etcd governor patroni postgresql postgres ha haproxy confd' +\
 
 EXTRAS_REQUIRE = {'aws': ['boto3'], 'etcd': ['python-etcd'], 'etcd3': ['python-etcd'],
                   'consul': ['python-consul'], 'exhibitor': ['kazoo'], 'zookeeper': ['kazoo'],
-                  'kubernetes': [], 'raft': ['pysyncobj', 'cryptography']}
+                  'kubernetes': [], 'raft': ['pysyncobj', 'cryptography'], 'jsonlogger': ['python-json-logger']}
 
 # Add here all kinds of additional classifiers as defined under
 # https://pypi.python.org/pypi?%3Aaction=list_classifiers
@@ -54,7 +54,8 @@ CONSOLE_SCRIPTS = ['patroni = patroni.__main__:main',
                    'patronictl = patroni.ctl:ctl',
                    'patroni_raft_controller = patroni.raft_controller:main',
                    "patroni_wale_restore = patroni.scripts.wale_restore:main",
-                   "patroni_aws = patroni.scripts.aws:main"]
+                   "patroni_aws = patroni.scripts.aws:main",
+                   "patroni_barman = patroni.scripts.barman.cli:main"]
 
 
 class _Command(Command):

tests/__init__.py

@@ -12,6 +12,7 @@ import patroni.psycopg as psycopg
 from patroni.dcs import Leader, Member
 from patroni.postgresql import Postgresql
 from patroni.postgresql.config import ConfigHandler
+from patroni.postgresql.mpp import get_mpp
 from patroni.utils import RetryFailedError, tzutc
 
 
@@ -25,8 +26,41 @@ mock_available_gucs = PropertyMock(return_value={
     'max_wal_senders', 'max_worker_processes', 'port', 'search_path', 'shared_preload_libraries',
     'stats_temp_directory', 'synchronous_standby_names', 'track_commit_timestamp', 'unix_socket_directories',
     'vacuum_cost_delay', 'vacuum_cost_limit', 'wal_keep_size', 'wal_level', 'wal_log_hints', 'zero_damaged_pages',
+    'autovacuum', 'wal_segment_size', 'wal_block_size', 'shared_buffers', 'wal_buffers',
 })
 
+GET_PG_SETTINGS_RESULT = [
+    ('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
+    ('wal_block_size', '8192', None, 'integer', 'internal'),
+    ('shared_buffers', '16384', '8kB', 'integer', 'postmaster'),
+    ('wal_buffers', '-1', '8kB', 'integer', 'postmaster'),
+    ('max_connections', '100', None, 'integer', 'postmaster'),
+    ('max_prepared_transactions', '200', None, 'integer', 'postmaster'),
+    ('max_worker_processes', '8', None, 'integer', 'postmaster'),
+    ('max_locks_per_transaction', '64', None, 'integer', 'postmaster'),
+    ('max_wal_senders', '5', None, 'integer', 'postmaster'),
+    ('search_path', 'public', None, 'string', 'user'),
+    ('port', '5432', None, 'integer', 'postmaster'),
+    ('listen_addresses', '127.0.0.2, 127.0.0.3', None, 'string', 'postmaster'),
+    ('autovacuum', 'on', None, 'bool', 'sighup'),
+    ('unix_socket_directories', '/tmp', None, 'string', 'postmaster'),
+    ('shared_preload_libraries', 'citus', None, 'string', 'postmaster'),
+    ('wal_keep_size', '128', 'MB', 'integer', 'sighup'),
+    ('cluster_name', 'batman', None, 'string', 'postmaster'),
+    ('vacuum_cost_delay', '200', 'ms', 'real', 'user'),
+    ('vacuum_cost_limit', '-1', None, 'integer', 'user'),
+    ('max_stack_depth', '2048', 'kB', 'integer', 'superuser'),
+    ('constraint_exclusion', '', None, 'enum', 'user'),
+    ('force_parallel_mode', '1', None, 'enum', 'user'),
+    ('zero_damaged_pages', 'off', None, 'bool', 'superuser'),
+    ('stats_temp_directory', '/tmp', None, 'string', 'sighup'),
+    ('track_commit_timestamp', 'off', None, 'bool', 'postmaster'),
+    ('wal_log_hints', 'on', None, 'bool', 'postmaster'),
+    ('hot_standby', 'on', None, 'bool', 'postmaster'),
+    ('max_replication_slots', '5', None, 'integer', 'postmaster'),
+    ('wal_level', 'logical', None, 'enum', 'postmaster'),
+]
+
 
 class MockResponse(object):
 
@@ -117,6 +151,8 @@ class MockCursor(object):
             self.results = [(False, 2)]
         elif sql.startswith('SELECT pg_catalog.pg_postmaster_start_time'):
             self.results = [(datetime.datetime.now(tzutc),)]
+        elif sql.endswith('AND pending_restart'):
+            self.results = []
         elif sql.startswith('SELECT name, pg_catalog.current_setting(name) FROM pg_catalog.pg_settings'):
             self.results = [('data_directory', 'data'),
                             ('hba_file', os.path.join('data', 'pg_hba.conf')),
@@ -133,22 +169,7 @@ class MockCursor(object):
                             ('archive_command', 'my archive command'),
                             ('cluster_name', 'my_cluster')]
         elif sql.startswith('SELECT name, setting'):
-            self.results = [('wal_segment_size', '2048', '8kB', 'integer', 'internal'),
-                            ('wal_block_size', '8192', None, 'integer', 'internal'),
-                            ('shared_buffers', '16384', '8kB', 'integer', 'postmaster'),
-                            ('wal_buffers', '-1', '8kB', 'integer', 'postmaster'),
-                            ('max_connections', '100', None, 'integer', 'postmaster'),
-                            ('max_prepared_transactions', '0', None, 'integer', 'postmaster'),
-                            ('max_worker_processes', '8', None, 'integer', 'postmaster'),
-                            ('max_locks_per_transaction', '64', None, 'integer', 'postmaster'),
-                            ('max_wal_senders', '5', None, 'integer', 'postmaster'),
-                            ('search_path', 'public', None, 'string', 'user'),
-                            ('port', '5433', None, 'integer', 'postmaster'),
-                            ('listen_addresses', '*', None, 'string', 'postmaster'),
-                            ('autovacuum', 'on', None, 'bool', 'sighup'),
-                            ('unix_socket_directories', '/tmp', None, 'string', 'postmaster')]
-        elif sql.startswith('SELECT COUNT(*) FROM pg_catalog.pg_settings'):
-            self.results = [(1,)]
+            self.results = GET_PG_SETTINGS_RESULT
         elif sql.startswith('IDENTIFY_SYSTEM'):
             self.results = [('1', 3, '0/402EEC0', '')]
         elif sql.startswith('TIMELINE_HISTORY '):
@@ -158,8 +179,12 @@ class MockCursor(object):
                                  b'3\t0/403DD98\tno recovery target specified\n')]
         elif sql.startswith('SELECT pg_catalog.citus_add_node'):
             self.results = [(2,)]
-        elif sql.startswith('SELECT nodeid, groupid'):
-            self.results = [(1, 0, 'host1', 5432, 'primary'), (2, 1, 'host2', 5432, 'primary')]
+        elif sql.startswith('SELECT groupid, nodename'):
+            self.results = [(0, 'host1', 5432, 'primary', 1),
+                            (0, '127.0.0.1', 5436, 'secondary', 2),
+                            (1, 'host4', 5432, 'primary', 3),
+                            (1, '127.0.0.1', 5437, 'secondary', 4),
+                            (1, '127.0.0.1', 5438, 'secondary', 5)]
         else:
             self.results = [(None, None, None, None, None, None, None, None, None, None)]
         self.rowcount = len(self.results)
@@ -218,11 +243,11 @@ class PostgresInit(unittest.TestCase):
     _PARAMETERS = {'wal_level': 'hot_standby', 'max_replication_slots': 5, 'f.oo': 'bar',
                    'search_path': 'public', 'hot_standby': 'on', 'max_wal_senders': 5,
                    'wal_keep_segments': 8, 'wal_log_hints': 'on', 'max_locks_per_transaction': 64,
-                   'max_worker_processes': 8, 'max_connections': 100, 'max_prepared_transactions': 0,
+                   'max_worker_processes': 8, 'max_connections': 100, 'max_prepared_transactions': 200,
                    'track_commit_timestamp': 'off', 'unix_socket_directories': '/tmp',
-                   'trigger_file': 'bla', 'stats_temp_directory': '/tmp', 'zero_damaged_pages': '',
+                   'trigger_file': 'bla', 'stats_temp_directory': '/tmp', 'zero_damaged_pages': 'off',
                    'force_parallel_mode': '1', 'constraint_exclusion': '',
-                   'max_stack_depth': 'Z', 'vacuum_cost_limit': -1, 'vacuum_cost_delay': 200}
+                   'max_stack_depth': 2048, 'vacuum_cost_limit': -1, 'vacuum_cost_delay': 200}
 
     @patch('patroni.psycopg._connect', psycopg_connect)
     @patch('patroni.postgresql.CallbackExecutor', Mock())
@@ -232,23 +257,24 @@ class PostgresInit(unittest.TestCase):
     @patch.object(Postgresql, 'get_postgres_role_from_data_directory', Mock(return_value='primary'))
     def setUp(self):
         data_dir = os.path.join('data', 'test0')
-        self.p = Postgresql({'name': 'postgresql0', 'scope': 'batman', 'data_dir': data_dir,
-                             'config_dir': data_dir, 'retry_timeout': 10,
-                             'krbsrvname': 'postgres', 'pgpass': os.path.join(data_dir, 'pgpass0'),
-                             'listen': '127.0.0.2, 127.0.0.3:5432',
-                             'connect_address': '127.0.0.2:5432', 'proxy_address': '127.0.0.2:5433',
-                             'authentication': {'superuser': {'username': 'foo', 'password': 'test'},
-                                                'replication': {'username': '', 'password': 'rep-pass'},
-                                                'rewind': {'username': 'rewind', 'password': 'test'}},
-                             'remove_data_directory_on_rewind_failure': True,
-                             'use_pg_rewind': True, 'pg_ctl_timeout': 'bla', 'use_unix_socket': True,
-                             'parameters': self._PARAMETERS,
-                             'recovery_conf': {'foo': 'bar'},
-                             'pg_hba': ['host all all 0.0.0.0/0 md5'],
-                             'pg_ident': ['krb realm postgres'],
-                             'callbacks': {'on_start': 'true', 'on_stop': 'true', 'on_reload': 'true',
-                                           'on_restart': 'true', 'on_role_change': 'true'},
-                             'citus': {'group': 0, 'database': 'citus'}})
+        config = {'name': 'postgresql0', 'scope': 'batman', 'data_dir': data_dir,
+                  'config_dir': data_dir, 'retry_timeout': 10,
+                  'krbsrvname': 'postgres', 'pgpass': os.path.join(data_dir, 'pgpass0'),
+                  'listen': '127.0.0.2, 127.0.0.3:5432',
+                  'connect_address': '127.0.0.2:5432', 'proxy_address': '127.0.0.2:5433',
+                  'authentication': {'superuser': {'username': 'foo', 'password': 'test'},
+                                     'replication': {'username': '', 'password': 'rep-pass'},
+                                     'rewind': {'username': 'rewind', 'password': 'test'}},
+                  'remove_data_directory_on_rewind_failure': True,
+                  'use_pg_rewind': True, 'pg_ctl_timeout': 'bla', 'use_unix_socket': True,
+                  'parameters': self._PARAMETERS,
+                  'recovery_conf': {'foo': 'bar'},
+                  'pg_hba': ['host all all 0.0.0.0/0 md5'],
+                  'pg_ident': ['krb realm postgres'],
+                  'callbacks': {'on_start': 'true', 'on_stop': 'true', 'on_reload': 'true',
+                                'on_restart': 'true', 'on_role_change': 'true'},
+                  'citus': {'group': 0, 'database': 'citus'}}
+        self.p = Postgresql(config, get_mpp(config))
 
 
 class BaseTestPostgresql(PostgresInit):

tests/test_api.py

@@ -8,11 +8,12 @@ from io import BytesIO as IO
 from mock import Mock, PropertyMock, patch
 from socketserver import ThreadingMixIn
 
+from patroni import global_config
 from patroni.api import RestApiHandler, RestApiServer
-from patroni.config import GlobalConfig
 from patroni.dcs import ClusterConfig, Member
 from patroni.exceptions import PostgresConnectionException
 from patroni.ha import _MemberStatus
+from patroni.postgresql.config import get_param_diff
 from patroni.psycopg import OperationalError
 from patroni.utils import RetryFailedError, tzutc
 
@@ -54,13 +55,13 @@ class MockPostgresql:
     major_version = 90600
     sysid = 'dummysysid'
     scope = 'dummy'
-    pending_restart = True
+    pending_restart_reason = {}
     wal_name = 'wal'
     lsn_name = 'lsn'
     wal_flush = '_flush'
     POSTMASTER_START_TIME = 'pg_catalog.pg_postmaster_start_time()'
     TL_LSN = 'CASE WHEN pg_catalog.pg_is_in_recovery()'
-    citus_handler = Mock()
+    mpp_handler = Mock()
 
     @staticmethod
     def postmaster_start_time():
@@ -148,16 +149,9 @@ class MockLogger(object):
     records_lost = 1
 
 
-class MockConfig(object):
-
-    def get_global_config(self, _):
-        return GlobalConfig({})
-
-
 class MockPatroni(object):
 
     ha = MockHa()
-    config = MockConfig()
     postgresql = ha.state_handler
     dcs = Mock()
     logger = MockLogger()
@@ -209,9 +203,10 @@ class TestRestApiHandler(unittest.TestCase):
     _authorization = '\nAuthorization: Basic dGVzdDp0ZXN0'
 
     def test_do_GET(self):
+        MockPostgresql.pending_restart_reason = {'max_connections': get_param_diff('200', '100')}
         MockPatroni.dcs.cluster.last_lsn = 20
         MockPatroni.dcs.cluster.sync.members = [MockPostgresql.name]
-        with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=True)):
+        with patch.object(global_config.__class__, 'is_synchronous_mode', PropertyMock(return_value=True)):
             MockRestApiServer(RestApiHandler, 'GET /replica')
         MockRestApiServer(RestApiHandler, 'GET /replica?lag=1M')
         MockRestApiServer(RestApiHandler, 'GET /replica?lag=10MB')
@@ -234,7 +229,7 @@ class TestRestApiHandler(unittest.TestCase):
         with patch.object(MockHa, 'is_leader', Mock(return_value=True)):
             MockRestApiServer(RestApiHandler, 'GET /replica')
             MockRestApiServer(RestApiHandler, 'GET /read-only-sync')
-            with patch.object(GlobalConfig, 'is_standby_cluster', Mock(return_value=True)):
+            with patch.object(global_config.__class__, 'is_standby_cluster', Mock(return_value=True)):
                 MockRestApiServer(RestApiHandler, 'GET /standby_leader')
         MockPatroni.dcs.cluster = None
         with patch.object(RestApiHandler, 'get_postgresql_status', Mock(return_value={'role': 'primary'})):
@@ -244,8 +239,8 @@ class TestRestApiHandler(unittest.TestCase):
         self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /primary'))
         with patch.object(RestApiServer, 'query', Mock(return_value=[('', 1, '', '', '', '', False, None, None, '')])):
             self.assertIsNotNone(MockRestApiServer(RestApiHandler, 'GET /patroni'))
-        with patch.object(GlobalConfig, 'is_standby_cluster', Mock(return_value=True)), \
-                patch.object(GlobalConfig, 'is_paused', Mock(return_value=True)):
+        with patch.object(global_config.__class__, 'is_standby_cluster', Mock(return_value=True)), \
+                patch.object(global_config.__class__, 'is_paused', Mock(return_value=True)):
             MockRestApiServer(RestApiHandler, 'GET /standby_leader')
 
         # test tags
@@ -475,7 +470,7 @@ class TestRestApiHandler(unittest.TestCase):
                 request = make_request(role='primary', postgres_version='9.5.2')
                 MockRestApiServer(RestApiHandler, request)
 
-        with patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
+        with patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)):
             MockRestApiServer(RestApiHandler, make_request(schedule='2016-08-42 12:45TZ+1', role='primary'))
             # Valid timeout
             MockRestApiServer(RestApiHandler, make_request(timeout='60s'))
@@ -537,7 +532,7 @@ class TestRestApiHandler(unittest.TestCase):
 
         # Switchover in pause mode
         with patch.object(RestApiHandler, 'write_response') as response_mock, \
-             patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
+             patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)):
             MockRestApiServer(RestApiHandler, request)
             response_mock.assert_called_with(
                 400, 'Switchover is possible only to a specific candidate in a paused state')
@@ -546,7 +541,8 @@ class TestRestApiHandler(unittest.TestCase):
         for is_synchronous_mode, response in (
                 (True, 'switchover is not possible: can not find sync_standby'),
                 (False, 'switchover is not possible: cluster does not have members except leader')):
-            with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \
+            with patch.object(global_config.__class__, 'is_synchronous_mode',
+                              PropertyMock(return_value=is_synchronous_mode)), \
                  patch.object(RestApiHandler, 'write_response') as response_mock:
                 MockRestApiServer(RestApiHandler, request)
                 response_mock.assert_called_with(412, response)
@@ -571,7 +567,8 @@ class TestRestApiHandler(unittest.TestCase):
         cluster.sync.matches.return_value = False
         for is_synchronous_mode, response in (
                 (True, 'candidate name does not match with sync_standby'), (False, 'candidate does not exists')):
-            with patch.object(GlobalConfig, 'is_synchronous_mode', PropertyMock(return_value=is_synchronous_mode)), \
+            with patch.object(global_config.__class__, 'is_synchronous_mode',
+                              PropertyMock(return_value=is_synchronous_mode)), \
                  patch.object(RestApiHandler, 'write_response') as response_mock:
                 MockRestApiServer(RestApiHandler, request)
                 response_mock.assert_called_with(412, response)
@@ -632,7 +629,7 @@ class TestRestApiHandler(unittest.TestCase):
 
         # Schedule in paused mode
         with patch.object(RestApiHandler, 'write_response') as response_mock, \
-             patch.object(GlobalConfig, 'is_paused', PropertyMock(return_value=True)):
+             patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)):
             dcs.manual_failover.return_value = False
             MockRestApiServer(RestApiHandler, request)
             response_mock.assert_called_with(400, "Can't schedule switchover in the paused state")
@@ -678,6 +675,12 @@ class TestRestApiHandler(unittest.TestCase):
         MockRestApiServer(RestApiHandler, post + '0\n\n')
         MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}')
 
+    @patch.object(MockHa, 'is_leader', Mock(return_value=True))
+    def test_do_POST_mpp(self):
+        post = 'POST /mpp HTTP/1.0' + self._authorization + '\nContent-Length: '
+        MockRestApiServer(RestApiHandler, post + '0\n\n')
+        MockRestApiServer(RestApiHandler, post + '14\n\n{"leader":"1"}')
+
 
 class TestRestApiServer(unittest.TestCase):
 

tests/test_barman.py

@@ -0,0 +1,765 @@
+import logging
+import mock
+from mock import MagicMock, Mock, patch
+import unittest
+from urllib3.exceptions import MaxRetryError
+
+from patroni.scripts.barman.cli import main
+from patroni.scripts.barman.config_switch import (ExitCode as BarmanConfigSwitchExitCode, _should_skip_switch,
+                                                  _switch_config, run_barman_config_switch)
+from patroni.scripts.barman.recover import ExitCode as BarmanRecoverExitCode, _restore_backup, run_barman_recover
+from patroni.scripts.barman.utils import ApiNotOk, OperationStatus, PgBackupApi, RetriesExceeded, set_up_logging
+
+
+API_URL = "http://localhost:7480"
+BARMAN_SERVER = "my_server"
+BARMAN_MODEL = "my_model"
+BACKUP_ID = "backup_id"
+SSH_COMMAND = "ssh postgres@localhost"
+DATA_DIRECTORY = "/path/to/pgdata"
+LOOP_WAIT = 10
+RETRY_WAIT = 2
+MAX_RETRIES = 5
+
+
+# stuff from patroni.scripts.barman.utils
+
+@patch("logging.basicConfig")
+def test_set_up_logging(mock_log_config):
+    log_file = "/path/to/some/file.log"
+    set_up_logging(log_file)
+    mock_log_config.assert_called_once_with(filename=log_file, level=logging.INFO,
+                                            format="%(asctime)s %(levelname)s: %(message)s")
+
+
+class TestPgBackupApi(unittest.TestCase):
+
+    @patch.object(PgBackupApi, "_ensure_api_ok", Mock())
+    @patch("patroni.scripts.barman.utils.PoolManager", MagicMock())
+    def setUp(self):
+        self.api = PgBackupApi(API_URL, None, None, RETRY_WAIT, MAX_RETRIES)
+        # Reset the mock as the same instance is used across tests
+        self.api._http.request.reset_mock()
+        self.api._http.request.side_effect = None
+
+    def test__build_full_url(self):
+        self.assertEqual(self.api._build_full_url("/some/path"), f"{API_URL}/some/path")
+
+    @patch("json.loads")
+    def test__deserialize_response(self, mock_json_loads):
+        mock_response = MagicMock()
+        self.assertIsNotNone(self.api._deserialize_response(mock_response))
+        mock_json_loads.assert_called_once_with(mock_response.data.decode("utf-8"))
+
+    @patch("json.dumps")
+    def test__serialize_request(self, mock_json_dumps):
+        body = "some_body"
+        ret = self.api._serialize_request(body)
+        self.assertIsNotNone(ret)
+        mock_json_dumps.assert_called_once_with(body)
+        mock_json_dumps.return_value.encode.assert_called_once_with("utf-8")
+
+    @patch.object(PgBackupApi, "_deserialize_response", Mock(return_value="test"))
+    def test__get_request(self):
+        mock_request = self.api._http.request
+
+        # with no error
+        self.assertEqual(self.api._get_request("/some/path"), "test")
+        mock_request.assert_called_once_with("GET", f"{API_URL}/some/path")
+
+        # with MaxRetryError
+        http_error = MaxRetryError(self.api._http, f"{API_URL}/some/path")
+        mock_request.side_effect = http_error
+
+        with self.assertRaises(RetriesExceeded) as exc:
+            self.assertIsNone(self.api._get_request("/some/path"))
+
+        self.assertEqual(
+            str(exc.exception),
+            "Failed to perform a GET request to http://localhost:7480/some/path"
+        )
+
+    @patch.object(PgBackupApi, "_deserialize_response", Mock(return_value="test"))
+    @patch.object(PgBackupApi, "_serialize_request")
+    def test__post_request(self, mock_serialize):
+        mock_request = self.api._http.request
+
+        # with no error
+        self.assertEqual(self.api._post_request("/some/path", "some body"), "test")
+        mock_serialize.assert_called_once_with("some body")
+        mock_request.assert_called_once_with("POST", f"{API_URL}/some/path", body=mock_serialize.return_value,
+                                             headers={"Content-Type": "application/json"})
+
+        # with HTTPError
+        http_error = MaxRetryError(self.api._http, f"{API_URL}/some/path")
+        mock_request.side_effect = http_error
+
+        with self.assertRaises(RetriesExceeded) as exc:
+            self.assertIsNone(self.api._post_request("/some/path", "some body"))
+
+        self.assertEqual(
+            str(exc.exception),
+            f"Failed to perform a POST request to http://localhost:7480/some/path with {mock_serialize.return_value}"
+        )
+
+    @patch.object(PgBackupApi, "_get_request")
+    def test__ensure_api_ok(self, mock_get_request):
+        # API ok
+        mock_get_request.return_value = "OK"
+        self.assertIsNone(self.api._ensure_api_ok())
+
+        # API not ok
+        mock_get_request.return_value = "random"
+
+        with self.assertRaises(ApiNotOk) as exc:
+            self.assertIsNone(self.api._ensure_api_ok())
+
+        self.assertEqual(
+            str(exc.exception),
+            "pg-backup-api is currently not up and running at http://localhost:7480: random",
+        )
+
+    @patch("patroni.scripts.barman.utils.OperationStatus")
+    @patch("logging.warning")
+    @patch("time.sleep")
+    @patch.object(PgBackupApi, "_get_request")
+    def test_get_operation_status(self, mock_get_request, mock_sleep, mock_logging, mock_op_status):
+        # well formed response
+        mock_get_request.return_value = {"status": "some status"}
+        mock_op_status.__getitem__.return_value = "SOME_STATUS"
+        self.assertEqual(self.api.get_operation_status(BARMAN_SERVER, "some_id"), "SOME_STATUS")
+        mock_get_request.assert_called_once_with(f"servers/{BARMAN_SERVER}/operations/some_id")
+        mock_sleep.assert_not_called()
+        mock_logging.assert_not_called()
+        mock_op_status.__getitem__.assert_called_once_with("some status")
+
+        # malformed response
+        mock_get_request.return_value = {"statuss": "some status"}
+
+        with self.assertRaises(RetriesExceeded) as exc:
+            self.api.get_operation_status(BARMAN_SERVER, "some_id")
+
+        self.assertEqual(str(exc.exception),
+                         "Maximum number of retries exceeded for method PgBackupApi.get_operation_status.")
+
+        self.assertEqual(mock_sleep.call_count, self.api.max_retries)
+        mock_sleep.assert_has_calls([mock.call(self.api.retry_wait)] * self.api.max_retries)
+
+        self.assertEqual(mock_logging.call_count, self.api.max_retries)
+        for i in range(mock_logging.call_count):
+            call_args = mock_logging.call_args_list[i][0]
+            self.assertEqual(len(call_args), 5)
+            self.assertEqual(call_args[0], "Attempt %d of %d on method %s failed with %r.")
+            self.assertEqual(call_args[1], i + 1)
+            self.assertEqual(call_args[2], self.api.max_retries)
+            self.assertEqual(call_args[3], "PgBackupApi.get_operation_status")
+            self.assertIsInstance(call_args[4], KeyError)
+            self.assertEqual(call_args[4].args, ('status',))
+
+    @patch("logging.warning")
+    @patch("time.sleep")
+    @patch.object(PgBackupApi, "_post_request")
+    def test_create_recovery_operation(self, mock_post_request, mock_sleep, mock_logging):
+        # well formed response
+        mock_post_request.return_value = {"operation_id": "some_id"}
+        self.assertEqual(
+            self.api.create_recovery_operation(BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY),
+            "some_id",
+        )
+        mock_sleep.assert_not_called()
+        mock_logging.assert_not_called()
+        mock_post_request.assert_called_once_with(
+            f"servers/{BARMAN_SERVER}/operations",
+            {
+                "type": "recovery",
+                "backup_id": BACKUP_ID,
+                "remote_ssh_command": SSH_COMMAND,
+                "destination_directory": DATA_DIRECTORY,
+            }
+        )
+
+        # malformed response
+        mock_post_request.return_value = {"operation_idd": "some_id"}
+
+        with self.assertRaises(RetriesExceeded) as exc:
+            self.api.create_recovery_operation(BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY)
+
+        self.assertEqual(str(exc.exception),
+                         "Maximum number of retries exceeded for method PgBackupApi.create_recovery_operation.")
+
+        self.assertEqual(mock_sleep.call_count, self.api.max_retries)
+
+        mock_sleep.assert_has_calls([mock.call(self.api.retry_wait)] * self.api.max_retries)
+
+        self.assertEqual(mock_logging.call_count, self.api.max_retries)
+        for i in range(mock_logging.call_count):
+            call_args = mock_logging.call_args_list[i][0]
+            self.assertEqual(len(call_args), 5)
+            self.assertEqual(call_args[0], "Attempt %d of %d on method %s failed with %r.")
+            self.assertEqual(call_args[1], i + 1)
+            self.assertEqual(call_args[2], self.api.max_retries)
+            self.assertEqual(call_args[3], "PgBackupApi.create_recovery_operation")
+            self.assertIsInstance(call_args[4], KeyError)
+            self.assertEqual(call_args[4].args, ('operation_id',))
+
+    @patch("logging.warning")
+    @patch("time.sleep")
+    @patch.object(PgBackupApi, "_post_request")
+    def test_create_config_switch_operation(self, mock_post_request, mock_sleep, mock_logging):
+        # well formed response -- sample 1
+        mock_post_request.return_value = {"operation_id": "some_id"}
+        self.assertEqual(
+            self.api.create_config_switch_operation(BARMAN_SERVER, BARMAN_MODEL, None),
+            "some_id",
+        )
+        mock_sleep.assert_not_called()
+        mock_logging.assert_not_called()
+        mock_post_request.assert_called_once_with(
+            f"servers/{BARMAN_SERVER}/operations",
+            {
+                "type": "config_switch",
+                "model_name": BARMAN_MODEL,
+            }
+        )
+
+        # well formed response -- sample 2
+        mock_post_request.reset_mock()
+
+        self.assertEqual(
+            self.api.create_config_switch_operation(BARMAN_SERVER, None, True),
+            "some_id",
+        )
+        mock_sleep.assert_not_called()
+        mock_logging.assert_not_called()
+        mock_post_request.assert_called_once_with(
+            f"servers/{BARMAN_SERVER}/operations",
+            {
+                "type": "config_switch",
+                "reset": True,
+            }
+        )
+
+        # malformed response
+        mock_post_request.return_value = {"operation_idd": "some_id"}
+
+        with self.assertRaises(RetriesExceeded) as exc:
+            self.api.create_config_switch_operation(BARMAN_SERVER, BARMAN_MODEL, None)
+
+        self.assertEqual(str(exc.exception),
+                         "Maximum number of retries exceeded for method PgBackupApi.create_config_switch_operation.")
+
+        self.assertEqual(mock_sleep.call_count, self.api.max_retries)
+
+        mock_sleep.assert_has_calls([mock.call(self.api.retry_wait)] * self.api.max_retries)
+
+        self.assertEqual(mock_logging.call_count, self.api.max_retries)
+        for i in range(mock_logging.call_count):
+            call_args = mock_logging.call_args_list[i][0]
+            self.assertEqual(len(call_args), 5)
+            self.assertEqual(call_args[0], "Attempt %d of %d on method %s failed with %r.")
+            self.assertEqual(call_args[1], i + 1)
+            self.assertEqual(call_args[2], self.api.max_retries)
+            self.assertEqual(call_args[3], "PgBackupApi.create_config_switch_operation")
+            self.assertIsInstance(call_args[4], KeyError)
+            self.assertEqual(call_args[4].args, ('operation_id',))
+
+
+# stuff from patroni.scripts.barman.recover
+
+
+class TestBarmanRecover(unittest.TestCase):
+
+    def setUp(self):
+        self.api = MagicMock()
+        # Reset the mock as the same instance is used across tests
+        self.api._http.request.reset_mock()
+        self.api._http.request.side_effect = None
+
+    @patch("time.sleep")
+    @patch("logging.info")
+    @patch("logging.error")
+    def test__restore_backup(self, mock_log_error, mock_log_info, mock_sleep):
+        mock_create_op = self.api.create_recovery_operation
+        mock_get_status = self.api.get_operation_status
+
+        # successful fast restore
+        mock_create_op.return_value = "some_id"
+        mock_get_status.return_value = OperationStatus.DONE
+
+        self.assertEqual(
+            _restore_backup(self.api, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT),
+            BarmanRecoverExitCode.RECOVERY_DONE,
+        )
+
+        mock_create_op.assert_called_once_with(BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY)
+        mock_get_status.assert_called_once_with(BARMAN_SERVER, "some_id")
+        mock_log_info.assert_has_calls([
+            mock.call("Created the recovery operation with ID %s", "some_id"),
+            mock.call("Recovery operation finished successfully."),
+        ])
+        mock_log_error.assert_not_called()
+        mock_sleep.assert_not_called()
+
+        # successful slow restore
+        mock_create_op.reset_mock()
+        mock_get_status.reset_mock()
+        mock_log_info.reset_mock()
+        mock_get_status.side_effect = [OperationStatus.IN_PROGRESS] * 20 + [OperationStatus.DONE]
+
+        self.assertEqual(
+            _restore_backup(self.api, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT),
+            BarmanRecoverExitCode.RECOVERY_DONE,
+        )
+
+        mock_create_op.assert_called_once()
+
+        self.assertEqual(mock_get_status.call_count, 21)
+        mock_get_status.assert_has_calls([mock.call(BARMAN_SERVER, "some_id")] * 21)
+
+        self.assertEqual(mock_log_info.call_count, 22)
+        mock_log_info.assert_has_calls([mock.call("Created the recovery operation with ID %s", "some_id")]
+                                       + [mock.call("Recovery operation %s is still in progress", "some_id")] * 20
+                                       + [mock.call("Recovery operation finished successfully.")])
+
+        mock_log_error.assert_not_called()
+
+        self.assertEqual(mock_sleep.call_count, 20)
+        mock_sleep.assert_has_calls([mock.call(LOOP_WAIT)] * 20)
+
+        # failed fast restore
+        mock_create_op.reset_mock()
+        mock_get_status.reset_mock()
+        mock_log_info.reset_mock()
+        mock_sleep.reset_mock()
+        mock_get_status.side_effect = None
+        mock_get_status.return_value = OperationStatus.FAILED
+
+        self.assertEqual(
+            _restore_backup(self.api, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT),
+            BarmanRecoverExitCode.RECOVERY_FAILED,
+        )
+
+        mock_create_op.assert_called_once()
+        mock_get_status.assert_called_once_with(BARMAN_SERVER, "some_id")
+        mock_log_info.assert_has_calls([
+            mock.call("Created the recovery operation with ID %s", "some_id"),
+        ])
+        mock_log_error.assert_has_calls([
+            mock.call("Recovery operation failed."),
+        ])
+        mock_sleep.assert_not_called()
+
+        # failed slow restore
+        mock_create_op.reset_mock()
+        mock_get_status.reset_mock()
+        mock_log_info.reset_mock()
+        mock_log_error.reset_mock()
+        mock_sleep.reset_mock()
+        mock_get_status.side_effect = [OperationStatus.IN_PROGRESS] * 20 + [OperationStatus.FAILED]
+
+        self.assertEqual(
+            _restore_backup(self.api, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT),
+            BarmanRecoverExitCode.RECOVERY_FAILED,
+        )
+
+        mock_create_op.assert_called_once()
+
+        self.assertEqual(mock_get_status.call_count, 21)
+        mock_get_status.assert_has_calls([mock.call(BARMAN_SERVER, "some_id")] * 21)
+
+        self.assertEqual(mock_log_info.call_count, 21)
+        mock_log_info.assert_has_calls([mock.call("Created the recovery operation with ID %s", "some_id")]
+                                       + [mock.call("Recovery operation %s is still in progress", "some_id")] * 20)
+
+        mock_log_error.assert_has_calls([
+            mock.call("Recovery operation failed."),
+        ])
+
+        self.assertEqual(mock_sleep.call_count, 20)
+        mock_sleep.assert_has_calls([mock.call(LOOP_WAIT)] * 20)
+
+        # create retries exceeded
+        mock_log_info.reset_mock()
+        mock_log_error.reset_mock()
+        mock_sleep.reset_mock()
+        mock_create_op.side_effect = RetriesExceeded()
+        mock_get_status.side_effect = None
+
+        self.assertEqual(
+            _restore_backup(self.api, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT),
+            BarmanRecoverExitCode.HTTP_ERROR,
+        )
+
+        mock_log_info.assert_not_called()
+        mock_log_error.assert_called_once_with("An issue was faced while trying to create a recovery operation: %r",
+                                               mock_create_op.side_effect)
+        mock_sleep.assert_not_called()
+
+        # get status retries exceeded
+        mock_create_op.reset_mock()
+        mock_create_op.side_effect = None
+        mock_log_error.reset_mock()
+        mock_log_info.reset_mock()
+        mock_get_status.side_effect = RetriesExceeded
+
+        self.assertEqual(
+            _restore_backup(self.api, BARMAN_SERVER, BACKUP_ID, SSH_COMMAND, DATA_DIRECTORY, LOOP_WAIT),
+            BarmanRecoverExitCode.HTTP_ERROR,
+        )
+
+        mock_log_info.assert_called_once_with("Created the recovery operation with ID %s", "some_id")
+        mock_log_error.assert_called_once_with("Maximum number of retries exceeded, exiting.")
+        mock_sleep.assert_not_called()
+
+
+class TestBarmanRecoverCli(unittest.TestCase):
+
+    @patch("patroni.scripts.barman.recover._restore_backup")
+    def test_run_barman_recover(self, mock_rb):
+        api = MagicMock()
+        args = MagicMock()
+
+        # successful execution
+        mock_rb.return_value = BarmanRecoverExitCode.RECOVERY_DONE
+
+        self.assertEqual(
+            run_barman_recover(api, args),
+            BarmanRecoverExitCode.RECOVERY_DONE,
+        )
+
+        mock_rb.assert_called_once_with(api, args.barman_server, args.backup_id,
+                                        args.ssh_command, args.data_directory,
+                                        args.loop_wait)
+
+        # failed execution
+        mock_rb.reset_mock()
+
+        mock_rb.return_value = BarmanRecoverExitCode.RECOVERY_FAILED
+
+        self.assertEqual(
+            run_barman_recover(api, args),
+            BarmanRecoverExitCode.RECOVERY_FAILED,
+        )
+
+        mock_rb.assert_called_once_with(api, args.barman_server, args.backup_id,
+                                        args.ssh_command, args.data_directory,
+                                        args.loop_wait)
+
+
+# stuff from patroni.scripts.barman.config_switch
+
+
+class TestBarmanConfigSwitch(unittest.TestCase):
+
+    def setUp(self):
+        self.api = MagicMock()
+        # Reset the mock as the same instance is used across tests
+        self.api._http.request.reset_mock()
+        self.api._http.request.side_effect = None
+
+    @patch("time.sleep")
+    @patch("logging.info")
+    @patch("logging.error")
+    def test__switch_config(self, mock_log_error, mock_log_info, mock_sleep):
+        mock_create_op = self.api.create_config_switch_operation
+        mock_get_status = self.api.get_operation_status
+
+        # successful fast config-switch
+        mock_create_op.return_value = "some_id"
+        mock_get_status.return_value = OperationStatus.DONE
+
+        self.assertEqual(
+            _switch_config(self.api, BARMAN_SERVER, BARMAN_MODEL, None),
+            BarmanConfigSwitchExitCode.CONFIG_SWITCH_DONE,
+        )
+
+        mock_create_op.assert_called_once_with(BARMAN_SERVER, BARMAN_MODEL, None)
+        mock_get_status.assert_called_once_with(BARMAN_SERVER, "some_id")
+        mock_log_info.assert_has_calls([
+            mock.call("Created the config switch operation with ID %s", "some_id"),
+            mock.call("Config switch operation finished successfully."),
+        ])
+        mock_log_error.assert_not_called()
+        mock_sleep.assert_not_called()
+
+        # successful slow config-switch
+        mock_create_op.reset_mock()
+        mock_get_status.reset_mock()
+        mock_log_info.reset_mock()
+        mock_get_status.side_effect = [OperationStatus.IN_PROGRESS] * 20 + [OperationStatus.DONE]
+
+        self.assertEqual(
+            _switch_config(self.api, BARMAN_SERVER, BARMAN_MODEL, None),
+            BarmanConfigSwitchExitCode.CONFIG_SWITCH_DONE,
+        )
+
+        mock_create_op.assert_called_once_with(BARMAN_SERVER, BARMAN_MODEL, None)
+
+        self.assertEqual(mock_get_status.call_count, 21)
+        mock_get_status.assert_has_calls([mock.call(BARMAN_SERVER, "some_id")] * 21)
+
+        self.assertEqual(mock_log_info.call_count, 22)
+        mock_log_info.assert_has_calls([mock.call("Created the config switch operation with ID %s", "some_id")]
+                                       + [mock.call("Config switch operation %s is still in progress", "some_id")] * 20
+                                       + [mock.call("Config switch operation finished successfully.")])
+
+        mock_log_error.assert_not_called()
+
+        self.assertEqual(mock_sleep.call_count, 20)
+        mock_sleep.assert_has_calls([mock.call(5)] * 20)
+
+        # failed fast config-switch
+        mock_create_op.reset_mock()
+        mock_get_status.reset_mock()
+        mock_log_info.reset_mock()
+        mock_sleep.reset_mock()
+        mock_get_status.side_effect = None
+        mock_get_status.return_value = OperationStatus.FAILED
+
+        self.assertEqual(
+            _switch_config(self.api, BARMAN_SERVER, BARMAN_MODEL, None),
+            BarmanConfigSwitchExitCode.CONFIG_SWITCH_FAILED,
+        )
+
+        mock_create_op.assert_called_once()
+        mock_get_status.assert_called_once_with(BARMAN_SERVER, "some_id")
+        mock_log_info.assert_called_once_with("Created the config switch operation with ID %s", "some_id")
+        mock_log_error.assert_called_once_with("Config switch operation failed.")
+        mock_sleep.assert_not_called()
+
+        # failed slow config-switch
+        mock_create_op.reset_mock()
+        mock_get_status.reset_mock()
+        mock_log_info.reset_mock()
+        mock_log_error.reset_mock()
+        mock_sleep.reset_mock()
+        mock_get_status.side_effect = [OperationStatus.IN_PROGRESS] * 20 + [OperationStatus.FAILED]
+
+        self.assertEqual(
+            _switch_config(self.api, BARMAN_SERVER, BARMAN_MODEL, None),
+            BarmanConfigSwitchExitCode.CONFIG_SWITCH_FAILED,
+        )
+
+        mock_create_op.assert_called_once()
+
+        self.assertEqual(mock_get_status.call_count, 21)
+        mock_get_status.assert_has_calls([mock.call(BARMAN_SERVER, "some_id")] * 21)
+
+        self.assertEqual(mock_log_info.call_count, 21)
+        mock_log_info.assert_has_calls([mock.call("Created the config switch operation with ID %s", "some_id")]
+                                       + [mock.call("Config switch operation %s is still in progress", "some_id")] * 20)
+
+        mock_log_error.assert_called_once_with("Config switch operation failed.")
+
+        self.assertEqual(mock_sleep.call_count, 20)
+        mock_sleep.assert_has_calls([mock.call(5)] * 20)
+
+        # create retries exceeded
+        mock_log_info.reset_mock()
+        mock_log_error.reset_mock()
+        mock_sleep.reset_mock()
+        mock_create_op.side_effect = RetriesExceeded()
+        mock_get_status.side_effect = None
+
+        self.assertEqual(
+            _switch_config(self.api, BARMAN_SERVER, BARMAN_MODEL, None),
+            BarmanConfigSwitchExitCode.HTTP_ERROR,
+        )
+
+        mock_log_info.assert_not_called()
+        mock_log_error.assert_called_once_with("An issue was faced while trying to create a config switch operation: "
+                                               "%r",
+                                               mock_create_op.side_effect)
+        mock_sleep.assert_not_called()
+
+        # get status retries exceeded
+        mock_create_op.reset_mock()
+        mock_create_op.side_effect = None
+        mock_log_error.reset_mock()
+        mock_get_status.side_effect = RetriesExceeded
+
+        self.assertEqual(
+            _switch_config(self.api, BARMAN_SERVER, BARMAN_MODEL, None),
+            BarmanConfigSwitchExitCode.HTTP_ERROR,
+        )
+
+        mock_log_info.assert_called_once_with("Created the config switch operation with ID %s", "some_id")
+        mock_log_error.assert_called_once_with("Maximum number of retries exceeded, exiting.")
+        mock_sleep.assert_not_called()
+
+
+class TestBarmanConfigSwitchCli(unittest.TestCase):
+
+    def test__should_skip_switch(self):
+        args = MagicMock()
+
+        for role, switch_when, expected in [
+            ("master", "promoted", False),
+            ("master", "demoted", True),
+            ("master", "always", False),
+
+            ("primary", "promoted", False),
+            ("primary", "demoted", True),
+            ("primary", "always", False),
+
+            ("promoted", "promoted", False),
+            ("promoted", "demoted", True),
+            ("promoted", "always", False),
+
+            ("standby_leader", "promoted", True),
+            ("standby_leader", "demoted", True),
+            ("standby_leader", "always", False),
+
+            ("replica", "promoted", True),
+            ("replica", "demoted", False),
+            ("replica", "always", False),
+
+            ("demoted", "promoted", True),
+            ("demoted", "demoted", False),
+            ("demoted", "always", False),
+        ]:
+            args.role = role
+            args.switch_when = switch_when
+            self.assertEqual(_should_skip_switch(args), expected)
+
+    @patch("patroni.scripts.barman.config_switch._should_skip_switch")
+    @patch("patroni.scripts.barman.config_switch._switch_config")
+    @patch("logging.error")
+    @patch("logging.info")
+    def test_run_barman_config_switch(self, mock_log_info, mock_log_error, mock_sc, mock_skip):
+        api = MagicMock()
+        args = MagicMock()
+        args.reset = None
+
+        # successful execution
+        mock_skip.return_value = False
+        mock_sc.return_value = BarmanConfigSwitchExitCode.CONFIG_SWITCH_DONE
+
+        self.assertEqual(
+            run_barman_config_switch(api, args),
+            BarmanConfigSwitchExitCode.CONFIG_SWITCH_DONE,
+        )
+
+        mock_sc.assert_called_once_with(api, args.barman_server, args.barman_model,
+                                        args.reset)
+
+        # failed execution
+        mock_sc.reset_mock()
+
+        mock_sc.return_value = BarmanConfigSwitchExitCode.CONFIG_SWITCH_FAILED
+
+        self.assertEqual(
+            run_barman_config_switch(api, args),
+            BarmanConfigSwitchExitCode.CONFIG_SWITCH_FAILED,
+        )
+
+        mock_sc.assert_called_once_with(api, args.barman_server, args.barman_model,
+                                        args.reset)
+
+        # skipped execution
+        mock_sc.reset_mock()
+        mock_skip.return_value = True
+
+        self.assertEqual(
+            run_barman_config_switch(api, args),
+            BarmanConfigSwitchExitCode.CONFIG_SWITCH_SKIPPED
+        )
+
+        mock_sc.assert_not_called()
+        mock_log_info.assert_called_once_with("Config switch operation was skipped (role=%s, "
+                                              "switch_when=%s).", args.role, args.switch_when)
+        mock_log_error.assert_not_called()
+
+        # invalid args -- sample 1
+        mock_skip.return_value = False
+        args = MagicMock()
+        args.barman_server = BARMAN_SERVER
+        args.barman_model = BARMAN_MODEL
+        args.reset = True
+
+        self.assertEqual(
+            run_barman_config_switch(api, args),
+            BarmanConfigSwitchExitCode.INVALID_ARGS,
+        )
+
+        mock_log_error.assert_called_once_with("One, and only one among 'barman_model' ('%s') and 'reset' "
+                                               "('%s') should be given", BARMAN_MODEL, True)
+        api.assert_not_called()
+
+        # invalid args -- sample 2
+        args = MagicMock()
+        args.barman_server = BARMAN_SERVER
+        args.barman_model = None
+        args.reset = None
+
+        mock_log_error.reset_mock()
+        api.reset_mock()
+
+        self.assertEqual(
+            run_barman_config_switch(api, args),
+            BarmanConfigSwitchExitCode.INVALID_ARGS,
+        )
+
+        mock_log_error.assert_called_once_with("One, and only one among 'barman_model' ('%s') and 'reset' "
+                                               "('%s') should be given", None, None)
+        api.assert_not_called()
+
+
+# stuff from patroni.scripts.barman.cli
+
+
+class TestMain(unittest.TestCase):
+
+    @patch("patroni.scripts.barman.cli.PgBackupApi")
+    @patch("patroni.scripts.barman.cli.set_up_logging")
+    @patch("patroni.scripts.barman.cli.ArgumentParser")
+    def test_main(self, mock_arg_parse, mock_set_up_log, mock_api):
+        # sub-command specified
+        args = MagicMock()
+        args.func.return_value = 0
+        mock_arg_parse.return_value.parse_known_args.return_value = (args, None)
+
+        with self.assertRaises(SystemExit) as exc:
+            main()
+
+        mock_arg_parse.assert_called_once()
+        mock_set_up_log.assert_called_once_with(args.log_file)
+        mock_api.assert_called_once_with(args.api_url, args.cert_file,
+                                         args.key_file, args.retry_wait,
+                                         args.max_retries)
+        mock_arg_parse.return_value.print_help.assert_not_called()
+        args.func.assert_called_once_with(mock_api.return_value, args)
+        self.assertEqual(exc.exception.code, 0)
+
+        # Issue in the API
+        mock_arg_parse.reset_mock()
+        mock_set_up_log.reset_mock()
+        mock_api.reset_mock()
+        mock_api.side_effect = ApiNotOk()
+
+        with self.assertRaises(SystemExit) as exc:
+            main()
+
+        mock_arg_parse.assert_called_once()
+        mock_set_up_log.assert_called_once_with(args.log_file)
+        mock_api.assert_called_once_with(args.api_url, args.cert_file,
+                                         args.key_file, args.retry_wait,
+                                         args.max_retries)
+        mock_arg_parse.return_value.print_help.assert_not_called()
+        self.assertEqual(exc.exception.code, -2)
+
+        # sub-command not specified
+        mock_arg_parse.reset_mock()
+        mock_set_up_log.reset_mock()
+        mock_api.reset_mock()
+        delattr(args, "func")
+        mock_api.side_effect = None
+
+        with self.assertRaises(SystemExit) as exc:
+            main()
+
+        mock_arg_parse.assert_called_once()
+        mock_set_up_log.assert_called_once_with(args.log_file)
+        mock_api.assert_not_called()
+        mock_arg_parse.return_value.print_help.assert_called_once_with()
+        self.assertEqual(exc.exception.code, -1)

tests/test_bootstrap.py

@@ -4,10 +4,11 @@ import sys
 from mock import Mock, PropertyMock, patch
 
 from patroni.async_executor import CriticalTask
+from patroni.collections import CaseInsensitiveDict
 from patroni.postgresql import Postgresql
 from patroni.postgresql.bootstrap import Bootstrap
 from patroni.postgresql.cancellable import CancellableSubprocess
-from patroni.postgresql.config import ConfigHandler
+from patroni.postgresql.config import ConfigHandler, get_param_diff
 
 from . import psycopg_connect, BaseTestPostgresql, mock_available_gucs
 
@@ -142,6 +143,16 @@ class TestBootstrap(BaseTestPostgresql):
                     (), error_handler
                 ),
                 ["--key=value with spaces"])
+            # not allowed options in list of dicts/strs are filtered out
+            self.assertEqual(
+                self.b.process_user_options(
+                    'pg_basebackup',
+                    [{'checkpoint': 'fast'}, {'dbname': 'dbname=postgres'}, 'gzip', {'label': 'standby'}, 'verbose'],
+                    ('dbname', 'verbose'),
+                    print
+                ),
+                ['--checkpoint=fast', '--gzip', '--label=standby'],
+            )
 
     @patch.object(CancellableSubprocess, 'call', Mock())
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
@@ -179,10 +190,17 @@ class TestBootstrap(BaseTestPostgresql):
     @patch.object(Postgresql, 'controldata', Mock(return_value={'Database cluster state': 'in production'}))
     def test_custom_bootstrap(self, mock_cancellable_subprocess_call):
         self.p.config._config.pop('pg_hba')
-        config = {'method': 'foo', 'foo': {'command': 'bar'}}
+        config = {'method': 'foo', 'foo': {'command': 'bar --arg1=val1'}}
 
         mock_cancellable_subprocess_call.return_value = 1
         self.assertFalse(self.b.bootstrap(config))
+        self.assertEqual(mock_cancellable_subprocess_call.call_args_list[0][0][0],
+                         ['bar', '--arg1=val1', '--scope=batman', '--datadir=' + os.path.join('data', 'test0')])
+
+        mock_cancellable_subprocess_call.reset_mock()
+        config['foo']['no_params'] = 1
+        self.assertFalse(self.b.bootstrap(config))
+        self.assertEqual(mock_cancellable_subprocess_call.call_args_list[0][0][0], ['bar', '--arg1=val1'])
 
         mock_cancellable_subprocess_call.return_value = 0
         with patch('multiprocessing.Process', Mock(side_effect=Exception("42"))), \
@@ -228,8 +246,9 @@ class TestBootstrap(BaseTestPostgresql):
         self.assertTrue(task.result)
 
         self.b.bootstrap(config)
-        with patch.object(Postgresql, 'pending_restart', PropertyMock(return_value=True)), \
-                patch.object(Postgresql, 'restart', Mock()) as mock_restart:
+        with patch.object(Postgresql, 'pending_restart_reason',
+                          PropertyMock(CaseInsensitiveDict({'max_connections': get_param_diff('200', '100')}))), \
+             patch.object(Postgresql, 'restart', Mock()) as mock_restart:
             self.b.post_bootstrap({}, task)
             mock_restart.assert_called_once()
 

tests/test_citus.py

@@ -1,25 +1,30 @@
 import time
-from mock import Mock, patch
-from patroni.postgresql.citus import CitusHandler
+import unittest
+
+from copy import deepcopy
+from mock import Mock, patch, PropertyMock
+from typing import List
+from patroni.postgresql.mpp.citus import CitusHandler, PgDistGroup, PgDistNode
+from patroni.psycopg import ProgrammingError
 
 from . import BaseTestPostgresql, MockCursor, psycopg_connect, SleepException
 from .test_ha import get_cluster_initialized_with_leader
 
 
-@patch('patroni.postgresql.citus.Thread', Mock())
+@patch('patroni.postgresql.mpp.citus.Thread', Mock())
 @patch('patroni.psycopg.connect', psycopg_connect)
 class TestCitus(BaseTestPostgresql):
 
     def setUp(self):
         super(TestCitus, self).setUp()
-        self.c = self.p.citus_handler
+        self.c = self.p.mpp_handler
         self.cluster = get_cluster_initialized_with_leader()
         self.cluster.workers[1] = self.cluster
 
     @patch('time.time', Mock(side_effect=[100, 130, 160, 190, 220, 250, 280, 310, 340, 370]))
-    @patch('patroni.postgresql.citus.logger.exception', Mock(side_effect=SleepException))
-    @patch('patroni.postgresql.citus.logger.warning')
-    @patch('patroni.postgresql.citus.PgDistNode.wait', Mock())
+    @patch('patroni.postgresql.mpp.citus.logger.exception', Mock(side_effect=SleepException))
+    @patch('patroni.postgresql.mpp.citus.logger.warning')
+    @patch('patroni.postgresql.mpp.citus.PgDistTask.wait', Mock())
     @patch.object(CitusHandler, 'is_alive', Mock(return_value=True))
     def test_run(self, mock_logger_warning):
         # `before_demote` or `before_promote` REST API calls starting a
@@ -31,18 +36,18 @@ class TestCitus(BaseTestPostgresql):
 
         self.c.handle_event(self.cluster, {'type': 'before_demote', 'group': 1,
                                            'leader': 'leader', 'timeout': 30, 'cooldown': 10})
-        self.c.add_task('after_promote', 2, 'postgres://host3:5432/postgres')
+        self.c.add_task('after_promote', 2, self.cluster, self.cluster.leader_name, 'postgres://host3:5432/postgres')
         self.assertRaises(SleepException, self.c.run)
         mock_logger_warning.assert_called_once()
         self.assertTrue(mock_logger_warning.call_args[0][0].startswith('Rolling back transaction'))
-        self.assertTrue(repr(mock_logger_warning.call_args[0][1]).startswith('PgDistNode'))
+        self.assertTrue(repr(mock_logger_warning.call_args[0][1]).startswith('PgDistTask'))
 
     @patch.object(CitusHandler, 'is_alive', Mock(return_value=False))
     @patch.object(CitusHandler, 'start', Mock())
-    def test_sync_pg_dist_node(self):
+    def test_sync_meta_data(self):
         with patch.object(CitusHandler, 'is_enabled', Mock(return_value=False)):
-            self.c.sync_pg_dist_node(self.cluster)
-        self.c.sync_pg_dist_node(self.cluster)
+            self.c.sync_meta_data(self.cluster)
+        self.c.sync_meta_data(self.cluster)
 
     def test_handle_event(self):
         self.c.handle_event(self.cluster, {})
@@ -51,62 +56,71 @@ class TestCitus(BaseTestPostgresql):
                                                'leader': 'leader', 'timeout': 30, 'cooldown': 10})
 
     def test_add_task(self):
-        with patch('patroni.postgresql.citus.logger.error') as mock_logger, \
-                patch('patroni.postgresql.citus.urlparse', Mock(side_effect=Exception)):
-            self.c.add_task('', 1, None)
+        with patch('patroni.postgresql.mpp.citus.logger.error') as mock_logger, \
+                patch('patroni.postgresql.mpp.citus.urlparse', Mock(side_effect=Exception)):
+            self.c.add_task('', 1, self.cluster, '', None)
             mock_logger.assert_called_once()
 
-        with patch('patroni.postgresql.citus.logger.debug') as mock_logger:
-            self.c.add_task('before_demote', 1, 'postgres://host:5432/postgres', 30)
+        with patch('patroni.postgresql.mpp.citus.logger.debug') as mock_logger:
+            self.c.add_task('before_demote', 1, self.cluster,
+                            self.cluster.leader_name, 'postgres://host:5432/postgres', 30)
             mock_logger.assert_called_once()
             self.assertTrue(mock_logger.call_args[0][0].startswith('Adding the new task:'))
 
-        with patch('patroni.postgresql.citus.logger.debug') as mock_logger:
-            self.c.add_task('before_promote', 1, 'postgres://host:5432/postgres', 30)
+        with patch('patroni.postgresql.mpp.citus.logger.debug') as mock_logger:
+            self.c.add_task('before_promote', 1, self.cluster,
+                            self.cluster.leader_name, 'postgres://host:5432/postgres', 30)
             mock_logger.assert_called_once()
             self.assertTrue(mock_logger.call_args[0][0].startswith('Overriding existing task:'))
 
         # add_task called from sync_pg_dist_node should not override already scheduled or in flight task until deadline
-        self.assertIsNotNone(self.c.add_task('after_promote', 1, 'postgres://host:5432/postgres', 30))
-        self.assertIsNone(self.c.add_task('after_promote', 1, 'postgres://host:5432/postgres'))
+        self.assertIsNotNone(self.c.add_task('after_promote', 1, self.cluster,
+                                             self.cluster.leader_name, 'postgres://host:5432/postgres', 30))
+        self.assertIsNone(self.c.add_task('after_promote', 1, self.cluster,
+                                          self.cluster.leader_name, 'postgres://host:5432/postgres'))
         self.c._in_flight = self.c._tasks.pop()
         self.c._in_flight.deadline = self.c._in_flight.timeout + time.time()
-        self.assertIsNone(self.c.add_task('after_promote', 1, 'postgres://host:5432/postgres'))
+        self.assertIsNone(self.c.add_task('after_promote', 1, self.cluster,
+                                          self.cluster.leader_name, 'postgres://host:5432/postgres'))
         self.c._in_flight.deadline = 0
-        self.assertIsNotNone(self.c.add_task('after_promote', 1, 'postgres://host:5432/postgres'))
+        self.assertIsNotNone(self.c.add_task('after_promote', 1, self.cluster,
+                                             self.cluster.leader_name, 'postgres://host:5432/postgres'))
 
         # If there is no transaction in progress and cached pg_dist_node matching desired state task should not be added
         self.c._schedule_load_pg_dist_node = False
-        self.c._pg_dist_node[self.c._in_flight.group] = self.c._in_flight
+        self.c._pg_dist_group[self.c._in_flight.groupid] = self.c._in_flight
         self.c._in_flight = None
-        self.assertIsNone(self.c.add_task('after_promote', 1, 'postgres://host:5432/postgres'))
+        self.assertIsNone(self.c.add_task('after_promote', 1, self.cluster,
+                                          self.cluster.leader_name, 'postgres://host:5432/postgres'))
 
     def test_pick_task(self):
-        self.c.add_task('after_promote', 1, 'postgres://host2:5432/postgres')
-        with patch.object(CitusHandler, 'process_task') as mock_process_task:
+        self.c.add_task('after_promote', 0, self.cluster, self.cluster.leader_name, 'postgres://host1:5432/postgres')
+        with patch.object(CitusHandler, 'update_node') as mock_update_node:
             self.c.process_tasks()
-            # process_task() shouln't be called because pick_task double checks with _pg_dist_node
-            mock_process_task.assert_not_called()
+            # process_task() shouln't be called because pick_task double checks with _pg_dist_group
+            mock_update_node.assert_not_called()
 
     def test_process_task(self):
-        self.c.add_task('after_promote', 0, 'postgres://host2:5432/postgres')
-        task = self.c.add_task('before_promote', 1, 'postgres://host4:5432/postgres', 30)
+        self.c.add_task('after_promote', 1, self.cluster, self.cluster.leader_name, 'postgres://host2:5432/postgres')
+        task = self.c.add_task('before_promote', 1, self.cluster,
+                               self.cluster.leader_name, 'postgres://host4:5432/postgres', 30)
         self.c.process_tasks()
         self.assertTrue(task._event.is_set())
 
         # the after_promote should result only in COMMIT
-        task = self.c.add_task('after_promote', 1, 'postgres://host4:5432/postgres', 30)
+        task = self.c.add_task('after_promote', 1, self.cluster,
+                               self.cluster.leader_name, 'postgres://host4:5432/postgres', 30)
         with patch.object(CitusHandler, 'query') as mock_query:
             self.c.process_tasks()
             mock_query.assert_called_once()
             self.assertEqual(mock_query.call_args[0][0], 'COMMIT')
 
     def test_process_tasks(self):
-        self.c.add_task('after_promote', 0, 'postgres://host2:5432/postgres')
+        self.c.add_task('after_promote', 0, self.cluster, self.cluster.leader_name, 'postgres://host2:5432/postgres')
         self.c.process_tasks()
 
-        self.c.add_task('after_promote', 0, 'postgres://host3:5432/postgres')
-        with patch('patroni.postgresql.citus.logger.error') as mock_logger, \
+        self.c.add_task('after_promote', 0, self.cluster, self.cluster.leader_name, 'postgres://host3:5432/postgres')
+        with patch('patroni.postgresql.mpp.citus.logger.error') as mock_logger, \
                 patch.object(CitusHandler, 'query', Mock(side_effect=Exception)):
             self.c.process_tasks()
             mock_logger.assert_called_once()
@@ -115,18 +129,19 @@ class TestCitus(BaseTestPostgresql):
     def test_on_demote(self):
         self.c.on_demote()
 
-    @patch('patroni.postgresql.citus.logger.error')
+    @patch('patroni.postgresql.mpp.citus.logger.error')
     @patch.object(MockCursor, 'execute', Mock(side_effect=Exception))
-    def test_load_pg_dist_node(self, mock_logger):
-        # load_pg_dist_node() triggers, query fails and exception is property handled
+    def test_load_pg_dist_group(self, mock_logger):
+        # load_pg_dist_group) triggers, query fails and exception is property handled
         self.c.process_tasks()
-        self.assertTrue(self.c._schedule_load_pg_dist_node)
+        self.assertTrue(self.c._schedule_load_pg_dist_group)
         mock_logger.assert_called_once()
         self.assertTrue(mock_logger.call_args[0][0].startswith('Exception when executing query'))
-        self.assertTrue(mock_logger.call_args[0][1].startswith('SELECT nodeid, groupid, '))
+        self.assertTrue(mock_logger.call_args[0][1].startswith('SELECT groupid, nodename, '))
 
     def test_wait(self):
-        task = self.c.add_task('before_demote', 1, 'postgres://host:5432/postgres', 30)
+        task = self.c.add_task('before_demote', 1, self.cluster,
+                               self.cluster.leader_name, u'postgres://host:5432/postgres', 30)
         task._event.wait = Mock()
         task.wait()
 
@@ -140,10 +155,6 @@ class TestCitus(BaseTestPostgresql):
         self.assertEqual(parameters['wal_level'], 'logical')
         self.assertEqual(parameters['citus.local_hostname'], '/tmp')
 
-    def test_bootstrap(self):
-        self.c._config = None
-        self.c.bootstrap()
-
     def test_ignore_replication_slot(self):
         self.assertFalse(self.c.ignore_replication_slot({'name': 'foo', 'type': 'physical',
                                                          'database': 'bar', 'plugin': 'wal2json'}))
@@ -161,3 +172,222 @@ class TestCitus(BaseTestPostgresql):
                                                          'type': 'logical', 'database': 'citus', 'plugin': 'pgoutput'}))
         self.assertTrue(self.c.ignore_replication_slot({'name': 'citus_shard_split_slot_1_2_3',
                                                         'type': 'logical', 'database': 'citus', 'plugin': 'citus'}))
+
+    @patch('patroni.postgresql.mpp.citus.logger.debug')
+    @patch('patroni.postgresql.mpp.citus.connect', psycopg_connect)
+    @patch('patroni.postgresql.mpp.citus.quote_ident', Mock())
+    def test_bootstrap_duplicate_database(self, mock_logger):
+        with patch.object(MockCursor, 'execute', Mock(side_effect=ProgrammingError)):
+            self.assertRaises(ProgrammingError, self.c.bootstrap)
+        with patch.object(MockCursor, 'execute', Mock(side_effect=[ProgrammingError, None, None, None])), \
+                patch.object(ProgrammingError, 'diag') as mock_diag:
+            type(mock_diag).sqlstate = PropertyMock(return_value='42P04')
+            self.c.bootstrap()
+        mock_logger.assert_called_once()
+        self.assertTrue(mock_logger.call_args[0][0].startswith('Exception when creating database'))
+
+
+class TestGroupTransition(unittest.TestCase):
+    nodeid = 100
+
+    def map_to_sql(self, group: int, transition: PgDistNode) -> str:
+        if transition.role not in ('primary', 'demoted', 'secondary'):
+            return "citus_remove_node('{0}', {1})".format(transition.host, transition.port)
+        elif transition.nodeid:
+            host = transition.host + ('-demoted' if transition.role == 'demoted' else '')
+            return "citus_update_node({0}, '{1}', {2})".format(transition.nodeid, host, transition.port)
+        else:
+            transition.nodeid = self.nodeid
+            self.nodeid += 1
+
+            return "citus_add_node('{0}', {1}, {2}, '{3}')".format(transition.host, transition.port,
+                                                                   group, transition.role)
+
+    def check_transitions(self, old_topology: PgDistGroup, new_topology: PgDistGroup,
+                          expected_transitions: List[str]) -> None:
+        check_topology = deepcopy(old_topology)
+
+        transitions: List[str] = []
+        for node in new_topology.transition(old_topology):
+            self.assertTrue(node not in check_topology or (check_topology.get(node) or node).role == 'demoted')
+            old_node = node.nodeid and next(iter(v for v in check_topology if v.nodeid == node.nodeid), None)
+            if old_node:
+                check_topology.discard(old_node)
+            transitions.append(self.map_to_sql(new_topology.groupid, node))
+            check_topology.add(node)
+        self.assertEqual(transitions, expected_transitions)
+
+    def test_new_topology(self):
+        old = PgDistGroup(0)
+        new = PgDistGroup(0, {PgDistNode('1', 5432, 'primary'),
+                              PgDistNode('2', 5432, 'secondary')})
+        expected = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=100),
+                                   PgDistNode('2', 5432, 'secondary', nodeid=101)})
+        self.check_transitions(old, new,
+                               ["citus_add_node('1', 5432, 0, 'primary')",
+                                "citus_add_node('2', 5432, 0, 'secondary')"])
+        self.assertTrue(new.equals(expected, True))
+
+    def test_switchover(self):
+        old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1),
+                              PgDistNode('2', 5432, 'secondary', nodeid=2)})
+        new = PgDistGroup(0, {PgDistNode('1', 5432, 'secondary'),
+                              PgDistNode('2', 5432, 'primary')})
+        expected = PgDistGroup(0, {PgDistNode('2', 5432, 'primary', nodeid=1),
+                                   PgDistNode('1', 5432, 'secondary', nodeid=2)})
+        self.check_transitions(old, new,
+                               ["citus_update_node(1, '1-demoted', 5432)",
+                                "citus_update_node(2, '1', 5432)",
+                                "citus_update_node(1, '2', 5432)"])
+        self.assertTrue(new.equals(expected, True))
+
+    def test_failover(self):
+        old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1),
+                              PgDistNode('2', 5432, 'secondary', nodeid=2)})
+        new = PgDistGroup(0, {PgDistNode('2', 5432, 'primary')})
+        expected = PgDistGroup(0, {PgDistNode('2', 5432, 'primary', nodeid=1),
+                                   PgDistNode('1', 5432, 'secondary', nodeid=2)})
+        self.check_transitions(old, new,
+                               ["citus_update_node(1, '1-demoted', 5432)",
+                                "citus_update_node(2, '1', 5432)",
+                                "citus_update_node(1, '2', 5432)"])
+        self.assertTrue(new.equals(expected, True))
+
+    def test_failover_and_new_secondary(self):
+        old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1),
+                              PgDistNode('2', 5432, 'secondary', nodeid=2)})
+        new = PgDistGroup(0, {PgDistNode('2', 5432, 'primary'),
+                              PgDistNode('3', 5432, 'secondary')})
+        expected = PgDistGroup(0, {PgDistNode('2', 5432, 'primary', nodeid=1),
+                                   PgDistNode('3', 5432, 'secondary', nodeid=2)})
+        # the secondary record is used to add the new standby and primary record is updated with the new hostname
+        self.check_transitions(old, new, ["citus_update_node(2, '3', 5432)", "citus_update_node(1, '2', 5432)"])
+        self.assertTrue(new.equals(expected, True))
+
+    def test_switchover_and_new_secondary_primary_gone(self):
+        old = PgDistGroup(0, {PgDistNode('1', 5432, 'demoted', nodeid=1),
+                              PgDistNode('2', 5432, 'secondary', nodeid=2)})
+        new = PgDistGroup(0, {PgDistNode('2', 5432, 'primary'),
+                              PgDistNode('3', 5432, 'secondary')})
+        expected = PgDistGroup(0, {PgDistNode('2', 5432, 'primary', nodeid=1),
+                                   PgDistNode('3', 5432, 'secondary', nodeid=2)})
+        # the secondary record is used to add the new standby and primary record is updated with the new hostname
+        self.check_transitions(old, new, ["citus_update_node(2, '3', 5432)", "citus_update_node(1, '2', 5432)"])
+        self.assertTrue(new.equals(expected, True))
+
+    def test_secondary_replaced(self):
+        old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1),
+                              PgDistNode('2', 5432, 'secondary', nodeid=2)})
+        new = PgDistGroup(0, {PgDistNode('1', 5432, 'primary'),
+                              PgDistNode('3', 5432, 'secondary')})
+        expected = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1),
+                                   PgDistNode('3', 5432, 'secondary', nodeid=2)})
+        self.check_transitions(old, new, ["citus_update_node(2, '3', 5432)"])
+        self.assertTrue(new.equals(expected, True))
+
+    def test_secondary_repmoved(self):
+        old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1),
+                              PgDistNode('2', 5432, 'secondary', nodeid=2),
+                              PgDistNode('3', 5432, 'secondary', nodeid=3)})
+        new = PgDistGroup(0, {PgDistNode('1', 5432, 'primary'),
+                              PgDistNode('3', 5432, 'secondary')})
+        expected = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1),
+                                   PgDistNode('3', 5432, 'secondary', nodeid=3)})
+        self.check_transitions(old, new, ["citus_remove_node('2', 5432)"])
+        self.assertTrue(new.equals(expected, True))
+
+    def test_switchover_and_secondary_removed(self):
+        old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1),
+                              PgDistNode('2', 5432, 'secondary', nodeid=2),
+                              PgDistNode('3', 5432, 'secondary', nodeid=3)})
+        new = PgDistGroup(0, {PgDistNode('1', 5432, 'secondary'),
+                              PgDistNode('2', 5432, 'primary')})
+        expected = PgDistGroup(0, {PgDistNode('1', 5432, 'secondary', nodeid=2),
+                                   PgDistNode('2', 5432, 'primary', nodeid=1),
+                                   PgDistNode('3', 5432, 'secondary', nodeid=3)})
+        self.check_transitions(old, new,
+                               ["citus_update_node(1, '1-demoted', 5432)",
+                                "citus_update_node(2, '1', 5432)",
+                                "citus_update_node(1, '2', 5432)"])
+        self.assertTrue(new.equals(expected, True))
+
+    def test_switchover_and_new_secondary(self):
+        old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1),
+                              PgDistNode('2', 5432, 'secondary', nodeid=2)})
+        new = PgDistGroup(0, {PgDistNode('1', 5432, 'secondary'),
+                              PgDistNode('2', 5432, 'primary'),
+                              PgDistNode('3', 5432, 'secondary')})
+        expected = PgDistGroup(0, {PgDistNode('1', 5432, 'secondary', nodeid=2),
+                                   PgDistNode('2', 5432, 'primary', nodeid=1)})
+        self.check_transitions(old, new,
+                               ["citus_update_node(1, '1-demoted', 5432)",
+                                "citus_update_node(2, '1', 5432)",
+                                "citus_update_node(1, '2', 5432)"])
+        self.assertTrue(new.equals(expected, True))
+
+    def test_failover_to_new_node_secondary_remains(self):
+        old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1),
+                              PgDistNode('2', 5432, 'secondary', nodeid=2)})
+        new = PgDistGroup(0, {PgDistNode('2', 5432, 'secondary'),
+                              PgDistNode('3', 5432, 'primary')})
+        expected = PgDistGroup(0, {PgDistNode('3', 5432, 'primary', nodeid=1),
+                                   PgDistNode('2', 5432, 'secondary', nodeid=2)})
+        self.check_transitions(old, new, ["citus_update_node(1, '3', 5432)"])
+        self.assertTrue(new.equals(expected, True))
+
+    def test_failover_to_new_node_secondary_removed(self):
+        old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1),
+                              PgDistNode('2', 5432, 'secondary', nodeid=2)})
+        new = PgDistGroup(0, {PgDistNode('3', 5432, 'primary')})
+        expected = PgDistGroup(0, {PgDistNode('3', 5432, 'primary', nodeid=1),
+                                   PgDistNode('2', 5432, 'secondary', nodeid=2)})
+        # the secondary record needs to be removed before we update the primary record
+        self.check_transitions(old, new, ["citus_update_node(1, '3', 5432)"])
+        self.assertTrue(new.equals(expected, True))
+
+    def test_switchover_to_new_node_and_secondary_removed(self):
+        old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1),
+                              PgDistNode('2', 5432, 'secondary', nodeid=2)})
+        new = PgDistGroup(0, {PgDistNode('1', 5432, 'secondary'),
+                              PgDistNode('3', 5432, 'primary')})
+        expected = PgDistGroup(0, {PgDistNode('3', 5432, 'primary', nodeid=1),
+                                   PgDistNode('1', 5432, 'secondary', nodeid=2)})
+        self.check_transitions(old, new, ["citus_update_node(1, '3', 5432)", "citus_update_node(2, '1', 5432)"])
+        self.assertTrue(new.equals(expected, True))
+
+    def test_switchover_with_pause(self):
+        old = PgDistGroup(0, {PgDistNode('1', 5432, 'primary', nodeid=1),
+                              PgDistNode('2', 5432, 'secondary', nodeid=2)})
+        new = PgDistGroup(0, {PgDistNode('1', 5432, 'demoted')})
+        expected = PgDistGroup(0, {PgDistNode('1', 5432, 'demoted', nodeid=1),
+                                   PgDistNode('2', 5432, 'secondary', nodeid=2)})
+        self.check_transitions(old, new, ["citus_update_node(1, '1-demoted', 5432)"])
+        self.assertTrue(new.equals(expected, True))
+
+    def test_switchover_after_paused_connections(self):
+        old = PgDistGroup(0, {PgDistNode('1', 5432, 'demoted', nodeid=1),
+                              PgDistNode('2', 5432, 'secondary', nodeid=2)})
+        new = PgDistGroup(0, {PgDistNode('2', 5432, 'primary')})
+        expected = PgDistGroup(0, {PgDistNode('1', 5432, 'secondary', nodeid=2),
+                                   PgDistNode('2', 5432, 'primary', nodeid=1)})
+        self.check_transitions(old, new, ["citus_update_node(2, '1', 5432)", "citus_update_node(1, '2', 5432)"])
+        self.assertTrue(new.equals(expected, True))
+
+    def test_switchover_to_new_node_after_paused_connections(self):
+        old = PgDistGroup(0, {PgDistNode('1', 5432, 'demoted', nodeid=1),
+                              PgDistNode('2', 5432, 'secondary', nodeid=2)})
+        new = PgDistGroup(0, {PgDistNode('3', 5432, 'primary')})
+        expected = PgDistGroup(0, {PgDistNode('1', 5432, 'secondary', nodeid=2),
+                                   PgDistNode('3', 5432, 'primary', nodeid=1)})
+        self.check_transitions(old, new, ["citus_update_node(1, '3', 5432)", "citus_update_node(2, '1', 5432)"])
+        self.assertTrue(new.equals(expected, True))
+
+    def test_switchover_to_new_node_after_paused_connections_secondary_added(self):
+        old = PgDistGroup(0, {PgDistNode('1', 5432, 'demoted', nodeid=1),
+                              PgDistNode('2', 5432, 'secondary', nodeid=2)})
+        new = PgDistGroup(0, {PgDistNode('4', 5432, 'secondary'),
+                              PgDistNode('3', 5432, 'primary')})
+        expected = PgDistGroup(0, {PgDistNode('4', 5432, 'secondary', nodeid=2),
+                                   PgDistNode('3', 5432, 'primary', nodeid=1)})
+        self.check_transitions(old, new, ["citus_update_node(1, '3', 5432)", "citus_update_node(2, '4', 5432)"])
+        self.assertTrue(new.equals(expected, True))

tests/test_config.py

@@ -5,7 +5,11 @@ import io
 
 from copy import deepcopy
 from mock import MagicMock, Mock, patch
-from patroni.config import Config, ConfigParseError, GlobalConfig
+
+from patroni import global_config
+from patroni.config import ClusterConfig, Config, ConfigParseError
+
+from .test_ha import get_cluster_initialized_with_only_leader
 
 
 class TestConfig(unittest.TestCase):
@@ -31,6 +35,7 @@ class TestConfig(unittest.TestCase):
             'PATRONI_NAMESPACE': '/patroni/',
             'PATRONI_SCOPE': 'batman2',
             'PATRONI_LOGLEVEL': 'ERROR',
+            'PATRONI_LOG_FORMAT': '["message", {"levelname": "level"}]',
             'PATRONI_LOG_LOGGERS': 'patroni.postmaster: WARNING, urllib3: DEBUG',
             'PATRONI_LOG_FILE_NUM': '5',
             'PATRONI_CITUS_DATABASE': 'citus',
@@ -155,48 +160,38 @@ class TestConfig(unittest.TestCase):
     @patch('patroni.config.logger')
     def test__validate_failover_tags(self, mock_logger, mock_get):
         """Ensures that only one of `nofailover` or `failover_priority` can be provided"""
-        mock_logger.warning.reset_mock()
-        config = Config("postgres0.yml")
         # Providing one of `nofailover` or `failover_priority` is fine
-        just_nofailover = {"nofailover": True}
-        mock_get.side_effect = [just_nofailover] * 2
-        self.assertIsNone(config._validate_failover_tags())
-        mock_logger.warning.assert_not_called()
-        just_failover_priority = {"failover_priority": 1}
-        mock_get.side_effect = [just_failover_priority] * 2
-        self.assertIsNone(config._validate_failover_tags())
-        mock_logger.warning.assert_not_called()
+        for single_param in ({"nofailover": True}, {"failover_priority": 1}, {"failover_priority": 0}):
+            mock_get.side_effect = [single_param] * 2
+            self.assertIsNone(self.config._validate_failover_tags())
+            mock_logger.warning.assert_not_called()
+
         # Providing both `nofailover` and `failover_priority` is fine if consistent
-        consistent_false = {"nofailover": False, "failover_priority": 1}
-        mock_get.side_effect = [consistent_false] * 2
-        self.assertIsNone(config._validate_failover_tags())
-        mock_logger.warning.assert_not_called()
-        consistent_true = {"nofailover": True, "failover_priority": 0}
-        mock_get.side_effect = [consistent_true] * 2
-        self.assertIsNone(config._validate_failover_tags())
-        mock_logger.warning.assert_not_called()
+        for consistent_state in (
+            {"nofailover": False, "failover_priority": 1},
+            {"nofailover": True, "failover_priority": 0},
+            {"nofailover": "False", "failover_priority": 0}
+        ):
+            mock_get.side_effect = [consistent_state] * 2
+            self.assertIsNone(self.config._validate_failover_tags())
+            mock_logger.warning.assert_not_called()
+
         # Providing both inconsistently should log a warning
-        inconsistent_false = {"nofailover": False, "failover_priority": 0}
-        mock_get.side_effect = [inconsistent_false] * 2
-        self.assertIsNone(config._validate_failover_tags())
-        mock_logger.warning.assert_called_once_with(
-            'Conflicting configuration between nofailover: %s and failover_priority: %s.'
-            + ' Defaulting to nofailover: %s',
-            False,
-            0,
-            False
-        )
-        mock_logger.warning.reset_mock()
-        inconsistent_true = {"nofailover": True, "failover_priority": 1}
-        mock_get.side_effect = [inconsistent_true] * 2
-        self.assertIsNone(config._validate_failover_tags())
-        mock_logger.warning.assert_called_once_with(
-            'Conflicting configuration between nofailover: %s and failover_priority: %s.'
-            + ' Defaulting to nofailover: %s',
-            True,
-            1,
-            True
-        )
+        for inconsistent_state in (
+            {"nofailover": False, "failover_priority": 0},
+            {"nofailover": True, "failover_priority": 1},
+            {"nofailover": "False", "failover_priority": 1},
+            {"nofailover": "", "failover_priority": 0}
+        ):
+            mock_get.side_effect = [inconsistent_state] * 2
+            self.assertIsNone(self.config._validate_failover_tags())
+            mock_logger.warning.assert_called_once_with(
+                'Conflicting configuration between nofailover: %s and failover_priority: %s.'
+                + ' Defaulting to nofailover: %s',
+                inconsistent_state['nofailover'],
+                inconsistent_state['failover_priority'],
+                inconsistent_state['nofailover'])
+            mock_logger.warning.reset_mock()
 
     def test__process_postgresql_parameters(self):
         expected_params = {
@@ -248,4 +243,6 @@ class TestConfig(unittest.TestCase):
     def test_global_config_is_synchronous_mode(self):
         # we should ignore synchronous_mode setting in a standby cluster
         config = {'standby_cluster': {'host': 'some_host'}, 'synchronous_mode': True}
-        self.assertFalse(GlobalConfig(config).is_synchronous_mode)
+        cluster = get_cluster_initialized_with_only_leader(cluster_config=ClusterConfig(1, config, 1))
+        test_config = global_config.from_cluster(cluster)
+        self.assertFalse(test_config.is_synchronous_mode)

tests/test_config_generator.py

@@ -62,9 +62,10 @@ class TestGenerateConfig(unittest.TestCase):
             'scope': self.environ['PATRONI_SCOPE'],
             'name': HOSTNAME,
             'log': {
+                'type': PatroniLogger.DEFAULT_TYPE,
+                'format': PatroniLogger.DEFAULT_FORMAT,
                 'level': PatroniLogger.DEFAULT_LEVEL,
                 'traceback_level': PatroniLogger.DEFAULT_TRACEBACK_LEVEL,
-                'format': PatroniLogger.DEFAULT_FORMAT,
                 'max_queue_size': PatroniLogger.DEFAULT_MAX_QUEUE_SIZE
             },
             'restapi': {
@@ -141,6 +142,7 @@ class TestGenerateConfig(unittest.TestCase):
                 'noloadbalance': False,
                 'clonefrom': True,
                 'nosync': False,
+                'nostream': False
             }
         }
         patch_config(self.config, conf)

tests/test_consul.py

@@ -3,8 +3,10 @@ import unittest
 
 from consul import ConsulException, NotFound
 from mock import Mock, PropertyMock, patch
+from patroni.dcs import get_dcs
 from patroni.dcs.consul import AbstractDCS, Cluster, Consul, ConsulInternalError, \
     ConsulError, ConsulClient, HTTPClient, InvalidSessionTTL, InvalidSession, RetryFailedError
+from patroni.postgresql.mpp import get_mpp
 from . import SleepException
 
 
@@ -91,13 +93,17 @@ class TestConsul(unittest.TestCase):
     @patch.object(consul.Consul.KV, 'get', kv_get)
     @patch.object(consul.Consul.KV, 'delete', Mock())
     def setUp(self):
-        Consul({'ttl': 30, 'scope': 't', 'name': 'p', 'url': 'https://l:1', 'retry_timeout': 10,
-                'verify': 'on', 'key': 'foo', 'cert': 'bar', 'cacert': 'buz', 'token': 'asd', 'dc': 'dc1',
-                'register_service': True})
-        Consul({'ttl': 30, 'scope': 't_', 'name': 'p', 'url': 'https://l:1', 'retry_timeout': 10,
-                'verify': 'on', 'cert': 'bar', 'cacert': 'buz', 'register_service': True})
-        self.c = Consul({'ttl': 30, 'scope': 'test', 'name': 'postgresql1', 'host': 'localhost:1', 'retry_timeout': 10,
-                         'register_service': True, 'service_check_tls_server_name': True})
+        self.assertIsInstance(get_dcs({'ttl': 30, 'scope': 't', 'name': 'p', 'retry_timeout': 10,
+                                       'consul': {'url': 'https://l:1', 'verify': 'on',
+                                                  'key': 'foo', 'cert': 'bar', 'cacert': 'buz',
+                                                  'token': 'asd', 'dc': 'dc1', 'register_service': True}}), Consul)
+        self.assertIsInstance(get_dcs({'ttl': 30, 'scope': 't_', 'name': 'p', 'retry_timeout': 10,
+                                       'consul': {'url': 'https://l:1', 'verify': 'on',
+                                                  'cert': 'bar', 'cacert': 'buz', 'register_service': True}}), Consul)
+        self.c = get_dcs({'ttl': 30, 'scope': 'test', 'name': 'postgresql1', 'retry_timeout': 10,
+                          'consul': {'host': 'localhost:1', 'register_service': True,
+                                     'service_check_tls_server_name': True}})
+        self.assertIsInstance(self.c, Consul)
         self.c._base_path = 'service/good'
         self.c.get_cluster()
 
@@ -130,7 +136,7 @@ class TestConsul(unittest.TestCase):
         self.assertIsInstance(self.c.get_cluster(), Cluster)
 
     def test__get_citus_cluster(self):
-        self.c._citus_group = '0'
+        self.c._mpp = get_mpp({'citus': {'group': 0, 'database': 'postgres'}})
         cluster = self.c.get_cluster()
         self.assertIsInstance(cluster, Cluster)
         self.assertIsInstance(cluster.workers[1], Cluster)
@@ -154,10 +160,8 @@ class TestConsul(unittest.TestCase):
         self.c.set_ttl(20)
         self.c._do_refresh_session = Mock()
         self.assertFalse(self.c.take_leader())
-        with patch('time.time', Mock(side_effect=[0, 100])):
-            self.assertRaises(ConsulError, self.c.take_leader)
-        with patch('time.time', Mock(side_effect=[0, 0, 0, 0, 0, 0, 100])):
-            self.assertRaises(ConsulError, self.c.take_leader)
+        with patch('time.time', Mock(side_effect=[0, 0, 0, 100, 100])):
+            self.assertFalse(self.c.take_leader())
 
     @patch.object(consul.Consul.KV, 'put', Mock(return_value=True))
     def test_set_failover_value(self):

tests/test_ctl.py

@@ -1,3 +1,4 @@
+import click
 import etcd
 import mock
 import os
@@ -6,10 +7,13 @@ import unittest
 from click.testing import CliRunner
 from datetime import datetime, timedelta
 from mock import patch, Mock, PropertyMock
+from patroni import global_config
 from patroni.ctl import ctl, load_config, output_members, get_dcs, parse_dcs, \
     get_all_members, get_any_member, get_cursor, query_member, PatroniCtlException, apply_config_changes, \
     format_config_for_editing, show_diff, invoke_editor, format_pg_version, CONFIG_FILE_PATH, PatronictlPrettyTable
-from patroni.dcs.etcd import AbstractEtcdClientWithFailover, Cluster, Failover
+from patroni.dcs import Cluster, Failover
+from patroni.postgresql.config import get_param_diff
+from patroni.postgresql.mpp import get_mpp
 from patroni.psycopg import OperationalError
 from patroni.utils import tzutc
 from prettytable import PrettyTable, ALL
@@ -21,26 +25,26 @@ from .test_ha import get_cluster_initialized_without_leader, get_cluster_initial
     get_cluster_initialized_with_only_leader, get_cluster_not_initialized_without_leader, get_cluster, Member
 
 
-DEFAULT_CONFIG = {
-    'scope': 'alpha',
-    'restapi': {'listen': '::', 'certfile': 'a'},
-    'ctl': {'certfile': 'a'},
-    'etcd': {'host': 'localhost:2379'},
-    'citus': {'database': 'citus', 'group': 0},
-    'postgresql': {'data_dir': '.', 'pgpass': './pgpass', 'parameters': {}, 'retry_timeout': 5}
-}
+def get_default_config(*args):
+    return {
+        'scope': 'alpha',
+        'restapi': {'listen': '::', 'certfile': 'a'},
+        'ctl': {'certfile': 'a'},
+        'etcd': {'host': 'localhost:2379', 'retry_timeout': 10, 'ttl': 30},
+        'citus': {'database': 'citus', 'group': 0},
+        'postgresql': {'data_dir': '.', 'pgpass': './pgpass', 'parameters': {}, 'retry_timeout': 5}
+    }
 
 
-@patch('patroni.ctl.load_config', Mock(return_value=DEFAULT_CONFIG))
+@patch.object(PoolManager, 'request', Mock(return_value=MockResponse()))
+@patch('patroni.ctl.load_config', get_default_config)
+@patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
 class TestCtl(unittest.TestCase):
     TEST_ROLES = ('master', 'primary', 'leader')
 
     @patch('socket.getaddrinfo', socket_getaddrinfo)
-    @patch.object(AbstractEtcdClientWithFailover, '_get_machines_list', Mock(return_value=['http://remotehost:2379']))
     def setUp(self):
         self.runner = CliRunner()
-        self.e = get_dcs({'etcd': {'ttl': 30, 'host': 'ok:2379', 'retry_timeout': 10},
-                          'citus': {'group': 0}}, 'foo', None)
 
     @patch('patroni.ctl.logging.debug')
     def test_load_config(self, mock_logger_debug):
@@ -66,29 +70,31 @@ class TestCtl(unittest.TestCase):
 
     @patch('patroni.psycopg.connect', psycopg_connect)
     def test_get_cursor(self):
-        for role in self.TEST_ROLES:
-            self.assertIsNone(get_cursor({}, get_cluster_initialized_without_leader(), None, {}, role=role))
-            self.assertIsNotNone(get_cursor({}, get_cluster_initialized_with_leader(), None, {}, role=role))
+        with click.Context(click.Command('query')) as ctx:
+            ctx.obj = {'__config': {}, '__mpp': get_mpp({})}
+            for role in self.TEST_ROLES:
+                self.assertIsNone(get_cursor(get_cluster_initialized_without_leader(), None, {}, role=role))
+                self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), None, {}, role=role))
 
-        # MockCursor returns pg_is_in_recovery as false
-        self.assertIsNone(get_cursor({}, get_cluster_initialized_with_leader(), None, {}, role='replica'))
+            # MockCursor returns pg_is_in_recovery as false
+            self.assertIsNone(get_cursor(get_cluster_initialized_with_leader(), None, {}, role='replica'))
 
-        self.assertIsNotNone(get_cursor({}, get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, role='any'))
+            self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, role='any'))
 
-        # Mutually exclusive options
-        with self.assertRaises(PatroniCtlException) as e:
-            get_cursor({}, get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, member_name='other',
-                       role='replica')
+            # Mutually exclusive options
+            with self.assertRaises(PatroniCtlException) as e:
+                get_cursor(get_cluster_initialized_with_leader(), None, {'dbname': 'foo'}, member_name='other',
+                           role='replica')
 
-        self.assertEqual(str(e.exception), '--role and --member are mutually exclusive options')
+            self.assertEqual(str(e.exception), '--role and --member are mutually exclusive options')
 
-        # Invalid member provided
-        self.assertIsNone(get_cursor({}, get_cluster_initialized_with_leader(), None, {'dbname': 'foo'},
-                                     member_name='invalid'))
+            # Invalid member provided
+            self.assertIsNone(get_cursor(get_cluster_initialized_with_leader(), None, {'dbname': 'foo'},
+                                         member_name='invalid'))
 
-        # Valid member provided
-        self.assertIsNotNone(get_cursor({}, get_cluster_initialized_with_leader(), None, {'dbname': 'foo'},
-                                        member_name='other'))
+            # Valid member provided
+            self.assertIsNotNone(get_cursor(get_cluster_initialized_with_leader(), None, {'dbname': 'foo'},
+                                            member_name='other'))
 
     def test_parse_dcs(self):
         assert parse_dcs(None) is None
@@ -102,23 +108,20 @@ class TestCtl(unittest.TestCase):
         self.assertRaises(PatroniCtlException, parse_dcs, 'invalid://test')
 
     def test_output_members(self):
-        scheduled_at = datetime.now(tzutc) + timedelta(seconds=600)
-        cluster = get_cluster_initialized_with_leader(Failover(1, 'foo', 'bar', scheduled_at))
-        del cluster.members[1].data['conn_url']
-        for fmt in ('pretty', 'json', 'yaml', 'topology'):
-            self.assertIsNone(output_members({}, cluster, name='abc', fmt=fmt))
+        with click.Context(click.Command('list')) as ctx:
+            ctx.obj = {'__config': {}, '__mpp': get_mpp({})}
+            scheduled_at = datetime.now(tzutc) + timedelta(seconds=600)
+            cluster = get_cluster_initialized_with_leader(Failover(1, 'foo', 'bar', scheduled_at))
+            del cluster.members[1].data['conn_url']
+            for fmt in ('pretty', 'json', 'yaml', 'topology'):
+                self.assertIsNone(output_members(cluster, name='abc', fmt=fmt))
 
-        with patch('click.echo') as mock_echo:
-            self.assertIsNone(output_members({}, cluster, name='abc', fmt='tsv'))
-            self.assertEqual(mock_echo.call_args[0][0], 'abc\tother\t\tReplica\trunning\t\tunknown')
-
-    @patch('patroni.ctl.get_dcs')
-    @patch.object(PoolManager, 'request', Mock(return_value=MockResponse()))
-    def test_switchover(self, mock_get_dcs):
-        mock_get_dcs.return_value = self.e
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
-        mock_get_dcs.return_value.set_failover_value = Mock()
+            with patch('click.echo') as mock_echo:
+                self.assertIsNone(output_members(cluster, name='abc', fmt='tsv'))
+                self.assertEqual(mock_echo.call_args[0][0], 'abc\tother\t\tReplica\trunning\t\tunknown')
 
+    @patch('patroni.dcs.AbstractDCS.set_failover_value', Mock())
+    def test_switchover(self):
         # Confirm
         result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
         self.assertEqual(result.exit_code, 0)
@@ -147,7 +150,7 @@ class TestCtl(unittest.TestCase):
         self.assertEqual(result.exit_code, 0)
 
         # Scheduled in pause mode
-        with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
+        with patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)):
             result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0',
                                               '--force', '--scheduled', '2015-01-01T12:00:00'])
             self.assertEqual(result.exit_code, 1)
@@ -156,7 +159,8 @@ class TestCtl(unittest.TestCase):
         # Target and source are equal
         result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nleader\n\ny')
         self.assertEqual(result.exit_code, 1)
-        self.assertIn('Switchover target and source are the same', result.output)
+        self.assertIn("Candidate ['other']", result.output)
+        self.assertIn('Member leader is already the leader of cluster dummy', result.output)
 
         # Candidate is not a member of the cluster
         result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nReality\n\ny')
@@ -180,12 +184,12 @@ class TestCtl(unittest.TestCase):
         self.assertIn('Member dummy is not the leader of cluster dummy', result.output)
 
         # Errors while sending Patroni REST API request
-        with patch.object(PoolManager, 'request', Mock(side_effect=Exception)):
+        with patch('patroni.ctl.request_patroni', Mock(side_effect=Exception)):
             result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'],
                                         input='leader\nother\n2300-01-01T12:23:00\ny')
             self.assertIn('falling back to DCS', result.output)
 
-        with patch.object(PoolManager, 'request') as mock_api_request:
+        with patch('patroni.ctl.request_patroni') as mock_api_request:
             mock_api_request.return_value.status = 500
             result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
             self.assertIn('Switchover failed', result.output)
@@ -196,64 +200,65 @@ class TestCtl(unittest.TestCase):
             self.assertIn('Switchover failed', result.output)
 
         # No members available
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_only_leader
-        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
-        self.assertEqual(result.exit_code, 1)
-        self.assertIn('No candidates found to switchover to', result.output)
+        with patch('patroni.dcs.AbstractDCS.get_cluster',
+                   Mock(return_value=get_cluster_initialized_with_only_leader())):
+            result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
+            self.assertEqual(result.exit_code, 1)
+            self.assertIn('No candidates found to switchover to', result.output)
 
         # No leader available
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_without_leader
-        result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
-        self.assertEqual(result.exit_code, 1)
-        self.assertIn('This cluster has no leader', result.output)
+        with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_without_leader())):
+            result = self.runner.invoke(ctl, ['switchover', 'dummy', '--group', '0'], input='leader\nother\n\ny')
+            self.assertEqual(result.exit_code, 1)
+            self.assertIn('This cluster has no leader', result.output)
 
         # Citus cluster, no group number specified
         result = self.runner.invoke(ctl, ['switchover', 'dummy', '--force'], input='\n')
         self.assertEqual(result.exit_code, 1)
         self.assertIn('For Citus clusters the --group must me specified', result.output)
 
-    @patch('patroni.ctl.get_dcs')
-    @patch.object(PoolManager, 'request', Mock(return_value=MockResponse()))
-    @patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse()))
-    def test_failover(self, mock_get_dcs):
-        mock_get_dcs.return_value.set_failover_value = Mock()
-
+    @patch('patroni.dcs.AbstractDCS.set_failover_value', Mock())
+    def test_failover(self):
         # No candidate specified
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
         result = self.runner.invoke(ctl, ['failover', 'dummy'], input='0\n')
         self.assertIn('Failover could be performed only to a specific candidate', result.output)
 
-        cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
+        # Candidate is the same as the leader
+        result = self.runner.invoke(ctl, ['failover', 'dummy', '--group', '0'], input='leader\n')
+        self.assertIn("Candidate ['other']", result.output)
+        self.assertIn('Member leader is already the leader of cluster dummy', result.output)
 
         # Temp test to check a fallback to switchover if leader is specified
         with patch('patroni.ctl._do_failover_or_switchover') as failover_func_mock:
             result = self.runner.invoke(ctl, ['failover', '--leader', 'leader', 'dummy'], input='0\n')
             self.assertIn('Supplying a leader name using this command is deprecated', result.output)
-            failover_func_mock.assert_called_once_with(
-                DEFAULT_CONFIG, 'switchover', 'dummy', None, 'leader', None, False)
+            failover_func_mock.assert_called_once_with('switchover', 'dummy', None, 'leader', None, False)
 
-        # Failover to an async member in sync mode (confirm)
+        cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
         cluster.members.append(Member(0, 'async', 28, {'api_url': 'http://127.0.0.1:8012/patroni'}))
         cluster.config.data['synchronous_mode'] = True
-        mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster)
-        result = self.runner.invoke(ctl, ['failover', 'dummy', '--group', '0', '--candidate', 'async'], input='y\ny')
-        self.assertIn('Are you sure you want to failover to the asynchronous node async', result.output)
+        with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=cluster)):
+            # Failover to an async member in sync mode (confirm)
+            result = self.runner.invoke(ctl,
+                                        ['failover', 'dummy', '--group', '0', '--candidate', 'async'], input='y\ny')
+            self.assertIn('Are you sure you want to failover to the asynchronous node async', result.output)
+            self.assertEqual(result.exit_code, 0)
 
-        # Failover to an async member in sync mode (abort)
-        mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster)
-        result = self.runner.invoke(ctl, ['failover', 'dummy', '--group', '0', '--candidate', 'async'], input='N')
-        self.assertEqual(result.exit_code, 1)
+            # Failover to an async member in sync mode (abort)
+            result = self.runner.invoke(ctl, ['failover', 'dummy', '--group', '0', '--candidate', 'async'], input='N')
+            self.assertEqual(result.exit_code, 1)
+            self.assertIn('Aborting failover', result.output)
 
-    @patch('patroni.dcs.dcs_modules', Mock(return_value=['patroni.dcs.dummy', 'patroni.dcs.etcd']))
+    @patch('patroni.dynamic_loader.iter_modules', Mock(return_value=['patroni.dcs.dummy', 'patroni.dcs.etcd']))
     def test_get_dcs(self):
-        self.assertRaises(PatroniCtlException, get_dcs, {'dummy': {}}, 'dummy', 0)
+        with click.Context(click.Command('list')) as ctx:
+            ctx.obj = {'__config': {'dummy': {}}, '__mpp': get_mpp({})}
+            self.assertRaises(PatroniCtlException, get_dcs, 'dummy', 0)
 
     @patch('patroni.psycopg.connect', psycopg_connect)
     @patch('patroni.ctl.query_member', Mock(return_value=([['mock column']], None)))
-    @patch('patroni.ctl.get_dcs')
     @patch.object(etcd.Client, 'read', etcd_read)
-    def test_query(self, mock_get_dcs):
-        mock_get_dcs.return_value = self.e
+    def test_query(self):
         # Mutually exclusive
         for role in self.TEST_ROLES:
             result = self.runner.invoke(ctl, ['query', 'alpha', '--member', 'abc', '--role', role])
@@ -286,31 +291,29 @@ class TestCtl(unittest.TestCase):
     def test_query_member(self):
         with patch('patroni.ctl.get_cursor', Mock(return_value=MockConnect().cursor())):
             for role in self.TEST_ROLES:
-                rows = query_member({}, None, None, None, None, role, 'SELECT pg_catalog.pg_is_in_recovery()', {})
+                rows = query_member(None, None, None, None, role, 'SELECT pg_catalog.pg_is_in_recovery()', {})
                 self.assertTrue('False' in str(rows))
 
             with patch.object(MockCursor, 'execute', Mock(side_effect=OperationalError('bla'))):
-                rows = query_member({}, None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
+                rows = query_member(None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
 
         with patch('patroni.ctl.get_cursor', Mock(return_value=None)):
             # No role nor member given -- generic message
-            rows = query_member({}, None, None, None, None, None, 'SELECT pg_catalog.pg_is_in_recovery()', {})
+            rows = query_member(None, None, None, None, None, 'SELECT pg_catalog.pg_is_in_recovery()', {})
             self.assertTrue('No connection is available' in str(rows))
 
             # Member given -- message pointing to member
-            rows = query_member({}, None, None, None, 'foo', None, 'SELECT pg_catalog.pg_is_in_recovery()', {})
+            rows = query_member(None, None, None, 'foo', None, 'SELECT pg_catalog.pg_is_in_recovery()', {})
             self.assertTrue('No connection to member foo' in str(rows))
 
             # Role given -- message pointing to role
-            rows = query_member({}, None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
+            rows = query_member(None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
             self.assertTrue('No connection to role replica' in str(rows))
 
         with patch('patroni.ctl.get_cursor', Mock(side_effect=OperationalError('bla'))):
-            rows = query_member({}, None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
+            rows = query_member(None, None, None, None, 'replica', 'SELECT pg_catalog.pg_is_in_recovery()', {})
 
-    @patch('patroni.ctl.get_dcs')
-    def test_dsn(self, mock_get_dcs):
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+    def test_dsn(self):
         result = self.runner.invoke(ctl, ['dsn', 'alpha'])
         assert 'host=127.0.0.1 port=5435' in result.output
 
@@ -323,11 +326,8 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['dsn', 'alpha', '--member', 'dummy'])
         assert result.exit_code == 1
 
-    @patch.object(PoolManager, 'request')
-    @patch('patroni.ctl.get_dcs')
-    def test_reload(self, mock_get_dcs, mock_post):
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
-
+    @patch('patroni.ctl.request_patroni')
+    def test_reload(self, mock_post):
         result = self.runner.invoke(ctl, ['reload', 'alpha'], input='y')
         assert 'Failed: reload for member' in result.output
 
@@ -339,10 +339,8 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['reload', 'alpha'], input='y')
         assert 'Reload request received for member' in result.output
 
-    @patch.object(PoolManager, 'request')
-    @patch('patroni.ctl.get_dcs')
-    def test_restart_reinit(self, mock_get_dcs, mock_post):
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+    @patch('patroni.ctl.request_patroni')
+    def test_restart_reinit(self, mock_post):
         mock_post.return_value.status = 503
         result = self.runner.invoke(ctl, ['restart', 'alpha'], input='now\ny\n')
         assert 'Failed: restart for' in result.output
@@ -382,7 +380,7 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
         assert 'Failed: flush scheduled restart' in result.output
 
-        with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
+        with patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)):
             result = self.runner.invoke(ctl,
                                         ['restart', 'alpha', 'other', '--force', '--scheduled', '2300-10-01T14:30'])
             assert result.exit_code == 1
@@ -417,12 +415,10 @@ class TestCtl(unittest.TestCase):
         assert 'Failed: another restart is already' in result.output
         assert result.exit_code == 0
 
-    @patch('patroni.ctl.get_dcs')
-    def test_remove(self, mock_get_dcs):
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+    def test_remove(self):
         result = self.runner.invoke(ctl, ['remove', 'dummy'], input='\n')
         assert 'For Citus clusters the --group must me specified' in result.output
-        result = self.runner.invoke(ctl, ['-k', 'remove', 'alpha', '--group', '0'], input='alpha\nstandby')
+        result = self.runner.invoke(ctl, ['remove', 'alpha', '--group', '0'], input='alpha\nstandby')
         assert 'Please confirm' in result.output
         assert 'You are about to remove all' in result.output
         # Not typing an exact confirmation
@@ -440,37 +436,36 @@ class TestCtl(unittest.TestCase):
         assert result.exit_code == 0
 
     def test_ctl(self):
-        self.runner.invoke(ctl, ['list'])
-
         result = self.runner.invoke(ctl, ['--help'])
         assert 'Usage:' in result.output
 
     def test_get_any_member(self):
-        for role in self.TEST_ROLES:
-            self.assertIsNone(get_any_member({}, get_cluster_initialized_without_leader(), None, role=role))
+        with click.Context(click.Command('list')) as ctx:
+            ctx.obj = {'__config': {}, '__mpp': get_mpp({})}
+            for role in self.TEST_ROLES:
+                self.assertIsNone(get_any_member(get_cluster_initialized_without_leader(), None, role=role))
 
-            m = get_any_member({}, get_cluster_initialized_with_leader(), None, role=role)
-            self.assertEqual(m.name, 'leader')
+                m = get_any_member(get_cluster_initialized_with_leader(), None, role=role)
+                self.assertEqual(m.name, 'leader')
 
     def test_get_all_members(self):
-        for role in self.TEST_ROLES:
-            self.assertEqual(list(get_all_members({}, get_cluster_initialized_without_leader(), None, role=role)), [])
+        with click.Context(click.Command('list')) as ctx:
+            ctx.obj = {'__config': {}, '__mpp': get_mpp({})}
+            for role in self.TEST_ROLES:
+                self.assertEqual(list(get_all_members(get_cluster_initialized_without_leader(), None, role=role)), [])
 
-            r = list(get_all_members({}, get_cluster_initialized_with_leader(), None, role=role))
+                r = list(get_all_members(get_cluster_initialized_with_leader(), None, role=role))
+                self.assertEqual(len(r), 1)
+                self.assertEqual(r[0].name, 'leader')
+
+            r = list(get_all_members(get_cluster_initialized_with_leader(), None, role='replica'))
             self.assertEqual(len(r), 1)
-            self.assertEqual(r[0].name, 'leader')
+            self.assertEqual(r[0].name, 'other')
 
-        r = list(get_all_members({}, get_cluster_initialized_with_leader(), None, role='replica'))
-        self.assertEqual(len(r), 1)
-        self.assertEqual(r[0].name, 'other')
-
-        self.assertEqual(len(list(get_all_members({}, get_cluster_initialized_without_leader(),
-                                                  None, role='replica'))), 2)
-
-    @patch('patroni.ctl.get_dcs')
-    def test_members(self, mock_get_dcs):
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+            self.assertEqual(len(list(get_all_members(get_cluster_initialized_without_leader(),
+                                                      None, role='replica'))), 2)
 
+    def test_members(self):
         result = self.runner.invoke(ctl, ['list'])
         assert '127.0.0.1' in result.output
         assert result.exit_code == 0
@@ -479,127 +474,115 @@ class TestCtl(unittest.TestCase):
         result = self.runner.invoke(ctl, ['list', '--group', '0'])
         assert 'Citus cluster: alpha (group: 0, 12345678901) -' in result.output
 
-        with patch('patroni.ctl.load_config', Mock(return_value={'scope': 'alpha'})):
+        config = get_default_config()
+        del config['citus']
+        with patch('patroni.ctl.load_config', Mock(return_value=config)):
             result = self.runner.invoke(ctl, ['list'])
             assert 'Cluster: alpha (12345678901) -' in result.output
 
         with patch('patroni.ctl.load_config', Mock(return_value={})):
             self.runner.invoke(ctl, ['list'])
 
-    @patch('patroni.ctl.get_dcs')
-    def test_list_extended(self, mock_get_dcs):
-        mock_get_dcs.return_value = self.e
-        cluster = get_cluster_initialized_with_leader(sync=('leader', 'other'))
-        mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster)
+        cluster = get_cluster_initialized_with_leader()
+        cluster.members[1].data['pending_restart'] = True
+        cluster.members[1].data['pending_restart_reason'] = {'param': get_param_diff('', 'very l' + 'o' * 34 + 'ng')}
+        with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=cluster)):
+            for cmd in ('list', 'topology'):
+                result = self.runner.invoke(ctl, [cmd, 'dummy'])
+                self.assertIn('param: [hidden - too long]', result.output)
 
+            result = self.runner.invoke(ctl, ['list', 'dummy', '-f', 'tsv'])
+            self.assertIn('param: ->very l' + 'o' * 34 + 'ng', result.output)
+
+            cluster.members[1].data['pending_restart_reason'] = {'param': get_param_diff('', 'new')}
+            result = self.runner.invoke(ctl, ['list', 'dummy'])
+            self.assertIn('param: ->new', result.output)
+
+    def test_list_extended(self):
         result = self.runner.invoke(ctl, ['list', 'dummy', '--extended', '--timestamp'])
         assert '2100' in result.output
         assert 'Scheduled restart' in result.output
 
-    @patch('patroni.ctl.get_dcs')
-    def test_topology(self, mock_get_dcs):
-        mock_get_dcs.return_value = self.e
+    def test_topology(self):
         cluster = get_cluster_initialized_with_leader()
-        cascade_member = Member(0, 'cascade', 28, {'conn_url': 'postgres://replicator:[email protected]:5437/postgres',
-                                                   'api_url': 'http://127.0.0.1:8012/patroni',
-                                                   'state': 'running',
-                                                   'tags': {'replicatefrom': 'other'},
-                                                   })
-        cascade_member_wrong_tags = Member(0, 'wrong_cascade', 28,
-                                           {'conn_url': 'postgres://replicator:[email protected]:5438/postgres',
-                                            'api_url': 'http://127.0.0.1:8013/patroni',
-                                            'state': 'running',
-                                            'tags': {'replicatefrom': 'nonexistinghost'},
-                                            })
-        cluster.members.append(cascade_member)
-        cluster.members.append(cascade_member_wrong_tags)
-        mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster)
-        result = self.runner.invoke(ctl, ['topology', 'dummy'])
-        assert '+\n|     0 | leader          | 127.0.0.1:5435 | Leader  |' in result.output
-        assert '|\n|     0 | + other         | 127.0.0.1:5436 | Replica |' in result.output
-        assert '|\n|     0 |   + cascade     | 127.0.0.1:5437 | Replica |' in result.output
-        assert '|\n|     0 | + wrong_cascade | 127.0.0.1:5438 | Replica |' in result.output
+        cluster.members.append(Member(0, 'cascade', 28,
+                                      {'conn_url': 'postgres://replicator:[email protected]:5437/postgres',
+                                       'api_url': 'http://127.0.0.1:8012/patroni', 'state': 'running',
+                                       'tags': {'replicatefrom': 'other'}}))
+        cluster.members.append(Member(0, 'wrong_cascade', 28,
+                                      {'conn_url': 'postgres://replicator:[email protected]:5438/postgres',
+                                       'api_url': 'http://127.0.0.1:8013/patroni', 'state': 'running',
+                                       'tags': {'replicatefrom': 'nonexistinghost'}}))
+        with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=cluster)):
+            result = self.runner.invoke(ctl, ['topology', 'dummy'])
+            assert '+\n|     0 | leader          | 127.0.0.1:5435 | Leader  |' in result.output
+            assert '|\n|     0 | + other         | 127.0.0.1:5436 | Replica |' in result.output
+            assert '|\n|     0 |   + cascade     | 127.0.0.1:5437 | Replica |' in result.output
+            assert '|\n|     0 | + wrong_cascade | 127.0.0.1:5438 | Replica |' in result.output
 
-        cluster = get_cluster_initialized_without_leader()
-        mock_get_dcs.return_value.get_cluster = Mock(return_value=cluster)
-        result = self.runner.invoke(ctl, ['topology', 'dummy'])
-        assert '+\n|     0 | + leader | 127.0.0.1:5435 | Replica |' in result.output
-        assert '|\n|     0 | + other  | 127.0.0.1:5436 | Replica |' in result.output
-
-    @patch('patroni.ctl.get_dcs')
-    @patch.object(PoolManager, 'request', Mock(return_value=MockResponse()))
-    def test_flush_restart(self, mock_get_dcs):
-        mock_get_dcs.return_value = self.e
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_without_leader())):
+            result = self.runner.invoke(ctl, ['topology', 'dummy'])
+            assert '+\n|     0 | + leader | 127.0.0.1:5435 | Replica |' in result.output
+            assert '|\n|     0 | + other  | 127.0.0.1:5436 | Replica |' in result.output
 
+    @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
+    def test_flush_restart(self):
         for role in self.TEST_ROLES:
-            result = self.runner.invoke(ctl, ['-k', 'flush', 'dummy', 'restart', '-r', role], input='y')
+            result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '-r', role], input='y')
             assert 'No scheduled restart' in result.output
 
         result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '--force'])
         assert 'Success: flush scheduled restart' in result.output
-        with patch.object(PoolManager, 'request', return_value=MockResponse(404)):
+        with patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse(404))):
             result = self.runner.invoke(ctl, ['flush', 'dummy', 'restart', '--force'])
             assert 'Failed: flush scheduled restart' in result.output
 
-    @patch('patroni.ctl.get_dcs')
-    @patch.object(PoolManager, 'request', Mock(return_value=MockResponse()))
-    def test_flush_switchover(self, mock_get_dcs):
-        mock_get_dcs.return_value = self.e
-
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
-        result = self.runner.invoke(ctl, ['flush', 'dummy', 'switchover'])
-        assert 'No pending scheduled switchover' in result.output
+    def test_flush_switchover(self):
+        with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader())):
+            result = self.runner.invoke(ctl, ['flush', 'dummy', 'switchover'])
+            assert 'No pending scheduled switchover' in result.output
 
         scheduled_at = datetime.now(tzutc) + timedelta(seconds=600)
-        mock_get_dcs.return_value.get_cluster = Mock(
-            return_value=get_cluster_initialized_with_leader(Failover(1, 'a', 'b', scheduled_at)))
-        result = self.runner.invoke(ctl, ['flush', 'dummy', 'switchover'])
-        assert result.output.startswith('Success: ')
+        with patch('patroni.dcs.AbstractDCS.get_cluster',
+                   Mock(return_value=get_cluster_initialized_with_leader(Failover(1, 'a', 'b', scheduled_at)))):
+            result = self.runner.invoke(ctl, ['-k', 'flush', 'dummy', 'switchover'])
+            assert result.output.startswith('Success: ')
 
-        mock_get_dcs.return_value.manual_failover = Mock()
-        with patch.object(PoolManager, 'request', side_effect=[MockResponse(409), Exception]):
-            result = self.runner.invoke(ctl, ['flush', 'dummy', 'switchover'])
-            assert 'Could not find any accessible member of cluster' in result.output
+            with patch('patroni.ctl.request_patroni', side_effect=[MockResponse(409), Exception]), \
+                    patch('patroni.dcs.AbstractDCS.manual_failover', Mock()):
+                result = self.runner.invoke(ctl, ['flush', 'dummy', 'switchover'])
+                assert 'Could not find any accessible member of cluster' in result.output
 
-    @patch.object(PoolManager, 'request')
-    @patch('patroni.ctl.get_dcs')
     @patch('patroni.ctl.polling_loop', Mock(return_value=[1]))
-    def test_pause_cluster(self, mock_get_dcs, mock_post):
-        mock_get_dcs.return_value = self.e
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+    def test_pause_cluster(self):
+        with patch('patroni.ctl.request_patroni', Mock(return_value=MockResponse(500))):
+            result = self.runner.invoke(ctl, ['pause', 'dummy'])
+            assert 'Failed' in result.output
 
-        mock_post.return_value.status = 500
-        result = self.runner.invoke(ctl, ['pause', 'dummy'])
-        assert 'Failed' in result.output
-
-        mock_post.return_value.status = 200
-        with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
+        with patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)):
             result = self.runner.invoke(ctl, ['pause', 'dummy'])
             assert 'Cluster is already paused' in result.output
 
         result = self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
         assert "'pause' request sent" in result.output
-        mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
-                                                                  get_cluster(None, None, [], None, None)])
-        self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
-        member = Member(1, 'other', 28, {})
-        mock_get_dcs.return_value.get_cluster = Mock(side_effect=[get_cluster_initialized_with_leader(),
-                                                                  get_cluster(None, None, [member], None, None)])
-        self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
 
-    @patch.object(PoolManager, 'request')
-    @patch('patroni.ctl.get_dcs')
-    def test_resume_cluster(self, mock_get_dcs, mock_post):
-        mock_get_dcs.return_value = self.e
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+        with patch('patroni.dcs.AbstractDCS.get_cluster',
+                   Mock(side_effect=[get_cluster_initialized_with_leader(), get_cluster(None, None, [], None, None)])):
+            self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
+        with patch('patroni.dcs.AbstractDCS.get_cluster',
+                   Mock(side_effect=[get_cluster_initialized_with_leader(),
+                                     get_cluster(None, None, [Member(1, 'other', 28, {})], None, None)])):
+            self.runner.invoke(ctl, ['pause', 'dummy', '--wait'])
 
+    @patch('patroni.ctl.request_patroni')
+    @patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=get_cluster_initialized_with_leader()))
+    def test_resume_cluster(self, mock_post):
         mock_post.return_value.status = 200
-        with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=False)):
+        with patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=False)):
             result = self.runner.invoke(ctl, ['resume', 'dummy'])
             assert 'Cluster is not paused' in result.output
 
-        with patch('patroni.config.GlobalConfig.is_paused', PropertyMock(return_value=True)):
+        with patch.object(global_config.__class__, 'is_paused', PropertyMock(return_value=True)):
             result = self.runner.invoke(ctl, ['resume', 'dummy'])
             assert 'Success' in result.output
 
@@ -701,67 +684,53 @@ class TestCtl(unittest.TestCase):
             with patch('shutil.which', Mock(return_value=e)):
                 self.assertRaises(PatroniCtlException, invoke_editor, 'foo: bar\n', 'test')
 
-    @patch('patroni.ctl.get_dcs')
-    def test_show_config(self, mock_get_dcs):
-        mock_get_dcs.return_value = self.e
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+    def test_show_config(self):
         self.runner.invoke(ctl, ['show-config', 'dummy'])
 
-    @patch('patroni.ctl.get_dcs')
     @patch('subprocess.call', Mock(return_value=0))
-    def test_edit_config(self, mock_get_dcs):
-        mock_get_dcs.return_value = self.e
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
-        mock_get_dcs.return_value.set_config_value = Mock(return_value=False)
+    def test_edit_config(self):
         os.environ['EDITOR'] = 'true'
         self.runner.invoke(ctl, ['edit-config', 'dummy'])
         self.runner.invoke(ctl, ['edit-config', 'dummy', '-s', 'foo=bar'])
         self.runner.invoke(ctl, ['edit-config', 'dummy', '--replace', 'postgres0.yml'])
         self.runner.invoke(ctl, ['edit-config', 'dummy', '--apply', '-'], input='foo: bar')
         self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar')
-        mock_get_dcs.return_value.set_config_value.return_value = True
-        self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar')
-        mock_get_dcs.return_value.get_cluster = Mock(return_value=Cluster.empty())
-        result = self.runner.invoke(ctl, ['edit-config', 'dummy'])
-        assert result.exit_code == 1
-        assert 'The config key does not exist in the cluster dummy' in result.output
+        with patch('patroni.dcs.etcd.Etcd.set_config_value', Mock(return_value=True)):
+            self.runner.invoke(ctl, ['edit-config', 'dummy', '--force', '--apply', '-'], input='foo: bar')
+        with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=Cluster.empty())):
+            result = self.runner.invoke(ctl, ['edit-config', 'dummy'])
+            assert result.exit_code == 1
+            assert 'The config key does not exist in the cluster dummy' in result.output
 
-    @patch('patroni.ctl.get_dcs')
-    def test_version(self, mock_get_dcs):
-        mock_get_dcs.return_value = self.e
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
-        with patch.object(PoolManager, 'request') as mocked:
-            result = self.runner.invoke(ctl, ['version'])
-            assert 'patronictl version' in result.output
-            mocked.return_value.data = b'{"patroni":{"version":"1.2.3"},"server_version": 100001}'
-            result = self.runner.invoke(ctl, ['version', 'dummy'])
-            assert '1.2.3' in result.output
-        with patch.object(PoolManager, 'request', Mock(side_effect=Exception)):
-            result = self.runner.invoke(ctl, ['version', 'dummy'])
-            assert 'failed to get version' in result.output
+    @patch('patroni.ctl.request_patroni')
+    def test_version(self, mock_request):
+        result = self.runner.invoke(ctl, ['version'])
+        assert 'patronictl version' in result.output
+        mock_request.return_value.data = b'{"patroni":{"version":"1.2.3"},"server_version": 100001}'
+        result = self.runner.invoke(ctl, ['version', 'dummy'])
+        assert '1.2.3' in result.output
+        mock_request.side_effect = Exception
+        result = self.runner.invoke(ctl, ['version', 'dummy'])
+        assert 'failed to get version' in result.output
 
-    @patch('patroni.ctl.get_dcs')
-    def test_history(self, mock_get_dcs):
-        mock_get_dcs.return_value.get_cluster = Mock()
-        mock_get_dcs.return_value.get_cluster.return_value.history.lines = [[1, 67176, 'no recovery target specified']]
-        result = self.runner.invoke(ctl, ['history'])
-        assert 'Reason' in result.output
+    def test_history(self):
+        with patch('patroni.dcs.AbstractDCS.get_cluster') as mock_get_cluster:
+            mock_get_cluster.return_value.history.lines = [[1, 67176, 'no recovery target specified']]
+            result = self.runner.invoke(ctl, ['history'])
+            assert 'Reason' in result.output
 
     def test_format_pg_version(self):
         self.assertEqual(format_pg_version(100001), '10.1')
         self.assertEqual(format_pg_version(90605), '9.6.5')
 
-    @patch('patroni.ctl.get_dcs')
-    def test_get_members(self, mock_get_dcs):
-        mock_get_dcs.return_value = self.e
-        mock_get_dcs.return_value.get_cluster = get_cluster_not_initialized_without_leader
-        result = self.runner.invoke(ctl, ['reinit', 'dummy'])
-        assert "cluster doesn\'t have any members" in result.output
+    def test_get_members(self):
+        with patch('patroni.dcs.AbstractDCS.get_cluster',
+                   Mock(return_value=get_cluster_not_initialized_without_leader())):
+            result = self.runner.invoke(ctl, ['reinit', 'dummy'])
+            assert "cluster doesn\'t have any members" in result.output
 
     @patch('time.sleep', Mock())
-    @patch('patroni.ctl.get_dcs')
-    def test_reinit_wait(self, mock_get_dcs):
-        mock_get_dcs.return_value.get_cluster = get_cluster_initialized_with_leader
+    def test_reinit_wait(self):
         with patch.object(PoolManager, 'request') as mocked:
             mocked.side_effect = [Mock(data=s, status=200) for s in
                                   [b"reinitialize", b'{"state":"creating replica"}', b'{"state":"running"}']]

tests/test_etcd.py

@@ -5,8 +5,10 @@ import unittest
 
 from dns.exception import DNSException
 from mock import Mock, PropertyMock, patch
+from patroni.dcs import get_dcs
 from patroni.dcs.etcd import AbstractDCS, EtcdClient, Cluster, Etcd, EtcdError, DnsCachingResolver
 from patroni.exceptions import DCSError
+from patroni.postgresql.mpp import get_mpp
 from patroni.utils import Retry
 from urllib3.exceptions import ReadTimeoutError
 
@@ -138,8 +140,9 @@ class TestClient(unittest.TestCase):
     @patch.object(EtcdClient, '_get_machines_list',
                   Mock(return_value=['http://localhost:2379', 'http://localhost:4001']))
     def setUp(self):
-        self.etcd = Etcd({'namespace': '/patroni/', 'ttl': 30, 'retry_timeout': 3,
-                          'srv': 'test', 'scope': 'test', 'name': 'foo'})
+        self.etcd = get_dcs({'namespace': '/patroni/', 'ttl': 30, 'retry_timeout': 3,
+                             'etcd': {'srv': 'test'}, 'scope': 'test', 'name': 'foo'})
+        self.assertIsInstance(self.etcd, Etcd)
         self.client = self.etcd._client
         self.client.http.request = http_request
         self.client.http.request_encode_body = http_request
@@ -235,7 +238,7 @@ class TestEtcd(unittest.TestCase):
                   Mock(return_value=['http://localhost:2379', 'http://localhost:4001']))
     def setUp(self):
         self.etcd = Etcd({'namespace': '/patroni/', 'ttl': 30, 'retry_timeout': 10,
-                          'host': 'localhost:2379', 'scope': 'test', 'name': 'foo'})
+                          'host': 'localhost:2379', 'scope': 'test', 'name': 'foo'}, get_mpp({}))
 
     def test_base_path(self):
         self.assertEqual(self.etcd._base_path, '/patroni/test')
@@ -270,10 +273,12 @@ class TestEtcd(unittest.TestCase):
         self.assertRaises(EtcdError, self.etcd.get_cluster)
 
     def test__get_citus_cluster(self):
-        self.etcd._citus_group = '0'
+        self.etcd._mpp = get_mpp({'citus': {'group': 0, 'database': 'postgres'}})
         cluster = self.etcd.get_cluster()
         self.assertIsInstance(cluster, Cluster)
         self.assertIsInstance(cluster.workers[1], Cluster)
+        self.etcd._base_path = '/service/nocluster'
+        self.assertTrue(self.etcd.get_cluster().is_empty())
 
     def test_touch_member(self):
         self.assertFalse(self.etcd.touch_member(''))

tests/test_etcd3.py

@@ -4,10 +4,12 @@ import unittest
 import urllib3
 
 from mock import Mock, PropertyMock, patch
+from patroni.dcs import get_dcs
 from patroni.dcs.etcd import DnsCachingResolver
 from patroni.dcs.etcd3 import PatroniEtcd3Client, Cluster, Etcd3, Etcd3Client, \
-    Etcd3Error, Etcd3ClientError, ReAuthenticateMode, RetryFailedError, InvalidAuthToken, Unavailable, \
+    Etcd3Error, Etcd3ClientError, RetryFailedError, InvalidAuthToken, Unavailable, \
     Unknown, UnsupportedEtcdVersion, UserEmpty, AuthFailed, AuthOldRevision, base64_encode
+from patroni.postgresql.mpp import get_mpp
 from threading import Thread
 
 from . import SleepException, MockResponse
@@ -80,9 +82,9 @@ class BaseTestEtcd3(unittest.TestCase):
     @patch.object(Thread, 'start', Mock())
     @patch.object(urllib3.PoolManager, 'urlopen', mock_urlopen)
     def setUp(self):
-        self.etcd3 = Etcd3({'namespace': '/patroni/', 'ttl': 30, 'retry_timeout': 10,
-                            'host': 'localhost:2378', 'scope': 'test', 'name': 'foo',
-                            'username': 'etcduser', 'password': 'etcdpassword'})
+        self.etcd3 = get_dcs({'namespace': '/patroni/', 'ttl': 30, 'retry_timeout': 10, 'name': 'foo', 'scope': 'test',
+                              'etcd3': {'host': 'localhost:2378', 'username': 'etcduser', 'password': 'etcdpassword'}})
+        self.assertIsInstance(self.etcd3, Etcd3)
         self.client = self.etcd3._client
         self.kv_cache = self.client._kv_cache
 
@@ -164,12 +166,14 @@ class TestPatroniEtcd3Client(BaseTestEtcd3):
             retry = self.etcd3._retry.copy()
             with patch('time.time', Mock(side_effect=[0, 10, 20, 30, 40])):
                 self.assertRaises(InvalidAuthToken, retry, self.client.deleteprefix, 'foo', retry=retry)
+            with patch('time.time', Mock(side_effect=[0, 10])):
+                self.assertRaises(InvalidAuthToken, self.client.deleteprefix, 'foo')
             self.client.username = None
-            self.client._reauthenticate_reason = ReAuthenticateMode.NOT_REQUIRED
+            self.client._reauthenticate = False
             retry = self.etcd3._retry.copy()
             self.assertRaises(InvalidAuthToken, retry, self.client.deleteprefix, 'foo', retry=retry)
             mock_urlopen.return_value.content = '{"code":3,"error":"etcdserver: revision of auth store is old"}'
-            self.client._reauthenticate_reason = ReAuthenticateMode.NOT_REQUIRED
+            self.client._reauthenticate = False
             self.assertRaises(AuthOldRevision, retry, self.client.deleteprefix, 'foo', retry=retry)
 
     def test__handle_server_response(self):
@@ -236,7 +240,7 @@ class TestEtcd3(BaseTestEtcd3):
             self.assertRaises(Etcd3Error, self.etcd3.get_cluster)
 
     def test__get_citus_cluster(self):
-        self.etcd3._citus_group = '0'
+        self.etcd3._mpp = get_mpp({'citus': {'group': 0, 'database': 'postgres'}})
         cluster = self.etcd3.get_cluster()
         self.assertIsInstance(cluster, Cluster)
         self.assertIsInstance(cluster.workers[1], Cluster)
@@ -269,8 +273,8 @@ class TestEtcd3(BaseTestEtcd3):
 
     def test_attempt_to_acquire_leader(self):
         self.assertFalse(self.etcd3.attempt_to_acquire_leader())
-        with patch('time.time', Mock(side_effect=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 200])):
-            self.assertRaises(Etcd3Error, self.etcd3.attempt_to_acquire_leader)
+        with patch('time.time', Mock(side_effect=[0, 0, 0, 0, 0, 100, 200])):
+            self.assertFalse(self.etcd3.attempt_to_acquire_leader())
         with patch('time.time', Mock(side_effect=[0, 100, 200, 300, 400])):
             self.assertRaises(Etcd3Error, self.etcd3.attempt_to_acquire_leader)
         with patch.object(PatroniEtcd3Client, 'put', Mock(return_value=False)):

tests/test_exhibitor.py

@@ -2,6 +2,7 @@ import unittest
 import urllib3
 
 from mock import Mock, patch
+from patroni.dcs import get_dcs
 from patroni.dcs.exhibitor import ExhibitorEnsembleProvider, Exhibitor
 from patroni.dcs.zookeeper import ZooKeeperError
 
@@ -26,8 +27,9 @@ class TestExhibitor(unittest.TestCase):
         status=200, body=b'{"servers":["127.0.0.1","127.0.0.2","127.0.0.3"],"port":2181}')))
     @patch('patroni.dcs.zookeeper.PatroniKazooClient', MockKazooClient)
     def setUp(self):
-        self.e = Exhibitor({'hosts': ['localhost', 'exhibitor'], 'port': 8181, 'scope': 'test',
-                            'name': 'foo', 'ttl': 30, 'retry_timeout': 10})
+        self.e = get_dcs({'exhibitor': {'hosts': ['localhost', 'exhibitor'], 'port': 8181},
+                          'scope': 'test', 'name': 'foo', 'ttl': 30, 'retry_timeout': 10})
+        self.assertIsInstance(self.e, Exhibitor)
 
     @patch.object(ExhibitorEnsembleProvider, 'poll', Mock(return_value=True))
     @patch.object(MockKazooClient, 'get_children', Mock(side_effect=Exception))

tests/test_ha.py

@@ -4,6 +4,7 @@ import os
 import sys
 
 from mock import Mock, MagicMock, PropertyMock, patch, mock_open
+from patroni import global_config
 from patroni.collections import CaseInsensitiveSet
 from patroni.config import Config
 from patroni.dcs import Cluster, ClusterConfig, Failover, Leader, Member, get_dcs, Status, SyncState, TimelineHistory
@@ -150,6 +151,7 @@ zookeeper:
         self.api.connection_string = 'http://127.0.0.1:8008'
         self.clonefrom = None
         self.nosync = False
+        self.nostream = False
         self.scheduled_restart = {'schedule': future_restart_time,
                                   'postmaster_start_time': str(postmaster_start_time)}
         self.watchdog = Watchdog(self.config)
@@ -196,7 +198,7 @@ def run_async(self, func, args=()):
 @patch('patroni.async_executor.AsyncExecutor.busy', PropertyMock(return_value=False))
 @patch('patroni.async_executor.AsyncExecutor.run_async', run_async)
 @patch('patroni.postgresql.rewind.Thread', Mock())
-@patch('patroni.postgresql.citus.CitusHandler.start', Mock())
+@patch('patroni.postgresql.mpp.citus.CitusHandler.start', Mock())
 @patch('subprocess.call', Mock(return_value=0))
 @patch('time.sleep', Mock())
 class TestHa(PostgresInit):
@@ -217,6 +219,7 @@ class TestHa(PostgresInit):
         self.ha = Ha(MockPatroni(self.p, self.e))
         self.ha.old_cluster = self.e.get_cluster()
         self.ha.cluster = get_cluster_initialized_without_leader()
+        global_config.update(self.ha.cluster)
         self.ha.load_cluster_from_dcs = Mock()
 
     def test_update_lock(self):
@@ -251,8 +254,8 @@ class TestHa(PostgresInit):
     @patch('patroni.dcs.etcd.Etcd.initialize', return_value=True)
     def test_bootstrap_as_standby_leader(self, initialize):
         self.p.data_directory_empty = true
-        self.ha.cluster = get_cluster_not_initialized_without_leader(cluster_config=ClusterConfig(0, {}, 0))
-        self.ha.patroni.config._dynamic_configuration = {"standby_cluster": {"port": 5432}}
+        self.ha.cluster = get_cluster_not_initialized_without_leader(
+            cluster_config=ClusterConfig(1, {"standby_cluster": {"port": 5432}}, 1))
         self.assertEqual(self.ha.run_cycle(), 'trying to bootstrap a new standby leader')
 
     def test_bootstrap_waiting_for_standby_leader(self):
@@ -318,7 +321,6 @@ class TestHa(PostgresInit):
             self.ha.state_handler.cancellable._process = Mock()
             self.ha._crash_recovery_started -= 600
             self.ha.cluster.config.data.update({'maximum_lag_on_failover': 10})
-            self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
             self.assertEqual(self.ha.run_cycle(), 'terminated crash recovery because of startup timeout')
 
     @patch.object(Rewind, 'ensure_clean_shutdown', Mock())
@@ -470,6 +472,11 @@ class TestHa(PostgresInit):
         self.p.is_primary = false
         self.assertEqual(self.ha.run_cycle(), 'not promoting because failed to update leader lock in DCS')
 
+    def test_get_node_to_follow_nostream(self):
+        self.ha.patroni.nostream = True
+        self.ha.cluster = get_cluster_initialized_with_leader()
+        self.assertEqual(self.ha._get_node_to_follow(self.ha.cluster), None)
+
     @patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
     def test_follow(self):
         self.p.is_primary = false
@@ -509,7 +516,7 @@ class TestHa(PostgresInit):
     def test_check_failsafe_topology(self):
         self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
         self.ha.cluster = get_cluster_initialized_with_leader_and_failsafe()
-        self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
+        global_config.update(self.ha.cluster)
         self.ha.dcs._last_failsafe = self.ha.cluster.failsafe
         self.assertEqual(self.ha.run_cycle(), 'demoting self because DCS is not accessible and I was a leader')
         self.ha.state_handler.name = self.ha.cluster.leader.name
@@ -529,7 +536,7 @@ class TestHa(PostgresInit):
     def test_no_dcs_connection_primary_failsafe(self):
         self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
         self.ha.cluster = get_cluster_initialized_with_leader_and_failsafe()
-        self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
+        global_config.update(self.ha.cluster)
         self.ha.dcs._last_failsafe = self.ha.cluster.failsafe
         self.ha.state_handler.name = self.ha.cluster.leader.name
         self.assertEqual(self.ha.run_cycle(),
@@ -546,7 +553,7 @@ class TestHa(PostgresInit):
     def test_no_dcs_connection_replica_failsafe(self):
         self.ha.load_cluster_from_dcs = Mock(side_effect=DCSError('Etcd is not responding properly'))
         self.ha.cluster = get_cluster_initialized_with_leader_and_failsafe()
-        self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
+        global_config.update(self.ha.cluster)
         self.ha.update_failsafe({'name': 'leader', 'api_url': 'http://127.0.0.1:8008/patroni',
                                  'conn_url': 'postgres://127.0.0.1:5432/postgres', 'slots': {'foo': 1000}})
         self.p.is_primary = false
@@ -589,8 +596,8 @@ class TestHa(PostgresInit):
         self.assertEqual(self.ha.bootstrap(), 'failed to acquire initialize lock')
 
     @patch('patroni.psycopg.connect', psycopg_connect)
-    @patch('patroni.postgresql.citus.connect', psycopg_connect)
-    @patch('patroni.postgresql.citus.quote_ident', Mock())
+    @patch('patroni.postgresql.mpp.citus.connect', psycopg_connect)
+    @patch('patroni.postgresql.mpp.citus.quote_ident', Mock())
     @patch.object(Postgresql, 'connection', Mock(return_value=None))
     def test_bootstrap_initialized_new_cluster(self):
         self.ha.cluster = get_cluster_not_initialized_without_leader()
@@ -611,8 +618,8 @@ class TestHa(PostgresInit):
         self.assertRaises(PatroniFatalException, self.ha.post_bootstrap)
 
     @patch('patroni.psycopg.connect', psycopg_connect)
-    @patch('patroni.postgresql.citus.connect', psycopg_connect)
-    @patch('patroni.postgresql.citus.quote_ident', Mock())
+    @patch('patroni.postgresql.mpp.citus.connect', psycopg_connect)
+    @patch('patroni.postgresql.mpp.citus.quote_ident', Mock())
     @patch.object(Postgresql, 'connection', Mock(return_value=None))
     def test_bootstrap_release_initialize_key_on_watchdog_failure(self):
         self.ha.cluster = get_cluster_not_initialized_without_leader()
@@ -655,7 +662,7 @@ class TestHa(PostgresInit):
     @patch.object(ConfigHandler, 'replace_pg_hba', Mock())
     @patch.object(ConfigHandler, 'replace_pg_ident', Mock())
     @patch.object(PostmasterProcess, 'start', Mock(return_value=MockPostmaster()))
-    @patch('patroni.postgresql.citus.CitusHandler.is_coordinator', Mock(return_value=False))
+    @patch('patroni.postgresql.mpp.AbstractMPPHandler.is_coordinator', Mock(return_value=False))
     def test_worker_restart(self):
         self.ha.has_lock = true
         self.ha.patroni.request = Mock()
@@ -690,7 +697,7 @@ class TestHa(PostgresInit):
             self.ha.is_paused = true
             self.assertEqual(self.ha.run_cycle(), 'PAUSE: restart in progress')
 
-    @patch('patroni.postgresql.citus.CitusHandler.is_coordinator', Mock(return_value=False))
+    @patch('patroni.postgresql.mpp.AbstractMPPHandler.is_coordinator', Mock(return_value=False))
     def test_manual_failover_from_leader(self):
         self.ha.has_lock = true  # I am the leader
 
@@ -729,7 +736,7 @@ class TestHa(PostgresInit):
                              ('Member %s exceeds maximum replication lag', 'b'))
             self.ha.cluster.members.pop()
 
-    @patch('patroni.postgresql.citus.CitusHandler.is_coordinator', Mock(return_value=False))
+    @patch('patroni.postgresql.mpp.AbstractMPPHandler.is_coordinator', Mock(return_value=False))
     def test_manual_switchover_from_leader(self):
         self.ha.has_lock = true  # I am the leader
 
@@ -766,11 +773,10 @@ class TestHa(PostgresInit):
         with patch('patroni.ha.logger.info') as mock_info:
             self.ha.fetch_node_status = get_node_status(wal_position=1)
             self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5})
-            self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
             self.assertEqual(self.ha.run_cycle(), 'no action. I am (postgresql0), the leader with the lock')
             self.assertEqual(mock_info.call_args_list[0][0], ('Member %s exceeds maximum replication lag', 'leader'))
 
-    @patch('patroni.postgresql.citus.CitusHandler.is_coordinator', Mock(return_value=False))
+    @patch('patroni.postgresql.mpp.AbstractMPPHandler.is_coordinator', Mock(return_value=False))
     def test_scheduled_switchover_from_leader(self):
         self.ha.has_lock = true  # I am the leader
 
@@ -1032,7 +1038,7 @@ class TestHa(PostgresInit):
     def test__is_healthiest_node(self):
         self.p.is_primary = false
         self.ha.cluster = get_cluster_initialized_without_leader(sync=('postgresql1', self.p.name))
-        self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
+        global_config.update(self.ha.cluster)
         self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
         self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
@@ -1049,7 +1055,7 @@ class TestHa(PostgresInit):
         with patch.object(Ha, 'is_synchronous_mode', Mock(return_value=True)):
             self.assertTrue(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         self.ha.cluster.config.data.update({'maximum_lag_on_failover': 5})
-        self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
+        global_config.update(self.ha.cluster)
         with patch('patroni.postgresql.Postgresql.last_operation', return_value=1):
             self.assertFalse(self.ha._is_healthiest_node(self.ha.old_cluster.members))
         with patch('patroni.postgresql.Postgresql.replica_cached_timeline', return_value=None):
@@ -1272,7 +1278,6 @@ class TestHa(PostgresInit):
         self.p.is_running = false
         self.ha.cluster = get_cluster_initialized_with_leader(sync=(self.p.name, 'other'))
         self.ha.cluster.config.data.update({'synchronous_mode': True, 'primary_start_timeout': 0})
-        self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
         self.ha.has_lock = true
         self.ha.update_lock = true
         self.ha.fetch_node_status = get_node_status()  # accessible, in_recovery
@@ -1282,13 +1287,13 @@ class TestHa(PostgresInit):
     def test_primary_stop_timeout(self):
         self.assertEqual(self.ha.primary_stop_timeout(), None)
         self.ha.cluster.config.data.update({'primary_stop_timeout': 30})
-        self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
+        global_config.update(self.ha.cluster)
         with patch.object(Ha, 'is_synchronous_mode', Mock(return_value=True)):
             self.assertEqual(self.ha.primary_stop_timeout(), 30)
         with patch.object(Ha, 'is_synchronous_mode', Mock(return_value=False)):
             self.assertEqual(self.ha.primary_stop_timeout(), None)
             self.ha.cluster.config.data['primary_stop_timeout'] = None
-            self.ha.global_config = self.ha.patroni.config.get_global_config(self.ha.cluster)
+            global_config.update(self.ha.cluster)
             self.assertEqual(self.ha.primary_stop_timeout(), None)
 
     @patch('patroni.postgresql.Postgresql.follow')
@@ -1380,8 +1385,8 @@ class TestHa(PostgresInit):
         # Test sync set to '*' when synchronous_mode_strict is enabled
         mock_set_sync.reset_mock()
         self.p.sync_handler.current_state = Mock(return_value=(CaseInsensitiveSet(), CaseInsensitiveSet()))
-        with patch('patroni.config.GlobalConfig.is_synchronous_mode_strict', PropertyMock(return_value=True)):
-            self.ha.run_cycle()
+        self.ha.cluster.config.data['synchronous_mode_strict'] = True
+        self.ha.run_cycle()
         mock_set_sync.assert_called_once_with(CaseInsensitiveSet('*'))
 
     def test_sync_replication_become_primary(self):
@@ -1514,7 +1519,6 @@ class TestHa(PostgresInit):
 
     @patch('patroni.postgresql.mtime', Mock(return_value=1588316884))
     @patch('builtins.open', Mock(side_effect=Exception))
-    @patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
     def test_restore_cluster_config(self):
         self.ha.cluster.config.data.clear()
         self.ha.has_lock = true
@@ -1532,7 +1536,7 @@ class TestHa(PostgresInit):
         self.ha.is_leader = true
 
         def stop(*args, **kwargs):
-            kwargs['on_shutdown'](123)
+            kwargs['on_shutdown'](123, 120)
 
         self.p.stop = stop
         self.ha.shutdown()
@@ -1540,7 +1544,7 @@ class TestHa(PostgresInit):
         self.ha.is_failover_possible = true
         self.ha.shutdown()
 
-    @patch('patroni.postgresql.citus.CitusHandler.is_coordinator', Mock(return_value=False))
+    @patch('patroni.postgresql.mpp.AbstractMPPHandler.is_coordinator', Mock(return_value=False))
     def test_shutdown_citus_worker(self):
         self.ha.is_leader = true
         self.p.is_running = Mock(side_effect=[Mock(), False])
@@ -1581,6 +1585,11 @@ class TestHa(PostgresInit):
         self.p.is_primary = false
         self.ha.run_cycle()
         exit_mock.assert_called_once_with(1)
+        self.p.set_role('replica')
+        self.ha.dcs.initialize = Mock()
+        with patch.object(Postgresql, 'cb_called', PropertyMock(return_value=True)):
+            self.assertEqual(self.ha.run_cycle(), 'promoted self to leader by acquiring session lock')
+        self.ha.dcs.initialize.assert_not_called()
 
     @patch.object(Cluster, 'is_unlocked', Mock(return_value=False))
     def test_after_pause(self):
@@ -1647,15 +1656,16 @@ class TestHa(PostgresInit):
         self.assertRaises(DCSError, self.ha.acquire_lock)
         self.assertFalse(self.ha.acquire_lock())
 
-    @patch('patroni.postgresql.citus.CitusHandler.is_coordinator', Mock(return_value=False))
+    @patch('patroni.postgresql.mpp.AbstractMPPHandler.is_coordinator', Mock(return_value=False))
     def test_notify_citus_coordinator(self):
         self.ha.patroni.request = Mock()
-        self.ha.notify_citus_coordinator('before_demote')
+        self.ha.notify_mpp_coordinator('before_demote')
         self.ha.patroni.request.assert_called_once()
         self.assertEqual(self.ha.patroni.request.call_args[1]['timeout'], 30)
         self.ha.patroni.request = Mock(side_effect=Exception)
         with patch('patroni.ha.logger.warning') as mock_logger:
-            self.ha.notify_citus_coordinator('before_promote')
+            self.ha.notify_mpp_coordinator('before_promote')
             self.assertEqual(self.ha.patroni.request.call_args[1]['timeout'], 2)
             mock_logger.assert_called()
-            self.assertTrue(mock_logger.call_args[0][0].startswith('Request to Citus coordinator'))
+            self.assertTrue(mock_logger.call_args[0][0].startswith('Request to %s coordinator leader'))
+            self.assertEqual(mock_logger.call_args[0][1], 'Citus')

tests/test_kubernetes.py

@@ -8,14 +8,17 @@ import unittest
 import urllib3
 
 from mock import Mock, PropertyMock, mock_open, patch
+from patroni.dcs import get_dcs
 from patroni.dcs.kubernetes import Cluster, k8s_client, k8s_config, K8sConfig, K8sConnectionFailed, \
     K8sException, K8sObject, Kubernetes, KubernetesError, KubernetesRetriableException, \
     Retry, RetryFailedError, SERVICE_HOST_ENV_NAME, SERVICE_PORT_ENV_NAME
+from patroni.postgresql.mpp import get_mpp
 from threading import Thread
 from . import MockResponse, SleepException
 
 
 def mock_list_namespaced_config_map(*args, **kwargs):
+    k8s_group_label = get_mpp({'citus': {'group': 0, 'database': 'postgres'}}).k8s_group_label
     metadata = {'resource_version': '1', 'labels': {'f': 'b'}, 'name': 'test-config',
                 'annotations': {'initialize': '123', 'config': '{}'}}
     items = [k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata))]
@@ -26,16 +29,16 @@ def mock_list_namespaced_config_map(*args, **kwargs):
     items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
     metadata.update({'name': 'test-sync', 'annotations': {'leader': 'p-0'}})
     items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
-    metadata.update({'name': 'test-0-leader', 'labels': {Kubernetes._CITUS_LABEL: '0'},
+    metadata.update({'name': 'test-0-leader', 'labels': {k8s_group_label: '0'},
                      'annotations': {'optime': '1234x', 'leader': 'p-0', 'ttl': '30s', 'slots': '{', 'failsafe': '{'}})
     items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
-    metadata.update({'name': 'test-0-config', 'labels': {Kubernetes._CITUS_LABEL: '0'},
+    metadata.update({'name': 'test-0-config', 'labels': {k8s_group_label: '0'},
                      'annotations': {'initialize': '123', 'config': '{}'}})
     items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
-    metadata.update({'name': 'test-1-leader', 'labels': {Kubernetes._CITUS_LABEL: '1'},
+    metadata.update({'name': 'test-1-leader', 'labels': {k8s_group_label: '1'},
                      'annotations': {'leader': 'p-3', 'ttl': '30s'}})
     items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
-    metadata.update({'name': 'test-2-config', 'labels': {Kubernetes._CITUS_LABEL: '2'}, 'annotations': {}})
+    metadata.update({'name': 'test-2-config', 'labels': {k8s_group_label: '2'}, 'annotations': {}})
     items.append(k8s_client.V1ConfigMap(metadata=k8s_client.V1ObjectMeta(**metadata)))
 
     metadata = k8s_client.V1ObjectMeta(resource_version='1')
@@ -60,7 +63,8 @@ def mock_list_namespaced_endpoints(*args, **kwargs):
 
 
 def mock_list_namespaced_pod(*args, **kwargs):
-    metadata = k8s_client.V1ObjectMeta(resource_version='1', labels={'f': 'b', Kubernetes._CITUS_LABEL: '1'},
+    k8s_group_label = get_mpp({'citus': {'group': 0, 'database': 'postgres'}}).k8s_group_label
+    metadata = k8s_client.V1ObjectMeta(resource_version='1', labels={'f': 'b', k8s_group_label: '1'},
                                        name='p-0', annotations={'status': '{}'},
                                        uid='964dfeae-e79b-4476-8a5a-1920b5c2a69d')
     status = k8s_client.V1PodStatus(pod_ip='10.0.0.1')
@@ -76,7 +80,7 @@ def mock_namespaced_kind(*args, **kwargs):
 
 
 def mock_load_k8s_config(self, *args, **kwargs):
-    self._server = ''
+    self._server = 'http://localhost'
 
 
 class TestK8sConfig(unittest.TestCase):
@@ -225,11 +229,12 @@ class BaseTestKubernetes(unittest.TestCase):
     @patch.object(k8s_client.CoreV1Api, 'list_namespaced_pod', mock_list_namespaced_pod, create=True)
     @patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map, create=True)
     def setUp(self, config=None):
-        config = config or {}
-        config.update(ttl=30, scope='test', name='p-0', loop_wait=10, group=0,
-                      retry_timeout=10, labels={'f': 'b'}, bypass_api_service=True)
-        self.k = Kubernetes(config)
-        self.k._citus_group = None
+        config = {'ttl': 30, 'scope': 'test', 'name': 'p-0', 'loop_wait': 10, 'retry_timeout': 10,
+                  'kubernetes': {'labels': {'f': 'b'}, 'bypass_api_service': True, **(config or {})},
+                  'citus': {'group': 0, 'database': 'postgres'}}
+        self.k = get_dcs(config)
+        self.assertIsInstance(self.k, Kubernetes)
+        self.k._mpp = get_mpp({})
         self.assertRaises(AttributeError, self.k._pods._build_cache)
         self.k._pods._is_ready = True
         self.assertRaises(TypeError, self.k._kinds._build_cache)
@@ -237,6 +242,7 @@ class BaseTestKubernetes(unittest.TestCase):
         self.k.get_cluster()
 
 
+@patch('urllib3.PoolManager.request', Mock())
 @patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', mock_namespaced_kind, create=True)
 class TestKubernetesConfigMaps(BaseTestKubernetes):
 
@@ -254,18 +260,31 @@ class TestKubernetesConfigMaps(BaseTestKubernetes):
             self.assertRaises(KubernetesError, self.k.get_cluster)
 
     def test__get_citus_cluster(self):
-        self.k._citus_group = '0'
+        self.k._mpp = get_mpp({'citus': {'group': 0, 'database': 'postgres'}})
         cluster = self.k.get_cluster()
         self.assertIsInstance(cluster, Cluster)
         self.assertIsInstance(cluster.workers[1], Cluster)
 
     @patch('patroni.dcs.kubernetes.logger.error')
-    def test_get_citus_coordinator(self, mock_logger):
-        self.assertIsInstance(self.k.get_citus_coordinator(), Cluster)
-        with patch.object(Kubernetes, '_cluster_loader', Mock(side_effect=Exception)):
-            self.assertIsNone(self.k.get_citus_coordinator())
+    def test_get_mpp_coordinator(self, mock_logger):
+        self.assertIsInstance(self.k.get_mpp_coordinator(), Cluster)
+        with patch.object(Kubernetes, '_postgresql_cluster_loader', Mock(side_effect=Exception)):
+            self.assertIsNone(self.k.get_mpp_coordinator())
             mock_logger.assert_called()
-            self.assertTrue(mock_logger.call_args[0][0].startswith('Failed to load Citus coordinator'))
+            self.assertEqual(mock_logger.call_args[0][0], 'Failed to load %s coordinator cluster from Kubernetes: %r')
+            self.assertEqual(mock_logger.call_args[0][1], 'Null')
+            self.assertIsInstance(mock_logger.call_args[0][2], KubernetesError)
+
+    @patch('patroni.dcs.kubernetes.logger.error')
+    def test_get_citus_coordinator(self, mock_logger):
+        self.k._mpp = get_mpp({'citus': {'group': 0, 'database': 'postgres'}})
+        self.assertIsInstance(self.k.get_mpp_coordinator(), Cluster)
+        with patch.object(Kubernetes, '_postgresql_cluster_loader', Mock(side_effect=Exception)):
+            self.assertIsNone(self.k.get_mpp_coordinator())
+            mock_logger.assert_called()
+            self.assertEqual(mock_logger.call_args[0][0], 'Failed to load %s coordinator cluster from Kubernetes: %r')
+            self.assertEqual(mock_logger.call_args[0][1], 'Citus')
+            self.assertIsInstance(mock_logger.call_args[0][2], KubernetesError)
 
     def test_attempt_to_acquire_leader(self):
         with patch.object(k8s_client.CoreV1Api, 'patch_namespaced_config_map', create=True) as mock_patch:
@@ -356,6 +375,7 @@ class TestKubernetesConfigMaps(BaseTestKubernetes):
         mock_warning.assert_called_once()
 
 
+@patch('urllib3.PoolManager.request', Mock())
 class TestKubernetesEndpointsNoPodIP(BaseTestKubernetes):
     @patch.object(k8s_client.CoreV1Api, 'list_namespaced_endpoints', mock_list_namespaced_endpoints, create=True)
     def setUp(self, config=None):
@@ -370,6 +390,7 @@ class TestKubernetesEndpointsNoPodIP(BaseTestKubernetes):
         self.assertEqual(args[2].subsets[0].addresses[0].ip, '10.0.0.1')
 
 
+@patch('urllib3.PoolManager.request', Mock())
 class TestKubernetesEndpoints(BaseTestKubernetes):
 
     @patch.object(k8s_client.CoreV1Api, 'list_namespaced_endpoints', mock_list_namespaced_endpoints, create=True)
@@ -460,13 +481,14 @@ def mock_watch(*args):
     return urllib3.HTTPResponse()
 
 
+@patch('urllib3.PoolManager.request', Mock())
 class TestCacheBuilder(BaseTestKubernetes):
 
     @patch.object(k8s_client.CoreV1Api, 'list_namespaced_config_map', mock_list_namespaced_config_map, create=True)
     @patch('patroni.dcs.kubernetes.ObjectCache._watch', mock_watch)
     @patch.object(urllib3.HTTPResponse, 'read_chunked')
     def test__build_cache(self, mock_read_chunked):
-        self.k._citus_group = '0'
+        self.k._mpp = get_mpp({'citus': {'group': 0, 'database': 'postgres'}})
         mock_read_chunked.return_value = [json.dumps(
             {'type': 'MODIFIED', 'object': {'metadata': {
                 'name': self.k.config_path, 'resourceVersion': '2', 'annotations': {self.k._CONFIG: 'foo'}}}}

tests/test_log.py

@@ -3,12 +3,23 @@ import os
 import sys
 import unittest
 import yaml
+from io import StringIO
 
 from mock import Mock, patch
 from patroni.config import Config
 from patroni.log import PatroniLogger
 from queue import Queue, Full
 
+try:
+    from pythonjsonlogger import jsonlogger
+
+    jsonlogger.JsonFormatter(None, None, rename_fields={}, static_fields={})
+    json_formatter_is_available = True
+
+    import json  # we need json.loads() function
+except Exception:
+    json_formatter_is_available = False
+
 _LOG = logging.getLogger(__name__)
 
 
@@ -72,3 +83,201 @@ class TestPatroniLogger(unittest.TestCase):
         _LOG.info('blabla')
         logger.shutdown()
         self.assertEqual(logger.records_lost, 0)
+
+    def test_json_list_format(self):
+        config = {
+            'type': 'json',
+            'format': [
+                {'asctime': '@timestamp'},
+                {'levelname': 'level'},
+                'message'
+            ],
+            'static_fields': {
+                'app': 'patroni'
+            }
+        }
+
+        test_message = 'test json logging in case of list format'
+
+        with patch('sys.stderr', StringIO()) as stderr_output:
+            logger = PatroniLogger()
+            logger.reload_config(config)
+
+            _LOG.info(test_message)
+            if json_formatter_is_available:
+                target_log = json.loads(stderr_output.getvalue().split('\n')[-2])
+
+                self.assertIn('@timestamp', target_log)
+                self.assertEqual(target_log['message'], test_message)
+                self.assertEqual(target_log['level'], 'INFO')
+                self.assertEqual(target_log['app'], 'patroni')
+                self.assertEqual(len(target_log), len(config['format']) + len(config['static_fields']))
+
+    def test_json_str_format(self):
+        config = {
+            'type': 'json',
+            'format': '%(asctime)s %(levelname)s %(message)s',
+            'static_fields': {
+                'app': 'patroni'
+            }
+        }
+
+        test_message = 'test json logging in case of string format'
+
+        with patch('sys.stderr', StringIO()) as stderr_output:
+            logger = PatroniLogger()
+            logger.reload_config(config)
+
+            _LOG.info(test_message)
+            if json_formatter_is_available:
+                target_log = json.loads(stderr_output.getvalue().split('\n')[-2])
+
+                self.assertIn('asctime', target_log)
+                self.assertEqual(target_log['message'], test_message)
+                self.assertEqual(target_log['levelname'], 'INFO')
+                self.assertEqual(target_log['app'], 'patroni')
+
+    def test_plain_format(self):
+        config = {
+            'type': 'plain',
+            'format': '[%(asctime)s] %(levelname)s %(message)s',
+        }
+
+        test_message = 'test plain logging'
+
+        with patch('sys.stderr', StringIO()) as stderr_output:
+            logger = PatroniLogger()
+            logger.reload_config(config)
+
+            _LOG.info(test_message)
+            target_log = stderr_output.getvalue()
+
+        self.assertRegex(target_log, fr'^\[.*\] INFO {test_message}$')
+
+    def test_dateformat(self):
+        config = {
+            'format': '[%(asctime)s] %(message)s',
+            'dateformat': '%Y-%m-%dT%H:%M:%S'
+        }
+
+        test_message = 'test date format'
+
+        with patch('sys.stderr', StringIO()) as stderr_output:
+            logger = PatroniLogger()
+            logger.reload_config(config)
+
+            _LOG.info(test_message)
+            target_log = stderr_output.getvalue()
+
+        self.assertRegex(target_log, r'\[\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\]')
+
+    def test_invalid_dateformat(self):
+        config = {
+            'format': '[%(asctime)s] %(message)s',
+            'dateformat': 5
+        }
+
+        with self.assertLogs() as captured_log:
+            logger = PatroniLogger()
+            logger.reload_config(config)
+
+            captured_log_level = captured_log.records[0].levelname
+            captured_log_message = captured_log.records[0].message
+
+            self.assertEqual(captured_log_level, 'WARNING')
+            self.assertRegex(
+                captured_log_message,
+                fr'Expected log dateformat to be a string, but got "{type(config["dateformat"])}"'
+            )
+
+    def test_invalid_plain_format(self):
+        config = {
+            'type': 'plain',
+            'format': ['message']
+        }
+
+        with self.assertLogs() as captured_log:
+            logger = PatroniLogger()
+            logger.reload_config(config)
+
+            captured_log_level = captured_log.records[0].levelname
+            captured_log_message = captured_log.records[0].message
+
+            self.assertEqual(captured_log_level, 'WARNING')
+            self.assertRegex(
+                captured_log_message,
+                r'Expected log format to be a string when log type is plain, but got ".*"'
+            )
+
+    def test_invalid_json_format(self):
+        config = {
+            'type': 'json',
+            'format': {
+                'asctime': 'timestamp',
+                'message': 'message'
+            }
+        }
+
+        with self.assertLogs() as captured_log:
+            logger = PatroniLogger()
+            logger.reload_config(config)
+
+            captured_log_level = captured_log.records[0].levelname
+            captured_log_message = captured_log.records[0].message
+
+            self.assertEqual(captured_log_level, 'WARNING')
+            self.assertRegex(captured_log_message, r'Expected log format to be a string or a list, but got ".*"')
+
+        with self.assertLogs() as captured_log:
+            config['format'] = [['levelname']]
+            logger.reload_config(config)
+
+            captured_log_level = captured_log.records[0].levelname
+            captured_log_message = captured_log.records[0].message
+
+            self.assertEqual(captured_log_level, 'WARNING')
+            self.assertRegex(
+                captured_log_message,
+                r'Expected each item of log format to be a string or dictionary, but got ".*"'
+            )
+
+        with self.assertLogs() as captured_log:
+            config['format'] = ['message', {'asctime': ['timestamp']}]
+            logger.reload_config(config)
+
+            captured_log_level = captured_log.records[0].levelname
+            captured_log_message = captured_log.records[0].message
+
+            self.assertEqual(captured_log_level, 'WARNING')
+            self.assertRegex(captured_log_message, r'Expected renamed log field to be a string, but got ".*"')
+
+    def test_fail_to_use_python_json_logger(self):
+        with self.assertLogs() as captured_log:
+            logger = PatroniLogger()
+            with patch('builtins.__import__', Mock(side_effect=ImportError)):
+                logger.reload_config({'type': 'json'})
+
+            captured_log_level = captured_log.records[0].levelname
+            captured_log_message = captured_log.records[0].message
+
+            self.assertEqual(captured_log_level, 'ERROR')
+            self.assertRegex(
+                captured_log_message,
+                r'Failed to import "python-json-logger" library: .*. Falling back to the plain logger'
+            )
+
+        with self.assertLogs() as captured_log:
+            logger = PatroniLogger()
+            pythonjsonlogger = Mock()
+            pythonjsonlogger.jsonlogger.JsonFormatter = Mock(side_effect=Exception)
+            with patch('builtins.__import__', Mock(return_value=pythonjsonlogger)):
+                logger.reload_config({'type': 'json'})
+
+            captured_log_level = captured_log.records[0].levelname
+            captured_log_message = captured_log.records[0].message
+
+            self.assertEqual(captured_log_level, 'ERROR')
+            self.assertRegex(
+                captured_log_message,
+                r'Failed to initialize JsonFormatter: .*. Falling back to the plain logger'
+            )

tests/test_mpp.py

@@ -0,0 +1,52 @@
+from typing import Any
+from patroni.exceptions import PatroniException
+from patroni.postgresql.mpp import AbstractMPP, get_mpp, Null
+
+from . import BaseTestPostgresql
+from .test_ha import get_cluster_initialized_with_leader
+
+
+class TestMPP(BaseTestPostgresql):
+
+    def setUp(self):
+        super(TestMPP, self).setUp()
+        self.cluster = get_cluster_initialized_with_leader()
+
+    def test_get_handler_impl_exception(self):
+        class DummyMPP(AbstractMPP):
+            def __init__(self) -> None:
+                super().__init__({})
+
+            @staticmethod
+            def validate_config(config: Any) -> bool:
+                return True
+
+            @property
+            def group(self) -> None:
+                return None
+
+            @property
+            def coordinator_group_id(self) -> None:
+                return None
+
+            @property
+            def type(self) -> str:
+                return "dummy"
+
+        mpp = DummyMPP()
+        self.assertRaises(PatroniException, mpp.get_handler_impl, self.p)
+
+    def test_null_handler(self):
+        config = {}
+        mpp = get_mpp(config)
+        self.assertIsInstance(mpp, Null)
+        self.assertIsNone(mpp.group)
+        self.assertTrue(mpp.validate_config(config))
+        nullHandler = mpp.get_handler_impl(self.p)
+        self.assertIsNone(nullHandler.handle_event(self.cluster, {}))
+        self.assertIsNone(nullHandler.sync_meta_data(self.cluster))
+        self.assertIsNone(nullHandler.on_demote())
+        self.assertIsNone(nullHandler.schedule_cache_rebuild())
+        self.assertIsNone(nullHandler.bootstrap())
+        self.assertIsNone(nullHandler.adjust_postgres_gucs({}))
+        self.assertFalse(nullHandler.ignore_replication_slot({}))

tests/test_patroni.py

@@ -45,7 +45,7 @@ class MockFrozenImporter(object):
 @patch('time.sleep', Mock())
 @patch('subprocess.call', Mock(return_value=0))
 @patch('patroni.psycopg.connect', psycopg_connect)
-@patch('urllib3.connection.HTTPConnection.connect', Mock(side_effect=Exception))
+@patch('urllib3.PoolManager.request', Mock(side_effect=Exception))
 @patch.object(ConfigHandler, 'append_pg_hba', Mock())
 @patch.object(ConfigHandler, 'write_postgresql_conf', Mock())
 @patch.object(ConfigHandler, 'write_recovery_conf', Mock())
@@ -69,7 +69,7 @@ class TestPatroni(unittest.TestCase):
             self.assertRaises(SystemExit, _main)
 
     @patch('pkgutil.iter_importers', Mock(return_value=[MockFrozenImporter()]))
-    @patch('urllib3.connection.HTTPConnection.connect', Mock(side_effect=Exception))
+    @patch('urllib3.PoolManager.request', Mock(side_effect=Exception))
     @patch('sys.frozen', Mock(return_value=True), create=True)
     @patch.object(HTTPServer, '__init__', Mock())
     @patch.object(etcd.Client, 'read', etcd_read)
@@ -154,6 +154,7 @@ class TestPatroni(unittest.TestCase):
         self.p.api.start = Mock()
         self.p.logger.start = Mock()
         self.p.config._dynamic_configuration = {}
+        self.assertRaises(SleepException, self.p.run)
         with patch('patroni.dcs.Cluster.is_unlocked', Mock(return_value=True)):
             self.assertRaises(SleepException, self.p.run)
         with patch('patroni.config.Config.reload_local_configuration', Mock(return_value=False)):
@@ -174,6 +175,20 @@ class TestPatroni(unittest.TestCase):
         self.p.next_run = time.time() - self.p.dcs.loop_wait - 1
         self.p.schedule_next_run()
 
+    def test__filter_tags(self):
+        tags = {'noloadbalance': False, 'clonefrom': False, 'nosync': False, 'smth': 'random'}
+        self.assertEqual(self.p._filter_tags(tags), {'smth': 'random'})
+
+        tags['clonefrom'] = True
+        tags['smth'] = False
+        self.assertEqual(self.p._filter_tags(tags), {'clonefrom': True, 'smth': False})
+
+        tags = {'nofailover': False, 'failover_priority': 0}
+        self.assertEqual(self.p._filter_tags(tags), tags)
+
+        tags = {'nofailover': True, 'failover_priority': 1}
+        self.assertEqual(self.p._filter_tags(tags), tags)
+
     def test_noloadbalance(self):
         self.p.tags['noloadbalance'] = True
         self.assertTrue(self.p.noloadbalance)
@@ -185,9 +200,11 @@ class TestPatroni(unittest.TestCase):
             # Setting `nofailover: True` has precedence
             (True, 0, True),
             (True, 1, True),
+            ('False', 1, True),  # because we use bool() for the value
             # Similarly, setting `nofailover: False` has precedence
             (False, 0, False),
             (False, 1, False),
+            ('', 0, False),
             # Only when we have `nofailover: None` should we got based on priority
             (None, 0, True),
             (None, 1, False),
@@ -232,6 +249,16 @@ class TestPatroni(unittest.TestCase):
         self.p.tags['nosync'] = None
         self.assertFalse(self.p.nosync)
 
+    def test_nostream(self):
+        self.p.tags['nostream'] = 'True'
+        self.assertTrue(self.p.nostream)
+        self.p.tags['nostream'] = 'None'
+        self.assertFalse(self.p.nostream)
+        self.p.tags['nostream'] = 'foo'
+        self.assertFalse(self.p.nostream)
+        self.p.tags['nostream'] = ''
+        self.assertFalse(self.p.nostream)
+
     @patch.object(Thread, 'join', Mock())
     def test_shutdown(self):
         self.p.api.shutdown = Mock(side_effect=Exception)
@@ -273,8 +300,8 @@ class TestPatroni(unittest.TestCase):
         )
         with patch('patroni.dcs.AbstractDCS.get_cluster', Mock(return_value=bad_cluster)):
             # If the api of the running node cannot be reached, this implies unique name
-            with patch('urllib3.connection.HTTPConnection.connect', Mock(side_effect=ConnectionError)):
+            with patch('urllib3.PoolManager.request', Mock(side_effect=ConnectionError)):
                 self.assertIsNone(self.p.ensure_unique_name())
             # Only if the api of the running node is reachable do we throw an error
-            with patch('urllib3.connection.HTTPConnection.connect', Mock()):
+            with patch('urllib3.PoolManager.request', Mock()):
                 self.assertRaises(SystemExit, self.p.ensure_unique_name)